From c3e6876a8f21da8aec195bfa2d76d65121eca2da Mon Sep 17 00:00:00 2001 From: Massimiliano Fasi Date: Wed, 12 Aug 2015 10:55:35 +0100 Subject: [PATCH 0001/1117] Make the algorithm for real powers of a matrix robust --- base/linalg/dense.jl | 56 ++++++-- base/linalg/diagonal.jl | 10 +- base/linalg/symmetric.jl | 57 +++++++- base/linalg/triangular.jl | 295 ++++++++++++++++++++++++++++---------- test/linalg/dense.jl | 34 +++++ 5 files changed, 354 insertions(+), 98 deletions(-) diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index e62731e1b528e..ffae38ec2d57c 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -150,17 +150,50 @@ kron(a::AbstractVector, b::AbstractVector)=vec(kron(reshape(a,length(a),1),resha kron(a::AbstractMatrix, b::AbstractVector)=kron(a,reshape(b,length(b),1)) kron(a::AbstractVector, b::AbstractMatrix)=kron(reshape(a,length(a),1),b) -^(A::Matrix, p::Integer) = p < 0 ? inv(A^-p) : Base.power_by_squaring(A,p) - -function ^(A::Matrix, p::Number) +# Matrix power +^(A::AbstractMatrix, p::Integer) = p < 0 ? Base.power_by_squaring(inv(A),-p) : Base.power_by_squaring(A,p) +function ^{T}(A::AbstractMatrix{T}, p::Real) + # For integer powers, use repeated squaring if isinteger(p) return A^Integer(real(p)) end - checksquare(A) - v, X = eig(A) - any(v.<0) && (v = complex(v)) - Xinv = ishermitian(A) ? X' : inv(X) - (X * Diagonal(v.^p)) * Xinv + + # If possible, use diagonalization + if issymmetric(A) && T <: Real + return full(Symmetric(A)^p) + end + if ishermitian(A) + return full(Hermitian(A)^p) + end + + # Otherwise, use Schur decomposition + n = checksquare(A) + if istriu(A) + retmat = full(UpperTriangular(A)^p) + d = diag(A) + else + S,Q,d = schur(complex(A)) + R = UpperTriangular(S)^p + retmat = Q * R * Q' + end + + # Check whether the matrix has nonpositive real eigs + np_real_eigs = false + for i = 1:n + if imag(d[i]) < eps() && real(d[i]) <= 0 + np_real_eigs = true + break + end + end + if np_real_eigs + warn("Matrix with nonpositive real eigenvalues, a nonprincipal matrix power will be returned.") + end + + if isreal(A) && ~np_real_eigs + return real(retmat) + else + return retmat + end end # Matrix exponential @@ -264,7 +297,7 @@ function rcswap!{T<:Number}(i::Integer, j::Integer, X::StridedMatrix{T}) end """ - logm(A::StridedMatrix) + logm(A{T}::StridedMatrix{T}) If `A` has no negative real eigenvalue, compute the principal matrix logarithm of `A`, i.e. the unique matrix ``X`` such that ``e^X = A`` and ``-\\pi < Im(\\lambda) < \\pi`` for all @@ -282,8 +315,11 @@ triangular factor. [^AHR13]: Awad H. Al-Mohy, Nicholas J. Higham and Samuel D. Relton, "Computing the Fréchet derivative of the matrix logarithm and estimating the condition number", SIAM Journal on Scientific Computing, 35(4), 2013, C394-C410. [doi:10.1137/120885991](http://dx.doi.org/10.1137/120885991) """ -function logm(A::StridedMatrix) +function logm{T}(A::StridedMatrix{T}) # If possible, use diagonalization + if issymmetric(A) && T <: Real + return full(logm(Symmetric(A))) + end if ishermitian(A) return full(logm(Hermitian(A))) end diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 170a05e40deee..8fe5105c2b5ab 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -230,9 +230,13 @@ end # identity matrices via eye(Diagonal{type},n) eye{T}(::Type{Diagonal{T}}, n::Int) = Diagonal(ones(T,n)) -expm(D::Diagonal) = Diagonal(exp(D.diag)) -logm(D::Diagonal) = Diagonal(log(D.diag)) -sqrtm(D::Diagonal) = Diagonal(sqrt(D.diag)) +# Matrix functions +^(D::Diagonal, p::Real) = Diagonal((D.diag).^p) +for (funm, func) in ([:expm,:exp], [:sqrtm,:sqrt], [:logm,:log]) + @eval begin + ($funm)(D::Diagonal) = Diagonal(($func)(D.diag)) + end +end #Linear solver function A_ldiv_B!(D::Diagonal, B::StridedVecOrMat) diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index 9308eb4d654f5..e1aac4a916f8e 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -199,8 +199,55 @@ function svdvals!{T<:Real,S}(A::Union{Hermitian{T,S}, Symmetric{T,S}, Hermitian{ return sort!(vals, rev = true) end -#Matrix-valued functions -function expm(A::Symmetric) +#Matrix functions +function ^{T<:Real}(A::Symmetric{T}, p::Integer) + if p < 0 + return Symmetric(Base.power_by_squaring(inv(A), -p)) + else + return Symmetric(Base.power_by_squaring(A, p)) + end +end +function ^{T<:Real}(A::Symmetric{T}, p::Real) + F = eigfact(full(A)) + if isposdef(F) + retmat = (F.vectors * Diagonal((F.values).^p)) * F.vectors' + else + retmat = (F.vectors * Diagonal((complex(F.values)).^p)) * F.vectors' + end + return Symmetric(retmat) +end +function ^(A::Hermitian, p::Integer) + n = checksquare(A) + if p < 0 + retmat = Base.power_by_squaring(inv(A), -p) + else + retmat = Base.power_by_squaring(A, p) + end + for i = 1:n + retmat[i,i] = real(retmat[i,i]) + end + return Hermitian(retmat) +end +function ^{T}(A::Hermitian{T}, p::Real) + n = checksquare(A) + F = eigfact(A) + if isposdef(F) + retmat = (F.vectors * Diagonal((F.values).^p)) * F.vectors' + if T <: Real + return Hermitian(retmat) + else + for i = 1:n + retmat[i,i] = real(retmat[i,i]) + end + return Hermitian(retmat) + end + else + retmat = (F.vectors * Diagonal((complex(F.values).^p))) * F.vectors' + return retmat + end +end + +function expm{T<:Real}(A::Symmetric{T}) F = eigfact(A) return Symmetric((F.vectors * Diagonal(exp(F.values))) * F.vectors') end @@ -219,10 +266,8 @@ function expm{T}(A::Hermitian{T}) end for (funm, func) in ([:logm,:log], [:sqrtm,:sqrt]) - @eval begin - - function ($funm)(A::Symmetric) + function ($funm){T<:Real}(A::Symmetric{T}) F = eigfact(A) if isposdef(F) retmat = (F.vectors * Diagonal(($func)(F.values))) * F.vectors' @@ -250,7 +295,5 @@ for (funm, func) in ([:logm,:log], [:sqrtm,:sqrt]) return retmat end end - end - end diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 30110711cd6d7..9f66a23bd3ac1 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -1573,18 +1573,105 @@ At_mul_Bt(A::AbstractTriangular, B::AbstractTriangular) = At_mul_B(A, B.') At_mul_Bt(A::AbstractTriangular, B::AbstractMatrix) = At_mul_B(A, B.') At_mul_Bt(A::AbstractMatrix, B::AbstractTriangular) = A_mul_Bt(A.', B) +function ^(A::UpperTriangular, p::Integer) + if p < 0 + return UpperTriangular(Base.power_by_squaring(inv(A), -p)) + else + return UpperTriangular(Base.power_by_squaring(A, p)) + end +end +function ^(A::UpperTriangular, p::Real) + # For integer powers, use repeated squaring + if isinteger(p) + return A^Integer(real(p)) + end + + # General UpperTriangular matrices + p1 = p - floor(p) + if cond(A) >= exp(log(p1/(1 - p1)) / p1) + return powm(A,p1) * A^floor(Integer, p) + else + return powm(A,p - ceil(p)) * A^ceil(Integer, p) + end +end +# Complex matrix power for upper triangular factor, see: +# Higham and Lin, "A Schur-Padé algorithm for fractional powers of a Matrix", +# SIAM J. Matrix Anal. & Appl., 32 (3), (2011) 1056–1078. +# Higham and Lin, "An improved Schur-Padé algorithm for fractional powers of +# a matrix and their Fréchet derivatives", SIAM. J. Matrix Anal. & Appl., +# 34(3), (2013) 1341–1360. +function powm{T<:Union{Float32,Float64,Complex{Float32},Complex{Float64}}}(A0::UpperTriangular{T}, p::Real) + + if abs(p) >= 1 + ArgumentError("p must be a real number in (-1,1), got $p") + end + + theta = [1.53e-5, 2.25e-3, 1.92e-2, 6.08e-2, 1.25e-1, 2.03e-1, 2.84e-1] + n = checksquare(A0) + + A, m, s = invsquaring(A0,theta) + A = I - A + + # Compute accurate diagonal of I - T + sqrt_diag!(A0,A,s) + for i = 1:n + A[i,i] = -A[i,i] + end + # Compute the Padé approximant + c = 0.5 * (p - m) / (2 * m - 1) + triu!(A) + S = c * A + Stmp = similar(S) + for j = m-1:-1:1 + j4 = 4 * j + c = (-p - j) / (j4 + 2) + for i = 1:n + @inbounds S[i,i] = S[i,i] + 1 + end + copy!(Stmp,S) + scale!(S,A,c) + A_ldiv_B!(Stmp,S.data) + + c = (p - j) / (j4 - 2) + for i = 1:n + @inbounds S[i,i] = S[i,i] + 1 + end + copy!(Stmp,S) + scale!(S,A,c) + A_ldiv_B!(Stmp,S.data) + end + for i = 1:n + S[i,i] = S[i,i] + 1 + end + copy!(Stmp,S) + scale!(S,A,-p) + A_ldiv_B!(Stmp,S.data) + for i = 1:n + @inbounds S[i,i] = S[i,i] + 1 + end + + blockpower!(A0,S,p/(2^s)) + for m = 1:s + A_mul_B!(Stmp.data,S,S) + copy!(S,Stmp) + blockpower!(A0,S,p/(2^(s-m))) + end + return S +end +^(A::LowerTriangular, p::Integer) = ^(A.', p::Integer).' +powm(A::LowerTriangular, p::Real) = powm(A.', p::Real).' + # Complex matrix logarithm for the upper triangular factor, see: # Al-Mohy and Higham, "Improved inverse scaling and squaring algorithms for -# the matrix logarithm", SIAM J. Sci. Comput., 34(4), (2012), pp. C153-C169. +# the matrix logarithm", SIAM J. Sci. Comput., 34(4), (2012), pp. C153–C169. # Al-Mohy, Higham and Relton, "Computing the Frechet derivative of the matrix -# logarithm and estimating the condition number", SIAM J. Sci. Comput., 35(4), -# (2013), C394-C410. +# logarithm and estimating the condition number", SIAM J. Sci. Comput., +# 35(4), (2013), C394–C410. # # Based on the code available at http://eprints.ma.man.ac.uk/1851/02/logm.zip, # Copyright (c) 2011, Awad H. Al-Mohy and Nicholas J. Higham # Julia version relicensed with permission from original authors -function logm{T<:Union{Float64,Complex{Float64}}}(A0::UpperTriangular{T}) - maxsqrt = 100 +function logm{T<:Union{Float32,Float64,Complex{Float32},Complex{Float64}}}(A0::UpperTriangular{T}) theta = [1.586970738772063e-005, 2.313807884242979e-003, 1.938179313533253e-002, @@ -1592,15 +1679,110 @@ function logm{T<:Union{Float64,Complex{Float64}}}(A0::UpperTriangular{T}) 1.276404810806775e-001, 2.060962623452836e-001, 2.879093714241194e-001] + + n = checksquare(A0) + + A, m, s = invsquaring(A0,theta) + + # Compute accurate diagonal of A - I + sqrt_diag!(A0, A, s) + + # Compute the Gauss-Legendre quadrature formula + x, w = Base.QuadGK.gauss(Float64, m) + for i = 1:m + @inbounds x[i] += 0.5(x[i] + 1) + end + scale!(w, 0.5) + + # Compute the Padé approximation + Y = UpperTriangular(zeros(T, n, n)) + D = similar(A) + Dtmp = similar(A) + triu!(Dtmp) + for k = 1:m + scale!(D,A,x[k]) + for i = 1:n + @inbounds D[i,i] = D[i,i] + 1 + end + scale!(Dtmp,A,w[k]) + A_rdiv_B!(Dtmp.data,D) + for j = 1:n + for i = 1:j + @inbounds Y[i,j] = Y[i,j] + Dtmp[i,j] + end + end + end + + # Scale back + scale!(2^s,Y.data) + + # Compute accurate diagonal and superdiagonal of log(A) + for k = 1:n-1 + Ak = complex(A0[k,k]) + Akp1 = complex(A0[k+1,k+1]) + logAk = log(Ak) + logAkp1 = log(Akp1) + Y[k,k] = logAk + Y[k+1,k+1] = logAkp1 + if Ak == Akp1 + Y[k,k+1] = A0[k,k+1] / Ak + elseif 2 * abs(Ak) < abs(Akp1) || 2 * abs(Akp1) < abs(Ak) + Y[k,k+1] = A0[k,k+1] * (logAkp1 - logAk) / (Akp1 - Ak) + else + w = atanh((Akp1 - Ak)/(Akp1 + Ak) + im * pi * unw(logAkp1-logAk)) + Y[k,k+1] = 2 * A0[k,k+1] * w / (Akp1 - Ak) + end + end + if isreal(Y) + return UpperTriangular(real(Y)) + else + return UpperTriangular(Y) + end +end +logm(A::LowerTriangular) = logm(A.').' + +# Auxiliary functions for logm and matrix power + +# Compute accurate diagonal of A = A0^s - I +# Al-Mohy, "A more accurate Briggs method for the logarithm", +# Numer. Algorithms, 59, (2012), 393–402. +function sqrt_diag!(A0::UpperTriangular, A::UpperTriangular, s) + n = checksquare(A0) + for i = 1:n + a = complex(A0[i,i]) + if s == 0 + A[i,i] = a - 1 + else + s0 = s + if imag(a) >= 0 && real(a) <= 0 && a != 0 + a = sqrt(a) + s0 = s - 1 + end + z0 = a - 1 + a = sqrt(a) + r = 1 + a + for j = 1:s0-1 + a = sqrt(a) + r = r * (1 + a) + end + A[i,i] = z0 / r + end + end +end + +# Repeatedly compute the square roots of A so that in the end its +# eigenvalues are close enough to the positive real line +function invsquaring(A0::UpperTriangular, theta) + maxsqrt = 100 tmax = size(theta, 1) - n = size(A0, 1) - A = copy(A0) + n = checksquare(A0) + A = complex(copy(A0)) p = 0 m = 0 # Compute repeated roots - d = diag(A) - dm1 = Array{T}(n) + d = complex(diag(A)) + dm1 = similar(d, n) s = 0 for i = 1:n dm1[i] = d[i] - 1. @@ -1644,6 +1826,7 @@ function logm{T<:Union{Float64,Complex{Float64}}}(A0::UpperTriangular{T}) end if j <= 6 m = j + foundm = true break elseif alpha3 / 2 <= theta[5] && p < 2 more = true @@ -1663,12 +1846,14 @@ function logm{T<:Union{Float64,Complex{Float64}}}(A0::UpperTriangular{T}) break end end + foundm = true break end end if s == maxsqrt m = tmax + foundm = true break end A = sqrtm(A) @@ -1678,13 +1863,26 @@ function logm{T<:Union{Float64,Complex{Float64}}}(A0::UpperTriangular{T}) # Compute accurate superdiagonal of T p = 1 / 2^s - for k = 1:n-1 - Ak = A0[k,k] - Akp1 = A0[k+1,k+1] + A = complex(A) + blockpower!(A, A0, p) + return A,m,s + +end + +# Compute accurate diagonal and superdiagonal of A = A0^p +function blockpower!(A::UpperTriangular, A0::UpperTriangular, p) + n = checksquare(A0) + @inbounds for k = 1:n-1 + + Ak = complex(A0[k,k]) + Akp1 = complex(A0[k+1,k+1]) + Akp = Ak^p Akp1p = Akp1^p + A[k,k] = Akp A[k+1,k+1] = Akp1p + if Ak == Akp1 A[k,k+1] = p * A0[k,k+1] * Ak^(p-1) elseif 2 * abs(Ak) < abs(Akp1) || 2 * abs(Akp1) < abs(Ak) @@ -1692,77 +1890,18 @@ function logm{T<:Union{Float64,Complex{Float64}}}(A0::UpperTriangular{T}) else logAk = log(Ak) logAkp1 = log(Akp1) - w = atanh((Akp1 - Ak)/(Akp1 + Ak)) + im*pi*ceil((imag(logAkp1-logAk)-pi)/(2*pi)) - dd = 2 * exp(p*(logAk+logAkp1)/2) * sinh(p*w) / (Akp1 - Ak) + w = atanh((Akp1 - Ak)/(Akp1 + Ak)) + im * pi * unw(logAkp1-logAk) + dd = 2 * exp(p*(logAk+logAkp1)/2) * sinh(p*w) / (Akp1 - Ak); A[k,k+1] = A0[k,k+1] * dd end end +end - # Compute accurate diagonal of T - for i = 1:n - a = A0[i,i] - if s == 0 - r = a - 1 - end - s0 = s - if angle(a) >= pi / 2 - a = sqrt(a) - s0 = s - 1 - end - z0 = a - 1 - a = sqrt(a) - r = 1 + a - for j = 1:s0-1 - a = sqrt(a) - r = r * (1 + a) - end - A[i,i] = z0 / r - end - - # Compute the Gauss-Legendre quadrature formula - R = zeros(Float64, m, m) - for i = 1:m - 1 - R[i,i+1] = i / sqrt((2 * i)^2 - 1) - R[i+1,i] = R[i,i+1] - end - x,V = eig(R) - w = Array{Float64}(m) - for i = 1:m - x[i] = (x[i] + 1) / 2 - w[i] = V[1,i]^2 - end - - # Compute the Padé approximation - Y = zeros(T, n, n) - for k = 1:m - Y = Y + w[k] * (A / (x[k] * A + I)) - end - - # Scale back - scale!(2^s, Y) - - # Compute accurate diagonal and superdiagonal of log(T) - for k = 1:n-1 - Ak = A0[k,k] - Akp1 = A0[k+1,k+1] - logAk = log(Ak) - logAkp1 = log(Akp1) - Y[k,k] = logAk - Y[k+1,k+1] = logAkp1 - if Ak == Akp1 - Y[k,k+1] = A0[k,k+1] / Ak - elseif 2 * abs(Ak) < abs(Akp1) || 2 * abs(Akp1) < abs(Ak) - Y[k,k+1] = A0[k,k+1] * (logAkp1 - logAk) / (Akp1 - Ak) - else - w = atanh((Akp1 - Ak)/(Akp1 + Ak) + im*pi*(ceil((imag(logAkp1-logAk) - pi)/(2*pi)))) - Y[k,k+1] = 2 * A0[k,k+1] * w / (Akp1 - Ak) - end - end - - return UpperTriangular(Y) +# Unwinding number +unw(x::Real) = 0 +unw(x::Number) = ceil((imag(x) - pi) / (2 * pi)) -end -logm(A::LowerTriangular) = logm(A.').' +# End of auxiliary functions for logm and matrix power function sqrtm{T}(A::UpperTriangular{T}) n = checksquare(A) diff --git a/test/linalg/dense.jl b/test/linalg/dense.jl index eb69d862392ec..155cb5d3445f4 100644 --- a/test/linalg/dense.jl +++ b/test/linalg/dense.jl @@ -438,6 +438,40 @@ eA10 = [ 1.0+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0i 0.0+0.0im 0.0+0.0im 0.0+0.0im 1.0+0.0im] @test_approx_eq expm(A10) eA10 +for elty in (Float64, Complex{Float64}) + A11 = convert(Matrix{elty}, [1 2 3; 4 7 1; 2 1 4]) + + OLD_STDERR = STDERR + rd,wr = redirect_stderr() + + @test A11^(1/2) ≈ sqrtm(A11) + s = readline(rd) + @test contains(s, "WARNING: Matrix with nonpositive real eigenvalues, a nonprincipal matrix power will be returned.") + + @test A11^(-1/2) ≈ inv(sqrtm(A11)) + s = readline(rd) + @test contains(s, "WARNING: Matrix with nonpositive real eigenvalues, a nonprincipal matrix power will be returned.") + + @test A11^(3/4) ≈ sqrtm(A11) * sqrtm(sqrtm(A11)) + s = readline(rd) + @test contains(s, "WARNING: Matrix with nonpositive real eigenvalues, a nonprincipal matrix power will be returned.") + + @test A11^(-3/4) ≈ inv(A11) * sqrtm(sqrtm(A11)) + s = readline(rd) + @test contains(s, "WARNING: Matrix with nonpositive real eigenvalues, a nonprincipal matrix power will be returned.") + + @test A11^(17/8) ≈ A11^2 * sqrtm(sqrtm(sqrtm(A11))) + s = readline(rd) + @test contains(s, "WARNING: Matrix with nonpositive real eigenvalues, a nonprincipal matrix power will be returned.") + + @test A11^(-17/8) ≈ inv(A11^2 * sqrtm(sqrtm(sqrtm(A11)))) + s = readline(rd) + @test contains(s, "WARNING: Matrix with nonpositive real eigenvalues, a nonprincipal matrix power will be returned.") + + redirect_stderr(OLD_STDERR) +end + + # issue #7181 A = [ 1 5 9 2 6 10 From a81426e7150940f82f9b58e671e7dd117aebc664 Mon Sep 17 00:00:00 2001 From: goelakash Date: Wed, 24 Feb 2016 09:21:36 +0530 Subject: [PATCH 0002/1117] Added performace tip for parentheses in expressions to docs --- doc/manual/performance-tips.rst | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index f1932563ac9c8..b1b1bbfa353a2 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -947,6 +947,51 @@ Taken to its extreme, pre-allocation can make your code uglier, so performance measurements and some judgment may be required. +Use parentheses in arithmetic operations +---------------------------------------- + +If your code has a long arithmetic operation involving `+` or `*` operators, +then consider using parentheses to chain up to four or five operations together. +This is to avoid the splatting penalty from a longer list of arguments to function +return statement. + +Without parentheses: + + const k = zeros(20) + function test_mem() + return k[1] + k[2] + k[3] + k[4] + k[5] + 2.0 * k[6] + k[7] + k[8] + k[9] + k[10] + end + + function test(n::Int64) + ret = 0.0 + for i = 1:n + ret += test_mem() + end + ret + end + @time test(100000000) + + 5.017971 seconds (900.00 M allocations: 13.411 GB, 15.04% gc time) + +With: + + const k = zeros(20) + function test_mem() + return (k[1] + k[2] + k[3] + k[4] + k[5]) + 2.0 * k[6] + k[7] + k[8] + k[9] + k[10] + end + + function test(n::Int64) + ret = 0.0 + for i = 1:n + ret += test_mem() + end + ret + end + @time test(100000000) + + 0.302478 seconds (5.26 k allocations: 248.985 KB) + + Avoid string interpolation for I/O ---------------------------------- From ae379334ee5cfb661335a0d66aa2051f29d7ffc5 Mon Sep 17 00:00:00 2001 From: goelakash Date: Wed, 25 May 2016 17:59:06 +0530 Subject: [PATCH 0003/1117] Double backticks and topic title fix --- doc/manual/performance-tips.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index b1b1bbfa353a2..f93a3e4df7227 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -947,10 +947,10 @@ Taken to its extreme, pre-allocation can make your code uglier, so performance measurements and some judgment may be required. -Use parentheses in arithmetic operations ----------------------------------------- +Use parentheses in long arithmetic operations +--------------------------------------------- -If your code has a long arithmetic operation involving `+` or `*` operators, +If your code has a long arithmetic operation involving ``+`` or ``*`` operators, then consider using parentheses to chain up to four or five operations together. This is to avoid the splatting penalty from a longer list of arguments to function return statement. From d4ec061124cf66df990cf4e572bd7db064556c06 Mon Sep 17 00:00:00 2001 From: Micah Smith Date: Wed, 25 May 2016 18:55:29 -0400 Subject: [PATCH 0004/1117] Comment default file mode --- src/support/ios.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/support/ios.c b/src/support/ios.c index d44afd2bc26ae..f05c6128d2f6a 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -892,7 +892,10 @@ ios_t *ios_file(ios_t *s, const char *fname, int rd, int wr, int create, int tru fd = _wopen(fname_w, flags | O_BINARY | O_NOINHERIT, _S_IREAD | _S_IWRITE); set_io_wait_begin(0); #else - fd = open_cloexec(fname, flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH /* 0666 */); + // The mode of the created file is (mode & ~umask), which resolves with + // default umask to u=rw,g=r,o=r + fd = open_cloexec(fname, flags, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); #endif s = ios_fd(s, fd, 1, 1); if (fd == -1) From e76add77cac2a17ecc0f45fda4f9283e79c9806c Mon Sep 17 00:00:00 2001 From: James Gilbert Date: Mon, 30 May 2016 15:13:35 +0100 Subject: [PATCH 0005/1117] Escape elementwise operators in :obj: links --- doc/manual/mathematical-operations.rst | 4 ++-- doc/manual/noteworthy-differences.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index 2f6ae1409a132..3a2a0050254b4 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -294,11 +294,11 @@ work on arrays. For example, ``0 .< A .< 1`` gives a boolean array whose entries are true where the corresponding elements of ``A`` are between 0 and 1. -The operator :obj:`.<` is intended for array objects; the operation +The operator :obj:`.\< ` is intended for array objects; the operation ``A .< B`` is valid only if ``A`` and ``B`` have the same dimensions. The operator returns an array with boolean entries and with the same dimensions as ``A`` and ``B``. Such operators are called *elementwise*; Julia offers a -suite of elementwise operators: :obj:`.*`, :obj:`.+`, etc. Some of the elementwise +suite of elementwise operators: :obj:`.* `, :obj:`.+ `, etc. Some of the elementwise operators can take a scalar operand such as the example ``0 .< A .< 1`` in the preceding paragraph. This notation means that the scalar operand should be replicated for each entry of diff --git a/doc/manual/noteworthy-differences.rst b/doc/manual/noteworthy-differences.rst index 605a6f2dd4e1e..5bb3ebd89b92e 100644 --- a/doc/manual/noteworthy-differences.rst +++ b/doc/manual/noteworthy-differences.rst @@ -147,7 +147,7 @@ noteworthy differences: multiplication in Julia, equivalent to R's ``A %*% B``. In R, this same notation would perform an element-wise (Hadamard) product. To get the element-wise multiplication operation, you need to write ``A .* B`` in Julia. -- Julia performs matrix transposition using the :obj:`.'` operator and conjugated +- Julia performs matrix transposition using the :obj:`.' ` operator and conjugated transposition using the :obj:`'` operator. Julia's ``A.'`` is therefore equivalent to R's ``t(A)``. - Julia does not require parentheses when writing ``if`` statements or From 1390b776f9abe945599f9ca6d05e402661497bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Sun, 22 May 2016 14:48:45 +0200 Subject: [PATCH 0006/1117] Add printing of kwargs... if the function accept variable number of keywords. --- base/methodshow.jl | 8 +++++++- src/julia-syntax.scm | 3 ++- test/show.jl | 8 ++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/base/methodshow.jl b/base/methodshow.jl index 928c8b09fe70a..fd9aa090b1f0b 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -78,7 +78,13 @@ function kwarg_decl(sig::ANY, kwtype::DataType) kwli = ccall(:jl_methtable_lookup, Any, (Any, Any), kwtype.name.mt, sig) if kwli !== nothing kwli = kwli::Method - return filter(x->!('#' in string(x)), kwli.lambda_template.slotnames[kwli.lambda_template.nargs+1:end]) + kws = filter(x->!('#' in string(x)), kwli.lambda_template.slotnames[kwli.lambda_template.nargs+1:end]) + # ensure the kwarg... is always printed last. The order of the arguments are not + # necessarily the same as defined in the function + i = findfirst(x -> endswith(string(x), "..."), kws) + i==0 && return kws + push!(kws, kws[i]) + return deleteat!(kws,i) end return () end diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index e85005b2f110d..6fd5e7e038857 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -411,7 +411,8 @@ sparams)) (keyword-sparam-names (map (lambda (s) (if (symbol? s) s (cadr s))) keyword-sparams))) - (let ((kw (gensy)) (i (gensy)) (ii (gensy)) (elt (gensy)) (rkw (gensy)) + (let ((kw (gensy)) (i (gensy)) (ii (gensy)) (elt (gensy)) + (rkw (if (null? restkw) '() (symbol (string (car restkw) "...")))) (mangled (symbol (string "#" (if name (undot-name name) 'call) "#" (string (current-julia-module-counter))))) (flags (map (lambda (x) (gensy)) vals))) diff --git a/test/show.jl b/test/show.jl index ff9d1ec960d92..2c0ae4d479531 100644 --- a/test/show.jl +++ b/test/show.jl @@ -358,10 +358,14 @@ end f5971(x, y...; z=1, w...) = nothing let repr = sprint(io -> show(io,"text/plain", methods(f5971))) - @test contains(repr, "f5971(x, y...; z)") + @test contains(repr, "f5971(x, y...; z, w...)") end let repr = sprint(io -> show(io,"text/html", methods(f5971))) - @test contains(repr, "f5971(x, y...; z)") + @test contains(repr, "f5971(x, y...; z, w...)") +end +f16580(x, y...; z=1, w=y+x, q...) = nothing +let repr = sprint(io -> show(io,"text/html", methods(f16580))) + @test contains(repr, "f16580(x, y...; z, w, q...)") end if isempty(Base.GIT_VERSION_INFO.commit) From 8ffd4ea7c52804de36095ef2fcc65d62321a5383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Sun, 22 May 2016 16:13:04 +0200 Subject: [PATCH 0007/1117] Method completion with keywords. --- base/REPLCompletions.jl | 11 ++++++++--- test/replcompletions.jl | 8 ++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/base/REPLCompletions.jl b/base/REPLCompletions.jl index c056987ae25d9..207be534a0520 100644 --- a/base/REPLCompletions.jl +++ b/base/REPLCompletions.jl @@ -323,10 +323,15 @@ function complete_methods(ex_org::Expr) out = String[] t_in = Tuple{Core.Typeof(func), args_ex...} # Input types na = length(args_ex)+1 - for method in methods(func) + ml = methods(func) + kwtype = isdefined(ml.mt, :kwsorter) ? Nullable{DataType}(typeof(ml.mt.kwsorter)) : Nullable{DataType}() + io = IOBuffer() + for method in ml # Check if the method's type signature intersects the input types - typeintersect(Tuple{method.sig.parameters[1 : min(na, end)]...}, t_in) != Union{} && - push!(out,string(method)) + if typeintersect(Tuple{method.sig.parameters[1 : min(na, end)]...}, t_in) != Union{} + show(io, method, kwtype=kwtype) + push!(out, takebuf_string(io)) + end end return out end diff --git a/test/replcompletions.jl b/test/replcompletions.jl index f7d9d88531b7f..0bc56bb4cc11b 100644 --- a/test/replcompletions.jl +++ b/test/replcompletions.jl @@ -43,6 +43,8 @@ module CompletionFoo const a=x->x test6()=[a, a] + kwtest(; x=1, y=2, w...) = pass + array = [1, 1] varfloat = 0.1 @@ -325,6 +327,12 @@ c, r, res = test_complete(s) @test length(c) == 2 ################################################################# +s = "CompletionFoo.kwtest( " +c, r, res = test_complete(s) +@test !res +@test length(c) == 1 +@test contains(c[1], "x, y, w...") + # Test of inference based getfield completion s = "\"\"." c,r = test_complete(s) From 962cea799db39e82aee040b59e29124fc46cdee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Sun, 29 May 2016 12:44:43 +0200 Subject: [PATCH 0008/1117] Throw method error instead of throwing a generic error upon keyword mismatch --- base/error.jl | 2 +- src/julia-syntax.scm | 3 ++- test/keywordargs.jl | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/base/error.jl b/base/error.jl index 994dde48f23a1..70a771d754545 100644 --- a/base/error.jl +++ b/base/error.jl @@ -27,7 +27,7 @@ backtrace() = ccall(:jl_backtrace_from_here, Array{Ptr{Void},1}, (Int32,), false catch_backtrace() = ccall(:jl_get_backtrace, Array{Ptr{Void},1}, ()) ## keyword arg lowering generates calls to this ## -kwerr(kw) = error("unrecognized keyword argument \"", kw, "\"") +kwerr(kw, args...) = throw(MethodError(typeof(args[1]).name.mt.kwsorter, (kw,args...))) ## system error handling ## diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 6fd5e7e038857..c98074a4511e9 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -507,7 +507,8 @@ ,else))) (if (null? restkw) ;; if no rest kw, give error for unrecognized - `(call (top kwerr) ,elt) + `(call (top kwerr) ,kw ,@(map arg-name pargl),@(if (null? vararg) '() + (list `(... ,(arg-name (car vararg)))))) ;; otherwise add to rest keywords `(ccall 'jl_array_ptr_1d_push Void (tuple Any Any) ,rkw (tuple ,elt diff --git a/test/keywordargs.jl b/test/keywordargs.jl index ce2c979f051af..eff8844ec7fa4 100644 --- a/test/keywordargs.jl +++ b/test/keywordargs.jl @@ -9,7 +9,7 @@ kwf1(ones; tens=0, hundreds=0) = ones + 10*tens + 100*hundreds @test kwf1(3, tens=7, hundreds=2) == 273 @test_throws MethodError kwf1() # no method, too few args -@test_throws ErrorException kwf1(1, z=0) # unsupported keyword +@test_throws MethodError kwf1(1, z=0) # unsupported keyword @test_throws MethodError kwf1(1, 2) # no method, too many positional args # keyword args plus varargs @@ -18,7 +18,7 @@ kwf2(x, rest...; y=1) = (x, y, rest) @test isequal(kwf2(0,1,2), (0, 1, (1,2))) @test isequal(kwf2(0,1,2,y=88), (0, 88, (1,2))) @test isequal(kwf2(0,y=88,1,2), (0, 88, (1,2))) -@test_throws ErrorException kwf2(0, z=1) +@test_throws MethodError kwf2(0, z=1) @test_throws MethodError kwf2(y=1) # keyword arg with declared type @@ -191,7 +191,7 @@ let f = (x;a=1,b=2)->(x, a, b) @test f(0) === (0, 1, 2) @test f(1,a=10,b=20) === (1,10,20) @test f(0,b=88) === (0, 1, 88) - @test_throws ErrorException f(0,z=1) + @test_throws MethodError f(0,z=1) end @test ((a=2)->10a)(3) == 30 @test ((a=2)->10a)() == 20 From eb9ff74c71e8ab43e9f79884982e5d32e8028df2 Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Tue, 14 Jun 2016 10:31:21 +0530 Subject: [PATCH 0009/1117] implement copy and == for MersenneTwister (fix #15698) --- base/dSFMT.jl | 12 ++++++++++-- base/random.jl | 33 +++++++++++++++++++++++++++------ test/random.jl | 22 ++++++++++++++++++++++ 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/base/dSFMT.jl b/base/dSFMT.jl index b78b1b1931c7b..b0eb6e7d41089 100644 --- a/base/dSFMT.jl +++ b/base/dSFMT.jl @@ -2,6 +2,8 @@ module dSFMT +import Base: copy, copy!, == + export DSFMT_state, dsfmt_get_min_array_size, dsfmt_get_idstring, dsfmt_init_gen_rand, dsfmt_init_by_array, dsfmt_gv_init_by_array, dsfmt_fill_array_close_open!, dsfmt_fill_array_close1_open2!, @@ -21,10 +23,16 @@ const JPOLY1e21 = "e172e20c5d2de26b567c0cace9e7c6cc4407bd5ffcd22ca59d37b73d54fd type DSFMT_state val::Vector{Int32} - DSFMT_state() = new(Array{Int32}(JN32)) - DSFMT_state(val::Vector{Int32}) = new(val) + + DSFMT_state(val::Vector{Int32} = zeros(Int32, JN32)) = + new(length(val) == JN32 ? val : throw(DomainError())) end +copy!(dst::DSFMT_state, src::DSFMT_state) = (copy!(dst.val, src.val); dst) +copy(src::DSFMT_state) = DSFMT_state(copy(src.val)) + +==(s1::DSFMT_state, s2::DSFMT_state) = s1.val == s2.val + function dsfmt_get_idstring() idstring = ccall((:dsfmt_get_idstring,:libdSFMT), Ptr{UInt8}, diff --git a/base/random.jl b/base/random.jl index c14fd9a96b389..1267f58d53717 100644 --- a/base/random.jl +++ b/base/random.jl @@ -4,7 +4,7 @@ module Random using Base.dSFMT using Base.GMP: GMP_VERSION, Limb -import Base.copymutable +import Base: copymutable, copy, copy!, == export srand, rand, rand!, @@ -64,16 +64,37 @@ rand(rng::RandomDevice, ::Type{CloseOpen}) = rand(rng, Close1Open2) - 1.0 const MTCacheLength = dsfmt_get_min_array_size() type MersenneTwister <: AbstractRNG + seed::Vector{UInt32} state::DSFMT_state vals::Vector{Float64} idx::Int - seed::Vector{UInt32} - MersenneTwister(state::DSFMT_state, seed) = new(state, Array{Float64}(MTCacheLength), MTCacheLength, seed) - MersenneTwister(seed) = srand(new(DSFMT_state(), Array{Float64}(MTCacheLength)), seed) - MersenneTwister() = MersenneTwister(0) + function MersenneTwister(seed, state, vals, idx) + length(vals) == MTCacheLength && 0 <= idx <= MTCacheLength || throw(DomainError()) + new(seed, state, vals, idx) + end +end + +MersenneTwister(seed::Vector{UInt32}, state::DSFMT_state) = + MersenneTwister(seed, state, zeros(Float64, MTCacheLength), MTCacheLength) + +MersenneTwister(seed=0) = srand(MersenneTwister(Vector{UInt32}(), DSFMT_state()), seed) + +function copy!(dst::MersenneTwister, src::MersenneTwister) + copy!(resize!(dst.seed, length(src.seed)), src.seed) + copy!(dst.state, src.state) + copy!(dst.vals, src.vals) + dst.idx = src.idx + dst end +copy(src::MersenneTwister) = + MersenneTwister(copy(src.seed), copy(src.state), copy(src.vals), src.idx) + +==(r1::MersenneTwister, r2::MersenneTwister) = + r1.seed == r2.seed && r1.state == r2.state && isequal(r1.vals, r2.vals) && r1.idx == r2.idx + + ## Low level API for MersenneTwister @inline mt_avail(r::MersenneTwister) = MTCacheLength - r.idx @@ -117,7 +138,7 @@ function randjump(mt::MersenneTwister, jumps::Integer, jumppoly::AbstractString) push!(mts, mt) for i in 1:jumps-1 cmt = mts[end] - push!(mts, MersenneTwister(dSFMT.dsfmt_jump(cmt.state, jumppoly), cmt.seed)) + push!(mts, MersenneTwister(cmt.seed, dSFMT.dsfmt_jump(cmt.state, jumppoly))) end return mts end diff --git a/test/random.jl b/test/random.jl index 4fccc2f918323..a50fd03c5a9a3 100644 --- a/test/random.jl +++ b/test/random.jl @@ -414,3 +414,25 @@ end # test that the following is not an error (#16925) srand(typemax(UInt)) srand(typemax(UInt128)) + +# copy and == +let seed = rand(UInt32, 10) + r = MersenneTwister(seed) + @test r == MersenneTwister(seed) # r.vals should be all zeros + s = copy(r) + @test s == r && s !== r + skip, len = rand(0:2000, 2) + for j=1:skip + rand(r) + rand(s) + end + @test rand(r, len) == rand(s, len) + @test s == r +end + +# MersenneTwister initialization with invalid values +@test_throws DomainError Base.dSFMT.DSFMT_state(zeros(Int32, rand(0:Base.dSFMT.JN32-1))) +@test_throws DomainError MersenneTwister(zeros(UInt32, 1), Base.dSFMT.DSFMT_state(), + zeros(Float64, 10), 0) +@test_throws DomainError MersenneTwister(zeros(UInt32, 1), Base.dSFMT.DSFMT_state(), + zeros(Float64, Base.Random.MTCacheLength), -1) From 665ba4186e96e1c73ed01fad57168926fa4990df Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Tue, 14 Jun 2016 12:02:37 +0530 Subject: [PATCH 0010/1117] make srand(::MersenneTwister, seed::Vector{UInt32}) copy the seed --- base/random.jl | 2 +- test/random.jl | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/base/random.jl b/base/random.jl index 1267f58d53717..b8e8c90d0a220 100644 --- a/base/random.jl +++ b/base/random.jl @@ -126,7 +126,7 @@ end @inline rand_ui2x52_raw(r::MersenneTwister) = rand_ui52_raw(r) % UInt128 << 64 | rand_ui52_raw(r) function srand(r::MersenneTwister, seed::Vector{UInt32}) - r.seed = seed + copy!(resize!(r.seed, length(seed)), seed) dsfmt_init_by_array(r.state, r.seed) mt_setempty!(r) return r diff --git a/test/random.jl b/test/random.jl index a50fd03c5a9a3..1e824f98e9905 100644 --- a/test/random.jl +++ b/test/random.jl @@ -436,3 +436,11 @@ end zeros(Float64, 10), 0) @test_throws DomainError MersenneTwister(zeros(UInt32, 1), Base.dSFMT.DSFMT_state(), zeros(Float64, Base.Random.MTCacheLength), -1) + +# seed is private to MersenneTwister +let seed = rand(UInt32, 10) + r = MersenneTwister(seed) + @test r.seed == seed && r.seed !== seed + resize!(seed, 4) + @test r.seed != seed +end From 4a1db54e2153a39b8cb61f287db30d417ace2fdb Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Mon, 13 Jun 2016 11:12:39 +0200 Subject: [PATCH 0011/1117] Fix get(x::Nullable{Union{}}, y) and add tests for Nullable{Union{}} It failed since conversion of y to Union{} wasn't possible. Stop converting y for consistency with other get() methods. Add tests for Nullable() everywhere the behavior of a null is tested. --- base/docs/helpdb/Base.jl | 16 ---------------- base/nullable.jl | 18 ++++++++++++------ doc/stdlib/base.rst | 10 ++-------- test/nullable.jl | 24 ++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index d73f689a60a17..521a35f90b09e 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5432,22 +5432,6 @@ Compute the cotangent of `x`, where `x` is in radians. """ cot -""" - get(x) - -Attempt to access the value of the `Nullable` object, `x`. Returns the value if it is -present; otherwise, throws a `NullException`. -""" -get(x) - -""" - get(x, y) - -Attempt to access the value of the `Nullable{T}` object, `x`. Returns -the value if it is present; otherwise, returns `convert(T, y)`. -""" -get(x,y) - """ get(collection, key, default) diff --git a/base/nullable.jl b/base/nullable.jl index 310d0a5473731..df1228b60e65e 100644 --- a/base/nullable.jl +++ b/base/nullable.jl @@ -45,16 +45,22 @@ function show{T}(io::IO, x::Nullable{T}) end end -get(x::Nullable) = x.isnull ? throw(NullException()) : x.value - -@inline function get{T}(x::Nullable{T}, y) - if isbits(T) - ifelse(x.isnull, convert(T, y), x.value) +""" + get(x::Nullable[, y]) + +Attempt to access the value of `x`. Returns the value if it is present; +otherwise, returns `y` if provided, or throws a `NullException` if not. +""" +@inline function get{S,T}(x::Nullable{S}, y::T) + if isbits(S) + ifelse(x.isnull, y, x.value) else - x.isnull ? convert(T, y) : x.value + x.isnull ? y : x.value end end +get(x::Nullable) = x.isnull ? throw(NullException()) : x.value + isnull(x::Nullable) = x.isnull function isequal(x::Nullable, y::Nullable) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 125b378847b04..7e3e92c70930f 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -728,17 +728,11 @@ Nullables Wrap value ``x`` in an object of type ``Nullable``\ , which indicates whether a value is present. ``Nullable(x)`` yields a non-empty wrapper, and ``Nullable{T}()`` yields an empty instance of a wrapper that might contain a value of type ``T``\ . -.. function:: get(x) +.. function:: get(x::Nullable[, y]) .. Docstring generated from Julia source - Attempt to access the value of the ``Nullable`` object, ``x``\ . Returns the value if it is present; otherwise, throws a ``NullException``\ . - -.. function:: get(x, y) - - .. Docstring generated from Julia source - - Attempt to access the value of the ``Nullable{T}`` object, ``x``\ . Returns the value if it is present; otherwise, returns ``convert(T, y)``\ . + Attempt to access the value of ``x``\ . Returns the value if it is present; otherwise, returns ``y`` if provided, or throws a ``NullException`` if not. .. function:: isnull(x) diff --git a/test/nullable.jl b/test/nullable.jl index 3d4d0b3a11ac2..7ef888f2d87ed 100644 --- a/test/nullable.jl +++ b/test/nullable.jl @@ -144,12 +144,17 @@ for T in types @test get(x3) === one(T) end +@test_throws NullException get(Nullable()) + # get{S, T}(x::Nullable{S}, y::T) for T in types + x0 = Nullable() x1 = Nullable{T}() x2 = Nullable(zero(T)) x3 = Nullable(one(T)) + @test get(x0, zero(T)) === zero(T) + @test get(x0, one(T)) === one(T) @test get(x1, zero(T)) === zero(T) @test get(x1, one(T)) === one(T) @test get(x2, one(T)) === zero(T) @@ -167,13 +172,21 @@ for T in types @test isnull(x3) === false end +@test isnull(Nullable()) + # function isequal{S, T}(x::Nullable{S}, y::Nullable{T}) for T in types + x0 = Nullable() x1 = Nullable{T}() x2 = Nullable{T}() x3 = Nullable(zero(T)) x4 = Nullable(one(T)) + @test isequal(x0, x1) === true + @test isequal(x0, x2) === true + @test isequal(x0, x3) === false + @test isequal(x0, x4) === false + @test isequal(x1, x1) === true @test isequal(x1, x2) === true @test isequal(x1, x3) === false @@ -197,11 +210,17 @@ end # function =={S, T}(x::Nullable{S}, y::Nullable{T}) for T in types + x0 = Nullable() x1 = Nullable{T}() x2 = Nullable{T}() x3 = Nullable(zero(T)) x4 = Nullable(one(T)) + @test_throws NullException (x0 == x1) + @test_throws NullException (x0 == x2) + @test_throws NullException (x0 == x3) + @test_throws NullException (x0 == x4) + @test_throws NullException (x1 == x1) @test_throws NullException (x1 == x2) @test_throws NullException (x1 == x3) @@ -225,16 +244,21 @@ end # function hash(x::Nullable, h::UInt) for T in types + x0 = Nullable() x1 = Nullable{T}() x2 = Nullable{T}() x3 = Nullable(zero(T)) x4 = Nullable(one(T)) + @test isa(hash(x0), UInt) @test isa(hash(x1), UInt) @test isa(hash(x2), UInt) @test isa(hash(x3), UInt) @test isa(hash(x4), UInt) + @test hash(x0) == hash(x2) + @test hash(x0) != hash(x3) + @test hash(x0) != hash(x4) @test hash(x1) == hash(x2) @test hash(x1) != hash(x3) @test hash(x1) != hash(x4) From d134c5bccc174852862477b9d9917de7c037be47 Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Wed, 15 Jun 2016 09:52:20 -0700 Subject: [PATCH 0012/1117] Always display header when `show`/`print`ing empty sparse matrices. --- base/sparse/sparsematrix.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 83d209c3cdc37..eff407d95293a 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -77,8 +77,8 @@ convenient iterating over a sparse matrix : nzrange(S::SparseMatrixCSC, col::Integer) = S.colptr[col]:(S.colptr[col+1]-1) function Base.show(io::IO, S::SparseMatrixCSC) - if get(io, :multiline, false) - print(io, S.m, "×", S.n, " sparse matrix with ", nnz(S), " ", eltype(S), " nonzero entries:") + if get(io, :multiline, false) || (nnz(S) == 0) + print(io, S.m, "×", S.n, " sparse matrix with ", nnz(S), " ", eltype(S), " nonzero entries", nnz(S) == 0 ? "" : ":") end limit::Bool = get(io, :limit, false) From 88059d2e04c4097d7835730ffe890c34ad563051 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 15 Jun 2016 15:18:29 -0400 Subject: [PATCH 0013/1117] don't generate debug info for compiler temp variables --- src/codegen.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 4d18e27392fa3..6da83a242f47a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4373,7 +4373,7 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func for(i=0; i < vinfoslen; i++) { jl_sym_t *s = (jl_sym_t*)jl_array_ptr_ref(lam->slotnames,i); jl_varinfo_t &varinfo = ctx.slots[i]; - if (varinfo.isArgument) + if (varinfo.isArgument || s == compiler_temp_sym || s == unused_sym) continue; #ifdef LLVM38 varinfo.dinfo = ctx.dbuilder->createAutoVariable( @@ -4469,7 +4469,7 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func if (!varinfo.isArgument) { // otherwise, just leave it in the input register Value *lv = alloc_local(i, &ctx); (void)lv; #ifdef LLVM36 - if (ctx.debug_enabled) { + if (ctx.debug_enabled && varinfo.dinfo) { assert((Metadata*)varinfo.dinfo->getType() != jl_pvalue_dillvmt); ctx.dbuilder->insertDeclare(lv, varinfo.dinfo, ctx.dbuilder->createExpression(), #ifdef LLVM37 @@ -4489,7 +4489,7 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func AllocaInst *av = new AllocaInst(T_pjlvalue, jl_symbol_name(s), /*InsertBefore*/ctx.ptlsStates); varinfo.memloc = av; #ifdef LLVM36 - if (ctx.debug_enabled) { + if (ctx.debug_enabled && varinfo.dinfo) { DIExpression *expr; if ((Metadata*)varinfo.dinfo->getType() == jl_pvalue_dillvmt) { expr = ctx.dbuilder->createExpression(); From ac6efba158e575464e82cfd00e558636ff778c05 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Thu, 9 Jun 2016 16:41:38 -0400 Subject: [PATCH 0014/1117] deprecate 3-argument convert to String --- base/deprecated.jl | 25 +++++++++++++++++++++++++ base/strings/string.jl | 22 ---------------------- test/strings/basic.jl | 8 -------- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index ded568ac7785b..03026cbc47a16 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -463,6 +463,31 @@ end @deprecate String(p::Union{Ptr{Int8},Ptr{UInt8}}) unsafe_string(p) @deprecate String(p::Union{Ptr{Int8},Ptr{UInt8}}, len::Integer) unsafe_string(p,len) +@deprecate( + convert(::Type{String}, a::Vector{UInt8}, invalids_as::AbstractString), + let a = a, invalids_as = invalids_as + l = length(a) + idx = 1 + iscopy = false + while idx <= l + if !is_valid_continuation(a[idx]) + nextidx = idx+1+utf8_trailing[a[idx]+1] + (nextidx <= (l+1)) && (idx = nextidx; continue) + end + !iscopy && (a = copy(a); iscopy = true) + endn = idx + while endn <= l + !is_valid_continuation(a[endn]) && break + endn += 1 + end + (endn > idx) && (endn -= 1) + splice!(a, idx:endn, invalids_as.data) + l = length(a) + end + String(a) + end +) + @deprecate ==(x::Char, y::Integer) UInt32(x) == y @deprecate ==(x::Integer, y::Char) x == UInt32(y) @deprecate isless(x::Char, y::Integer) UInt32(x) < y diff --git a/base/strings/string.jl b/base/strings/string.jl index da4ae45fb3e25..d9051d8cbc5a1 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -299,28 +299,6 @@ function convert(::Type{String}, dat::Vector{UInt8}) String(buf) end -function convert(::Type{String}, a::Vector{UInt8}, invalids_as::AbstractString) - l = length(a) - idx = 1 - iscopy = false - while idx <= l - if !is_valid_continuation(a[idx]) - nextidx = idx+1+utf8_trailing[a[idx]+1] - (nextidx <= (l+1)) && (idx = nextidx; continue) - end - !iscopy && (a = copy(a); iscopy = true) - endn = idx - while endn <= l - !is_valid_continuation(a[endn]) && break - endn += 1 - end - (endn > idx) && (endn -= 1) - splice!(a, idx:endn, invalids_as.data) - l = length(a) - end - String(a) -end - """ Converts an already validated vector of `UInt16` or `UInt32` to a `String` diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 789ce31c89150..2a8edc8821516 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -154,14 +154,6 @@ end @test lcfirst("")=="" @test lcfirst("*")=="*" -#more String tests -@test convert(String, UInt8[32,107,75], "*") == " kK" -@test convert(String, UInt8[132,107,75], "*") == "*kK" -@test convert(String, UInt8[32,107,75], "αβ") == " kK" -@test convert(String, UInt8[132,107,75], "αβ") == "αβkK" -@test convert(String, UInt8[], "*") == "" -@test convert(String, UInt8[255], "αβ") == "αβ" - # test AbstractString functions at beginning of string.jl immutable tstStringType <: AbstractString data::Array{UInt8,1} From 9f81520f2ce8acd4317428491ef63c0639914471 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Wed, 15 Jun 2016 15:36:46 -0400 Subject: [PATCH 0015/1117] move base/unicode/UnicodeErrors.jl to base/strings/errors.jl the base/unicode directory will go away soon --- base/{unicode/UnicodeError.jl => strings/errors.jl} | 0 base/strings/strings.jl | 1 + base/unicode/unicode.jl | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) rename base/{unicode/UnicodeError.jl => strings/errors.jl} (100%) diff --git a/base/unicode/UnicodeError.jl b/base/strings/errors.jl similarity index 100% rename from base/unicode/UnicodeError.jl rename to base/strings/errors.jl diff --git a/base/strings/strings.jl b/base/strings/strings.jl index 7ecc1b516062b..c6d7834640699 100644 --- a/base/strings/strings.jl +++ b/base/strings/strings.jl @@ -1,5 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +include("strings/errors.jl") include("strings/string.jl") include("strings/types.jl") include("strings/basic.jl") diff --git a/base/unicode/unicode.jl b/base/unicode/unicode.jl index 0a2fd9b230fda..a8bd91e1a5a81 100644 --- a/base/unicode/unicode.jl +++ b/base/unicode/unicode.jl @@ -1,6 +1,5 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -include("unicode/UnicodeError.jl") include("unicode/types.jl") include("unicode/checkstring.jl") include("unicode/utf16.jl") From 0209ed1e8861053ca70b5818f02eab014806ab3e Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Wed, 15 Jun 2016 16:32:51 -0400 Subject: [PATCH 0016/1117] change some tests to GenericString instead of Unicode strings --- test/random.jl | 4 ++-- test/regex.jl | 4 ++-- test/strings/basic.jl | 30 ++++++++++++++---------------- test/strings/io.jl | 1 - test/unicode/utf8.jl | 10 +++++++++- test/unicode/utf8proc.jl | 10 +++++++--- test/version.jl | 6 +++--- 7 files changed, 37 insertions(+), 28 deletions(-) diff --git a/test/random.jl b/test/random.jl index 1e824f98e9905..91358626f5601 100644 --- a/test/random.jl +++ b/test/random.jl @@ -216,8 +216,8 @@ u1 = uuid1() u4 = uuid4() @test uuid_version(u1) == 1 @test uuid_version(u4) == 4 -@test u1 == UUID(string(u1)) == UUID(utf16(string(u1))) == UUID(utf32(string(u1))) -@test u4 == UUID(string(u4)) == UUID(utf16(string(u4))) == UUID(utf32(string(u4))) +@test u1 == UUID(string(u1)) == UUID(GenericString(string(u1))) +@test u4 == UUID(string(u4)) == UUID(GenericString(string(u4))) @test u1 == UUID(UInt128(u1)) @test u4 == UUID(UInt128(u4)) @test uuid4(MersenneTwister()) == uuid4(MersenneTwister()) diff --git a/test/regex.jl b/test/regex.jl index be153a142cd82..636c25da954a7 100644 --- a/test/regex.jl +++ b/test/regex.jl @@ -35,8 +35,8 @@ show(buf, r"") @test ismatch(Regex("^a\0b\$"), "a\0b") # regex match / search string must be a String -@test_throws ArgumentError match(r"test", utf32("this is a test")) -@test_throws ArgumentError search(utf32("this is a test"), r"test") +@test_throws ArgumentError match(r"test", GenericString("this is a test")) +@test_throws ArgumentError search(GenericString("this is a test"), r"test") # Named subpatterns let m = match(r"(?.)(.)(?.)", "xyz") diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 2a8edc8821516..b88c39f67dfa9 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -37,7 +37,7 @@ str = "s\u2200" @test sizeof("\u2222") == 3 # issue #3597 -@test string(utf32(['T', 'e', 's', 't'])[1:1], "X") == "TX" +@test string(GenericString("Test")[1:1], "X") == "TX" for T = (UInt8,Int8,UInt16,Int16,UInt32,Int32,UInt64,Int64,UInt128,Int128,BigInt), b = 2:62, _ = 1:10 @@ -449,21 +449,19 @@ end @test ucfirst("abc") == "Abc" @test lcfirst("ABC") == "aBC" @test lcfirst("aBC") == "aBC" -@test ucfirst(utf32("")) == "" -@test lcfirst(utf32("")) == "" -@test ucfirst(utf32("a")) == "A" -@test lcfirst(utf32("A")) == "a" -@test lcfirst(utf32("a")) == "a" -@test ucfirst(utf32("A")) == "A" - -# issue # 11464: uppercase/lowercase of UTF16String becomes a String +@test ucfirst(GenericString("")) == "" +@test lcfirst(GenericString("")) == "" +@test ucfirst(GenericString("a")) == "A" +@test lcfirst(GenericString("A")) == "a" +@test lcfirst(GenericString("a")) == "a" +@test ucfirst(GenericString("A")) == "A" + +# issue # 11464: uppercase/lowercase of GenericString becomes a String str = "abcdef\uff\uffff\u10ffffABCDEF" @test typeof(uppercase("abcdef")) == String -@test typeof(uppercase(utf16(str))) == UTF16String -@test typeof(uppercase(utf32(str))) == UTF32String +@test typeof(uppercase(GenericString(str))) == String @test typeof(lowercase("ABCDEF")) == String -@test typeof(lowercase(utf16(str))) == UTF16String -@test typeof(lowercase(utf32(str))) == UTF32String +@test typeof(lowercase(GenericString(str))) == String foomap(ch) = (ch > Char(65)) foobar(ch) = Char(0xd800) @@ -483,7 +481,7 @@ foobaz(ch) = reinterpret(Char, typemax(UInt32)) # ascii works on ASCII strings and fails on non-ASCII strings @test ascii("Hello, world") == "Hello, world" @test typeof(ascii("Hello, world")) == String -@test ascii(utf32("Hello, world")) == "Hello, world" -@test typeof(ascii(utf32("Hello, world"))) == String +@test ascii(GenericString("Hello, world")) == "Hello, world" +@test typeof(ascii(GenericString("Hello, world"))) == String @test_throws ArgumentError ascii("Hello, ∀") -@test_throws ArgumentError ascii(utf32("Hello, ∀")) +@test_throws ArgumentError ascii(GenericString("Hello, ∀")) diff --git a/test/strings/io.jl b/test/strings/io.jl index a07662d1098f2..41df929335f39 100644 --- a/test/strings/io.jl +++ b/test/strings/io.jl @@ -253,4 +253,3 @@ join(myio, "", "", 1) @test Base.unindent("\n \tfoo",4) == "\n foo" @test Base.unindent("\n\t\n \tfoo",4) == "\n \n foo" @test Base.unindent("\n\tfoo\tbar",4) == "\n foo bar" - diff --git a/test/unicode/utf8.jl b/test/unicode/utf8.jl index c3037a7d624df..8827642ba9c3a 100644 --- a/test/unicode/utf8.jl +++ b/test/unicode/utf8.jl @@ -27,7 +27,15 @@ end @test reverse("a") == "a" @test reverse("abc") == "cba" @test reverse("xyz\uff\u800\uffff\U10ffff") == "\U10ffff\uffff\u800\uffzyx" -for str in (b"xyz\xc1", b"xyz\xd0", b"xyz\xe0", b"xyz\xed\x80", b"xyz\xf0", b"xyz\xf0\x80", b"xyz\xf0\x80\x80") +for str in [ + b"xyz\xc1", + b"xyz\xd0", + b"xyz\xe0", + b"xyz\xed\x80", + b"xyz\xf0", + b"xyz\xf0\x80", + b"xyz\xf0\x80\x80" +] @test_throws UnicodeError reverse(String(str)) end diff --git a/test/unicode/utf8proc.jl b/test/unicode/utf8proc.jl index 2a829a5717d55..1ecd4b35f5a80 100644 --- a/test/unicode/utf8proc.jl +++ b/test/unicode/utf8proc.jl @@ -256,7 +256,11 @@ let grphtest = (("b\u0300lahβlahb\u0302láh", ["b\u0300","l","a","h", end # up-to-date character widths (#3721, #6939) -@test charwidth('\U1f355') == strwidth("\U1f355") == strwidth(utf16("\U1f355")) == strwidth("\U1f355\u0302") == strwidth(utf16("\U1f355\u0302")) == 2 +@test charwidth('\U1f355') == 2 +@test strwidth("\U1f355") == 2 +@test strwidth(GenericString("\U1f355")) == 2 +@test strwidth("\U1f355\u0302") == 2 +@test strwidth(GenericString("\U1f355\u0302")) == 2 # handling of embedded NUL chars (#10958) @test length("\0w") == length("\0α") == 2 @@ -264,7 +268,7 @@ end @test normalize_string("\0W", casefold=true) == "\0w" # Make sure AbstractString case is covered (for utf8proc_map) -@test normalize_string(utf32("\u006e\u0303"), :NFC) == "\u00f1" +@test normalize_string(GenericString("\u006e\u0303"), :NFC) == "\u00f1" @test_throws ArgumentError normalize_string("\u006e\u0303", compose=false, compat=true) @test_throws ArgumentError normalize_string("\u006e\u0303", compose=false, stripmark=true) @@ -289,7 +293,7 @@ let str = ascii("This is a test") g = graphemes(str) h = hash(str) @test hash(g) == h - @test convert(UTF16String, g) == str + @test convert(GenericString, g) == str io = IOBuffer() show(io, g) check = "length-14 GraphemeIterator{String} for \"$str\"" diff --git a/test/version.jl b/test/version.jl index 3dbf92389e65a..8f0bb9ec73a09 100644 --- a/test/version.jl +++ b/test/version.jl @@ -237,6 +237,6 @@ io = IOBuffer() @test VersionNumber(true, 0x2) == v"1.2" @test VersionNumber(true, 0x2, Int128(3)) == v"1.2.3" @test VersionNumber(true, 0x2, Int128(3)) == v"1.2.3" -@test VersionNumber(true, 0x2, Int128(3), (utf16("rc"), 0x1)) == v"1.2.3-rc.1" -@test VersionNumber(true, 0x2, Int128(3), (utf16("rc"), 0x1)) == v"1.2.3-rc.1" -@test VersionNumber(true, 0x2, Int128(3), (), (utf16("sp"), 0x2)) == v"1.2.3+sp.2" +@test VersionNumber(true, 0x2, Int128(3), (GenericString("rc"), 0x1)) == v"1.2.3-rc.1" +@test VersionNumber(true, 0x2, Int128(3), (GenericString("rc"), 0x1)) == v"1.2.3-rc.1" +@test VersionNumber(true, 0x2, Int128(3), (), (GenericString("sp"), 0x2)) == v"1.2.3+sp.2" From 9bbfae25e0a9c4876d6e763956ffdc1c5545af9c Mon Sep 17 00:00:00 2001 From: kshyatt Date: Wed, 15 Jun 2016 16:38:09 -0700 Subject: [PATCH 0017/1117] Add example for `eigs` to match `svds` --- base/linalg/arnoldi.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index 155ef14be7686..81f34ac4267d8 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -132,7 +132,14 @@ The following keyword arguments are supported: iterations `niter` and the number of matrix vector multiplications `nmult`, as well as the final residual vector `resid`. -**note** +**Example** + +```julia +X = sprand(10, 5, 0.2) +eigs(X, nsv = 2, tol = 1e-3) +``` + +**Note** The `sigma` and `which` keywords interact: the description of eigenvalues searched for by `which` do _not_ necessarily refer to the eigenvalue problem ``Av = Bv\\lambda``, but rather From 26dccf1c7ffa6772c6d57a6c8cc8dcd48278692b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jun 2016 19:37:20 -0500 Subject: [PATCH 0018/1117] Update TimeZones.jl URL TimeZones.jl recently moved to the new GitHub organization: JuliaTime. --- doc/manual/dates.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/dates.rst b/doc/manual/dates.rst index 5b1ef2a0c4410..bf589a46fad9f 100644 --- a/doc/manual/dates.rst +++ b/doc/manual/dates.rst @@ -8,7 +8,7 @@ The :mod:`Dates` module provides two types for working with dates: :class:`Date` and :class:`DateTime`, representing day and millisecond precision, respectively; both are subtypes of the abstract :class:`TimeType`. The motivation for distinct types is simple: some operations are much simpler, both in terms of code and mental reasoning, when the complexities of greater precision don't have to be dealt with. For example, since the :class:`Date` type only resolves to the precision of a single date (i.e. no hours, minutes, or seconds), normal considerations for time zones, daylight savings/summer time, and leap seconds are unnecessary and avoided. -Both :class:`Date` and :class:`DateTime` are basically immutable ``Int64`` wrappers. The single ``instant`` field of either type is actually a ``UTInstant{P}`` type, which represents a continuously increasing machine timeline based on the UT second [1]_. The :class:`DateTime` type is *timezone-unaware* (in Python parlance) or is analogous to a *LocalDateTime* in Java 8. Additional time zone functionality can be added through the `Timezones.jl package `_, which compiles the `Olsen Time Zone Database `_. Both :class:`Date` and :class:`DateTime` are based on the ISO 8601 standard, which follows the proleptic Gregorian calendar. One note is that the ISO 8601 standard is particular about BC/BCE dates. In general, the last day of the BC/BCE era, 1-12-31 BC/BCE, was followed by 1-1-1 AD/CE, thus no year zero exists. The ISO standard, however, states that 1 BC/BCE is year zero, so ``0000-12-31`` is the day before ``0001-01-01``, and year ``-0001`` (yes, negative one for the year) is 2 BC/BCE, year ``-0002`` is 3 BC/BCE, etc. +Both :class:`Date` and :class:`DateTime` are basically immutable ``Int64`` wrappers. The single ``instant`` field of either type is actually a ``UTInstant{P}`` type, which represents a continuously increasing machine timeline based on the UT second [1]_. The :class:`DateTime` type is *timezone-unaware* (in Python parlance) or is analogous to a *LocalDateTime* in Java 8. Additional time zone functionality can be added through the `TimeZones.jl package `_, which compiles the `IANA time zone database `_. Both :class:`Date` and :class:`DateTime` are based on the ISO 8601 standard, which follows the proleptic Gregorian calendar. One note is that the ISO 8601 standard is particular about BC/BCE dates. In general, the last day of the BC/BCE era, 1-12-31 BC/BCE, was followed by 1-1-1 AD/CE, thus no year zero exists. The ISO standard, however, states that 1 BC/BCE is year zero, so ``0000-12-31`` is the day before ``0001-01-01``, and year ``-0001`` (yes, negative one for the year) is 2 BC/BCE, year ``-0002`` is 3 BC/BCE, etc. .. [1] The notion of the UT second is actually quite fundamental. There are basically two different notions of time generally accepted, one based on the physical rotation of the earth (one full rotation = 1 day), the other based on the SI second (a fixed, constant value). These are radically different! Think about it, a "UT second", as defined relative to the rotation of the earth, may have a different absolute length depending on the day! Anyway, the fact that :class:`Date` and :class:`DateTime` are based on UT seconds is a simplifying, yet honest assumption so that things like leap seconds and all their complexity can be avoided. This basis of time is formally called `UT `_ or UT1. Basing types on the UT second basically means that every minute has 60 seconds and every day has 24 hours and leads to more natural calculations when working with calendar dates. From 05a76604ec5c6e135aeec1eba01967607e1876cb Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 15 Jun 2016 22:18:53 -0400 Subject: [PATCH 0019/1117] Add cmdlineargs tests for --precompiled this would have at least caught #16921 in local testing on platforms other than Windows --- test/cmdlineargs.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index a0372e2fdc4ac..05a698b067571 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -274,3 +274,9 @@ end # Make sure `julia --lisp` doesn't break run(pipeline(DevNull, `$(Base.julia_cmd()) --lisp`, DevNull)) + +# --precompiled={yes|no} +let exename = `$(Base.julia_cmd())` + @test readchomp(`$exename --precompiled=yes -E "Bool(Base.JLOptions().use_precompiled)"`) == "true" + @test readchomp(`$exename --precompiled=no -E "Bool(Base.JLOptions().use_precompiled)"`) == "false" +end From 86ec9f5f0f323b63d9a4eb43dc60e51da4c626f7 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Wed, 15 Jun 2016 16:38:22 -0700 Subject: [PATCH 0020/1117] Move documentation out of helpdb, add more Documented constructors for special matrix types inline. Added some info about argument rules. --- base/docs/helpdb/Base.jl | 28 ------------ base/linalg/bidiag.jl | 22 ++++++++++ base/linalg/diagonal.jl | 10 +++++ base/linalg/symmetric.jl | 34 ++++++++++++++ base/linalg/tridiag.jl | 19 +++++++- doc/stdlib/linalg.rst | 95 ++++++++++++++++++++++++++++++++++++---- 6 files changed, 169 insertions(+), 39 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 521a35f90b09e..c83f0730feab6 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -3901,15 +3901,6 @@ Compute the logarithm of `x` to base 2. Throws `DomainError` for negative `Real` """ log2 -""" - SymTridiagonal(d, du) - -Construct a real symmetric tridiagonal matrix from the diagonal and upper diagonal, -respectively. The result is of type `SymTridiagonal` and provides efficient specialized -eigensolvers, but may be converted into a regular matrix with [`full`](:func:`full`). -""" -SymTridiagonal - """ colon(start, [step], stop) @@ -5382,16 +5373,6 @@ All metadata and dbg.* calls are removed from the printed bitcode. Use code_llvm """ code_llvm -""" - Bidiagonal(dv, ev, isupper) - -Constructs an upper (`isupper=true`) or lower (`isupper=false`) bidiagonal matrix using the -given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` -and provides efficient specialized linear solvers, but may be converted into a regular -matrix with [`full`](:func:`full`). -""" -Bidiagonal - """ notify(condition, val=nothing; all=true, error=false) @@ -7524,15 +7505,6 @@ Return ``x^{1/3}``. The prefix operator `∛` is equivalent to `cbrt`. """ cbrt -""" - Tridiagonal(dl, d, du) - -Construct a tridiagonal matrix from the lower diagonal, diagonal, and upper diagonal, -respectively. The result is of type `Tridiagonal` and provides efficient specialized linear -solvers, but may be converted into a regular matrix with [`full`](:func:`full`). -""" -Tridiagonal - """ findprev(A, i) diff --git a/base/linalg/bidiag.jl b/base/linalg/bidiag.jl index 2e62a742950ae..9a0785e2e8f35 100644 --- a/base/linalg/bidiag.jl +++ b/base/linalg/bidiag.jl @@ -12,9 +12,25 @@ type Bidiagonal{T} <: AbstractMatrix{T} new(dv, ev, isupper) end end +""" + Bidiagonal(dv, ev, isupper) + +Constructs an upper (`isupper=true`) or lower (`isupper=false`) bidiagonal matrix using the +given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` +and provides efficient specialized linear solvers, but may be converted into a regular +matrix with [`full`](:func:`full`). `ev`'s length must be one less than the length of `dv`. +""" Bidiagonal{T}(dv::AbstractVector{T}, ev::AbstractVector{T}, isupper::Bool) = Bidiagonal{T}(collect(dv), collect(ev), isupper) Bidiagonal(dv::AbstractVector, ev::AbstractVector) = throw(ArgumentError("did you want an upper or lower Bidiagonal? Try again with an additional true (upper) or false (lower) argument.")) +""" + Bidiagonal(dv, ev, uplo) + +Constructs an upper (`uplo='U'`) or lower (`uplo='L'`) bidiagonal matrix using the +given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` +and provides efficient specialized linear solvers, but may be converted into a regular +matrix with [`full`](:func:`full`). `ev`'s length must be one less than the length of `dv`. +""" #Convert from BLAS uplo flag to boolean internal Bidiagonal(dv::AbstractVector, ev::AbstractVector, uplo::Char) = begin if uplo === 'U' @@ -31,6 +47,12 @@ function Bidiagonal{Td,Te}(dv::AbstractVector{Td}, ev::AbstractVector{Te}, isupp Bidiagonal(convert(Vector{T}, dv), convert(Vector{T}, ev), isupper) end +""" + Bidiagonal(A, uplo) + +Construct a `Bidiagonal` matrix from the main diagonal of `A` and +its first super- (if `isupper=true`) or sub-diagonal (if `isupper=false`). +""" Bidiagonal(A::AbstractMatrix, isupper::Bool)=Bidiagonal(diag(A), diag(A, isupper?1:-1), isupper) function getindex{T}(A::Bidiagonal{T}, i::Integer, j::Integer) diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 8fe5105c2b5ab..91c73aff73aa7 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -5,7 +5,17 @@ immutable Diagonal{T} <: AbstractMatrix{T} diag::Vector{T} end +""" + Diagonal(A::AbstractMatrix) + +Constructs a matrix from the diagonal of `A`. +""" Diagonal(A::AbstractMatrix) = Diagonal(diag(A)) +""" + Diagonal(V::AbstractVector) + +Constructs a matrix with `V` as its diagonal. +""" Diagonal(V::AbstractVector) = Diagonal(collect(V)) convert{T}(::Type{Diagonal{T}}, D::Diagonal{T}) = D diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index e1aac4a916f8e..4e5bc4de2864b 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -5,11 +5,45 @@ immutable Symmetric{T,S<:AbstractMatrix} <: AbstractMatrix{T} data::S uplo::Char end +""" + Symmetric(A, uplo=:U) + +Construct a `Symmetric` matrix from the upper (if `uplo = :U`) or lower (if `uplo = :L`) triangle of `A`. + +**Example** + +```julia +A = randn(10,10) +Supper = Symmetric(A) +Slower = Symmetric(A,:L) +eigfact(Supper) +``` + +`eigfact` will use a method specialized for matrices known to be symmetric. +Note that `Supper` will not be equal to `Slower` unless `A` is itself symmetric (e.g. if `A == A.'`). +""" Symmetric(A::AbstractMatrix, uplo::Symbol=:U) = (checksquare(A);Symmetric{eltype(A),typeof(A)}(A, char_uplo(uplo))) immutable Hermitian{T,S<:AbstractMatrix} <: AbstractMatrix{T} data::S uplo::Char end +""" + Hermitian(A, uplo=:U) + +Construct a `Hermitian` matrix from the upper (if `uplo = :U`) or lower (if `uplo = :L`) triangle of `A`. + +**Example** + +```julia +A = randn(10,10) +Hupper = Hermitian(A) +Hlower = Hermitian(A,:L) +eigfact(Hupper) +``` + +`eigfact` will use a method specialized for matrices known to be Hermitian. +Note that `Hupper` will not be equal to `Hlower` unless `A` is itself Hermitian (e.g. if `A == A'`). +""" function Hermitian(A::AbstractMatrix, uplo::Symbol=:U) n = checksquare(A) for i=1:n diff --git a/base/linalg/tridiag.jl b/base/linalg/tridiag.jl index 3c5fa0d2ddecf..6727d144a0c47 100644 --- a/base/linalg/tridiag.jl +++ b/base/linalg/tridiag.jl @@ -14,6 +14,13 @@ immutable SymTridiagonal{T} <: AbstractMatrix{T} end end +""" + SymTridiagonal(dv, ev) + +Construct a symmetric tridiagonal matrix from the diagonal and first sub/super-diagonal, +respectively. The result is of type `SymTridiagonal` and provides efficient specialized +eigensolvers, but may be converted into a regular matrix with [`full`](:func:`full`). +""" SymTridiagonal{T}(dv::Vector{T}, ev::Vector{T}) = SymTridiagonal{T}(dv, ev) function SymTridiagonal{Td,Te}(dv::AbstractVector{Td}, ev::AbstractVector{Te}) @@ -314,6 +321,14 @@ immutable Tridiagonal{T} <: AbstractMatrix{T} du2::Vector{T} # supsup-diagonal for pivoting end +""" + Tridiagonal(dl, d, du) + +Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, +respectively. The result is of type `Tridiagonal` and provides efficient specialized linear +solvers, but may be converted into a regular matrix with [`full`](:func:`full`). +The lengths of `dl` and `du` must be one less than the length of `d`. +""" # Basic constructor takes in three dense vectors of same type function Tridiagonal{T}(dl::Vector{T}, d::Vector{T}, du::Vector{T}) n = length(d) @@ -332,8 +347,8 @@ end """ Tridiagonal(A) -returns a `Tridiagonal` array based on (abstract) matrix `A`, using its lower diagonal, -diagonal, and upper diagonal. +returns a `Tridiagonal` array based on (abstract) matrix `A`, using its first lower diagonal, +main diagonal, and first upper diagonal. """ function Tridiagonal(A::AbstractMatrix) return Tridiagonal(diag(A,-1), diag(A), diag(A,+1)) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 71c2594b573ea..4763985c80596 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -59,6 +59,82 @@ Linear algebra functions in Julia are largely implemented by calling functions f Reconstruct the matrix ``A`` from the factorization ``F=factorize(A)``\ . +.. function:: Diagonal(A::AbstractMatrix) + + .. Docstring generated from Julia source + + Constructs a matrix from the diagonal of ``A``\ . + +.. function:: Diagonal(V::AbstractVector) + + .. Docstring generated from Julia source + + Constructs a matrix with ``V`` as its diagonal. + +.. function:: Bidiagonal(dv, ev, isupper) + + .. Docstring generated from Julia source + + Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . + +.. function:: Bidiagonal(dv, ev, uplo) + + .. Docstring generated from Julia source + + Constructs an upper (``uplo='U'``\ ) or lower (``uplo='L'``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . + +.. function:: Bidiagonal(A, uplo) + + .. Docstring generated from Julia source + + Construct a ``Bidiagonal`` matrix from the main diagonal of ``A`` and its first super- (if ``isupper=true``\ ) or sub-diagonal (if ``isupper=false``\ ). + +.. function:: SymTridiagonal(dv, ev) + + .. Docstring generated from Julia source + + Construct a symmetric tridiagonal matrix from the diagonal and first sub/super-diagonal, respectively. The result is of type ``SymTridiagonal`` and provides efficient specialized eigensolvers, but may be converted into a regular matrix with :func:`full`\ . + +.. function:: Tridiagonal(dl, d, du) + + .. Docstring generated from Julia source + + Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . + +.. function:: Symmetric(A, uplo=:U) + + .. Docstring generated from Julia source + + Construct a ``Symmetric`` matrix from the upper (if ``uplo = :U``\ ) or lower (if ``uplo = :L``\ ) triangle of ``A``\ . + + **Example** + + .. code-block:: julia + + A = randn(10,10) + Supper = Symmetric(A) + Slower = Symmetric(A,:L) + eigfact(Supper) + + ``eigfact`` will use a method specialized for matrices known to be symmetric. Note that ``Supper`` will not be equal to ``Slower`` unless ``A`` is itself symmetric (e.g. if ``A == A.'``\ ). + +.. function:: Hermitian(A, uplo=:U) + + .. Docstring generated from Julia source + + Construct a ``Hermitian`` matrix from the upper (if ``uplo = :U``\ ) or lower (if ``uplo = :L``\ ) triangle of ``A``\ . + + **Example** + + .. code-block:: julia + + A = randn(10,10) + Hupper = Hermitian(A) + Hlower = Hermitian(A,:L) + eigfact(Hupper) + + ``eigfact`` will use a method specialized for matrices known to be Hermitian. Note that ``Hupper`` will not be equal to ``Hlower`` unless ``A`` is itself Hermitian (e.g. if ``A == A'``\ ). + .. function:: lu(A) -> L, U, p .. Docstring generated from Julia source @@ -822,19 +898,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Construct a tridiagonal matrix from the lower diagonal, diagonal, and upper diagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . + Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . .. function:: Bidiagonal(dv, ev, isupper) .. Docstring generated from Julia source - Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . - -.. function:: SymTridiagonal(d, du) - - .. Docstring generated from Julia source - - Construct a real symmetric tridiagonal matrix from the diagonal and upper diagonal, respectively. The result is of type ``SymTridiagonal`` and provides efficient specialized eigensolvers, but may be converted into a regular matrix with :func:`full`\ . + Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . .. function:: rank(M) @@ -1253,7 +1323,14 @@ Linear algebra functions in Julia are largely implemented by calling functions f ``eigs`` returns the ``nev`` requested eigenvalues in ``d``\ , the corresponding Ritz vectors ``v`` (only if ``ritzvec=true``\ ), the number of converged eigenvalues ``nconv``\ , the number of iterations ``niter`` and the number of matrix vector multiplications ``nmult``\ , as well as the final residual vector ``resid``\ . - **note** + **Example** + + .. code-block:: julia + + X = sprand(10, 5, 0.2) + eigs(X, nsv = 2, tol = 1e-3) + + **Note** The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalue problem :math:`Av = Bv\lambda`\ , but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``\ . From 886f8c638eb1f0be5155e8bde2ba9b5b58fe97a4 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 16 Jun 2016 07:43:03 -0700 Subject: [PATCH 0021/1117] turn bounds checking on when running tests on appveyor --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 67847ed881e66..be1fa0d55b214 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -54,4 +54,5 @@ build_script: test_script: - usr\bin\julia -e "versioninfo()" - copy usr\lib\julia\sys.ji local.ji && usr\bin\julia -J local.ji -e "true" && del local.ji - - cd test && ..\usr\bin\julia runtests.jl all && ..\usr\bin\julia runtests.jl libgit2-online pkg + - cd test && ..\usr\bin\julia --check-bounds=yes runtests.jl all && + ..\usr\bin\julia --check-bounds=yes runtests.jl libgit2-online pkg From 2493dafbf6f13d86715dbb7459797e6128fdeda8 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sun, 12 Jun 2016 18:41:24 -0400 Subject: [PATCH 0022/1117] Add LLVM patches to fix InstCombine loosing tbaa metadata Fixes #16894 --- deps/llvm.mk | 2 + .../llvm-D21271-instcombine-tbaa-3.7.patch | 97 ++++++++++++++++++ .../llvm-D21271-instcombine-tbaa-3.8.patch | 98 +++++++++++++++++++ 3 files changed, 197 insertions(+) create mode 100644 deps/patches/llvm-D21271-instcombine-tbaa-3.7.patch create mode 100644 deps/patches/llvm-D21271-instcombine-tbaa-3.8.patch diff --git a/deps/llvm.mk b/deps/llvm.mk index 0d6426e139f0d..8b835beec197b 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -429,6 +429,7 @@ $(eval $(call LLVM_PATCH,llvm-3.7.1_symlinks)) $(eval $(call LLVM_PATCH,llvm-3.8.0_bindir)) $(eval $(call LLVM_PATCH,llvm-D14260)) $(eval $(call LLVM_PATCH,llvm-nodllalias)) +$(eval $(call LLVM_PATCH,llvm-D21271-instcombine-tbaa-3.7)) else ifeq ($(LLVM_VER),3.8.0) $(eval $(call LLVM_PATCH,llvm-3.7.1_3)) $(eval $(call LLVM_PATCH,llvm-D14260)) @@ -445,6 +446,7 @@ $(eval $(call LLVM_PATCH,llvm-D17712)) $(eval $(call LLVM_PATCH,llvm-PR26180)) $(eval $(call LLVM_PATCH,llvm-PR27046)) $(eval $(call LLVM_PATCH,llvm-3.8.0_ppc64_SUBFC8)) +$(eval $(call LLVM_PATCH,llvm-D21271-instcombine-tbaa-3.8)) endif # LLVM_VER ifeq ($(LLVM_VER),3.7.1) diff --git a/deps/patches/llvm-D21271-instcombine-tbaa-3.7.patch b/deps/patches/llvm-D21271-instcombine-tbaa-3.7.patch new file mode 100644 index 0000000000000..1da6845dddc6c --- /dev/null +++ b/deps/patches/llvm-D21271-instcombine-tbaa-3.7.patch @@ -0,0 +1,97 @@ +From a3c3e1bd22fef6de6d31b3a80d773d1176b4ca8c Mon Sep 17 00:00:00 2001 +From: Yichao Yu +Date: Sun, 12 Jun 2016 17:14:38 -0400 +Subject: [PATCH] Fix `InstCombine` to not widen metadata on store-to-load + forwarding. + +The original check for load CSE or store-to-load forwarding is wrong +when the forwarded stored value happened to be a load. +--- + include/llvm/Analysis/Loads.h | 3 ++- + lib/Analysis/Loads.cpp | 5 ++++- + .../InstCombine/InstCombineLoadStoreAlloca.cpp | 6 ++++-- + test/Transforms/InstCombine/tbaa-store-to-load.ll | 17 +++++++++++++++++ + 4 files changed, 27 insertions(+), 4 deletions(-) + create mode 100644 test/Transforms/InstCombine/tbaa-store-to-load.ll + +diff --git a/include/llvm/Analysis/Loads.h b/include/llvm/Analysis/Loads.h +index 42667d2..5db3c2f 100644 +--- a/include/llvm/Analysis/Loads.h ++++ b/include/llvm/Analysis/Loads.h +@@ -50,7 +50,8 @@ Value *FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, + BasicBlock::iterator &ScanFrom, + unsigned MaxInstsToScan = 6, + AliasAnalysis *AA = nullptr, +- AAMDNodes *AATags = nullptr); ++ AAMDNodes *AATags = nullptr, ++ bool *IsLoadCSE = nullptr); + + } + +diff --git a/lib/Analysis/Loads.cpp b/lib/Analysis/Loads.cpp +index 624c5a1..baecd3f 100644 +--- a/lib/Analysis/Loads.cpp ++++ b/lib/Analysis/Loads.cpp +@@ -183,7 +183,8 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom, + Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, + BasicBlock::iterator &ScanFrom, + unsigned MaxInstsToScan, +- AliasAnalysis *AA, AAMDNodes *AATags) { ++ AliasAnalysis *AA, AAMDNodes *AATags, ++ bool *IsLoadCSE) { + if (MaxInstsToScan == 0) + MaxInstsToScan = ~0U; + +@@ -220,6 +221,8 @@ Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, + CastInst::isBitOrNoopPointerCastable(LI->getType(), AccessTy, DL)) { + if (AATags) + LI->getAAMetadata(*AATags); ++ if (IsLoadCSE) ++ *IsLoadCSE = true; + return LI; + } + +diff --git a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +index e3179db..f1af1f7 100644 +--- a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp ++++ b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +@@ -750,9 +750,11 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) { + // separated by a few arithmetic operations. + BasicBlock::iterator BBI = &LI; + AAMDNodes AATags; ++ bool IsLoadCSE = false; + if (Value *AvailableVal = FindAvailableLoadedValue(Op, LI.getParent(), BBI, +- 6, AA, &AATags)) { +- if (LoadInst *NLI = dyn_cast(AvailableVal)) { ++ 6, AA, &AATags, &IsLoadCSE)) { ++ if (IsLoadCSE) { ++ LoadInst *NLI = cast(AvailableVal); + unsigned KnownIDs[] = { + LLVMContext::MD_tbaa, + LLVMContext::MD_alias_scope, +diff --git a/test/Transforms/InstCombine/tbaa-store-to-load.ll b/test/Transforms/InstCombine/tbaa-store-to-load.ll +new file mode 100644 +index 0000000..707be73 +--- /dev/null ++++ b/test/Transforms/InstCombine/tbaa-store-to-load.ll +@@ -0,0 +1,17 @@ ++; RUN: opt -S -instcombine < %s 2>&1 | FileCheck %s ++ ++define i64 @f(i64* %p1, i64* %p2) { ++top: ++ ; check that the tbaa is preserved ++ ; CHECK-LABEL: @f( ++ ; CHECK: %v1 = load i64, i64* %p1, align 8, !tbaa !0 ++ ; CHECK: store i64 %v1, i64* %p2, align 8 ++ ; CHECK: ret i64 %v1 ++ %v1 = load i64, i64* %p1, align 8, !tbaa !0 ++ store i64 %v1, i64* %p2, align 8 ++ %v2 = load i64, i64* %p2, align 8 ++ ret i64 %v2 ++} ++ ++!0 = !{!1, !1, i64 0} ++!1 = !{!"load_tbaa"} +-- +2.8.3 + diff --git a/deps/patches/llvm-D21271-instcombine-tbaa-3.8.patch b/deps/patches/llvm-D21271-instcombine-tbaa-3.8.patch new file mode 100644 index 0000000000000..3792446ffd1b4 --- /dev/null +++ b/deps/patches/llvm-D21271-instcombine-tbaa-3.8.patch @@ -0,0 +1,98 @@ +From 3dc2f7e69ec0a65af503bb6264fa6674ecfeaf84 Mon Sep 17 00:00:00 2001 +From: Yichao Yu +Date: Sun, 12 Jun 2016 17:14:38 -0400 +Subject: [PATCH] Fix `InstCombine` to not widen metadata on store-to-load + forwarding. + +The original check for load CSE or store-to-load forwarding is wrong +when the forwarded stored value happened to be a load. +--- + include/llvm/Analysis/Loads.h | 3 ++- + lib/Analysis/Loads.cpp | 5 ++++- + .../InstCombine/InstCombineLoadStoreAlloca.cpp | 6 ++++-- + test/Transforms/InstCombine/tbaa-store-to-load.ll | 17 +++++++++++++++++ + 4 files changed, 27 insertions(+), 4 deletions(-) + create mode 100644 test/Transforms/InstCombine/tbaa-store-to-load.ll + +diff --git a/include/llvm/Analysis/Loads.h b/include/llvm/Analysis/Loads.h +index 939663b..a7fc035 100644 +--- a/include/llvm/Analysis/Loads.h ++++ b/include/llvm/Analysis/Loads.h +@@ -55,7 +55,8 @@ Value *FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, + BasicBlock::iterator &ScanFrom, + unsigned MaxInstsToScan = DefMaxInstsToScan, + AliasAnalysis *AA = nullptr, +- AAMDNodes *AATags = nullptr); ++ AAMDNodes *AATags = nullptr, ++ bool *IsLoadCSE = nullptr); + + } + +diff --git a/lib/Analysis/Loads.cpp b/lib/Analysis/Loads.cpp +index 4b2fa3c..f5705fa 100644 +--- a/lib/Analysis/Loads.cpp ++++ b/lib/Analysis/Loads.cpp +@@ -196,7 +196,8 @@ llvm::DefMaxInstsToScan("available-load-scan-limit", cl::init(6), cl::Hidden, + Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, + BasicBlock::iterator &ScanFrom, + unsigned MaxInstsToScan, +- AliasAnalysis *AA, AAMDNodes *AATags) { ++ AliasAnalysis *AA, AAMDNodes *AATags, ++ bool *IsLoadCSE) { + if (MaxInstsToScan == 0) + MaxInstsToScan = ~0U; + +@@ -233,6 +234,8 @@ Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, + CastInst::isBitOrNoopPointerCastable(LI->getType(), AccessTy, DL)) { + if (AATags) + LI->getAAMetadata(*AATags); ++ if (IsLoadCSE) ++ *IsLoadCSE = true; + return LI; + } + +diff --git a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +index 4d42658..8302ef8 100644 +--- a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp ++++ b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +@@ -800,10 +800,12 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) { + // separated by a few arithmetic operations. + BasicBlock::iterator BBI(LI); + AAMDNodes AATags; ++ bool IsLoadCSE = false; + if (Value *AvailableVal = + FindAvailableLoadedValue(Op, LI.getParent(), BBI, +- DefMaxInstsToScan, AA, &AATags)) { +- if (LoadInst *NLI = dyn_cast(AvailableVal)) { ++ DefMaxInstsToScan, AA, &AATags, &IsLoadCSE)) { ++ if (IsLoadCSE) { ++ LoadInst *NLI = cast(AvailableVal); + unsigned KnownIDs[] = { + LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope, + LLVMContext::MD_noalias, LLVMContext::MD_range, +diff --git a/test/Transforms/InstCombine/tbaa-store-to-load.ll b/test/Transforms/InstCombine/tbaa-store-to-load.ll +new file mode 100644 +index 0000000..707be73 +--- /dev/null ++++ b/test/Transforms/InstCombine/tbaa-store-to-load.ll +@@ -0,0 +1,17 @@ ++; RUN: opt -S -instcombine < %s 2>&1 | FileCheck %s ++ ++define i64 @f(i64* %p1, i64* %p2) { ++top: ++ ; check that the tbaa is preserved ++ ; CHECK-LABEL: @f( ++ ; CHECK: %v1 = load i64, i64* %p1, align 8, !tbaa !0 ++ ; CHECK: store i64 %v1, i64* %p2, align 8 ++ ; CHECK: ret i64 %v1 ++ %v1 = load i64, i64* %p1, align 8, !tbaa !0 ++ store i64 %v1, i64* %p2, align 8 ++ %v2 = load i64, i64* %p2, align 8 ++ ret i64 %v2 ++} ++ ++!0 = !{!1, !1, i64 0} ++!1 = !{!"load_tbaa"} +-- +2.8.3 + From efc2862412f73be7c9de6cb1bd2d2398a71385e4 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 16 Jun 2016 10:18:21 -0400 Subject: [PATCH 0023/1117] Clear Comdats in llvmcall --- src/ccall.cpp | 4 ++++ src/jitlayers.cpp | 13 ++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 9682a269fe132..1103b4e918ad5 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -528,6 +528,8 @@ class FunctionMover : public ValueMaterializer SmallVector Returns; llvm::CloneFunctionInto(NewF,F,VMap,true,Returns,"",NULL,NULL,this); + NewF->setComdat(nullptr); + NewF->setSection(""); } Function *CloneFunction(Function *F) @@ -552,6 +554,7 @@ class FunctionMover : public ValueMaterializer Function *NewF = destModule->getFunction(F->getName()); if (!NewF) { NewF = function_proto(F); + NewF->setComdat(nullptr); destModule->getFunctionList().push_back(NewF); } return NewF; @@ -609,6 +612,7 @@ class FunctionMover : public ValueMaterializer NULL, GV->getName()); newGV->copyAttributesFrom(GV); + newGV->setComdat(nullptr); if (GV->isDeclaration()) return newGV; if (!GV->getName().empty()) { diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index bc594847fb1c0..35c920878e84f 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1,5 +1,6 @@ // This file is a part of Julia. License is MIT: http://julialang.org/license +#include #include // Except for parts of this file which were copied from LLVM, under the UIUC license (marked below). @@ -398,9 +399,15 @@ class JuliaOJIT { if (F->isDeclaration()) { if (F->use_empty()) F->eraseFromParent(); - else - assert(F->isIntrinsic() || findUnmangledSymbol(F->getName()) || - SectionMemoryManager::getSymbolAddressInProcess(F->getName())); + else if (!(F->isIntrinsic() || + findUnmangledSymbol(F->getName()) || + SectionMemoryManager::getSymbolAddressInProcess( + F->getName()))) { + std::cerr << "FATAL ERROR: " + << "Symbol \"" << F->getName().str() << "\"" + << "not found"; + abort(); + } } } #endif From a48d0b7ff91e5aecba3dd2c03e759811bd34bc73 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Jun 2016 01:22:12 -0400 Subject: [PATCH 0024/1117] correct precise_container_types to handle TypeConstructor correctly --- base/essentials.jl | 25 +++++++++++++------------ base/inference.jl | 15 +++++++++------ base/tuple.jl | 2 +- test/inference.jl | 6 ++++++ 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/base/essentials.jl b/base/essentials.jl index 8ab11f6981433..3c2563abfd905 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -41,27 +41,28 @@ macro generated(f) end end +argtail(x, rest...) = rest +tail(x::Tuple) = argtail(x...) -tuple_type_head(::Type{Union{}}) = throw(MethodError(tuple_type_head, (Union{},))) -function tuple_type_head{T<:Tuple}(::Type{T}) +tuple_type_head(T::TypeConstructor) = tuple_type_head(T.body) +function tuple_type_head(T::DataType) @_pure_meta - T.parameters[1] + T.name === Tuple.name || throw(MethodError(tuple_type_head, (T,))) + return T.parameters[1] end - -isvarargtype(t::ANY) = isa(t,DataType)&&is((t::DataType).name,Vararg.name) -isvatuple(t::DataType) = (n = length(t.parameters); n > 0 && isvarargtype(t.parameters[n])) -unwrapva(t::ANY) = isvarargtype(t) ? t.parameters[1] : t - -function tuple_type_tail{T<:Tuple}(::Type{T}) +tuple_type_tail(T::TypeConstructor) = tuple_type_tail(T.body) +function tuple_type_tail(T::DataType) @_pure_meta + T.name === Tuple.name || throw(MethodError(tuple_type_tail, (T,))) if isvatuple(T) && length(T.parameters) == 1 return T end - Tuple{argtail(T.parameters...)...} + return Tuple{argtail(T.parameters...)...} end -argtail(x, rest...) = rest -tail(x::Tuple) = argtail(x...) +isvarargtype(t::ANY) = isa(t, DataType) && is((t::DataType).name, Vararg.name) +isvatuple(t::DataType) = (n = length(t.parameters); n > 0 && isvarargtype(t.parameters[n])) +unwrapva(t::ANY) = isvarargtype(t) ? t.parameters[1] : t convert{T<:Tuple{Any,Vararg{Any}}}(::Type{T}, x::Tuple{Any, Vararg{Any}}) = tuple(convert(tuple_type_head(T),x[1]), convert(tuple_type_tail(T), tail(x))...) diff --git a/base/inference.jl b/base/inference.jl index 7f3f0a0f942b1..58eb400eaba4b 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -873,16 +873,19 @@ function precise_container_types(args, types, vtypes::VarTable, sv) ai = args[i] ti = types[i] tti = widenconst(ti) - if isa(ai,Expr) && ai.head === :call && (abstract_evals_to_constant(ai.args[1], svec, vtypes, sv) || - abstract_evals_to_constant(ai.args[1], tuple, vtypes, sv)) + if isa(tti, TypeConstructor) + tti = tti.body + end + if isa(ai, Expr) && ai.head === :call && (abstract_evals_to_constant(ai.args[1], svec, vtypes, sv) || + abstract_evals_to_constant(ai.args[1], tuple, vtypes, sv)) aa = ai.args result[i] = Any[ (isa(aa[j],Expr) ? aa[j].typ : abstract_eval(aa[j],vtypes,sv)) for j=2:length(aa) ] if _any(isvarargtype, result[i]) return nothing end - elseif isa(ti, Union) + elseif isa(tti, Union) return nothing - elseif ti ⊑ Tuple + elseif tti <: Tuple if i == n if isvatuple(tti) && length(tti.parameters) == 1 result[i] = Any[Vararg{tti.parameters[1].parameters[1]}] @@ -894,7 +897,7 @@ function precise_container_types(args, types, vtypes::VarTable, sv) else return nothing end - elseif ti⊑AbstractArray && i==n + elseif tti <: AbstractArray && i == n result[i] = Any[Vararg{eltype(tti)}] else return nothing @@ -1264,7 +1267,7 @@ function tmerge(typea::ANY, typeb::ANY) typea, typeb = widenconst(typea), widenconst(typeb) typea === typeb && return typea if (typea <: Tuple) && (typeb <: Tuple) - if length(typea.parameters) == length(typeb.parameters) && !isvatuple(typea) && !isvatuple(typeb) + if isa(typea, DataType) && isa(typeb, DataType) && length(typea.parameters) == length(typeb.parameters) && !isvatuple(typea) && !isvatuple(typeb) return typejoin(typea, typeb) end return Tuple diff --git a/base/tuple.jl b/base/tuple.jl index b47bd59c5eb65..2e69bf217bff2 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -41,7 +41,7 @@ first(t::Tuple) = t[1] # eltype eltype(::Type{Tuple{}}) = Bottom -eltype{T,_}(::Type{NTuple{_,T}}) = T +eltype{T}(::Type{Tuple{Vararg{T}}}) = T # front (the converse of tail: it skips the last entry) diff --git a/test/inference.jl b/test/inference.jl index 1dcf0b881dcb2..d15ee68205f4b 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -252,3 +252,9 @@ end let g() = Int <: Real ? 1 : "" @test Base.return_types(g, Tuple{}) == [Int] end + +typealias NInt{N} Tuple{Vararg{Int, N}} +@test Base.eltype(NInt) === Int +fNInt(x::NInt) = (x...) +gNInt() = fNInt(x) +@test Base.return_types(gNInt, ()) == Any[NInt] From 154d02e37c8d39b08ad1c4b9494ede739cc0973d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Jun 2016 02:47:43 -0400 Subject: [PATCH 0025/1117] inference: remove duplicated handling of sparam_vals --- base/inference.jl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 58eb400eaba4b..e839334c5e0d5 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -75,18 +75,16 @@ type InferenceState inferred::Bool tfunc_bp::Union{TypeMapEntry, Void} - function InferenceState(linfo::LambdaInfo, atypes::ANY, sparams::SimpleVector, optimize::Bool) + function InferenceState(linfo::LambdaInfo, optimize::Bool) @assert isa(linfo.code,Array{Any,1}) linfo.inInference = true nslots = length(linfo.slotnames) nl = label_counter(linfo.code)+1 - if length(linfo.sparam_vals) > 0 - sp = linfo.sparam_vals - elseif isempty(sparams) && !isempty(linfo.sparam_syms) + if isempty(linfo.sparam_vals) && !isempty(linfo.sparam_syms) sp = svec(Any[ TypeVar(sym, Any, true) for sym in linfo.sparam_syms ]...) else - sp = sparams + sp = linfo.sparam_vals end if !isa(linfo.slottypes, Array) @@ -101,6 +99,7 @@ type InferenceState # initial types s[1] = Any[ VarState(Bottom,true) for i=1:nslots ] + atypes = linfo.specTypes la = linfo.nargs if la > 0 if linfo.isva @@ -1510,7 +1509,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr linfo = specialize_method(method, atypes, sparams) end # our stack frame inference context - frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), atypes, sparams, optimize) + frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), optimize) if cached frame.tfunc_bp = ccall(:jl_specializations_insert, Ref{TypeMapEntry}, (Any, Any, Any), method, atypes, linfo) end @@ -1554,7 +1553,7 @@ end function typeinf_ext(linfo::LambdaInfo) if isdefined(linfo, :def) # method lambda - infer this specialization via the method cache - (code, _t, _) = typeinf_edge(linfo.def, linfo.specTypes, svec(), true, true, true, linfo) + (code, _t, _) = typeinf_edge(linfo.def, linfo.specTypes, linfo.sparam_vals, true, true, true, linfo) if code.inferred linfo.inferred = true linfo.inInference = false @@ -1570,7 +1569,7 @@ function typeinf_ext(linfo::LambdaInfo) end else # toplevel lambda - infer directly - frame = InferenceState(linfo, linfo.specTypes, svec(), true) + frame = InferenceState(linfo, true) typeinf_loop(frame) @assert frame.inferred # TODO: deal with this better end From 9e477bc1cf0b8d6afca1f502912ab4e40eba27e1 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Jun 2016 02:49:10 -0400 Subject: [PATCH 0026/1117] fail jl_specializations_lookup more conservatively in the presence of incorrect UnionAll types previously, inserting a type and then looking up that exact type could fail, since it would compute the UnionAll in the wrong position and thus not consider the types to be equivalent --- src/gf.c | 4 ++-- src/typemap.c | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/gf.c b/src/gf.c index 88b12c7bedc3b..0f41429b307e0 100644 --- a/src/gf.c +++ b/src/gf.c @@ -119,7 +119,7 @@ JL_DLLEXPORT jl_typemap_entry_t *jl_specializations_insert(jl_method_t *m, jl_tu JL_DLLEXPORT jl_value_t *jl_specializations_lookup(jl_method_t *m, jl_tupletype_t *type) { - jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(m->specializations, type, NULL, 1, /*subtype*/0, /*offs*/0); + jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(m->specializations, type, NULL, 2, /*subtype*/0, /*offs*/0); if (!sf) return jl_nothing; return sf->func.value; @@ -127,7 +127,7 @@ JL_DLLEXPORT jl_value_t *jl_specializations_lookup(jl_method_t *m, jl_tupletype_ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_tupletype_t *type) { - jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(mt->defs, type, NULL, 1, /*subtype*/0, /*offs*/0); + jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(mt->defs, type, NULL, 2, /*subtype*/0, /*offs*/0); if (!sf) return jl_nothing; return sf->func.value; diff --git a/src/typemap.c b/src/typemap.c index 19a004499fd3e..f2b603face54e 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -514,8 +514,11 @@ int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs, int sigs_eq(jl_value_t *a, jl_value_t *b, int useenv) { - if (jl_has_typevars(a) || jl_has_typevars(b)) { - return jl_types_equal_generic(a,b,useenv); + // useenv == 0 : subtyping + ensure typevars correspond + // useenv == 1 : subtyping + ensure typevars correspond + fail if bound != bound in some typevar match + // useenv == 2 : ignore typevars (because UnionAll getting lost in intersection can cause jl_types_equal to fail in the wrong direction for some purposes) + if (useenv != 2 && (jl_has_typevars(a) || jl_has_typevars(b))) { + return jl_types_equal_generic(a, b, useenv); } return jl_subtype(a, b, 0) && jl_subtype(b, a, 0); } From 4f4cb4e6dbad33a0de5338b0abf170dbe8878b84 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 14 Jun 2016 00:35:20 -0400 Subject: [PATCH 0027/1117] avoid some redundant lookups & copies in inference --- base/inference.jl | 212 ++++++++++++++++++++----------------------- src/codegen.cpp | 2 +- src/gf.c | 57 +++++------- src/julia_internal.h | 2 +- 4 files changed, 120 insertions(+), 153 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index e839334c5e0d5..c74697d6b62a2 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -47,7 +47,6 @@ type InferenceState # info on the state of inference and the linfo linfo::LambdaInfo - destination::LambdaInfo # results need to be copied here when we finish nargs::Int stmt_types::Vector{Any} # return type @@ -73,11 +72,9 @@ type InferenceState inworkq::Bool optimize::Bool inferred::Bool - tfunc_bp::Union{TypeMapEntry, Void} function InferenceState(linfo::LambdaInfo, optimize::Bool) @assert isa(linfo.code,Array{Any,1}) - linfo.inInference = true nslots = length(linfo.slotnames) nl = label_counter(linfo.code)+1 @@ -156,12 +153,12 @@ type InferenceState inmodule = isdefined(linfo, :def) ? linfo.def.module : current_module() # toplevel thunks are inferred in the current module frame = new( sp, nl, Dict{SSAValue, Bool}(), inmodule, 0, false, - linfo, linfo, la, s, Union{}, W, n, + linfo, la, s, Union{}, W, n, cur_hand, handler_at, n_handlers, ssavalue_uses, ssavalue_init, ObjectIdDict(), #Dict{InferenceState, Vector{LineNum}}(), Vector{Tuple{InferenceState, Vector{LineNum}}}(), - false, false, false, optimize, false, nothing) + false, false, false, optimize, false) push!(active, frame) nactive[] += 1 return frame @@ -1387,25 +1384,25 @@ function newvar!(sv::InferenceState, typ) end # create a specialized LambdaInfo from a method -function specialize_method(method::Method, types::ANY, sp::SimpleVector) - li = ccall(:jl_get_specialized, Ref{LambdaInfo}, (Any, Any, Any), method, types, sp) - return li +function specialize_method(method::Method, types::ANY, sp::SimpleVector, cached) + if cached + return ccall(:jl_specializations_get_linfo, Ref{LambdaInfo}, (Any, Any, Any), method, types, sp) + else + return ccall(:jl_get_specialized, Ref{LambdaInfo}, (Any, Any, Any), method, types, sp) + end end # create copies of any field that type-inference might modify function unshare_linfo!(li::LambdaInfo) - if !isa(li.code, Array{Any,1}) + orig = li.def.lambda_template + if isa(li.code, Array{UInt8,1}) li.code = ccall(:jl_uncompress_ast, Any, (Any,Any), li, li.code) - else - li.code = copy_exprargs(li.code) + elseif li.code === orig.code + li.code = copy_exprargs(orig.code) end - li.slotnames = copy(li.slotnames) - li.slotflags = copy(li.slotflags) - if isa(li.slottypes, Array) - li.slottypes = copy(li.slottypes) - end - if isa(li.ssavaluetypes, Array) - li.ssavaluetypes = copy(li.ssavaluetypes) + if !li.def.isstaged + li.slotnames = copy(li.slotnames) + li.slotflags = copy(li.slotflags) end return li end @@ -1414,9 +1411,11 @@ end function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtree::Bool, optimize::Bool, cached::Bool, caller) local code = nothing local frame = nothing - # check cached specializations - # for an existing result stored there - if cached + if isa(caller, LambdaInfo) + code = caller + elseif cached + # check cached specializations + # for an existing result stored there if !is(method.specializations, nothing) code = ccall(:jl_specializations_lookup, Any, (Any, Any), method, atypes) if isa(code, Void) @@ -1436,89 +1435,80 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr code = nothing end end + end - if isa(code, LambdaInfo) && code.inInference - # inference on this signature may be in progress, - # find the corresponding frame in the active list - for infstate in active - infstate === nothing && continue - infstate = infstate::InferenceState - if code === infstate.linfo - frame = infstate - break + if caller === nothing && in_typeinf_loop + # if the caller needed the ast, but we are already in the typeinf loop + # then just return early -- we can't fulfill this request + # if the client was inlining, then this means we decided not to try to infer this + # particular signature (due to signature coarsening in abstract_call_gf_by_type) + # and attempting to force it now would be a bad idea (non terminating) + skip = true + if method.module == _topmod(method.module) || (isdefined(Main, :Base) && method.module == Main.Base) + # however, some gf have special tfunc and meaning they wouldn't have been inferred yet + # check the same conditions from abstract_call to detect this case + if method.name == :promote_type || method.name == :typejoin + skip = false + elseif method.name == :getindex || method.name == :next || method.name == :indexed_next + argtypes = atypes.parameters + if length(argtypes)>2 && argtypes[3] ⊑ Int + at2 = widenconst(argtypes[2]) + if (at2 <: Tuple || + (isa(at2, DataType) && isdefined(Main, :Base) && isdefined(Main.Base, :Pair) && + (at2::DataType).name === Main.Base.Pair.name)) + skip = false + end end end end + if skip + return (nothing, Union{}, false) + end end - if isa(caller, LambdaInfo) - code = caller - end - - if frame === nothing - # inference not started yet, make a new frame for a new lambda - # add lam to be inferred and record the edge - - if caller === nothing && in_typeinf_loop - # if the caller needed the ast, but we are already in the typeinf loop - # then just return early -- we can't fulfill this request - # if the client was inlining, then this means we decided not to try to infer this - # particular signature (due to signature coarsening in abstract_call_gf_by_type) - # and attempting to force it now would be a bad idea (non terminating) - skip = true - if method.module == _topmod(method.module) || (isdefined(Main, :Base) && method.module == Main.Base) - # however, some gf have special tfunc and meaning they wouldn't have been inferred yet - # check the same conditions from abstract_call to detect this case - if method.name == :promote_type || method.name == :typejoin - skip = false - elseif method.name == :getindex || method.name == :next || method.name == :indexed_next - argtypes = atypes.parameters - if length(argtypes)>2 && argtypes[3] ⊑ Int - at2 = widenconst(argtypes[2]) - if (at2 <: Tuple || - (isa(at2, DataType) && isdefined(Main, :Base) && isdefined(Main.Base, :Pair) && - (at2::DataType).name === Main.Base.Pair.name)) - skip = false - end - end - end - end - if skip - return (nothing, Union{}, false) - end + if isa(code, LambdaInfo) && code.code !== nothing + # reuse the existing code object + linfo = code + @assert typeseq(linfo.specTypes, atypes) + elseif method.isstaged + if !isleaftype(atypes) + # don't call staged functions on abstract types. + # (see issues #8504, #10230) + # we can't guarantee that their type behavior is monotonic. + return (nothing, Any, false) + end + try + # user code might throw errors – ignore them + linfo = specialize_method(method, atypes, sparams, cached) + catch + return (nothing, Any, false) end + else + linfo = specialize_method(method, atypes, sparams, cached) + end - if isa(code, LambdaInfo) && code.code !== nothing - # reuse the existing code object - linfo = code - @assert typeseq(linfo.specTypes, atypes) - elseif method.isstaged - if !isleaftype(atypes) - # don't call staged functions on abstract types. - # (see issues #8504, #10230) - # we can't guarantee that their type behavior is monotonic. - return (nothing, Any, false) - end - try - # user code might throw errors – ignore them - linfo = specialize_method(method, atypes, sparams) - catch - return (nothing, Any, false) + if linfo.inInference + # inference on this signature may be in progress, + # find the corresponding frame in the active list + for infstate in active + infstate === nothing && continue + infstate = infstate::InferenceState + if linfo === infstate.linfo + frame = infstate + break end - else - linfo = specialize_method(method, atypes, sparams) end - # our stack frame inference context + # TODO: this assertion seems iffy + assert(frame !== nothing) + else + # inference not started yet, make a new frame for a new lambda + linfo.inInference = true frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), optimize) - if cached - frame.tfunc_bp = ccall(:jl_specializations_insert, Ref{TypeMapEntry}, (Any, Any, Any), method, atypes, linfo) - end end frame = frame::InferenceState - if !isa(caller, Void) && !isa(caller, LambdaInfo) - # if we were called from inside inference, - # the caller will be the InferenceState object + if isa(caller, InferenceState) + # if we were called from inside inference, the caller will be the InferenceState object # for which the edge was required caller = caller::InferenceState if haskey(caller.edges, frame) @@ -1554,26 +1544,30 @@ function typeinf_ext(linfo::LambdaInfo) if isdefined(linfo, :def) # method lambda - infer this specialization via the method cache (code, _t, _) = typeinf_edge(linfo.def, linfo.specTypes, linfo.sparam_vals, true, true, true, linfo) - if code.inferred + if code.inferred && linfo !== code + # This case occurs when the IR for a function has been deleted. + # `code` will be a newly-created LambdaInfo, and we need to copy its + # contents to the existing one to copy the info to the method cache. linfo.inferred = true linfo.inInference = false - if linfo !== code - linfo.code = code.code - linfo.slotnames = code.slotnames - linfo.slottypes = code.slottypes - linfo.slotflags = code.slotflags - linfo.ssavaluetypes = code.ssavaluetypes - linfo.rettype = code.rettype - linfo.pure = code.pure - end - end + linfo.code = code.code + linfo.slotnames = code.slotnames + linfo.slottypes = code.slottypes + linfo.slotflags = code.slotflags + linfo.ssavaluetypes = code.ssavaluetypes + linfo.rettype = code.rettype + linfo.pure = code.pure + linfo.inlineable = code.inlineable + end + return code else # toplevel lambda - infer directly + linfo.inInference = true frame = InferenceState(linfo, true) typeinf_loop(frame) @assert frame.inferred # TODO: deal with this better + return linfo end - nothing end @@ -1947,20 +1941,6 @@ function finish(me::InferenceState) end me.linfo.rettype = me.bestguess - if me.destination !== me.linfo - out = me.destination - out.inferred = true - out.inInference = false - out.code = me.linfo.code - out.slotnames = me.linfo.slotnames - out.slottypes = me.linfo.slottypes - out.slotflags = me.linfo.slotflags - out.ssavaluetypes = me.linfo.ssavaluetypes - out.rettype = me.linfo.rettype - out.pure = me.linfo.pure - out.inlineable = me.linfo.inlineable - end - # lazy-delete the item from active for several reasons: # efficiency, correctness, and recursion-safety nactive[] -= 1 diff --git a/src/codegen.cpp b/src/codegen.cpp index 6da83a242f47a..edf2feb936af9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1141,7 +1141,7 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) if (linfo->code == jl_nothing) { // re-infer if we've deleted the code - jl_type_infer(linfo, 0); + linfo = jl_type_infer(linfo, 0); if (linfo->code == jl_nothing) { JL_GC_POP(); return NULL; diff --git a/src/gf.c b/src/gf.c index 0f41429b307e0..4de0b42c7ec47 100644 --- a/src/gf.c +++ b/src/gf.c @@ -111,10 +111,20 @@ static int8_t jl_cachearg_offset(jl_methtable_t *mt) /// ----- Insertion logic for special entries ----- /// -JL_DLLEXPORT jl_typemap_entry_t *jl_specializations_insert(jl_method_t *m, jl_tupletype_t *type, - jl_value_t *value) +JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp); + +// get or create the LambdaInfo for a specialization +JL_DLLEXPORT jl_lambda_info_t *jl_specializations_get_linfo(jl_method_t *m, jl_tupletype_t *type, jl_svec_t *sparams) { - return jl_typemap_insert(&m->specializations, (jl_value_t*)m, type, jl_emptysvec, NULL, jl_emptysvec, value, /*offs*/0, &tfunc_cache, NULL); + jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(m->specializations, type, NULL, 1, /*subtype*/0, /*offs*/0); + if (sf && jl_is_lambda_info(sf->func.value) && ((jl_lambda_info_t*)sf->func.value)->code != jl_nothing) + return (jl_lambda_info_t*)sf->func.value; + jl_lambda_info_t *li = jl_get_specialized(m, type, sparams); + JL_GC_PUSH1(&li); + // TODO: fuse lookup and insert steps + jl_typemap_insert(&m->specializations, (jl_value_t*)m, type, jl_emptysvec, NULL, jl_emptysvec, (jl_value_t*)li, 0, &tfunc_cache, NULL); + JL_GC_POP(); + return li; } JL_DLLEXPORT jl_value_t *jl_specializations_lookup(jl_method_t *m, jl_tupletype_t *type) @@ -154,8 +164,6 @@ jl_value_t *jl_mk_builtin_func(const char *name, jl_fptr_t fptr) return f; } -JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp); - jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method) { // one unspecialized version of a function can be shared among all cached specializations @@ -189,11 +197,11 @@ jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method) } /* - run type inference on lambda "li" in-place, for given argument types. - "def" is the original method definition of which this is an instance; - can be equal to "li->def" if not applicable. + run type inference on lambda "li" for given argument types. + if "li" has been inferred before but the IR was deleted, returns a + new LambdaInfo with the IR reconstituted. */ -void jl_type_infer(jl_lambda_info_t *li, int force) +jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force) { #ifdef ENABLE_INFERENCE jl_module_t *mod = NULL; @@ -207,7 +215,6 @@ void jl_type_infer(jl_lambda_info_t *li, int force) (mod != jl_core_module || !lastIn)))) { // avoid any potential recursion in calling jl_typeinf_func on itself JL_LOCK(&codegen_lock); // Might GC assert(li->inInference == 0); - li->inInference = 1; jl_value_t *fargs[2]; fargs[0] = (jl_value_t*)jl_typeinf_func; fargs[1] = (jl_value_t*)li; @@ -216,12 +223,13 @@ void jl_type_infer(jl_lambda_info_t *li, int force) jl_static_show_func_sig(JL_STDERR, (jl_value_t*)li->specTypes); jl_printf(JL_STDERR, "\n"); #endif - jl_value_t *info = jl_apply(fargs, 2); (void)info; + li = (jl_lambda_info_t*)jl_apply(fargs, 2); assert(li->def || li->inInference == 0); // if this is toplevel expr, make sure inference finished JL_UNLOCK(&codegen_lock); // Might GC } inInference = lastIn; #endif + return li; } static int get_spec_unspec_list(jl_typemap_entry_t *l, void *closure) @@ -610,19 +618,7 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca } // here we infer types and specialize the method - int from_specializations = 1; - // get a specialized version of the method (or reuse an existing one) - if (definition->specializations.unknown != jl_nothing) { - jl_value_t *li = jl_specializations_lookup(definition, type); - if (jl_typeof(li) == (jl_value_t*)jl_lambda_info_type) { - newmeth = (jl_lambda_info_t*)li; - } - } - - if (!newmeth) { - from_specializations = 0; - newmeth = jl_get_specialized(definition, type, sparams); - } + newmeth = jl_specializations_get_linfo(definition, type, sparams); if (cache_with_orig) { // if there is a need to cache with one of the original signatures, @@ -662,15 +658,7 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_typemap_insert(cache, parent, origtype, jl_emptysvec, type, guardsigs, (jl_value_t*)newmeth, jl_cachearg_offset(mt), &lambda_cache, NULL); - if (from_specializations) { - JL_UNLOCK(&codegen_lock); // Might GC - JL_GC_POP(); - return newmeth; - } - - if (newmeth->code != NULL && !newmeth->inferred && !newmeth->inInference) { - // notify type inference of the new method it should infer - jl_specializations_insert(definition, newmeth->specTypes, (jl_value_t*)newmeth); + if (!newmeth->inferred && !newmeth->inInference) { if (jl_options.compile_enabled != JL_OPTIONS_COMPILE_OFF) // don't bother with typeinf if compile is off if (jl_symbol_name(definition->name)[0] != '@') // don't bother with typeinf on macros jl_type_infer(newmeth, 0); @@ -705,9 +693,8 @@ static jl_lambda_info_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t * jl_lambda_info_t *nf; if (!cache) nf = jl_get_specialized(m, sig, env); - else { + else nf = cache_method(mt, &mt->cache, (jl_value_t*)mt, sig, tt, entry, env); - } JL_GC_POP(); return nf; } diff --git a/src/julia_internal.h b/src/julia_internal.h index c13925b82334f..712256f1b03cf 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -69,7 +69,7 @@ STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) return jv; } -void jl_type_infer(jl_lambda_info_t *li, int force); +jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force); void jl_generate_fptr(jl_lambda_info_t *li); void jl_compile_linfo(jl_lambda_info_t *li); jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li); From 71a1883e63eae50cdd033dabbb379924ed671920 Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Wed, 15 Jun 2016 11:42:45 -0700 Subject: [PATCH 0028/1117] Test, document, and export dropzeros[!] for both `SparseMatrixCSC`s and `SparseVector`s. --- base/exports.jl | 2 ++ base/sparse/sparse.jl | 2 +- base/sparse/sparsematrix.jl | 17 +++++++++++ base/sparse/sparsevector.jl | 18 ++++++++++++ doc/stdlib/arrays.rst | 31 ++++++++++++++++++++ test/sparsedir/sparse.jl | 52 +++++++++++++++++++++++++--------- test/sparsedir/sparsevector.jl | 39 +++++++++++++++++++++---- 7 files changed, 140 insertions(+), 21 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index a08da4b723892..61113e9d4e343 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -707,6 +707,8 @@ export # sparse full, + dropzeros, + dropzeros!, # bitarrays falses, diff --git a/base/sparse/sparse.jl b/base/sparse/sparse.jl index cb94702c31b57..9e262e840a5da 100644 --- a/base/sparse/sparse.jl +++ b/base/sparse/sparse.jl @@ -29,7 +29,7 @@ import Base: @get!, acos, acosd, acot, acotd, acsch, asech, asin, asind, asinh, import Base.Broadcast: eltype_plus, broadcast_shape export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, - SparseMatrixCSC, SparseVector, blkdiag, dense, droptol!, dropzeros!, etree, + SparseMatrixCSC, SparseVector, blkdiag, dense, droptol!, dropzeros!, dropzeros, etree, issparse, nonzeros, nzrange, rowvals, sparse, sparsevec, spdiagm, speye, spones, sprand, sprandn, spzeros, symperm, nnz diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index eff407d95293a..a7e195e35be96 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -795,7 +795,24 @@ end droptol!(A::SparseMatrixCSC, tol, trim::Bool = true) = fkeep!(A, (i, j, x) -> abs(x) > tol, trim) +""" + dropzeros!(A::SparseMatrixCSC, trim::Bool = true) + +Removes stored numerical zeros from `A`, optionally trimming resulting excess space from +`A.rowval` and `A.nzval` when `trim` is `true`. + +For an out-of-place version, see [`dropzeros`](:func:`Base.SparseArrays.dropzeros`). For +algorithmic information, see [`Base.SparseArrays.fkeep!`](:func:`Base.SparseArrays.fkeep!`). +""" dropzeros!(A::SparseMatrixCSC, trim::Bool = true) = fkeep!(A, (i, j, x) -> x != 0, trim) +""" + dropzeros(A::SparseMatrixCSC, trim::Bool = true) + +Generates a copy of `A` and removes stored numerical zeros from that copy, optionally +trimming excess space from the result's `rowval` and `nzval` arrays when `trim` is `true`. + +For an in-place version and algorithmic information, see [`dropzeros!`](:func:`Base.SparseArrays.dropzeros!`). +""" dropzeros(A::SparseMatrixCSC, trim::Bool = true) = dropzeros!(copy(A), trim) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index f7987a2b898da..d796a25159df3 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -1689,9 +1689,27 @@ end droptol!(x::SparseVector, tol, trim::Bool = true) = fkeep!(x, (i, x) -> abs(x) > tol, trim) +""" + dropzeros!(x::SparseVector, trim::Bool = true) + +Removes stored numerical zeros from `x`, optionally trimming resulting excess space from +`x.nzind` and `x.nzval` when `trim` is `true`. + +For an out-of-place version, see [`dropzeros`](:func:`Base.SparseArrays.dropzeros`). For +algorithmic information, see [`Base.SparseArrays.fkeep!`](:func:`Base.SparseArrays.fkeep!`). +""" dropzeros!(x::SparseVector, trim::Bool = true) = fkeep!(x, (i, x) -> x != 0, trim) +""" + dropzeros(x::SparseVector, trim::Bool = true) + +Generates a copy of `x` and removes numerical zeros from that copy, optionally trimming +excess space from the result's `nzind` and `nzval` arrays when `trim` is `true`. + +For an in-place version and algorithmic information, see [`dropzeros!`](:func:`Base.SparseArrays.dropzeros!`). +""" dropzeros(x::SparseVector, trim::Bool = true) = dropzeros!(copy(x), trim) + function _fillnonzero!{Tv,Ti}(arr::SparseMatrixCSC{Tv, Ti}, val) m, n = size(arr) resize!(arr.colptr, n+1) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 5c98822ca395c..9a4a5d203bb53 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1036,3 +1036,34 @@ dense counterparts. The following functions are specific to sparse arrays. end end +.. function:: dropzeros!(A::SparseMatrixCSC, trim::Bool = true) + + .. Docstring generated from Julia source + + Removes stored numerical zeros from ``A``\ , optionally trimming resulting excess space from ``A.rowval`` and ``A.nzval`` when ``trim`` is ``true``\ . + + For an out-of-place version, see :func:`Base.SparseArrays.dropzeros`\ . For algorithmic information, see :func:`Base.SparseArrays.fkeep!`\ . + +.. function:: dropzeros(A::SparseMatrixCSC, trim::Bool = true) + + .. Docstring generated from Julia source + + Generates a copy of ``A`` and removes stored numerical zeros from that copy, optionally trimming excess space from the result's ``rowval`` and ``nzval`` arrays when ``trim`` is ``true``\ . + + For an in-place version and algorithmic information, see :func:`Base.SparseArrays.dropzeros!`\ . + +.. function:: dropzeros!(x::SparseVector, trim::Bool = true) + + .. Docstring generated from Julia source + + Removes stored numerical zeros from ``x``\ , optionally trimming resulting excess space from ``x.nzind`` and ``x.nzval`` when ``trim`` is ``true``\ . + + For an out-of-place version, see :func:`Base.SparseArrays.dropzeros`\ . For algorithmic information, see :func:`Base.SparseArrays.fkeep!`\ . + +.. function:: dropzeros(x::SparseVector, trim::Bool = true) + + .. Docstring generated from Julia source + + Generates a copy of ``x`` and removes numerical zeros from that copy, optionally trimming excess space from the result's ``nzind`` and ``nzval`` arrays when ``trim`` is ``true``\ . + + For an in-place version and algorithmic information, see :func:`Base.SparseArrays.dropzeros!`\ . diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 5f4ae28caae14..2eb2ed5ad3634 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -365,11 +365,6 @@ mfe22 = eye(Float64, 2) # issue #5190 @test_throws ArgumentError sparsevec([3,5,7],[0.1,0.0,3.2],4) -# issue #5169 -@test nnz(sparse([1,1],[1,2],[0.0,-0.0])) == 2 -@test nnz(Base.SparseArrays.dropzeros!(sparse([1,1],[1,2],[0.0,-0.0]))) == 0 -# changed from `== 0` to `== 2` and added dropzeros! form in #14798, -# matching sparse()'s behavioral revision discussed in #12605, #9928, #9906, #6769 # issue #5386 K,J,V = findnz(SparseMatrixCSC(2,1,[1,3],[1,2],[1.0,0.0])) @@ -379,11 +374,6 @@ K,J,V = findnz(SparseMatrixCSC(2,1,[1,3],[1,2],[1.0,0.0])) A = speye(Bool, 5) @test find(A) == find(x -> x == true, A) == find(full(A)) -# issue #5437 -@test nnz(sparse([1,2,3],[1,2,3],[0.0,1.0,2.0])) == 3 -@test nnz(Base.SparseArrays.dropzeros!(sparse([1,2,3],[1,2,3],[0.0,1.0,2.0]))) == 2 -# changed from `== 2` to `== 3` and added dropzeros! form in #14798, -# matching sparse()'s behavioral revision discussed in #12605, #9928, #9906, #6769 # issue #5824 @test sprand(4,5,0.5).^0 == sparse(ones(4,5)) @@ -1063,10 +1053,44 @@ perm = randperm(10) @test Base.droptol!(A,0.01).colptr == [1,1,1,2,2,3,4,6,6,7,9] @test isequal(Base.droptol!(sparse([1], [1], [1]), 1), SparseMatrixCSC(1,1,Int[1,1],Int[],Int[])) -# dropzeros -A = sparse([1 2 3; 4 5 6; 7 8 9]) -A.nzval[2] = A.nzval[6] = A.nzval[7] = 0 -@test Base.dropzeros!(A).colptr == [1, 3, 5, 7] +# Test dropzeros[!] +let smalldim = 5, largedim = 10, nzprob = 0.4, targetnumposzeros = 5, targetnumnegzeros = 5 + for (m, n) in ((largedim, largedim), (smalldim, largedim), (largedim, smalldim)) + A = sprand(m, n, nzprob) + struczerosA = find(x -> x == 0, A) + poszerosinds = unique(rand(struczerosA, targetnumposzeros)) + negzerosinds = unique(rand(struczerosA, targetnumnegzeros)) + Aposzeros = setindex!(copy(A), 2, poszerosinds) + Anegzeros = setindex!(copy(A), -2, negzerosinds) + Abothsigns = setindex!(copy(Aposzeros), -2, negzerosinds) + map!(x -> x == 2 ? 0.0 : x, Aposzeros.nzval) + map!(x -> x == -2 ? -0.0 : x, Anegzeros.nzval) + map!(x -> x == 2 ? 0.0 : x == -2 ? -0.0 : x, Abothsigns.nzval) + for Awithzeros in (Aposzeros, Anegzeros, Abothsigns) + # Basic functionality / dropzeros! + @test Base.dropzeros!(copy(Awithzeros)) == A + @test Base.dropzeros!(copy(Awithzeros), false) == A + # Basic functionality / dropzeros + @test Base.SparseArrays.dropzeros(Awithzeros) == A + @test Base.SparseArrays.dropzeros(Awithzeros, false) == A + # Check trimming works as expected + @test length(Base.dropzeros!(copy(Awithzeros)).nzval) == length(A.nzval) + @test length(Base.dropzeros!(copy(Awithzeros)).rowval) == length(A.rowval) + @test length(Base.dropzeros!(copy(Awithzeros), false).nzval) == length(Awithzeros.nzval) + @test length(Base.dropzeros!(copy(Awithzeros), false).rowval) == length(Awithzeros.rowval) + end + end + # original lone dropzeros test + A = sparse([1 2 3; 4 5 6; 7 8 9]) + A.nzval[2] = A.nzval[6] = A.nzval[7] = 0 + @test Base.dropzeros!(A).colptr == [1, 3, 5, 7] + # test for issue #5169, modified for new behavior following #15242/#14798 + @test nnz(sparse([1, 1], [1, 2], [0.0, -0.0])) == 2 + @test nnz(Base.SparseArrays.dropzeros!(sparse([1, 1], [1, 2], [0.0, -0.0]))) == 0 + # test for issue #5437, modified for new behavior following #15242/#14798 + @test nnz(sparse([1, 2, 3], [1, 2, 3], [0.0, 1.0, 2.0])) == 3 + @test nnz(Base.SparseArrays.dropzeros!(sparse([1, 2, 3],[1, 2, 3],[0.0, 1.0, 2.0]))) == 2 +end #trace @test_throws DimensionMismatch trace(sparse(ones(5,6))) diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index fde3585357dd9..7d9f2680b97b4 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -903,12 +903,6 @@ let x = sparsevec(1:7, [3., 2., -1., 1., -2., -3., 3.], 7) Base.droptol!(xdrop, 3.) @test exact_equal(xdrop, SparseVector(7, Int[], Float64[])) - # dropzeros - xdrop = copy(x) - xdrop.nzval[[2, 4, 6]] = 0.0 - Base.SparseArrays.dropzeros!(xdrop) - @test exact_equal(xdrop, SparseVector(7, [1, 3, 5, 7], [3, -1., -2., 3.])) - xdrop = copy(x) # This will keep index 1, 3, 4, 7 in xdrop f_drop(i, x) = (abs(x) == 1.) || (i in [1, 7]) @@ -916,6 +910,39 @@ let x = sparsevec(1:7, [3., 2., -1., 1., -2., -3., 3.], 7) @test exact_equal(xdrop, SparseVector(7, [1, 3, 4, 7], [3., -1., 1., 3.])) end +# dropzeros[!] +let testdims = (10, 20, 30), nzprob = 0.4, targetnumposzeros = 5, targetnumnegzeros = 5 + for m in testdims + v = sprand(m, nzprob) + struczerosv = find(x -> x == 0, v) + poszerosinds = unique(rand(struczerosv, targetnumposzeros)) + negzerosinds = unique(rand(struczerosv, targetnumnegzeros)) + vposzeros = setindex!(copy(v), 2, poszerosinds) + vnegzeros = setindex!(copy(v), -2, negzerosinds) + vbothsigns = setindex!(copy(vposzeros), -2, negzerosinds) + map!(x -> x == 2 ? 0.0 : x, vposzeros.nzval) + map!(x -> x == -2 ? -0.0 : x, vnegzeros.nzval) + map!(x -> x == 2 ? 0.0 : x == -2 ? -0.0 : x, vbothsigns.nzval) + for vwithzeros in (vposzeros, vnegzeros, vbothsigns) + # Basic functionality / dropzeros! + @test dropzeros!(copy(vwithzeros)) == v + @test dropzeros!(copy(vwithzeros), false) == v + # Basic functionality / dropzeros + @test dropzeros(vwithzeros) == v + @test dropzeros(vwithzeros, false) == v + # Check trimming works as expected + @test length(dropzeros!(copy(vwithzeros)).nzval) == length(v.nzval) + @test length(dropzeros!(copy(vwithzeros)).nzind) == length(v.nzind) + @test length(dropzeros!(copy(vwithzeros), false).nzval) == length(vwithzeros.nzval) + @test length(dropzeros!(copy(vwithzeros), false).nzind) == length(vwithzeros.nzind) + end + end + # original dropzeros! test + xdrop = sparsevec(1:7, [3., 2., -1., 1., -2., -3., 3.], 7) + xdrop.nzval[[2, 4, 6]] = 0.0 + Base.SparseArrays.dropzeros!(xdrop) + @test exact_equal(xdrop, SparseVector(7, [1, 3, 5, 7], [3, -1., -2., 3.])) +end # It's tempting to share data between a SparseVector and a SparseArrays, # but if that's done, then modifications to one or the other will cause From 525c1e730b96aefac53b6e5776a6ef4ec3b7bc92 Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Thu, 16 Jun 2016 10:57:28 -0700 Subject: [PATCH 0029/1117] Add targeted test for issue #14816. --- test/sparsedir/sparse.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 2eb2ed5ad3634..706a050be7daf 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -1409,3 +1409,10 @@ end @test !issparse([rand(10,10) rand(10,10)]) @test !issparse([rand(10,10); rand(10,10)]) @test !issparse([rand(10,10) rand(10,10); rand(10,10) rand(10,10)]) + +# issue #14816 +let m = 5 + intmat = fill(1, m, m) + ltintmat = LowerTriangular(rand(1:5, m, m)) + @test isapprox(At_ldiv_B(ltintmat, sparse(intmat)), At_ldiv_B(ltintmat, intmat)) +end \ No newline at end of file From e56bca8b244e3061a31e312743be3ab71ea458da Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Thu, 16 Jun 2016 19:38:36 -0400 Subject: [PATCH 0030/1117] Base.transcode to replace utf8to16 and utf16to8 (#16974) --- base/c.jl | 10 ++++++---- base/env.jl | 4 ++-- base/file.jl | 4 ++-- base/filesystem.jl | 4 ++-- base/interactiveutil.jl | 2 +- base/libc.jl | 8 ++++---- base/path.jl | 4 ++-- test/misc.jl | 24 ++++++++++++------------ 8 files changed, 31 insertions(+), 29 deletions(-) diff --git a/base/c.jl b/base/c.jl index 13e1808fc5883..ee7eb1ab15618 100644 --- a/base/c.jl +++ b/base/c.jl @@ -106,13 +106,15 @@ if is_windows() function cwstring(s::AbstractString) bytes = String(s).data 0 in bytes && throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))")) - return push!(utf8to16(bytes), 0) + return push!(transcode(UInt16, bytes), 0) end end -# conversions between UTF-8 and UTF-16 for Windows APIs +# transcoding between data in UTF-8 and UTF-16 for Windows APIs -function utf8to16(src::Vector{UInt8}) +transcode{T<:Union{UInt8,UInt16}}(::Type{T}, src::Vector{T}) = src + +function transcode(::Type{UInt16}, src::Vector{UInt8}) dst = UInt16[] i, n = 1, length(src) n > 0 || return dst @@ -162,7 +164,7 @@ function utf8to16(src::Vector{UInt8}) return dst end -function utf16to8(src::Vector{UInt16}) +function transcode(::Type{UInt8}, src::Vector{UInt16}) dst = UInt8[] i, n = 1, length(src) n > 0 || return dst diff --git a/base/env.jl b/base/env.jl index f1e0076bc2b66..cb21ecbe0dcf4 100644 --- a/base/env.jl +++ b/base/env.jl @@ -19,7 +19,7 @@ function access_env(onError::Function, str::AbstractString) error(string("getenv: ", str, ' ', len, "-1 != ", ret, ": ", Libc.FormatMessage())) end pop!(val) # NUL - return String(utf16to8(val)) + return String(transcode(UInt8, val)) end function _setenv(svar::AbstractString, sval::AbstractString, overwrite::Bool=true) @@ -97,7 +97,7 @@ function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) buf = Array{UInt16}(len) unsafe_copy!(pointer(buf), pos, len) - env = String(utf16to8(buf)) + env = String(transcode(UInt8, buf)) m = match(r"^(=?[^=]+)=(.*)$"s, env) if m === nothing error("malformed environment entry: $env") diff --git a/base/file.jl b/base/file.jl index b7d2ffbd1a8a7..cf11560eb745b 100644 --- a/base/file.jl +++ b/base/file.jl @@ -203,7 +203,7 @@ function tempdir() error("GetTempPath failed: $(Libc.FormatMessage())") end resize!(temppath,lentemppath) - return String(utf16to8(temppath)) + return String(transcode(UInt8, temppath)) end tempname(uunique::UInt32=UInt32(0)) = tempname(tempdir(), uunique) const temp_prefix = cwstring("jl_") @@ -216,7 +216,7 @@ function tempname(temppath::AbstractString,uunique::UInt32) error("GetTempFileName failed: $(Libc.FormatMessage())") end resize!(tname,lentname) - return String(utf16to8(tname)) + return String(transcode(UInt8, tname)) end function mktemp(parent=tempdir()) filename = tempname(parent, UInt32(0)) diff --git a/base/filesystem.jl b/base/filesystem.jl index 17f2e35f5796f..c43f8bf92836b 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -42,8 +42,8 @@ export File, import Base: UVError, _sizeof_uv_fs, check_open, close, eof, eventloop, fd, isopen, nb_available, position, read, read!, readavailable, seek, seekend, show, - skip, stat, unsafe_read, unsafe_write, utf16to8, utf8to16, uv_error, - uvhandle, uvtype, write + skip, stat, unsafe_read, unsafe_write, transcode, uv_error, uvhandle, + uvtype, write if is_windows() import Base: cwstring diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index d78fffad41608..22821a8c4fd6f 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -150,7 +150,7 @@ elseif is_windows() len = 0 while unsafe_load(plock, len+1) != 0; len += 1; end # get Vector{UInt16}, transcode data to UTF-8, make a String of it - s = String(utf16to8(unsafe_wrap(Array, plock, len))) + s = String(transcode(UInt8, unsafe_wrap(Array, plock, len))) systemerror(:GlobalUnlock, 0==ccall((:GlobalUnlock, "kernel32"), stdcall, Cint, (Ptr{UInt16},), plock)) return s end diff --git a/base/libc.jl b/base/libc.jl index f7c662f3c6dac..6943d457ebde7 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -2,14 +2,14 @@ module Libc +import Base: transcode + export FILE, TmStruct, strftime, strptime, getpid, gethostname, free, malloc, calloc, realloc, - errno, strerror, flush_cstdio, systemsleep, time + errno, strerror, flush_cstdio, systemsleep, time, transcode if is_windows() export GetLastError, FormatMessage end -import Base: utf16to8 - include(string(length(Core.ARGS)>=2?Core.ARGS[2]:"","errno_h.jl")) # include($BUILDROOT/base/errno_h.jl) ## RawFD ## @@ -277,7 +277,7 @@ if is_windows() buf = Array{UInt16}(len) unsafe_copy!(pointer(buf), p, len) ccall(:LocalFree,stdcall,Ptr{Void},(Ptr{Void},),p) - return String(utf16to8(buf)) + return String(transcode(UInt8, buf)) end end diff --git a/base/path.jl b/base/path.jl index c50f3e386981f..85aec28aadb07 100644 --- a/base/path.jl +++ b/base/path.jl @@ -136,7 +136,7 @@ function realpath(path::AbstractString) systemerror(:realpath, n == 0) x = n < length(buf) # is the buffer big enough? resize!(buf, n) # shrink if x, grow if !x - x && return String(utf16to8(buf)) + x && return String(transcode(UInt8, buf)) end end @@ -150,7 +150,7 @@ function longpath(path::AbstractString) systemerror(:longpath, n == 0) x = n < length(buf) # is the buffer big enough? resize!(buf, n) # shrink if x, grow if !x - x && return String(utf16to8(buf)) + x && return String(transcode(UInt8, buf)) end end diff --git a/test/misc.jl b/test/misc.jl index 0fd15cdfb7969..e3ae6b72e8ce6 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -209,11 +209,11 @@ whos(IOBuffer(), Tmp14173) # warm up @test @allocated(whos(IOBuffer(), Tmp14173)) < 10000 ## test conversion from UTF-8 to UTF-16 (for Windows APIs) -import Base: utf8to16, utf16to8 +import Base.Libc: transcode # empty arrays -@test utf8to16(UInt8[]) == UInt16[] -@test utf16to8(UInt16[]) == UInt8[] +@test transcode(UInt16, UInt8[]) == UInt16[] +@test transcode(UInt8, UInt16[]) == UInt8[] # UTF-8-like sequences V8 = [ @@ -304,15 +304,15 @@ I8 = [(s,map(UInt16,s)) for s in X8] for (X,Y,Z) in ((V8,V8,V8), (I8,V8,I8), (V8,I8,V8), (V8,V8,I8), (I8,V8,V8)) for (a8, a16) in X - @test utf8to16(a8) == a16 + @test transcode(UInt16, a8) == a16 for (b8, b16) in Y ab8 = [a8; b8] ab16 = [a16; b16] - @test utf8to16(ab8) == ab16 + @test transcode(UInt16, ab8) == ab16 for (c8, c16) in Z abc8 = [ab8; c8] abc16 = [ab16; c16] - @test utf8to16(abc8) == abc16 + @test transcode(UInt16, abc8) == abc16 end end end @@ -359,18 +359,18 @@ I16 = [ for (X,Y,Z) in ((V16,V16,V16), (I16,V16,I16), (V16,I16,V16), (V16,V16,I16), (I16,V16,V16)) for (a16, a8) in X - @test utf16to8(a16) == a8 - @test utf8to16(a8) == a16 + @test transcode(UInt8, a16) == a8 + @test transcode(UInt16, a8) == a16 for (b16, b8) in Y ab16 = [a16; b16] ab8 = [a8; b8] - @test utf16to8(ab16) == ab8 - @test utf8to16(ab8) == ab16 + @test transcode(UInt8, ab16) == ab8 + @test transcode(UInt16, ab8) == ab16 for (c16, c8) in Z abc16 = [ab16; c16] abc8 = [ab8; c8] - @test utf16to8(abc16) == abc8 - @test utf8to16(abc8) == abc16 + @test transcode(UInt8, abc16) == abc8 + @test transcode(UInt16, abc8) == abc16 end end end From f513b17c0837a53e32065c1fdd31c4684a51e6c7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 31 May 2016 22:36:35 -0400 Subject: [PATCH 0031/1117] add a new Expr head for directly calling a LambdaInfo Expr(:invoke, LambdaInfo, call-args...) is a more primitive form of Expr(:call) for which the dispatch logic has been pre-determined this is not used by lowering, but is used by inference to allowing moving of this logic out of codegen also fix the tfunc for kwfunc, since it was all munged up and making the invoke inlining unhappy also fixes a number of bugs caused by deleting code not preserving various invariants or not testing correctly for them also fixes a number of bugs that have accumulated in --compile=all operation since that configuration stopped being tested on CI --- base/inference.jl | 77 +++++++++++++--------- base/show.jl | 45 ++++++++++++- base/stacktraces.jl | 28 +------- base/strings/string.jl | 5 ++ base/sysimg.jl | 2 +- base/unicode/checkstring.jl | 5 -- src/alloc.c | 3 +- src/cgutils.cpp | 6 +- src/codegen.cpp | 112 +++++++++++++++++++++---------- src/dump.c | 6 +- src/gf.c | 127 +++++++++++++++++++++++------------- src/interpreter.c | 17 +++++ src/intrinsics.cpp | 7 +- src/jltypes.c | 1 + src/julia.h | 4 +- src/julia_internal.h | 1 + src/threading.c | 3 +- src/toplevel.c | 2 +- 18 files changed, 296 insertions(+), 155 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index c74697d6b62a2..eb228bc2fd4b9 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -977,10 +977,8 @@ function abstract_call(f::ANY, fargs, argtypes::Vector{Any}, vtypes::VarTable, s elseif is(f,Core.kwfunc) if length(fargs) == 2 ft = widenconst(argtypes[2]) - if isa(ft,DataType) && !ft.abstract - if isdefined(ft.name.mt, :kwsorter) - return typeof(ft.name.mt.kwsorter) - end + if isa(ft,DataType) && isdefined(ft.name, :mt) && isdefined(ft.name.mt, :kwsorter) + return Const(ft.name.mt.kwsorter) end end return Any @@ -1017,7 +1015,7 @@ function abstract_call(f::ANY, fargs, argtypes::Vector{Any}, vtypes::VarTable, s return abstract_call_gf_by_type(f, atype, sv) end -function abstract_eval_call(e, vtypes::VarTable, sv::InferenceState) +function abstract_eval_call(e::Expr, vtypes::VarTable, sv::InferenceState) argtypes = Any[abstract_eval(a, vtypes, sv) for a in e.args] #print("call ", e.args[1], argtypes, "\n\n") for x in argtypes @@ -1548,16 +1546,17 @@ function typeinf_ext(linfo::LambdaInfo) # This case occurs when the IR for a function has been deleted. # `code` will be a newly-created LambdaInfo, and we need to copy its # contents to the existing one to copy the info to the method cache. - linfo.inferred = true - linfo.inInference = false + linfo.inInference = true linfo.code = code.code linfo.slotnames = code.slotnames linfo.slottypes = code.slottypes linfo.slotflags = code.slotflags linfo.ssavaluetypes = code.ssavaluetypes - linfo.rettype = code.rettype linfo.pure = code.pure linfo.inlineable = code.inlineable + ccall(:jl_set_lambda_rettype, Void, (Any, Any), linfo, code.rettype) + linfo.inferred = true + linfo.inInference = false end return code else @@ -1931,15 +1930,15 @@ function finish(me::InferenceState) # determine and cache inlineability me.linfo.inlineable = isinlineable(me.linfo) - me.linfo.inferred = true - me.linfo.inInference = false - me.linfo.pure = ispure if isdefined(me.linfo, :def) # compress code for non-toplevel thunks compressedtree = ccall(:jl_compress_ast, Any, (Any,Any), me.linfo, me.linfo.code) me.linfo.code = compressedtree end - me.linfo.rettype = me.bestguess + me.linfo.pure = ispure + ccall(:jl_set_lambda_rettype, Void, (Any, Any), me.linfo, me.bestguess) + me.linfo.inferred = true + me.linfo.inInference = false # lazy-delete the item from active for several reasons: # efficiency, correctness, and recursion-safety @@ -2310,9 +2309,10 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end # special-case inliners for known pure functions that compute types if isType(e.typ) && !has_typevars(e.typ.parameters[1],true) - if (is(f,apply_type) || is(f,fieldtype) || is(f,typeof) || + if (is(f, apply_type) || is(f, fieldtype) || is(f, typeof) || istopfunction(topmod, f, :typejoin) || istopfunction(topmod, f, :promote_type)) + # XXX: compute effect_free for the actual arguments if length(argexprs) < 2 || effect_free(argexprs[2], sv, true) return (e.typ.parameters[1],()) else @@ -2320,17 +2320,36 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end end end - if isa(f,IntrinsicFunction) || ft ⊑ IntrinsicFunction + if is(f, Core.kwfunc) && length(argexprs) == 2 && isa(e.typ, Const) + if effect_free(argexprs[2], sv, true) + return (e.typ.val, ()) + else + return (e.typ.val, Any[argexprs[2]]) + end + end + if isa(f, IntrinsicFunction) || ft ⊑ IntrinsicFunction || + isa(f, Builtin) || ft ⊑ Builtin return NF end - atype = argtypes_to_type(atypes) - if length(atype.parameters) - 1 > MAX_TUPLETYPE_LEN - atype = limit_tuple_type(atype) + atype_unlimited = argtypes_to_type(atypes) + if length(atype_unlimited.parameters) - 1 > MAX_TUPLETYPE_LEN + atype = limit_tuple_type(atype_unlimited) + else + atype = atype_unlimited + end + function invoke_NF() + # converts a :call to :invoke + cache_linfo = ccall(:jl_get_spec_lambda, Any, (Any,), atype_unlimited) + if cache_linfo !== nothing + e.head = :invoke + unshift!(e.args, cache_linfo) + end + return NF end meth = _methods_by_ftype(atype, 1) if meth === false || length(meth) != 1 - return NF + return invoke_NF() end meth = meth[1]::SimpleVector metharg = meth[1]::Type @@ -2364,7 +2383,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference if !inline_incompletematch_allowed || !isdefined(Main,:Base) # provide global disable if this optimization is not desirable # need Main.Base defined for MethodError - return NF + return invoke_NF() end end @@ -2395,9 +2414,10 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference # end (linfo, ty, inferred) = typeinf(method, metharg, methsp, false) - if !inferred || linfo === nothing - return NF - elseif !linfo.inlineable + if linfo === nothing || !inferred + return invoke_NF() + end + if !linfo.inlineable # TODO #= if incompletematch @@ -2420,15 +2440,15 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference body.args = Any[Expr(:return, newcall)] ast = Expr(:lambda, newnames, Any[[], locals, [], 0], body) else - return NF + return invoke_NF() end =# - return NF + return invoke_NF() elseif linfo.code === nothing (linfo, ty, inferred) = typeinf(method, metharg, methsp, true) end - if linfo === nothing || !inferred || !linfo.inlineable - return NF + if linfo === nothing || !inferred || !linfo.inlineable || (ast = linfo.code) === nothing + return invoke_NF() end na = linfo.nargs @@ -2468,10 +2488,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference methargs = metharg.parameters nm = length(methargs) - ast = linfo.code - if ast === nothing - return NF - elseif !isa(ast,Array{Any,1}) + if !isa(ast, Array{Any,1}) ast = ccall(:jl_uncompress_ast, Any, (Any,Any), linfo, ast) else ast = copy_exprargs(ast) diff --git a/base/show.jl b/base/show.jl index 492dda654f4b4..74ea4ea153449 100644 --- a/base/show.jl +++ b/base/show.jl @@ -276,7 +276,15 @@ end function show(io::IO, l::LambdaInfo) if isdefined(l, :def) - println(io, "LambdaInfo for ", l.def.name) + if (l === l.def.lambda_template) + print(io, "LambdaInfo template for ") + show(io, l.def) + println(io) + else + print(io, "LambdaInfo for ") + show_lambda_types(io, l.specTypes.parameters) + println(io) + end else println(io, "Toplevel LambdaInfo thunk") end @@ -946,11 +954,40 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) show(io, ex.head) for arg in args print(io, ", ") - show(io, arg) + if isa(arg, LambdaInfo) && isdefined(arg, :specTypes) + show_lambda_types(io, arg.specTypes.parameters) + else + show(io, arg) + end end print(io, "))") end show_type && show_expr_type(io, ex.typ, emphstate) + nothing +end + +function show_lambda_types(io::IO, sig::SimpleVector) + # print a method signature tuple + ft = sig[1] + if ft <: Function && isempty(ft.parameters) && + isdefined(ft.name.module, ft.name.mt.name) && + ft == typeof(getfield(ft.name.module, ft.name.mt.name)) + print(io, ft.name.mt.name) + elseif isa(ft, DataType) && is(ft.name, Type.name) && isleaftype(ft) + f = ft.parameters[1] + print(io, f) + else + print(io, "(::", ft, ")") + end + first = true + print(io, '(') + for i = 2:length(sig) # fixme (iter): `eachindex` with offset? + first || print(io, ", ") + first = false + print(io, "::", sig[i]) + end + print(io, ')') + nothing end function ismodulecall(ex::Expr) @@ -988,6 +1025,7 @@ function show(io::IO, tv::TypeVar) print(io, "<:") show(io, tv.ub) end + nothing end function dump(io::IO, x::SimpleVector, n::Int, indent) @@ -1007,6 +1045,7 @@ function dump(io::IO, x::SimpleVector, n::Int, indent) end end end + nothing end function dump(io::IO, x::ANY, n::Int, indent) @@ -1068,6 +1107,7 @@ function dump(io::IO, x::Array, n::Int, indent) end end end + nothing end dump(io::IO, x::Symbol, n::Int, indent) = print(io, typeof(x), " ", x) @@ -1089,6 +1129,7 @@ function dump(io::IO, x::DataType, n::Int, indent) end end end + nothing end # dumptype is for displaying abstract type hierarchies like Jameson diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 04eb029641073..9be7b64ca9195 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -162,32 +162,8 @@ function show_spec_linfo(io::IO, frame::StackFrame) end else linfo = get(frame.linfo) - params = - if isdefined(linfo, :specTypes) - linfo.specTypes.parameters - else - nothing - end - if params !== nothing - ft = params[1] - if ft <: Function && isempty(ft.parameters) && - isdefined(ft.name.module, ft.name.mt.name) && - ft == typeof(getfield(ft.name.module, ft.name.mt.name)) - print(io, ft.name.mt.name) - elseif isa(ft, DataType) && is(ft.name, Type.name) && isleaftype(ft) - f = ft.parameters[1] - print(io, f) - else - print(io, "(::", ft, ")") - end - first = true - print(io, '(') - for i = 2:length(params) - first || print(io, ", ") - first = false - print(io, "::", params[i]) - end - print(io, ')') + if isdefined(linfo, :specTypes) + Base.show_lambda_types(io, linfo.specTypes.parameters) else print(io, linfo.name) end diff --git a/base/strings/string.jl b/base/strings/string.jl index d9051d8cbc5a1..57dabebe6925a 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -9,6 +9,11 @@ ## basic UTF-8 decoding & iteration ## +is_surrogate_lead(c::Unsigned) = ((c & ~0x003ff) == 0xd800) +is_surrogate_trail(c::Unsigned) = ((c & ~0x003ff) == 0xdc00) +is_surrogate_codeunit(c::Unsigned) = ((c & ~0x007ff) == 0xd800) +is_valid_continuation(c) = ((c & 0xc0) == 0x80) + const utf8_offset = [ 0x00000000, 0x00003080, 0x000e2080, 0x03c82080, diff --git a/base/sysimg.jl b/base/sysimg.jl index 7f7067759f9ea..dac5f9c4b1727 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -136,6 +136,7 @@ include("iobuffer.jl") # strings & printing include("char.jl") +include("intfuncs.jl") include("strings/strings.jl") include("unicode/unicode.jl") include("parse.jl") @@ -151,7 +152,6 @@ using .Libc: getpid, gethostname, time include("libdl.jl") using .Libdl: DL_LOAD_PATH include("env.jl") -include("intfuncs.jl") # nullable types include("nullable.jl") diff --git a/base/unicode/checkstring.jl b/base/unicode/checkstring.jl index 50bece61b683d..72e9eb9d31062 100644 --- a/base/unicode/checkstring.jl +++ b/base/unicode/checkstring.jl @@ -3,11 +3,6 @@ ## Functions to check validity of UTF-8, UTF-16, and UTF-32 encoded strings, # and also to return information necessary to convert to other encodings -is_surrogate_lead(c::Unsigned) = ((c & ~0x003ff) == 0xd800) -is_surrogate_trail(c::Unsigned) = ((c & ~0x003ff) == 0xdc00) -is_surrogate_codeunit(c::Unsigned) = ((c & ~0x007ff) == 0xd800) -is_valid_continuation(c) = ((c & 0xc0) == 0x80) - ## Return flags for check_string function const UTF_LONG = 1 ##< Long encodings are present diff --git a/src/alloc.c b/src/alloc.c index 78afa45f542aa..5c3541b71200f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -71,7 +71,8 @@ jl_value_t *jl_memory_exception; jl_value_t *jl_readonlymemory_exception; union jl_typemap_t jl_cfunction_list; -jl_sym_t *call_sym; jl_sym_t *dots_sym; +jl_sym_t *call_sym; jl_sym_t *invoke_sym; +jl_sym_t *dots_sym; jl_sym_t *module_sym; jl_sym_t *slot_sym; jl_sym_t *empty_sym; jl_sym_t *export_sym; jl_sym_t *import_sym; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 72de27e164378..491ccddf3ee7b 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1355,12 +1355,12 @@ static Value *init_bits_value(Value *newv, Value *jt, Value *v, MDNode *tbaa) return newv; } static Value *as_value(Type *t, const jl_cgval_t&); -static Value *init_bits_cgval(Value *newv, const jl_cgval_t& v, MDNode *tbaa, Type *t, jl_codectx_t *ctx) +static Value *init_bits_cgval(Value *newv, const jl_cgval_t& v, MDNode *tbaa, jl_codectx_t *ctx) { Value *jt = literal_pointer_val(v.typ); if (v.ispointer()) { init_tag(newv, jt); - builder.CreateMemCpy(newv, data_pointer(v,ctx,PointerType::get(t,0)), jl_datatype_size(v.typ), sizeof(void*)); + builder.CreateMemCpy(newv, data_pointer(v, ctx, T_pint8), jl_datatype_size(v.typ), sizeof(void*)); return newv; } else { @@ -1530,7 +1530,7 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) return literal_pointer_val(jb->instance); } else { - box = init_bits_cgval(emit_allocobj(jl_datatype_size(jt)), vinfo, jb->mutabl ? tbaa_mutab : tbaa_immut, t, ctx); + box = init_bits_cgval(emit_allocobj(jl_datatype_size(jt)), vinfo, jb->mutabl ? tbaa_mutab : tbaa_immut, ctx); } if (gcrooted) { diff --git a/src/codegen.cpp b/src/codegen.cpp index edf2feb936af9..2603462ab1ad3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -370,6 +370,7 @@ static Function *jlcopyast_func; static Function *jltuple_func; static Function *jlnsvec_func; static Function *jlapplygeneric_func; +static Function *jlinvoke_func; static Function *jlapply2va_func; static Function *jlgetfield_func; static Function *jlmethod_func; @@ -868,7 +869,8 @@ static void to_function(jl_lambda_info_t *li) // if not inlineable, code won't be needed again if (JL_DELETE_NON_INLINEABLE && - li->def && li->inferred && !li->inlineable && !jl_options.outputji) { + li->def && li->inferred && !li->inlineable && + li != li->def->lambda_template && !jl_options.outputji) { li->code = jl_nothing; li->slottypes = jl_nothing; li->ssavaluetypes = jl_box_long(jl_array_len(li->ssavaluetypes)); jl_gc_wb(li, li->ssavaluetypes); @@ -1014,28 +1016,6 @@ extern "C" void jl_generate_fptr(jl_lambda_info_t *li) JL_UNLOCK(&codegen_lock); // Might GC } -extern "C" jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) -{ - if (li->inInference || li->inCompile) { - // if inference is running on this function, get a copy - // of the function to be compiled without inference and run. - assert(li->def != NULL); - li = jl_get_unspecialized(li); - } - if (li->fptr == NULL) { - if (li->functionObjectsDecls.functionObject == NULL) { - if (li->code == jl_nothing) - jl_type_infer(li, 0); - if (li->functionObjectsDecls.functionObject == NULL && li->code == jl_nothing) { - li = jl_get_unspecialized(li); - } - jl_compile_linfo(li); - } - jl_generate_fptr(li); - } - return li; -} - // this generates llvm code for the lambda info // (and adds it to the shadow module), but doesn't yet compile // or generate object code for it @@ -1138,17 +1118,22 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) JL_GC_POP(); return NULL; } + // make sure to compile this normally first, + // since `emit_function` doesn't handle recursive compilation correctly + linfo = jl_compile_for_dispatch(linfo); - if (linfo->code == jl_nothing) { - // re-infer if we've deleted the code - linfo = jl_type_infer(linfo, 0); + if (!getdeclarations) { if (linfo->code == jl_nothing) { - JL_GC_POP(); - return NULL; + // re-infer if we've deleted the code + // XXX: this only /partially/ corrupts linfo + // (by confusing the compiler about the + // validity of the code it already generated) + linfo = jl_type_infer(linfo, 0); + if (linfo->code == jl_nothing || linfo->inInference) { + JL_GC_POP(); + return NULL; + } } - } - - if (!getdeclarations) { // emit this function into a new module Function *f, *specf; jl_llvm_functions_t declarations; @@ -1185,7 +1170,6 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) return specf; } } - jl_compile_linfo(linfo); Function *llvmf; if (!getwrapper && linfo->functionObjectsDecls.specFunctionObject != NULL) { llvmf = (Function*)linfo->functionObjectsDecls.specFunctionObject; @@ -2713,15 +2697,53 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval expr_type(callexpr, ctx), ctx); } +static jl_cgval_t emit_invoke(jl_expr_t *ex, jl_codectx_t *ctx) +{ + jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); + size_t arglen = jl_array_dim0(ex->args); + size_t nargs = arglen - 1; + assert(arglen >= 2); + jl_lambda_info_t *li = (jl_lambda_info_t*)args[0]; + assert(jl_is_lambda_info(li)); + + jl_cgval_t result; + if (li->functionObjectsDecls.functionObject == NULL) { + assert(!li->inCompile); + if (li->code == jl_nothing && !li->inInference && li->inferred) { + // XXX: it was inferred in the past, so it's almost valid to re-infer it now + jl_type_infer(li, 0); + } + if (!li->inInference && li->inferred && li->code != jl_nothing) { + jl_compile_linfo(li); + } + } + Value *theFptr = (Value*)li->functionObjectsDecls.functionObject; + if (theFptr) { + jl_cgval_t fval = emit_expr(args[1], ctx); + result = emit_call_function_object(li, fval, theFptr, &args[1], nargs - 1, (jl_value_t*)ex, ctx); + } + else { + result = mark_julia_type(emit_jlcall(prepare_call(jlinvoke_func), literal_pointer_val((jl_value_t*)li), + &args[1], nargs, ctx), + true, expr_type((jl_value_t*)ex, ctx), ctx); + } + + if (result.typ == jl_bottom_type) { + CreateTrap(builder); + } + return result; +} + static jl_cgval_t emit_call(jl_expr_t *ex, jl_codectx_t *ctx) { jl_value_t *expr = (jl_value_t*)ex; jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); size_t arglen = jl_array_dim0(ex->args); size_t nargs = arglen - 1; + assert(arglen >= 1); Value *theFptr = NULL; - jl_cgval_t result; jl_value_t *aty = NULL; + jl_cgval_t result; jl_function_t *f = (jl_function_t*)static_eval(args[0], ctx, true); JL_GC_PUSH2(&f, &aty); @@ -2765,7 +2787,8 @@ static jl_cgval_t emit_call(jl_expr_t *ex, jl_codectx_t *ctx) jl_sprint((jl_value_t*)aty)); }*/ jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)aty); - if (li != NULL) { + if (li != NULL && !li->inInference) { + jl_compile_linfo(li); assert(li->functionObjectsDecls.functionObject != NULL); theFptr = (Value*)li->functionObjectsDecls.functionObject; jl_cgval_t fval; @@ -3207,6 +3230,9 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) builder.CreateCondBr(isfalse, ifnot, ifso); builder.SetInsertPoint(ifso); } + else if (head == invoke_sym) { + return emit_invoke(ex, ctx); + } else if (head == call_sym) { if (ctx->linfo->def) { // don't bother codegen constant-folding for toplevel jl_value_t *c = static_eval(expr, ctx, true, true); @@ -3532,9 +3558,18 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t jl_error("va_arg syntax not allowed for cfunction argument list"); const char *name = "cfunction"; + // try to look up this function for direct invoking jl_lambda_info_t *lam = jl_get_specialization1((jl_tupletype_t*)sigt); jl_value_t *astrt = (jl_value_t*)jl_any_type; + // infer it first, if necessary + if (lam && lam->inInference) + lam = NULL; // TODO: use emit_invoke framework to dispatch these + if (lam && (lam->code == jl_nothing || !lam->inferred)) + jl_type_infer(lam, 0); + if (lam && (lam->inInference || !lam->inferred)) + lam = NULL; // TODO: use emit_invoke framework to dispatch these if (lam != NULL) { + jl_compile_linfo(lam); name = jl_symbol_name(lam->def->name); astrt = lam->rettype; if (astrt != (jl_value_t*)jl_bottom_type && @@ -5360,6 +5395,15 @@ static void init_julia_llvm_env(Module *m) "jl_apply_generic", m); add_named_global(jlapplygeneric_func, &jl_apply_generic); + std::vector invokeargs(0); + invokeargs.push_back(T_pjlvalue); + invokeargs.push_back(T_ppjlvalue); + invokeargs.push_back(T_uint32); + jlinvoke_func = Function::Create(FunctionType::get(T_pjlvalue, invokeargs, false), + Function::ExternalLinkage, + "jl_invoke", m); + add_named_global(jlinvoke_func, &jl_invoke); + std::vector exp_args(0); exp_args.push_back(T_int1); expect_func = Intrinsic::getDeclaration(m, Intrinsic::expect, exp_args); diff --git a/src/dump.c b/src/dump.c index d75522ab8cb0d..4b26f2ad40947 100644 --- a/src/dump.c +++ b/src/dump.c @@ -867,7 +867,6 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_serialize_value(s, (jl_value_t*)li->sparam_syms); jl_serialize_value(s, (jl_value_t*)li->sparam_vals); jl_serialize_value(s, (jl_value_t*)li->specTypes); - jl_serialize_value(s, (jl_value_t*)li->unspecialized_ducttape); write_int8(s, li->inferred); write_int8(s, li->pure); write_int8(s, li->inlineable); @@ -1501,8 +1500,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_gc_wb(li, li->sparam_vals); li->specTypes = (jl_tupletype_t*)jl_deserialize_value(s, (jl_value_t**)&li->specTypes); if (li->specTypes) jl_gc_wb(li, li->specTypes); - li->unspecialized_ducttape = (jl_lambda_info_t*)jl_deserialize_value(s, (jl_value_t**)&li->unspecialized_ducttape); - if (li->unspecialized_ducttape) jl_gc_wb(li, li->unspecialized_ducttape); + li->unspecialized_ducttape = NULL; li->inferred = read_int8(s); li->pure = read_int8(s); li->inlineable = read_int8(s); @@ -2413,7 +2411,7 @@ void jl_init_serializer(void) // everything above here represents a class of object rather than only a literal jl_emptysvec, jl_emptytuple, jl_false, jl_true, jl_nothing, jl_any_type, - call_sym, goto_ifnot_sym, return_sym, body_sym, line_sym, + call_sym, invoke_sym, goto_ifnot_sym, return_sym, body_sym, line_sym, lambda_sym, jl_symbol("tuple"), assign_sym, // empirical list of very common symbols diff --git a/src/gf.c b/src/gf.c index 4de0b42c7ec47..98a50b4694296 100644 --- a/src/gf.c +++ b/src/gf.c @@ -24,6 +24,11 @@ extern "C" { #endif +JL_DLLEXPORT jl_value_t *jl_invoke(jl_lambda_info_t *meth, jl_value_t **args, uint32_t nargs) +{ + return jl_call_method_internal(meth, args, nargs); +} + /// ----- Handling for Julia callbacks ----- /// int in_pure_callback; @@ -232,6 +237,20 @@ jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force) return li; } +JL_DLLEXPORT void jl_set_lambda_rettype(jl_lambda_info_t *li, jl_value_t *rettype) +{ + // changing rettype changes the llvm signature, + // so clear all of the llvm state at the same time + assert(li->inInference); + li->rettype = rettype; + jl_gc_wb(li, rettype); + li->functionObjectsDecls.functionObject = NULL; + li->functionObjectsDecls.specFunctionObject = NULL; + li->functionID = 0; + li->specFunctionID = 0; + li->jlcall_api = 0; +} + static int get_spec_unspec_list(jl_typemap_entry_t *l, void *closure) { if (jl_is_lambda_info(l->func.value) && !l->func.linfo->inferred) @@ -658,11 +677,6 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_typemap_insert(cache, parent, origtype, jl_emptysvec, type, guardsigs, (jl_value_t*)newmeth, jl_cachearg_offset(mt), &lambda_cache, NULL); - if (!newmeth->inferred && !newmeth->inInference) { - if (jl_options.compile_enabled != JL_OPTIONS_COMPILE_OFF) // don't bother with typeinf if compile is off - if (jl_symbol_name(definition->name)[0] != '@') // don't bother with typeinf on macros - jl_type_infer(newmeth, 0); - } JL_UNLOCK(&codegen_lock); // Might GC if (definition->traced && jl_method_tracer) jl_call_tracer(jl_method_tracer, (jl_value_t*)newmeth); @@ -1037,6 +1051,38 @@ jl_lambda_info_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int include_ambiguous); +jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) +{ + if (li->functionObjectsDecls.functionObject == NULL) { + if (li->inInference || li->inCompile) { + // if inference is running on this function, get a copy + // of the function to be compiled without inference and run. + assert(li->def != NULL); + li = jl_get_unspecialized(li); + } + else { + if (li->code == jl_nothing || + (!li->inferred && li->def != NULL && jl_symbol_name(li->def->name)[0] != '@')) { + // don't bother with typeinf on macros or toplevel thunks + jl_type_infer(li, 0); + } + if (li->functionObjectsDecls.functionObject == NULL) { + if (li->inInference || li->inCompile || li->code == jl_nothing) { + // if inference is running on this function, get a copy + // of the function to be compiled without inference and run. + assert(li->def != NULL); + li = jl_get_unspecialized(li); + } + } + } + if (li->functionObjectsDecls.functionObject == NULL) { // check again, because jl_type_infer may have compiled it + jl_compile_linfo(li); + } + } + jl_generate_fptr(li); + return li; +} + // compile-time method lookup jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types) { @@ -1047,7 +1093,7 @@ jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types) // make sure exactly 1 method matches (issue #7302). int i; - for(i=0; i < jl_nparams(types); i++) { + for (i = 0; i < jl_nparams(types); i++) { jl_value_t *ti = jl_tparam(types, i); // if one argument type is DataType, multiple Type{} definitions // might match. also be conservative with tuples rather than trying @@ -1068,36 +1114,38 @@ jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types) JL_TRY { sf = jl_method_lookup_by_type(mt, types, 1, 1); } JL_CATCH { - goto not_found; - } - if (sf != NULL) { - jl_method_t *m = sf->def; - if (jl_has_call_ambiguities(types, m)) { - goto not_found; - } + sf = NULL; } - if (sf == NULL || sf->code == NULL || sf->inInference) - goto not_found; - if (sf->functionObjectsDecls.functionObject == NULL) { - if (sf->fptr != NULL) - goto not_found; - if (sf->code == jl_nothing) { - jl_type_infer(sf, 0); - if (sf->code == jl_nothing) - goto not_found; - } - jl_compile_linfo(sf); + if (sf == NULL || sf->code == NULL || + jl_has_call_ambiguities(types, sf->def)) { + sf = NULL; } JL_GC_POP(); return sf; - not_found: - JL_GC_POP(); - return NULL; } JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) { - return jl_get_specialization1(types) != NULL; + jl_lambda_info_t *li = jl_get_specialization1(types); + if (li == NULL) + return 0; + if (li->functionObjectsDecls.functionObject == NULL) { + assert(!li->inCompile); + if (li->inInference) + return 0; + if (li->code == jl_nothing || !li->inferred) + jl_type_infer(li, 0); + if (li->inInference || li->code == jl_nothing) + return 0; + jl_compile_linfo(li); + } + return 1; +} + +JL_DLLEXPORT jl_value_t *jl_get_spec_lambda(jl_tupletype_t *types) +{ + jl_value_t *li = (jl_value_t*)jl_get_specialization1(types); + return li ? li : jl_nothing; } int jl_has_call_ambiguities(jl_tupletype_t *types, jl_method_t *m) @@ -1159,9 +1207,9 @@ static int _compile_all_tvar_union(jl_tupletype_t *methsig, jl_svec_t *tvars) if (jl_is_leaf_type((jl_value_t*)methsig)) { // usually can create a specialized version of the function, // if the signature is already a leaftype - jl_lambda_info_t *spec = jl_get_specialization1(methsig); - if (spec) + if (jl_compile_hint(methsig)) { return 1; + } } return 0; } @@ -1192,7 +1240,7 @@ static int _compile_all_tvar_union(jl_tupletype_t *methsig, jl_svec_t *tvars) goto getnext; // signature wouldn't be callable / is invalid -- skip it } if (jl_is_leaf_type(sig)) { - if (jl_get_specialization1((jl_tupletype_t*)sig)) { + if (jl_compile_hint((jl_tupletype_t*)sig)) { if (!jl_has_typevars((jl_value_t*)sig)) goto getnext; // success } } @@ -1300,7 +1348,7 @@ static void _compile_all_deq(jl_array_t *found) jl_lambda_info_t *linfo = NULL; JL_GC_PUSH1(&linfo); for (found_i = 0; found_i < found_l; found_i++) { - if (found_i % (found_l / 300) == 0 || found_i == found_l - 1) // show 300 progress steps, to show progress without overwhelming log files + if (found_i % (1 + found_l / 300) == 0 || found_i == found_l - 1) // show 300 progress steps, to show progress without overwhelming log files jl_printf(JL_STDERR, " %d / %d\r", found_i + 1, found_l); jl_typemap_entry_t *ml = (jl_typemap_entry_t*)jl_array_ptr_ref(found, found_i); if (ml->func.value == NULL) @@ -1310,10 +1358,7 @@ static void _compile_all_deq(jl_array_t *found) if (jl_is_method(ml->func.value)) { // type infer a copy of the template, to avoid modifying the template itself templ = ml->func.method->lambda_template; - if (templ->unspecialized_ducttape) - linfo = templ->unspecialized_ducttape; // TODO: switch to using the ->tfunc field to store/retrieve this - else - linfo = jl_get_specialized(ml->func.method, ml->sig, jl_emptysvec); + linfo = jl_specializations_get_linfo(ml->func.method, ml->sig, jl_emptysvec); } else if (jl_is_lambda_info(ml->func.value)) { templ = ml->func.linfo; @@ -1323,14 +1368,9 @@ static void _compile_all_deq(jl_array_t *found) continue; // this should be unreachable } - if (!linfo->inferred) { + if (!linfo->inferred || linfo->code == jl_nothing) { // force this function to be recompiled jl_type_infer(linfo, 1); - linfo->functionObjectsDecls.functionObject = NULL; - linfo->functionObjectsDecls.specFunctionObject = NULL; - linfo->functionID = 0; - linfo->specFunctionID = 0; - linfo->jlcall_api = 0; } // keep track of whether all possible signatures have been cached (and thus whether it can skip trying to compile the template function) @@ -1345,13 +1385,12 @@ static void _compile_all_deq(jl_array_t *found) jl_compile_linfo(linfo); assert(linfo->functionID > 0); if (linfo != templ) { - // copy the function pointer back to the template + // copy the function pointer back to the lambda_template templ->functionObjectsDecls.functionObject = linfo->functionObjectsDecls.functionObject; templ->functionObjectsDecls.specFunctionObject = linfo->functionObjectsDecls.specFunctionObject; templ->functionID = linfo->functionID; templ->specFunctionID = linfo->specFunctionID; templ->jlcall_api = linfo->jlcall_api; - templ->unspecialized_ducttape = linfo; jl_gc_wb(templ, linfo); } } diff --git a/src/interpreter.c b/src/interpreter.c index 2c775124207ff..dc7f4f9d30679 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -67,6 +67,20 @@ static jl_value_t *do_call(jl_value_t **args, size_t nargs, interpreter_state *s return result; } +static jl_value_t *do_invoke(jl_value_t **args, size_t nargs, interpreter_state *s) +{ + jl_value_t **argv; + JL_GC_PUSHARGS(argv, nargs - 1); + size_t i; + for (i = 1; i < nargs; i++) + argv[i - 1] = eval(args[i], s); + jl_lambda_info_t *meth = (jl_lambda_info_t*)args[0]; + assert(jl_is_lambda_info(meth) && !meth->inInference); + jl_value_t *result = jl_call_method_internal(meth, argv, nargs - 1); + JL_GC_POP(); + return result; +} + jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e) { jl_value_t *v = jl_get_global(m, e); @@ -173,6 +187,9 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) if (ex->head == call_sym) { return do_call(args, nargs, s); } + else if (ex->head == invoke_sym) { + return do_invoke(args, nargs, s); + } else if (ex->head == new_sym) { jl_value_t *thetype = eval(args[0], s); jl_value_t *v=NULL; diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 2ac9c4c91dd6e..a16a774c2373f 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -505,7 +505,12 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx vx = builder.CreateBitCast(vx, llvmt); } - return mark_julia_type(vx, false, bt, ctx); + if (jl_is_leaf_type(bt)) + return mark_julia_type(vx, false, bt, ctx); + else + return mark_julia_type( + init_bits_value(emit_allocobj(nb), boxed(bt_value, ctx), vx, tbaa_immut), + true, bt, ctx); } // put a bits type tag on some value diff --git a/src/jltypes.c b/src/jltypes.c index 21990589d43da..3b889ee6665ac 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3809,6 +3809,7 @@ void jl_init_types(void) empty_sym = jl_symbol(""); call_sym = jl_symbol("call"); + invoke_sym = jl_symbol("invoke"); quote_sym = jl_symbol("quote"); inert_sym = jl_symbol("inert"); top_sym = jl_symbol("top"); diff --git a/src/julia.h b/src/julia.h index 1c5155bf38836..4604fbff2c0cc 100644 --- a/src/julia.h +++ b/src/julia.h @@ -545,7 +545,8 @@ extern JL_DLLEXPORT jl_value_t *jl_false; extern JL_DLLEXPORT jl_value_t *jl_nothing; // some important symbols -extern jl_sym_t *call_sym; extern jl_sym_t *empty_sym; +extern jl_sym_t *call_sym; extern jl_sym_t *invoke_sym; +extern jl_sym_t *empty_sym; extern jl_sym_t *dots_sym; extern jl_sym_t *vararg_sym; extern jl_sym_t *quote_sym; extern jl_sym_t *newvar_sym; extern jl_sym_t *top_sym; extern jl_sym_t *dot_sym; @@ -1379,6 +1380,7 @@ STATIC_INLINE int jl_vinfo_usedundef(uint8_t vi) // calling into julia --------------------------------------------------------- JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs); +JL_DLLEXPORT jl_value_t *jl_invoke(jl_lambda_info_t *meth, jl_value_t **args, uint32_t nargs); STATIC_INLINE jl_value_t *jl_apply(jl_value_t **args, uint32_t nargs) diff --git a/src/julia_internal.h b/src/julia_internal.h index 712256f1b03cf..91093cca95902 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -72,6 +72,7 @@ STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force); void jl_generate_fptr(jl_lambda_info_t *li); void jl_compile_linfo(jl_lambda_info_t *li); +JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types); jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li); // invoke (compiling if necessary) the jlcall function pointer for a method diff --git a/src/threading.c b/src/threading.c index c2dcba0f468c7..8acf03b726047 100644 --- a/src/threading.c +++ b/src/threading.c @@ -395,8 +395,7 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) int8_t gc_state = jl_gc_unsafe_enter(); JL_GC_PUSH1(&argtypes); argtypes = arg_type_tuple(jl_svec_data(args), jl_svec_len(args)); - jl_lambda_info_t *li = jl_get_specialization1(argtypes); - jl_generate_fptr(li); + jl_compile_hint(argtypes); threadwork.command = TI_THREADWORK_RUN; // TODO jb/functions: lookup and store jlcall fptr here diff --git a/src/toplevel.c b/src/toplevel.c index 06ce0ecc99cb2..69a0f55f41d3e 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -79,7 +79,7 @@ static void jl_module_load_time_initialize(jl_module_t *m) jl_value_t *tt = jl_is_type(f) ? (jl_value_t*)jl_wrap_Type(f) : jl_typeof(f); JL_GC_PUSH1(&tt); tt = (jl_value_t*)jl_apply_tuple_type_v(&tt, 1); - jl_get_specialization1((jl_tupletype_t*)tt); + jl_compile_hint((jl_tupletype_t*)tt); JL_GC_POP(); } } From b338cd217601e74835776af711136e449297dd21 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 1 Jun 2016 00:02:46 -0400 Subject: [PATCH 0032/1117] remove the ability for codegen to compute specialized call sites fix #16201 --- src/codegen.cpp | 59 +---------------------------------------------- test/inference.jl | 12 ++++++---- 2 files changed, 9 insertions(+), 62 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 2603462ab1ad3..aa84619a5e6ab 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2064,23 +2064,6 @@ static Value *emit_f_is(const jl_cgval_t &arg1, const jl_cgval_t &arg2, jl_codec #endif } -static jl_svec_t *call_arg_types(jl_value_t **args, size_t n, jl_codectx_t *ctx) -{ - jl_svec_t *t = jl_alloc_svec(n); - JL_GC_PUSH1(&t); - size_t i; - for(i=0; i < n; i++) { - jl_value_t *ty = expr_type(args[i], ctx); - if (!jl_is_leaf_type(ty)) { - t = NULL; - break; - } - jl_svecset(t, i, ty); - } - JL_GC_POP(); - return t; -} - static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, size_t nargs, jl_codectx_t *ctx, jl_value_t *expr) // returns true if the call has been handled @@ -2742,11 +2725,10 @@ static jl_cgval_t emit_call(jl_expr_t *ex, jl_codectx_t *ctx) size_t nargs = arglen - 1; assert(arglen >= 1); Value *theFptr = NULL; - jl_value_t *aty = NULL; jl_cgval_t result; jl_function_t *f = (jl_function_t*)static_eval(args[0], ctx, true); - JL_GC_PUSH2(&f, &aty); + JL_GC_PUSH1(&f); if (f != NULL) { // function is a compile-time constant if (jl_typeis(f, jl_intrinsic_type)) { @@ -2776,45 +2758,6 @@ static jl_cgval_t emit_call(jl_expr_t *ex, jl_codectx_t *ctx) } } - if (ctx->linfo->inferred) { - aty = (jl_value_t*)call_arg_types(args, arglen, ctx); - // attempt compile-time specialization for inferred types - if (aty != NULL) { - aty = (jl_value_t*)jl_apply_tuple_type((jl_svec_t*)aty); - /*if (trace) { - jl_printf(JL_STDOUT, "call %s%s\n", - jl_sprint(args[0]), - jl_sprint((jl_value_t*)aty)); - }*/ - jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)aty); - if (li != NULL && !li->inInference) { - jl_compile_linfo(li); - assert(li->functionObjectsDecls.functionObject != NULL); - theFptr = (Value*)li->functionObjectsDecls.functionObject; - jl_cgval_t fval; - if (f != NULL) { - // TODO jb/functions: avoid making too many roots here - if (!jl_is_globalref(args[0]) && !jl_is_symbol(args[0]) && - !jl_is_leaf_type(f)) { - if (ctx->linfo->def) - jl_add_linfo_root(ctx->linfo, f); - else // for toplevel thunks, just write the value back to the AST to root it - jl_array_ptr_set(ex->args, 0, f); - } - fval = mark_julia_const((jl_value_t*)f); - } - else { - fval = emit_expr(args[0], ctx); - } - if (ctx->linfo->def) // root this li in case it gets deleted from the cache in `f` - jl_add_linfo_root(ctx->linfo, (jl_value_t*)li); - result = emit_call_function_object(li, fval, theFptr, args, nargs, expr, ctx); - JL_GC_POP(); - return result; - } - } - } - // emit function and arguments nargs++; // add function to nargs count jl_cgval_t *anArg = (jl_cgval_t*)alloca(sizeof(jl_cgval_t) * nargs); diff --git a/test/inference.jl b/test/inference.jl index d15ee68205f4b..05216b4786c83 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -5,14 +5,14 @@ # issue 9770 @noinline x9770() = false function f9770(x) - if x9770() + return if x9770() g9770(:a, :foo) else x end end function g9770(x,y) - if isa(y, Symbol) + return if isa(y, Symbol) f9770(x) else g9770(:a, :foo) @@ -58,7 +58,7 @@ function g3182(t::DataType) # subtype of Type, but we cannot infer the T in Type{T} just # by knowing (at compile time) that the argument is a DataType. # however the ::Type{T} method should still match at run time. - f3182(t) + return f3182(t) end @test g3182(Complex) == 0 @@ -107,7 +107,7 @@ barTuple2() = fooTuple{tuple(:y)}() # issue #12476 function f12476(a) (k, v) = a - v + return v end @inferred f12476(1.0 => 1) @@ -143,14 +143,17 @@ f12826{I<:Integer}(v::Vector{I}) = v[1] # non-terminating inference, issue #14009 +# non-terminating codegen, issue #16201 type A14009{T}; end A14009{T}(a::T) = A14009{T}() f14009(a) = rand(Bool) ? f14009(A14009(a)) : a code_typed(f14009, (Int,)) +code_llvm(DevNull, f14009, (Int,)) type B14009{T}; end g14009(a) = g14009(B14009{a}) code_typed(g14009, (Type{Int},)) +code_llvm(DevNull, f14009, (Int,)) # issue #9232 @@ -164,6 +167,7 @@ result_type9232{T1<:Number,T2<:Number}(::Type{T1}, ::Type{T2}) = arithtype9232(T function g10878(x; kw...); end invoke_g10878() = invoke(g10878, Tuple{Any}, 1) @code_typed invoke_g10878() +code_llvm(DevNull, invoke_g10878, ()) # issue #10930 From 58c828e2d77f49b74d5ce297694f39cb9775b6f2 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Jun 2016 18:28:32 -0400 Subject: [PATCH 0033/1117] fix handling of TypeVars in sparams in codegen --- src/cgutils.cpp | 2 ++ src/codegen.cpp | 32 +++++++++++++++++++++----------- src/interpreter.c | 11 +++++++---- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 491ccddf3ee7b..600d40ce3b4ab 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -956,6 +956,8 @@ static jl_value_t *expr_type(jl_value_t *e, jl_codectx_t *ctx) if (idx >= jl_svec_len(ctx->linfo->sparam_vals)) return (jl_value_t*)jl_any_type; e = jl_svecref(ctx->linfo->sparam_vals, idx); + if (jl_is_typevar(e)) + return (jl_value_t*)jl_any_type; goto type_of_constant; } jl_value_t *typ = ((jl_expr_t*)e)->etype; diff --git a/src/codegen.cpp b/src/codegen.cpp index aa84619a5e6ab..a6950cb4b2557 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1642,8 +1642,12 @@ jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, } else if (e->head == static_parameter_sym) { size_t idx = jl_unbox_long(jl_exprarg(e,0)); - if (linfo && idx <= jl_svec_len(linfo->sparam_vals)) - return jl_svecref(linfo->sparam_vals, idx-1); + if (linfo && idx <= jl_svec_len(linfo->sparam_vals)) { + jl_value_t *e = jl_svecref(linfo->sparam_vals, idx - 1); + if (jl_is_typevar(e)) + return NULL; + return e; + } } return NULL; } @@ -2701,7 +2705,7 @@ static jl_cgval_t emit_invoke(jl_expr_t *ex, jl_codectx_t *ctx) } } Value *theFptr = (Value*)li->functionObjectsDecls.functionObject; - if (theFptr) { + if (theFptr && li->jlcall_api == 0) { jl_cgval_t fval = emit_expr(args[1], ctx); result = emit_call_function_object(li, fval, theFptr, &args[1], nargs - 1, (jl_value_t*)ex, ctx); } @@ -2861,15 +2865,16 @@ static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, static jl_cgval_t emit_sparam(size_t i, jl_codectx_t *ctx) { if (jl_svec_len(ctx->linfo->sparam_vals) > 0) { - return mark_julia_const(jl_svecref(ctx->linfo->sparam_vals, i)); - } - else { - assert(ctx->spvals_ptr != NULL); - Value *bp = builder.CreateConstInBoundsGEP1_32(LLVM37_param(T_pjlvalue) - builder.CreateBitCast(ctx->spvals_ptr, T_ppjlvalue), - i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); - return mark_julia_type(tbaa_decorate(tbaa_const, builder.CreateLoad(bp)), true, jl_any_type, ctx); + jl_value_t *e = jl_svecref(ctx->linfo->sparam_vals, i); + if (!jl_is_typevar(e)) { + return mark_julia_const(e); + } } + assert(ctx->spvals_ptr != NULL); + Value *bp = builder.CreateConstInBoundsGEP1_32(LLVM37_param(T_pjlvalue) + builder.CreateBitCast(ctx->spvals_ptr, T_ppjlvalue), + i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); + return mark_julia_type(tbaa_decorate(tbaa_const, builder.CreateLoad(bp)), true, jl_any_type, ctx); } static jl_cgval_t emit_global(jl_sym_t *sym, jl_codectx_t *ctx) @@ -4052,6 +4057,11 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func bool specsig = false; bool needsparams = jl_svec_len(lam->sparam_syms) != jl_svec_len(lam->sparam_vals); + for (i = 0; !needsparams && i < jl_svec_len(lam->sparam_vals); i++) { + jl_value_t *e = jl_svecref(lam->sparam_vals, i); + if (jl_is_typevar(e)) + needsparams = true; + } if (!va && !needsparams && lam->specTypes != jl_anytuple_type && lam->inferred) { // not vararg, consider specialized signature for(size_t i=0; i < jl_nparams(lam->specTypes); i++) { diff --git a/src/interpreter.c b/src/interpreter.c index dc7f4f9d30679..8d83eec54bd65 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -209,11 +209,14 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) ssize_t n = jl_unbox_long(args[0]); assert(n > 0); if (s->sparam_vals) - return jl_svecref(s->sparam_vals, n-1); + return jl_svecref(s->sparam_vals, n - 1); + if (n <= jl_svec_len(lam->sparam_vals)) { + jl_value_t *sp = jl_svecref(lam->sparam_vals, n - 1); + if (!jl_is_typevar(sp)) + return sp; + } // static parameter val unknown needs to be an error for ccall - if (n > jl_svec_len(lam->sparam_vals)) - jl_error("could not determine static parameter value"); - return jl_svecref(lam->sparam_vals, n-1); + jl_error("could not determine static parameter value"); } else if (ex->head == inert_sym) { return args[0]; From 74d376df4649bc5b638a4c295fd5547f996e580f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 16 Jun 2016 23:09:32 -0400 Subject: [PATCH 0034/1117] Don't clear slottypes when deleting IR --- src/codegen.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index a6950cb4b2557..e09a8db619bac 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -872,7 +872,6 @@ static void to_function(jl_lambda_info_t *li) li->def && li->inferred && !li->inlineable && li != li->def->lambda_template && !jl_options.outputji) { li->code = jl_nothing; - li->slottypes = jl_nothing; li->ssavaluetypes = jl_box_long(jl_array_len(li->ssavaluetypes)); jl_gc_wb(li, li->ssavaluetypes); li->slotflags = NULL; li->slotnames = NULL; From b8eb3954d148daa0657c72a5cd2683b07976832d Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Thu, 16 Jun 2016 21:05:10 -0700 Subject: [PATCH 0035/1117] Add backslash method specialized for `SparseMatrixCSC`s and a test checking that that specialized method gets called, a simple if inelegant fix for #16548. --- base/sparse/linalg.jl | 26 ++++++++++++++++++++++++++ test/sparsedir/sparse.jl | 5 ++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/base/sparse/linalg.jl b/base/sparse/linalg.jl index 1b24452007c97..05247ccf0343e 100644 --- a/base/sparse/linalg.jl +++ b/base/sparse/linalg.jl @@ -856,6 +856,32 @@ end scale!(A::SparseMatrixCSC, b::Number) = (scale!(A.nzval, b); A) scale!(b::Number, A::SparseMatrixCSC) = (scale!(b, A.nzval); A) +function (\)(A::SparseMatrixCSC, B::AbstractVecOrMat) + m, n = size(A) + if m == n + if istril(A) + if istriu(A) + return Diagonal(A) \ B + else + return LowerTriangular(A) \ B + end + elseif istriu(A) + return UpperTriangular(A) \ B + end + if ishermitian(A) + try + return cholfact(Hermitian(A)) \ B + catch e + isa(e, PosDefException) || rethrow(e) + return ldltfact(Hermitian(A)) \ B + end + end + return lufact(A) \ B + else + return qrfact(A) \ B + end +end + function factorize(A::SparseMatrixCSC) m, n = size(A) if m == n diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 706a050be7daf..d4b941b0744fa 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -1415,4 +1415,7 @@ let m = 5 intmat = fill(1, m, m) ltintmat = LowerTriangular(rand(1:5, m, m)) @test isapprox(At_ldiv_B(ltintmat, sparse(intmat)), At_ldiv_B(ltintmat, intmat)) -end \ No newline at end of file +end + +# Test temporary fix for issue #16548 in PR #16979. Brittle. Expect to remove with `\` revisions. +@test which(\, (SparseMatrixCSC, AbstractVecOrMat)).module == Base.SparseArrays \ No newline at end of file From 4584e36a82fc8c01c93f50b0021efb7cb9e77d07 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 16 Jun 2016 23:02:39 -0700 Subject: [PATCH 0036/1117] Fix a bunch of 'the the' typos --- base/docs/helpdb/Base.jl | 2 +- base/linalg/schur.jl | 4 ++-- base/linalg/triangular.jl | 2 +- base/managers.jl | 2 +- doc/manual/conversion-and-promotion.rst | 2 +- doc/stdlib/arrays.rst | 1 + doc/stdlib/linalg.rst | 4 ++-- doc/stdlib/sort.rst | 2 +- src/builtins.c | 2 +- test/perf/kernel/go_benchmark.c | 2 +- test/perf/kernel/go_benchmark.jl | 2 +- 11 files changed, 13 insertions(+), 12 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index c83f0730feab6..ab0d546231e1e 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4360,7 +4360,7 @@ position """ selectperm(v, k, [alg=,] [by=,] [lt=,] [rev=false]) -Return a partial permutation of the the vector `v`, according to the order specified by +Return a partial permutation of the vector `v`, according to the order specified by `by`, `lt` and `rev`, so that `v[output]` returns the first `k` (or range of adjacent values if `k` is a range) values of a fully sorted version of `v`. If `k` is a single index (Integer), an array of the first `k` indices is returned; if `k` is a range, an array of diff --git a/base/linalg/schur.jl b/base/linalg/schur.jl index 445b444db2455..45d679011102b 100644 --- a/base/linalg/schur.jl +++ b/base/linalg/schur.jl @@ -72,7 +72,7 @@ end Reorders the Schur factorization `F` of a matrix `A = Z*T*Z'` according to the logical array `select` returning the reordered factorization `F` object. The selected eigenvalues appear -in the leading diagonal of `F[:Schur]` and the the corresponding leading columns of +in the leading diagonal of `F[:Schur]` and the corresponding leading columns of `F[:vectors]` form an orthogonal/unitary basis of the corresponding right invariant subspace. In the real case, a complex conjugate pair of eigenvalues must be either both included or both excluded via `select`. @@ -91,7 +91,7 @@ ordschur!{Ty<:BlasFloat}(T::StridedMatrix{Ty}, Z::StridedMatrix{Ty}, select::Uni Reorders the Schur factorization of a real matrix `A = Z*T*Z'` according to the logical array `select` returning the reordered matrices `T` and `Z` as well as the vector of -eigenvalues `λ`. The selected eigenvalues appear in the leading diagonal of `T` and the the +eigenvalues `λ`. The selected eigenvalues appear in the leading diagonal of `T` and the corresponding leading columns of `Z` form an orthogonal/unitary basis of the corresponding right invariant subspace. In the real case, a complex conjugate pair of eigenvalues must be either both included or both excluded via `select`. diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 9f66a23bd3ac1..ccce61d38edb1 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -1557,7 +1557,7 @@ for (f, g) in ((:/, :A_rdiv_B!), (:A_rdiv_Bc, :A_rdiv_Bc!), (:A_rdiv_Bt, :A_rdiv end end -# If these are not defined, the they will fallback to the versions in matmul.jl +# If these are not defined, they will fallback to the versions in matmul.jl # and dispatch to generic_matmatmul! which is very costly to compile. The methods # below might compute an unnecessary copy. Eliminating the copy requires adding # all the promotion logic here once again. Since these methods are probably relatively diff --git a/base/managers.jl b/base/managers.jl index b53503e03898b..7164bee7f0721 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -201,7 +201,7 @@ function ssh_tunnel(user, host, bind_addr, port, sshflags) # the connection is forwarded to `port` on the remote server over the local port `localport` # the -f option backgrounds the ssh session # `sleep 60` command specifies that an alloted time of 60 seconds is allowed to start the - # remote julia process and estabilish the network connections specified the the process topology. + # remote julia process and establish the network connections specified by the process topology. # If no connections are made within 60 seconds, ssh will exit and an error will be printed on the # process that launched the remote process. ssh = `ssh -T -a -x -o ExitOnForwardFailure=yes` diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 7d37fc18d843b..8e3a06c2a2198 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -268,7 +268,7 @@ That is really all there is to using promotions. The rest is just a matter of clever application, the most typical "clever" application being the definition of catch-all methods for numeric operations like the arithmetic operators ``+``, ``-``, ``*`` and ``/``. Here are some of -the the catch-all method definitions given in +the catch-all method definitions given in `promotion.jl `_:: +(x::Number, y::Number) = +(promote(x,y)...) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 9a4a5d203bb53..ccad51321734d 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1067,3 +1067,4 @@ dense counterparts. The following functions are specific to sparse arrays. Generates a copy of ``x`` and removes numerical zeros from that copy, optionally trimming excess space from the result's ``nzind`` and ``nzval`` arrays when ``trim`` is ``true``\ . For an in-place version and algorithmic information, see :func:`Base.SparseArrays.dropzeros!`\ . + diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 4763985c80596..00e05696f5b32 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -614,7 +614,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Reorders the Schur factorization ``F`` of a matrix ``A = Z*T*Z'`` according to the logical array ``select`` returning the reordered factorization ``F`` object. The selected eigenvalues appear in the leading diagonal of ``F[:Schur]`` and the the corresponding leading columns of ``F[:vectors]`` form an orthogonal/unitary basis of the corresponding right invariant subspace. In the real case, a complex conjugate pair of eigenvalues must be either both included or both excluded via ``select``\ . + Reorders the Schur factorization ``F`` of a matrix ``A = Z*T*Z'`` according to the logical array ``select`` returning the reordered factorization ``F`` object. The selected eigenvalues appear in the leading diagonal of ``F[:Schur]`` and the corresponding leading columns of ``F[:vectors]`` form an orthogonal/unitary basis of the corresponding right invariant subspace. In the real case, a complex conjugate pair of eigenvalues must be either both included or both excluded via ``select``\ . .. function:: ordschur!(F::Schur, select::Union{Vector{Bool},BitVector}) -> F::Schur @@ -626,7 +626,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Reorders the Schur factorization of a real matrix ``A = Z*T*Z'`` according to the logical array ``select`` returning the reordered matrices ``T`` and ``Z`` as well as the vector of eigenvalues ``λ``\ . The selected eigenvalues appear in the leading diagonal of ``T`` and the the corresponding leading columns of ``Z`` form an orthogonal/unitary basis of the corresponding right invariant subspace. In the real case, a complex conjugate pair of eigenvalues must be either both included or both excluded via ``select``\ . + Reorders the Schur factorization of a real matrix ``A = Z*T*Z'`` according to the logical array ``select`` returning the reordered matrices ``T`` and ``Z`` as well as the vector of eigenvalues ``λ``\ . The selected eigenvalues appear in the leading diagonal of ``T`` and the corresponding leading columns of ``Z`` form an orthogonal/unitary basis of the corresponding right invariant subspace. In the real case, a complex conjugate pair of eigenvalues must be either both included or both excluded via ``select``\ . .. function:: ordschur!(T::StridedMatrix, Z::StridedMatrix, select::Union{Vector{Bool},BitVector}) -> T::StridedMatrix, Z::StridedMatrix, λ::Vector diff --git a/doc/stdlib/sort.rst b/doc/stdlib/sort.rst index 60ab3e1aa15be..671284119b9bc 100644 --- a/doc/stdlib/sort.rst +++ b/doc/stdlib/sort.rst @@ -205,7 +205,7 @@ Order-Related Functions .. Docstring generated from Julia source - Return a partial permutation of the the vector ``v``\ , according to the order specified by ``by``\ , ``lt`` and ``rev``\ , so that ``v[output]`` returns the first ``k`` (or range of adjacent values if ``k`` is a range) values of a fully sorted version of ``v``\ . If ``k`` is a single index (Integer), an array of the first ``k`` indices is returned; if ``k`` is a range, an array of those indices is returned. Note that the handling of integer values for ``k`` is different from ``select`` in that it returns a vector of ``k`` elements instead of just the ``k`` th element. Also note that this is equivalent to, but more efficient than, calling ``sortperm(...)[k]`` + Return a partial permutation of the vector ``v``\ , according to the order specified by ``by``\ , ``lt`` and ``rev``\ , so that ``v[output]`` returns the first ``k`` (or range of adjacent values if ``k`` is a range) values of a fully sorted version of ``v``\ . If ``k`` is a single index (Integer), an array of the first ``k`` indices is returned; if ``k`` is a range, an array of those indices is returned. Note that the handling of integer values for ``k`` is different from ``select`` in that it returns a vector of ``k`` elements instead of just the ``k`` th element. Also note that this is equivalent to, but more efficient than, calling ``sortperm(...)[k]`` .. function:: selectperm!(ix, v, k, [alg=,] [by=,] [lt=,] [rev=false,] [initialized=false]) diff --git a/src/builtins.c b/src/builtins.c index abe9de5a79c76..4c25cd8562ed6 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -480,7 +480,7 @@ JL_CALLABLE(jl_f__apply) newargs[0] = (jl_value_t*)arg_heap; newargs = jl_svec_data(arg_heap); } - // GC Note: here we assume that the the return value of `jl_svecref`, + // GC Note: here we assume that the return value of `jl_svecref`, // `jl_array_ptr_ref` will not be young if `arg_heap` becomes old // since they are allocated before `arg_heap`. Otherwise, // we need to add write barrier for !onstack diff --git a/test/perf/kernel/go_benchmark.c b/test/perf/kernel/go_benchmark.c index dfe12b8b09d2b..a1b2a93dfb52d 100644 --- a/test/perf/kernel/go_benchmark.c +++ b/test/perf/kernel/go_benchmark.c @@ -284,7 +284,7 @@ play_move(int i, int j, int color) * than one direction. */ if (on_board(ai, aj) && board[pos2] == color && !same_string(pos, pos2)) { - /* The strings are linked together simply by swapping the the + /* The strings are linked together simply by swapping the * next_stone pointers. */ int tmp = next_stone[pos2]; diff --git a/test/perf/kernel/go_benchmark.jl b/test/perf/kernel/go_benchmark.jl index 8c8ab39e47489..2f31b52a7bf0b 100644 --- a/test/perf/kernel/go_benchmark.jl +++ b/test/perf/kernel/go_benchmark.jl @@ -269,7 +269,7 @@ function play_move(board::Board, i::Int, j::Int, color::Int) # may happen if the same string neighbors the new stone in more # than one direction. if on_board(board, ai, aj) && board[pos2] == color && !same_string(board, pos, pos2) - # The strings are linked together simply by swapping the the + # The strings are linked together simply by swapping the # next_stone pointers. (board.next_stone[pos], board.next_stone[pos2]) = (board.next_stone[pos2], board.next_stone[pos]) end From 9acc0ccb8882715f99820bd5c118ba5a5222b1a6 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Thu, 16 Jun 2016 21:44:57 +0100 Subject: [PATCH 0037/1117] rename sub/slice -> view --- base/abstractarray.jl | 4 +- base/deprecated.jl | 3 ++ base/docs/helpdb/Base.jl | 12 +---- base/exports.jl | 3 +- base/iterator.jl | 2 +- base/linalg/arnoldi.jl | 4 +- base/linalg/arpack.jl | 6 +-- base/linalg/cholesky.jl | 4 +- base/linalg/eigen.jl | 4 +- base/linalg/lapack.jl | 2 +- base/linalg/qr.jl | 38 +++++++------- base/linalg/svd.jl | 2 +- base/loading.jl | 2 +- base/poll.jl | 2 +- base/sharedarray.jl | 4 +- base/show.jl | 2 +- base/sort.jl | 4 +- base/sparse/cholmod.jl | 2 +- base/sparse/linalg.jl | 4 +- base/sparse/sparsematrix.jl | 2 +- base/sparse/sparsevector.jl | 8 +-- base/strings/io.jl | 2 +- base/subarray.jl | 48 +++--------------- test/arrayops.jl | 34 ++++++------- test/blas.jl | 2 +- test/broadcast.jl | 6 +-- test/core.jl | 2 +- test/dsp.jl | 6 +-- test/fft.jl | 6 +-- test/hashing.jl | 2 +- test/linalg/arnoldi.jl | 2 +- test/linalg/bidiag.jl | 4 +- test/linalg/bunchkaufman.jl | 14 +++--- test/linalg/cholesky.jl | 2 +- test/linalg/dense.jl | 16 +++--- test/linalg/diagonal.jl | 8 +-- test/linalg/eigen.jl | 16 +++--- test/linalg/generic.jl | 18 +++---- test/linalg/givens.jl | 4 +- test/linalg/lq.jl | 2 +- test/linalg/lu.jl | 8 +-- test/linalg/matmul.jl | 82 +++++++++++++++--------------- test/linalg/qr.jl | 4 +- test/linalg/schur.jl | 10 ++-- test/linalg/svd.jl | 4 +- test/linalg/triangular.jl | 10 ++-- test/linalg/tridiag.jl | 6 +-- test/linalg/uniformscaling.jl | 12 ++--- test/numbers.jl | 2 +- test/offsetarray.jl | 12 ++--- test/perf/array/indexing.jl | 2 +- test/perf/kernel/laplace.jl | 2 +- test/reduce.jl | 4 +- test/serialize.jl | 6 +-- test/sorting.jl | 10 ++-- test/sparsedir/cholmod.jl | 4 +- test/sparsedir/sparse.jl | 2 +- test/staged.jl | 2 +- test/subarray.jl | 94 ++++++++++++++++------------------- test/unicode/utf32.jl | 2 +- 60 files changed, 269 insertions(+), 317 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index c500a68586ef0..e937e9ca4208a 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1416,7 +1416,7 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector) idx[d] = Colon() end - r1 = f(slice(A, idx...)) + r1 = f(view(A, idx...)) # determine result size and allocate Rsize = copy(dimsA) @@ -1448,7 +1448,7 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector) for i in 1:nidx idx[otherdims[i]] = ridx[otherdims[i]] = I.I[i] end - R[ridx...] = f(slice(A, idx...)) + R[ridx...] = f(view(A, idx...)) end end diff --git a/base/deprecated.jl b/base/deprecated.jl index 03026cbc47a16..d938921f4b65f 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -756,6 +756,9 @@ function first(::Colon) 1 end +@deprecate slice view +@deprecate sub view + # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index ab0d546231e1e..3d35be2c6cebf 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5383,14 +5383,14 @@ is raised as an exception in the woken tasks. notify """ - sub(A, inds...) + view(A, inds...) Like [`getindex`](:func:`getindex`), but returns a view into the parent array `A` with the given indices instead of making a copy. Calling [`getindex`](:func:`getindex`) or [`setindex!`](:func:`setindex!`) on the returned [`SubArray`](:obj:`SubArray`) computes the indices to the parent array on the fly without checking bounds. """ -sub +view """ expanduser(path::AbstractString) -> AbstractString @@ -7408,14 +7408,6 @@ julia> "Hello " * "world" """ Base.:(*)(s::AbstractString, t::AbstractString) -""" - slice(A, inds...) - -Returns a view of array `A` with the given indices like [`sub`](:func:`sub`), but drops all -dimensions indexed with scalars. -""" -slice - """ time() diff --git a/base/exports.jl b/base/exports.jl index 61113e9d4e343..e1c8a74cdb79e 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -587,7 +587,6 @@ export shuffle, shuffle!, size, - slice, slicedim, sort!, sort, @@ -601,7 +600,6 @@ export step, stride, strides, - sub, sub2ind, sum!, sum, @@ -612,6 +610,7 @@ export sum_kbn, vcat, vec, + view, zeros, # linear algebra diff --git a/base/iterator.jl b/base/iterator.jl index c3479f8fab74b..05359a4a14093 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -547,7 +547,7 @@ done(itr::PartitionIterator, state) = done(itr.c, state) function next{T<:Vector}(itr::PartitionIterator{T}, state) l = state r = min(state + itr.n-1, length(itr.c)) - return sub(itr.c, l:r), r + 1 + return view(itr.c, l:r), r + 1 end function next(itr::PartitionIterator, state) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index 81f34ac4267d8..138ef65ed2ac3 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -303,8 +303,8 @@ end function A_mul_B!{T,S}(u::StridedVector{T}, s::SVDOperator{T,S}, v::StridedVector{T}) a, b = s.m, length(v) - A_mul_B!(sub(u,1:a), s.X, sub(v,a+1:b)) # left singular vector - Ac_mul_B!(sub(u,a+1:b), s.X, sub(v,1:a)) # right singular vector + A_mul_B!(view(u,1:a), s.X, view(v,a+1:b)) # left singular vector + Ac_mul_B!(view(u,a+1:b), s.X, view(v,1:a)) # right singular vector u end size(s::SVDOperator) = s.m + s.n, s.m + s.n diff --git a/base/linalg/arpack.jl b/base/linalg/arpack.jl index a812ef3a3fb28..9b27aa3bc019b 100644 --- a/base/linalg/arpack.jl +++ b/base/linalg/arpack.jl @@ -53,8 +53,8 @@ function aupd_wrapper(T, matvecA!::Function, matvecB::Function, solveSI::Functio throw(ARPACKException(info[1])) end - x = sub(workd, ipntr[1]+zernm1) - y = sub(workd, ipntr[2]+zernm1) + x = view(workd, ipntr[1]+zernm1) + y = view(workd, ipntr[2]+zernm1) if mode == 1 # corresponds to dsdrv1, dndrv1 or zndrv1 if ido[1] == 1 matvecA!(y, x) @@ -89,7 +89,7 @@ function aupd_wrapper(T, matvecA!::Function, matvecB::Function, solveSI::Functio if ido[1] == -1 y[:] = solveSI(matvecB(x)) elseif ido[1] == 1 - y[:] = solveSI(sub(workd,ipntr[3]+zernm1)) + y[:] = solveSI(view(workd,ipntr[3]+zernm1)) elseif ido[1] == 2 y[:] = matvecB(x) elseif ido[1] == 99 diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index 611df8b838004..8265d694c160a 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -379,11 +379,11 @@ function A_ldiv_B!{T<:BlasFloat}(C::CholeskyPivoted{T}, B::StridedMatrix{T}) chkfullrank(C) n = size(C, 1) for i=1:size(B, 2) - permute!(sub(B, 1:n, i), C.piv) + permute!(view(B, 1:n, i), C.piv) end LAPACK.potrs!(C.uplo, C.factors, B) for i=1:size(B, 2) - ipermute!(sub(B, 1:n, i), C.piv) + ipermute!(view(B, 1:n, i), C.piv) end B end diff --git a/base/linalg/eigen.jl b/base/linalg/eigen.jl index 13fdf3b0b6423..e1edaabe3ffa8 100644 --- a/base/linalg/eigen.jl +++ b/base/linalg/eigen.jl @@ -35,7 +35,7 @@ function eigfact!{T<:BlasReal}(A::StridedMatrix{T}; permute::Bool=true, scale::B j = 1 while j <= n if WI[j] == 0 - evec[:,j] = slice(VR, :, j) + evec[:,j] = view(VR, :, j) else for i = 1:n evec[i,j] = VR[i,j] + im*VR[i,j+1] @@ -131,7 +131,7 @@ function eigfact!{T<:BlasReal}(A::StridedMatrix{T}, B::StridedMatrix{T}) j = 1 while j <= n if alphai[j] == 0 - vecs[:,j] = slice(vr, :, j) + vecs[:,j] = view(vr, :, j) else for i = 1:n vecs[i,j ] = vr[i,j] + im*vr[i,j+1] diff --git a/base/linalg/lapack.jl b/base/linalg/lapack.jl index 7a40910e07f73..62a8e1a8fba6c 100644 --- a/base/linalg/lapack.jl +++ b/base/linalg/lapack.jl @@ -1437,7 +1437,7 @@ for (gglse, elty) in ((:dgglse_, :Float64), work = Array{$elty}(lwork) end end - X, dot(sub(c, n - p + 1:m), sub(c, n - p + 1:m)) + X, dot(view(c, n - p + 1:m), view(c, n - p + 1:m)) end end end diff --git a/base/linalg/qr.jl b/base/linalg/qr.jl index 0a4459ba8d39c..42f4f77fd5056 100644 --- a/base/linalg/qr.jl +++ b/base/linalg/qr.jl @@ -28,20 +28,20 @@ function qrfactUnblocked!{T}(A::AbstractMatrix{T}) m, n = size(A) τ = zeros(T, min(m,n)) for k = 1:min(m - 1 + !(T<:Real), n) - x = slice(A, k:m, k) + x = view(A, k:m, k) τk = reflector!(x) τ[k] = τk - reflectorApply!(x, τk, slice(A, k:m, k + 1:n)) + reflectorApply!(x, τk, view(A, k:m, k + 1:n)) end QR(A, τ) end # Find index for columns with largest two norm function indmaxcolumn(A::StridedMatrix) - mm = norm(slice(A, :, 1)) + mm = norm(view(A, :, 1)) ii = 1 for i = 2:size(A, 2) - mi = norm(slice(A, :, i)) + mi = norm(view(A, :, i)) if abs(mi) > mm mm = mi ii = i @@ -57,7 +57,7 @@ function qrfactPivotedUnblocked!(A::StridedMatrix) for j = 1:min(m,n) # Find column with maximum norm in trailing submatrix - jm = indmaxcolumn(slice(A, j:m, j:n)) + j - 1 + jm = indmaxcolumn(view(A, j:m, j:n)) + j - 1 if jm != j # Flip elements in pivoting vector @@ -74,12 +74,12 @@ function qrfactPivotedUnblocked!(A::StridedMatrix) end # Compute reflector of columns j - x = slice(A, j:m, j) + x = view(A, j:m, j) τj = LinAlg.reflector!(x) τ[j] = τj # Update trailing submatrix with reflector - LinAlg.reflectorApply!(x, τj, sub(A, j:m, j+1:n)) + LinAlg.reflectorApply!(x, τj, view(A, j:m, j+1:n)) end return LinAlg.QRPivoted{eltype(A), typeof(A)}(A, τ, piv) end @@ -451,8 +451,8 @@ for (f1, f2) in ((:Ac_mul_B, :A_mul_B!), end end -A_ldiv_B!{T<:BlasFloat}(A::QRCompactWY{T}, b::StridedVector{T}) = (A_ldiv_B!(UpperTriangular(A[:R]), sub(Ac_mul_B!(A[:Q], b), 1:size(A, 2))); b) -A_ldiv_B!{T<:BlasFloat}(A::QRCompactWY{T}, B::StridedMatrix{T}) = (A_ldiv_B!(UpperTriangular(A[:R]), sub(Ac_mul_B!(A[:Q], B), 1:size(A, 2), 1:size(B, 2))); B) +A_ldiv_B!{T<:BlasFloat}(A::QRCompactWY{T}, b::StridedVector{T}) = (A_ldiv_B!(UpperTriangular(A[:R]), view(Ac_mul_B!(A[:Q], b), 1:size(A, 2))); b) +A_ldiv_B!{T<:BlasFloat}(A::QRCompactWY{T}, B::StridedMatrix{T}) = (A_ldiv_B!(UpperTriangular(A[:R]), view(Ac_mul_B!(A[:Q], B), 1:size(A, 2), 1:size(B, 2))); B) # Julia implementation similarly to xgelsy function A_ldiv_B!{T<:BlasFloat}(A::QRPivoted{T}, B::StridedMatrix{T}, rcond::Real) @@ -467,8 +467,8 @@ function A_ldiv_B!{T<:BlasFloat}(A::QRPivoted{T}, B::StridedMatrix{T}, rcond::Re xmax = ones(T, 1) tmin = tmax = ar while rnk < nr - tmin, smin, cmin = LAPACK.laic1!(2, xmin, tmin, sub(A.factors, 1:rnk, rnk + 1), A.factors[rnk + 1, rnk + 1]) - tmax, smax, cmax = LAPACK.laic1!(1, xmax, tmax, sub(A.factors, 1:rnk, rnk + 1), A.factors[rnk + 1, rnk + 1]) + tmin, smin, cmin = LAPACK.laic1!(2, xmin, tmin, view(A.factors, 1:rnk, rnk + 1), A.factors[rnk + 1, rnk + 1]) + tmax, smax, cmax = LAPACK.laic1!(1, xmax, tmax, view(A.factors, 1:rnk, rnk + 1), A.factors[rnk + 1, rnk + 1]) tmax*rcond > tmin && break push!(xmin, cmin) push!(xmax, cmax) @@ -479,10 +479,10 @@ function A_ldiv_B!{T<:BlasFloat}(A::QRPivoted{T}, B::StridedMatrix{T}, rcond::Re rnk += 1 end C, τ = LAPACK.tzrzf!(A.factors[1:rnk,:]) - A_ldiv_B!(UpperTriangular(C[1:rnk,1:rnk]),sub(Ac_mul_B!(getq(A),sub(B, 1:mA, 1:nrhs)),1:rnk,1:nrhs)) + A_ldiv_B!(UpperTriangular(C[1:rnk,1:rnk]),view(Ac_mul_B!(getq(A),view(B, 1:mA, 1:nrhs)),1:rnk,1:nrhs)) B[rnk+1:end,:] = zero(T) - LAPACK.ormrz!('L', eltype(B)<:Complex ? 'C' : 'T', C, τ, sub(B,1:nA,1:nrhs)) - B[1:nA,:] = sub(B, 1:nA, :)[invperm(A[:p]::Vector{BlasInt}),:] + LAPACK.ormrz!('L', eltype(B)<:Complex ? 'C' : 'T', C, τ, view(B,1:nA,1:nrhs)) + B[1:nA,:] = view(B, 1:nA, :)[invperm(A[:p]::Vector{BlasInt}),:] return B, rnk end A_ldiv_B!{T<:BlasFloat}(A::QRPivoted{T}, B::StridedVector{T}) = vec(A_ldiv_B!(A,reshape(B,length(B),1))) @@ -491,13 +491,13 @@ function A_ldiv_B!{T}(A::QR{T}, B::StridedMatrix{T}) m, n = size(A) minmn = min(m,n) mB, nB = size(B) - Ac_mul_B!(A[:Q], sub(B, 1:m, :)) + Ac_mul_B!(A[:Q], view(B, 1:m, :)) R = A[:R] @inbounds begin if n > m # minimum norm solution τ = zeros(T,m) for k = m:-1:1 # Trapezoid to triangular by elementary operation - x = slice(R, k, [k; m + 1:n]) + x = view(R, k, [k; m + 1:n]) τk = reflector!(x) τ[k] = τk' for i = 1:k - 1 @@ -513,7 +513,7 @@ function A_ldiv_B!{T}(A::QR{T}, B::StridedMatrix{T}) end end end - Base.A_ldiv_B!(UpperTriangular(sub(R, :, 1:minmn)), sub(B, 1:minmn, :)) + Base.A_ldiv_B!(UpperTriangular(view(R, :, 1:minmn)), view(B, 1:minmn, :)) if n > m # Apply elementary transformation to solution B[m + 1:mB,1:nB] = zero(T) for j = 1:nB @@ -536,12 +536,12 @@ end A_ldiv_B!(A::QR, B::StridedVector) = A_ldiv_B!(A, reshape(B, length(B), 1))[:] function A_ldiv_B!(A::QRPivoted, b::StridedVector) A_ldiv_B!(QR(A.factors,A.τ), b) - b[1:size(A.factors, 2)] = sub(b, 1:size(A.factors, 2))[invperm(A.jpvt)] + b[1:size(A.factors, 2)] = view(b, 1:size(A.factors, 2))[invperm(A.jpvt)] b end function A_ldiv_B!(A::QRPivoted, B::StridedMatrix) A_ldiv_B!(QR(A.factors, A.τ), B) - B[1:size(A.factors, 2),:] = sub(B, 1:size(A.factors, 2), :)[invperm(A.jpvt),:] + B[1:size(A.factors, 2),:] = view(B, 1:size(A.factors, 2), :)[invperm(A.jpvt),:] B end diff --git a/base/linalg/svd.jl b/base/linalg/svd.jl index a3f80b67aa465..896364c69d242 100644 --- a/base/linalg/svd.jl +++ b/base/linalg/svd.jl @@ -95,7 +95,7 @@ svdvals{T, Tr}(S::SVD{T, Tr}) = (S[:S])::Vector{Tr} # SVD least squares function A_ldiv_B!{Ta,Tb}(A::SVD{Ta}, B::StridedVecOrMat{Tb}) k = searchsortedlast(A.S, eps(real(Ta))*A.S[1], rev=true) - sub(A.Vt,1:k,:)' * (sub(A.S,1:k) .\ (sub(A.U,:,1:k)' * B)) + view(A.Vt,1:k,:)' * (view(A.S,1:k) .\ (view(A.U,:,1:k)' * B)) end # Generalized svd diff --git a/base/loading.jl b/base/loading.jl index 04406d0b4c8b1..99c049faf5593 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -60,7 +60,7 @@ elseif is_apple() continue end casepreserved_basename = - sub(buf, (header_size+1):(header_size+filename_length-1)) + view(buf, (header_size+1):(header_size+filename_length-1)) break end # Hack to compensate for inability to create a string from a subarray with no allocations. diff --git a/base/poll.jl b/base/poll.jl index f9a6036d1e31f..1425b6ee59be4 100644 --- a/base/poll.jl +++ b/base/poll.jl @@ -102,7 +102,7 @@ type _FDWatcher if fd.fd+1 > length(FDWatchers) old_len = length(FDWatchers) resize!(FDWatchers, fd.fd+1) - fill!(sub(FDWatchers, old_len+1:fd.fd+1), nothing) + fill!(view(FDWatchers, old_len+1:fd.fd+1), nothing) elseif FDWatchers[fd.fd + 1] !== nothing this = FDWatchers[fd.fd + 1]::_FDWatcher this.refcount = (this.refcount[1] + Int(readable), this.refcount[2] + Int(writable)) diff --git a/base/sharedarray.jl b/base/sharedarray.jl index ab46bd146f4cc..fa9d42412c966 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -327,7 +327,7 @@ function range_1dim(S::SharedArray, pidx) end end -sub_1dim(S::SharedArray, pidx) = sub(S.s, range_1dim(S, pidx)) +sub_1dim(S::SharedArray, pidx) = view(S.s, range_1dim(S, pidx)) function init_loc_flds{T,N}(S::SharedArray{T,N}) if myid() in S.pids @@ -342,7 +342,7 @@ function init_loc_flds{T,N}(S::SharedArray{T,N}) S.loc_subarr_1d = sub_1dim(S, S.pidx) else S.pidx = 0 - S.loc_subarr_1d = sub(Array{T}(ntuple(d->0,N)), 1:0) + S.loc_subarr_1d = view(Array{T}(ntuple(d->0,N)), 1:0) end end diff --git a/base/show.jl b/base/show.jl index 74ea4ea153449..e48497ac20dc9 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1479,7 +1479,7 @@ function show_nd(io::IO, a::AbstractArray, print_matrix, label_slices) for i = 1:(nd-1); print(io, "$(idxs[i]), "); end println(io, idxs[end], "] =") end - slice = sub(a, indices(a,1), indices(a,2), idxs...) + slice = view(a, indices(a,1), indices(a,2), idxs...) print_matrix(io, slice) print(io, idxs == map(last,tail) ? "" : "\n\n") @label skip diff --git a/base/sort.jl b/base/sort.jl index 1abe23f6fdbac..9b2bd7ea4e724 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -509,7 +509,7 @@ function sortrows(A::AbstractMatrix; kws...) T = slicetypeof(A, inds, :) rows = allocate_for(Vector{T}, A, shape(A, 1)) for i in inds - rows[i] = slice(A, i, :) + rows[i] = view(A, i, :) end p = sortperm(rows; kws..., order=Lexicographic) A[p,:] @@ -520,7 +520,7 @@ function sortcols(A::AbstractMatrix; kws...) T = slicetypeof(A, :, inds) cols = allocate_for(Vector{T}, A, shape(A, 2)) for i in inds - cols[i] = slice(A, :, i) + cols[i] = view(A, :, i) end p = sortperm(cols; kws..., order=Lexicographic) A[:,p] diff --git a/base/sparse/cholmod.jl b/base/sparse/cholmod.jl index ca90b2aa521b5..f596b26d40ae0 100644 --- a/base/sparse/cholmod.jl +++ b/base/sparse/cholmod.jl @@ -830,7 +830,7 @@ function (::Type{Sparse}){Tv<:VTypes}(m::Integer, n::Integer, # check if columns are sorted iss = true for i = 2:length(colptr) - if !issorted(sub(rowval, colptr[i - 1] + 1:colptr[i])) + if !issorted(view(rowval, colptr[i - 1] + 1:colptr[i])) iss = false break end diff --git a/base/sparse/linalg.jl b/base/sparse/linalg.jl index 05247ccf0343e..a80e5f4a6cb51 100644 --- a/base/sparse/linalg.jl +++ b/base/sparse/linalg.jl @@ -588,7 +588,7 @@ function normestinv{T}(A::SparseMatrixCSC{T}, t::Integer = min(2,maximum(size(A) X[1:n,1] = 1 for j = 2:t while true - _rand_pm1!(slice(X,1:n,j)) + _rand_pm1!(view(X,1:n,j)) yaux = X[1:n,j]' * X[1:n,1:j-1] if !_any_abs_eq(yaux,n) break @@ -649,7 +649,7 @@ function normestinv{T}(A::SparseMatrixCSC{T}, t::Integer = min(2,maximum(size(A) end end if repeated - _rand_pm1!(slice(S,1:n,j)) + _rand_pm1!(view(S,1:n,j)) else break end diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index a7e195e35be96..7e3bac45a3594 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -3127,7 +3127,7 @@ function spdiagm_internal(B, d) range = 1+i:numel+i I[range] = row+1:row+numel J[range] = col+1:col+numel - copy!(sub(V, range), vec) + copy!(view(V, range), vec) i += numel end diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index d796a25159df3..4bbad1ca948f7 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -1550,8 +1550,8 @@ for isunittri in (true, false), islowertri in (true, false) nzrange = $( (islowertri && !istrans) || (!islowertri && istrans) ? :(b.nzind[1]:b.n) : :(1:b.nzind[end]) ) - nzrangeviewr = sub(r, nzrange) - nzrangeviewA = $tritype(sub(A.data, nzrange, nzrange)) + nzrangeviewr = view(r, nzrange) + nzrangeviewA = $tritype(view(A.data, nzrange, nzrange)) ($ipfunc)(convert(AbstractArray{TAb}, nzrangeviewA), nzrangeviewr) end r @@ -1586,8 +1586,8 @@ for isunittri in (true, false), islowertri in (true, false) nzrange = $( (islowertri && !istrans) || (!islowertri && istrans) ? :(b.nzind[1]:b.n) : :(1:b.nzind[end]) ) - nzrangeviewbnz = sub(b.nzval, nzrange - b.nzind[1] + 1) - nzrangeviewA = $tritype(sub(A.data, nzrange, nzrange)) + nzrangeviewbnz = view(b.nzval, nzrange - b.nzind[1] + 1) + nzrangeviewA = $tritype(view(A.data, nzrange, nzrange)) ($func)(nzrangeviewA, nzrangeviewbnz) # could strip any miraculous zeros here perhaps end diff --git a/base/strings/io.jl b/base/strings/io.jl index 89a8c84b02ea0..199fe0eefc785 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -90,7 +90,7 @@ end # IOBuffer views of a (byte)string: IOBuffer(str::String) = IOBuffer(str.data) -IOBuffer(s::SubString{String}) = IOBuffer(sub(s.string.data, s.offset + 1 : s.offset + sizeof(s))) +IOBuffer(s::SubString{String}) = IOBuffer(view(s.string.data, s.offset + 1 : s.offset + sizeof(s))) # join is implemented using IO function join(io::IO, strings, delim, last) diff --git a/base/subarray.jl b/base/subarray.jl index aea6dcf80bfdd..ff155d3a49675 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -85,53 +85,27 @@ parentindexes(a::AbstractArray) = ntuple(i->1:size(a,i), ndims(a)) ## SubArray creation # Drops singleton dimensions (those indexed with a scalar) -function slice{T,N}(A::AbstractArray{T,N}, I::Vararg{ViewIndex,N}) +function view{T,N}(A::AbstractArray{T,N}, I::Vararg{ViewIndex,N}) @_inline_meta @boundscheck checkbounds(A, I...) - unsafe_slice(A, I...) + unsafe_view(A, I...) end -function slice(A::AbstractArray, i::ViewIndex) +function view(A::AbstractArray, i::ViewIndex) @_inline_meta @boundscheck checkbounds(A, i) - unsafe_slice(reshape(A, Val{1}), i) + unsafe_view(reshape(A, Val{1}), i) end -function slice{N}(A::AbstractArray, I::Vararg{ViewIndex,N}) # TODO: DEPRECATE FOR #14770 +function view{N}(A::AbstractArray, I::Vararg{ViewIndex,N}) # TODO: DEPRECATE FOR #14770 @_inline_meta @boundscheck checkbounds(A, I...) - unsafe_slice(reshape(A, Val{N}), I...) + unsafe_view(reshape(A, Val{N}), I...) end -function unsafe_slice{T,N}(A::AbstractArray{T,N}, I::Vararg{ViewIndex,N}) +function unsafe_view{T,N}(A::AbstractArray{T,N}, I::Vararg{ViewIndex,N}) @_inline_meta J = to_indexes(I...) SubArray(A, J, map(dimlength, index_shape(A, J...))) end -keep_leading_scalars(T::Tuple{}) = T -keep_leading_scalars(T::Tuple{Real, Vararg{Real}}) = T -keep_leading_scalars(T::Tuple{Real, Vararg{Any}}) = (@_inline_meta; (NoSlice(T[1]), keep_leading_scalars(tail(T))...)) -keep_leading_scalars(T::Tuple{Any, Vararg{Any}}) = (@_inline_meta; (T[1], keep_leading_scalars(tail(T))...)) - -function sub{T,N}(A::AbstractArray{T,N}, I::Vararg{ViewIndex,N}) - @_inline_meta - @boundscheck checkbounds(A, I...) - unsafe_sub(A, I...) -end -function sub(A::AbstractArray, i::ViewIndex) - @_inline_meta - @boundscheck checkbounds(A, i) - unsafe_sub(reshape(A, Val{1}), i) -end -function sub{N}(A::AbstractArray, I::Vararg{ViewIndex,N}) # TODO: DEPRECATE FOR #14770 - @_inline_meta - @boundscheck checkbounds(A, I...) - unsafe_sub(reshape(A, Val{N}), I...) -end -function unsafe_sub{T,N}(A::AbstractArray{T,N}, I::Vararg{ViewIndex,N}) - @_inline_meta - J = keep_leading_scalars(to_indexes(I...)) - SubArray(A, J, index_shape(A, J...)) -end - # Re-indexing is the heart of a view, transforming A[i, j][x, y] to A[i[x], j[y]] # # Recursively look through the heads of the parent- and sub-indexes, considering @@ -216,18 +190,12 @@ function setindex!(V::FastContiguousSubArray, x, i::Real) V end -function unsafe_slice{T,N}(V::SubArray{T,N}, I::Vararg{ViewIndex,N}) +function unsafe_view{T,N}(V::SubArray{T,N}, I::Vararg{ViewIndex,N}) @_inline_meta idxs = reindex(V, V.indexes, to_indexes(I...)) SubArray(V.parent, idxs, index_shape(V.parent, idxs...)) end -function unsafe_sub{T,N}(V::SubArray{T,N}, I::Vararg{ViewIndex,N}) - @_inline_meta - idxs = reindex(V, V.indexes, keep_leading_scalars(to_indexes(I...))) - SubArray(V.parent, idxs, index_shape(V.parent, idxs...)) -end - linearindexing{T<:FastSubArray}(::Type{T}) = LinearFast() linearindexing{T<:SubArray}(::Type{T}) = LinearSlow() diff --git a/test/arrayops.jl b/test/arrayops.jl index a556260ad942a..6a061d8011cc1 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -90,7 +90,7 @@ a = reshape(b, (2, 2, 2, 2, 2)) a = collect(reshape(1:5, 1, 5)) # reshaping linearfast SubArrays -s = sub(a, :, 2:4) +s = view(a, :, 2:4) r = reshape(s, (length(s),)) @test length(r) == 3 @test r[1] == 2 @@ -105,7 +105,7 @@ r = reshape(s, (length(s),)) @test Base.unsafe_convert(Ptr{Int}, r) == Base.unsafe_convert(Ptr{Int}, s) # reshaping linearslow SubArrays -s = sub(a, :, [2,3,5]) +s = view(a, :, [2,3,5]) r = reshape(s, length(s)) @test length(r) == 3 @test r[1] == 2 @@ -121,7 +121,7 @@ r = reshape(s, length(s)) r[2] = -1 @test a[3] == -1 a = zeros(0, 5) # an empty linearslow array -s = sub(a, :, [2,3,5]) +s = view(a, :, [2,3,5]) @test length(reshape(s, length(s))) == 0 @test reshape(1:5, (5,)) === 1:5 @@ -435,14 +435,14 @@ end # of a subarray a = rand(5,5) -s = sub(a,2:3,2:3) +s = view(a,2:3,2:3) p = permutedims(s, [2,1]) @test p[1,1]==a[2,2] && p[1,2]==a[3,2] @test p[2,1]==a[2,3] && p[2,2]==a[3,3] # of a non-strided subarray a = reshape(1:60, 3, 4, 5) -s = sub(a,:,[1,2,4],[1,5]) +s = view(a,:,[1,2,4],[1,5]) c = convert(Array, s) for p in ([1,2,3], [1,3,2], [2,1,3], [2,3,1], [3,1,2], [3,2,1]) @test permutedims(s, p) == permutedims(c, p) @@ -700,7 +700,7 @@ let @test R == [1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2] A = rand(4,4) - for s in Any[A[1:2:4, 1:2:4], sub(A, 1:2:4, 1:2:4)] + for s in Any[A[1:2:4, 1:2:4], view(A, 1:2:4, 1:2:4)] c = cumsum(s, 1) @test c[1,1] == A[1,1] @test c[2,1] == A[1,1]+A[3,1] @@ -932,9 +932,9 @@ end # fill @test fill!(Array{Float64}(1),-0.0)[1] === -0.0 A = ones(3,3) -S = sub(A, 2, 1:3) +S = view(A, 2, 1:3) fill!(S, 2) -S = sub(A, 1:2, 3) +S = view(A, 1:2, 3) fill!(S, 3) @test A == [1 1 3; 2 2 3; 1 1 1] rt = Base.return_types(fill!, Tuple{Array{Int32, 3}, UInt8}) @@ -1166,7 +1166,7 @@ end a = [1:5;] @test isa(Base.linearindexing(a), Base.LinearFast) -b = sub(a, :) +b = view(a, :) @test isa(Base.linearindexing(b), Base.LinearFast) @test isa(Base.linearindexing(trues(2)), Base.LinearFast) @test isa(Base.linearindexing(BitArray{2}), Base.LinearFast) @@ -1178,7 +1178,7 @@ for i = 1:10 @test mdsum(A) == 15 @test mdsum2(A) == 15 AA = reshape(aa, tuple(2, shp...)) - B = sub(AA, 1:1, ntuple(i->Colon(), i)...) + B = view(AA, 1:1, ntuple(i->Colon(), i)...) @test isa(Base.linearindexing(B), Base.IteratorsMD.LinearSlow) @test mdsum(B) == 15 @test mdsum2(B) == 15 @@ -1191,7 +1191,7 @@ for i = 2:10 A = reshape(a, tuple(shp...)) @test mdsum(A) == 55 @test mdsum2(A) == 55 - B = sub(A, ntuple(i->Colon(), i)...) + B = view(A, ntuple(i->Colon(), i)...) @test mdsum(B) == 55 @test mdsum2(B) == 55 insert!(shp, 2, 1) @@ -1202,10 +1202,10 @@ a = reshape([2]) @test mdsum2(a) == 2 a = ones(0,5) -b = sub(a, :, :) +b = view(a, :, :) @test mdsum(b) == 0 a = ones(5,0) -b = sub(a, :, :) +b = view(a, :, :) @test mdsum(b) == 0 a = copy(reshape(1:60, 3, 4, 5)) @@ -1219,7 +1219,7 @@ a[1,CartesianIndex{2}(3,4)] = -2 a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] = -3 @test a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] == -3 -a = sub(zeros(3, 4, 5), :, :, :) +a = view(zeros(3, 4, 5), :, :, :) a[CartesianIndex{3}(2,3,3)] = -1 @test a[CartesianIndex{3}(2,3,3)] == -1 a[1,CartesianIndex{2}(3,4)] = -2 @@ -1264,7 +1264,7 @@ a = spzeros(2,3) @test CartesianRange(size(a)) == eachindex(a) a[CartesianIndex{2}(2,3)] = 5 @test a[2,3] == 5 -b = sub(a, 1:2, 2:3) +b = view(a, 1:2, 2:3) b[CartesianIndex{2}(1,1)] = 7 @test a[1,2] == 7 @test 2*CartesianIndex{3}(1,2,3) == CartesianIndex{3}(2,4,6) @@ -1314,7 +1314,7 @@ R = CartesianRange((3,0)) @test @inferred(eachindex(Base.LinearSlow(),zeros(3),zeros(2,2),zeros(2,2,2),zeros(2,2))) == CartesianRange((3,2,2)) @test @inferred(eachindex(Base.LinearFast(),zeros(3),zeros(2,2),zeros(2,2,2),zeros(2,2))) == 1:8 -@test @inferred(eachindex(zeros(3),sub(zeros(3,3),1:2,1:2),zeros(2,2,2),zeros(2,2))) == CartesianRange((3,2,2)) +@test @inferred(eachindex(zeros(3),view(zeros(3,3),1:2,1:2),zeros(2,2,2),zeros(2,2))) == CartesianRange((3,2,2)) @test @inferred(eachindex(zeros(3),zeros(2,2),zeros(2,2,2),zeros(2,2))) == 1:8 @@ -1461,7 +1461,7 @@ Base.setindex!(A::LinSlowMatrix, v, i::Integer, j::Integer) = A.data[i,j] = v A = rand(3,5) B = LinSlowMatrix(A) -S = sub(A, :, :) +S = view(A, :, :) @test A == B @test B == A diff --git a/test/blas.jl b/test/blas.jl index b870ae59b12a7..f44a5ae900b51 100644 --- a/test/blas.jl +++ b/test/blas.jl @@ -100,7 +100,7 @@ for elty in [Float32, Float64, Complex64, Complex128] # nrm2, iamax, and asum for StridedVectors a = rand(elty,n) - b = slice(a,2:2:n,1) + b = view(a,2:2:n,1) @test BLAS.nrm2(b) ≈ norm(b) if elty <: Real @test BLAS.asum(b) ≈ sum(abs(b)) diff --git a/test/broadcast.jl b/test/broadcast.jl index 357711fbd585d..2484a43d29701 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -55,13 +55,13 @@ end function as_sub(x::AbstractVector) y = similar(x, eltype(x), tuple(([size(x)...]*2)...)) - y = sub(y, 2:2:length(y)) + y = view(y, 2:2:length(y)) y[:] = x[:] y end function as_sub(x::AbstractMatrix) y = similar(x, eltype(x), tuple(([size(x)...]*2)...)) - y = sub(y, 2:2:size(y,1), 2:2:size(y,2)) + y = view(y, 2:2:size(y,1), 2:2:size(y,2)) for j=1:size(x,2) for i=1:size(x,1) y[i,j] = x[i,j] @@ -71,7 +71,7 @@ function as_sub(x::AbstractMatrix) end function as_sub{T}(x::AbstractArray{T,3}) y = similar(x, eltype(x), tuple(([size(x)...]*2)...)) - y = sub(y, 2:2:size(y,1), 2:2:size(y,2), 2:2:size(y,3)) + y = view(y, 2:2:size(y,1), 2:2:size(y,2), 2:2:size(y,3)) for k=1:size(x,3) for j=1:size(x,2) for i=1:size(x,1) diff --git a/test/core.jl b/test/core.jl index 11c038ae1b0b6..0fb29bf35ca10 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2061,7 +2061,7 @@ function issue7897!(data, arr) end a = ones(UInt8, 10) -sa = sub(a,4:6) +sa = view(a,4:6) # This can throw an error, but shouldn't segfault try issue7897!(sa, zeros(10)) diff --git a/test/dsp.jl b/test/dsp.jl index 34aa579c14c12..a55ad5faa2a1c 100644 --- a/test/dsp.jl +++ b/test/dsp.jl @@ -48,7 +48,7 @@ if Base.fftw_vendor() != :mkl X = reshape([1,2,7,2,1,5,9,-1,3,4,6,9],3,4) Y = rand(17,14) Y[3:5,9:12] = X - sX = slice(Y,3:5,9:12) + sX = view(Y,3:5,9:12) true_Xdct = [ 13.856406460551018 -3.863239728836245 2.886751345948129 -0.274551994240164; -2.828427124746190 -2.184015211898548 -4.949747468305834 3.966116180118245; 4.898979485566356 -0.194137576915510 -2.857738033247041 2.731723009609389 ] @@ -87,8 +87,8 @@ if Base.fftw_vendor() != :mkl sXdct = dct(sX) psXdct = plan_dct(sX)*(sX) - sYdct! = copy(Y); sXdct! = slice(sYdct!,3:5,9:12); dct!(sXdct!) - psYdct! = copy(Y); psXdct! = slice(psYdct!,3:5,9:12); plan_dct!(psXdct!)*(psXdct!) + sYdct! = copy(Y); sXdct! = view(sYdct!,3:5,9:12); dct!(sXdct!) + psYdct! = copy(Y); psXdct! = view(psYdct!,3:5,9:12); plan_dct!(psXdct!)*(psXdct!) for i = 1:length(X) @test_approx_eq Xdct[i] true_Xdct[i] diff --git a/test/fft.jl b/test/fft.jl index a15f0ab474262..9ebc93c353dfc 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -29,7 +29,7 @@ true_fftd2_m4 = [ b = rand(17,14) b[3:6,9:12] = m4 -sm4 = slice(b,3:6,9:12) +sm4 = view(b,3:6,9:12) m3d = map(Float32,copy(reshape(1:5*3*2, 5, 3, 2))) true_fftd3_m3d = Array{Float32}(5, 3, 2) @@ -78,9 +78,9 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), sfftn_m4 = f(sm4) psfftn_m4 = pf(sm4)*sm4 sfft!n_b = map(Complex128,b) - sfft!n_m4 = slice(sfft!n_b,3:6,9:12); fft!(sfft!n_m4) + sfft!n_m4 = view(sfft!n_b,3:6,9:12); fft!(sfft!n_m4) psfft!n_b = map(Complex128,b) - psfft!n_m4 = slice(psfft!n_b,3:6,9:12); plan_fft!(psfft!n_m4)*psfft!n_m4 + psfft!n_m4 = view(psfft!n_b,3:6,9:12); plan_fft!(psfft!n_m4)*psfft!n_m4 for i = 1:length(m4) @test_approx_eq fft_m4[i] true_fft_m4[i] diff --git a/test/hashing.jl b/test/hashing.jl index 9ee196c119e1e..1919226f41d2a 100644 --- a/test/hashing.jl +++ b/test/hashing.jl @@ -83,7 +83,7 @@ end @test hash(:(X.x)) == hash(:(X.x)) @test hash(:(X.x)) != hash(:(X.y)) -@test hash([1,2]) == hash(sub([1,2,3,4],1:2)) +@test hash([1,2]) == hash(view([1,2,3,4],1:2)) # test explicit zeros in SparseMatrixCSC x = sprand(10, 10, 0.5) diff --git a/test/linalg/arnoldi.jl b/test/linalg/arnoldi.jl index 35eaad7107862..d9bc48fc00106 100644 --- a/test/linalg/arnoldi.jl +++ b/test/linalg/arnoldi.jl @@ -115,7 +115,7 @@ function A_mul_B!{T<:Base.LinAlg.BlasFloat}(rho2::StridedVector{T},Phi::CPM{T},r rho=reshape(rho,(size(Phi.kraus,3),size(Phi.kraus,3))) rho1=zeros(T,(size(Phi.kraus,1),size(Phi.kraus,1))) for s=1:size(Phi.kraus,2) - As=slice(Phi.kraus,:,s,:) + As=view(Phi.kraus,:,s,:) rho1+=As*rho*As' end return copy!(rho2,rho1) diff --git a/test/linalg/bidiag.jl b/test/linalg/bidiag.jl index 168790a2dddd1..6ea351a3c2a65 100644 --- a/test/linalg/bidiag.jl +++ b/test/linalg/bidiag.jl @@ -139,8 +139,8 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) b = bb c = cc else - b = sub(bb, 1:n) - c = sub(cc, 1:n, 1:2) + b = view(bb, 1:n) + c = view(cc, 1:n, 1:2) end end debug && println("Linear solver") diff --git a/test/linalg/bunchkaufman.jl b/test/linalg/bunchkaufman.jl index cf2dda7cc3fdd..7e3cbeef29e30 100644 --- a/test/linalg/bunchkaufman.jl +++ b/test/linalg/bunchkaufman.jl @@ -30,10 +30,10 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) a = a a2 = a2 else - a = sub(a, 1:n, 1:n) - a2 = sub(a2, 1:n, 1:n) - asym = sub(asym, 1:n, 1:n) - apd = sub(apd, 1:n, 1:n) + a = view(a, 1:n, 1:n) + a2 = view(a2, 1:n, 1:n) + asym = view(asym, 1:n, 1:n) + apd = view(apd, 1:n, 1:n) end ε = εa = eps(abs(float(one(eltya)))) @@ -43,7 +43,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) if btype == "Array" b = b else - b = sub(b, 1:n, 1:2) + b = view(b, 1:n, 1:2) end εb = eps(abs(float(one(eltyb)))) @@ -92,7 +92,7 @@ let if Astype == "Array" As = As else - As = sub(As, 1:n, 1:n) + As = view(As, 1:n, 1:n) end for rook in (false, true) @@ -110,6 +110,6 @@ let A = rand(6,5); A = complex(A'*A) # to avoid calling the real-lhs-complex-rhs method F = cholfact(A); v6 = rand(Complex128, 6) - v5 = sub(v6, 1:5) + v5 = view(v6, 1:5) @test F\v5 == F\v6[1:5] end diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index 0f8faf8e82963..f42a00e865c0c 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -116,7 +116,7 @@ debug && println("\ntype of a: ", eltya, " type of b: ", eltyb, "\n") if atype == "Array" b = Bs else - b = sub(Bs, 1:n, 1) + b = view(Bs, 1:n, 1) end # Test error bound on linear solver: LAWNS 14, Theorem 2.1 diff --git a/test/linalg/dense.jl b/test/linalg/dense.jl index 155cb5d3445f4..a3d8f6f1cfca6 100644 --- a/test/linalg/dense.jl +++ b/test/linalg/dense.jl @@ -159,7 +159,7 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com ## Vector x = ones(elty,10) - xs = sub(x,1:2:10) + xs = view(x,1:2:10) @test_approx_eq norm(x, -Inf) 1 @test_approx_eq norm(x, -1) 1/10 @test_approx_eq norm(x, 0) 10 @@ -199,11 +199,11 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com x = elty <: Integer ? convert(Vector{elty}, rand(1:10, nnorm)) : elty <: Complex ? convert(Vector{elty}, complex(randn(nnorm), randn(nnorm))) : convert(Vector{elty}, randn(nnorm)) - xs = sub(x,1:2:nnorm) + xs = view(x,1:2:nnorm) y = elty <: Integer ? convert(Vector{elty}, rand(1:10, nnorm)) : elty <: Complex ? convert(Vector{elty}, complex(randn(nnorm), randn(nnorm))) : convert(Vector{elty}, randn(nnorm)) - ys = sub(y,1:2:nnorm) + ys = view(y,1:2:nnorm) α = elty <: Integer ? randn() : elty <: Complex ? convert(elty, complex(randn(),randn())) : convert(elty, randn()) @@ -244,7 +244,7 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com end ## Matrix (Operator) A = ones(elty,10,10) - As = sub(A,1:5,1:5) + As = view(A,1:5,1:5) @test_approx_eq norm(A, 1) 10 elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test_approx_eq norm(A, 2) 10 @test_approx_eq norm(A, Inf) 10 @@ -256,11 +256,11 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com A = elty <: Integer ? convert(Matrix{elty}, rand(1:10, mmat, nmat)) : elty <: Complex ? convert(Matrix{elty}, complex(randn(mmat, nmat), randn(mmat, nmat))) : convert(Matrix{elty}, randn(mmat, nmat)) - As = sub(A,1:nmat,1:nmat) + As = view(A,1:nmat,1:nmat) B = elty <: Integer ? convert(Matrix{elty}, rand(1:10, mmat, nmat)) : elty <: Complex ? convert(Matrix{elty}, complex(randn(mmat, nmat), randn(mmat, nmat))) : convert(Matrix{elty}, randn(mmat, nmat)) - Bs = sub(B,1:nmat,1:nmat) + Bs = view(B,1:nmat,1:nmat) α = elty <: Integer ? randn() : elty <: Complex ? convert(elty, complex(randn(),randn())) : convert(elty, randn()) @@ -323,7 +323,7 @@ let A = [1 2 0 0; 0 1 0 0; 0 0 0 0; 0 0 0 0] Asq = sqrtm(A) @test_approx_eq Asq*Asq A - A2 = sub(A, 1:2, 1:2) + A2 = view(A, 1:2, 1:2) A2sq = sqrtm(A2) @test_approx_eq A2sq*A2sq A2 end @@ -544,5 +544,5 @@ end # stride1 a = rand(10) -b = slice(a,2:2:10) +b = view(a,2:2:10) @test Base.LinAlg.stride1(b) == 2 diff --git a/test/linalg/diagonal.jl b/test/linalg/diagonal.jl index 3ed0deed3a23d..be856257cef9d 100644 --- a/test/linalg/diagonal.jl +++ b/test/linalg/diagonal.jl @@ -62,8 +62,8 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) v = vv U = UU else - v = sub(vv, 1:n) - U = sub(UU, 1:n, 1:2) + v = view(vv, 1:n) + U = view(UU, 1:n, 1:2) end debug && println("Linear solve") @@ -81,7 +81,7 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) b = sparse(b) @test A_ldiv_B!(D,copy(b)) ≈ full(D)\full(b) @test_throws SingularException A_ldiv_B!(Diagonal(zeros(elty,n)),copy(b)) - b = sub(rand(elty,n),collect(1:n)) + b = view(rand(elty,n),collect(1:n)) b2 = copy(b) c = A_ldiv_B!(D,b) d = full(D)\b2 @@ -92,7 +92,7 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) b = rand(elty,n+1,n+1) b = sparse(b) @test_throws DimensionMismatch A_ldiv_B!(D,copy(b)) - b = sub(rand(elty,n+1),collect(1:n+1)) + b = view(rand(elty,n+1),collect(1:n+1)) @test_throws DimensionMismatch A_ldiv_B!(D,b) end diff --git a/test/linalg/eigen.jl b/test/linalg/eigen.jl index ab6aed4185a0c..89089f0f422fc 100644 --- a/test/linalg/eigen.jl +++ b/test/linalg/eigen.jl @@ -24,9 +24,9 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) if atype == "Array" a = aa else - a = sub(aa, 1:n, 1:n) - asym = sub(asym, 1:n, 1:n) - apd = sub(apd, 1:n, 1:n) + a = view(aa, 1:n, 1:n) + asym = view(asym, 1:n, 1:n) + apd = view(apd, 1:n, 1:n) end ε = εa = eps(abs(float(one(eltya)))) @@ -59,8 +59,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) asym_sg = asym[1:n1, 1:n1] a_sg = a[:,n1+1:n2] else - asym_sg = sub(asym, 1:n1, 1:n1) - a_sg = sub(a, 1:n, n1+1:n2) + asym_sg = view(asym, 1:n1, 1:n1) + a_sg = view(a, 1:n, n1+1:n2) end f = eigfact(asym_sg, a_sg'a_sg) @test_approx_eq asym_sg*f[:vectors] (a_sg'a_sg*f[:vectors]) * Diagonal(f[:values]) @@ -80,8 +80,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) a1_nsg = a[1:n1, 1:n1] a2_nsg = a[n1+1:n2, n1+1:n2] else - a1_nsg = sub(a, 1:n1, 1:n1) - a2_nsg = sub(a, n1+1:n2, n1+1:n2) + a1_nsg = view(a, 1:n1, 1:n1) + a2_nsg = view(a, n1+1:n2, n1+1:n2) end f = eigfact(a1_nsg, a2_nsg) @test_approx_eq a1_nsg*f[:vectors] (a2_nsg*f[:vectors]) * Diagonal(f[:values]) @@ -101,7 +101,7 @@ let aa = rand(200, 200) if atype == "Array" a = aa else - a = sub(aa, 1:n, 1:n) + a = view(aa, 1:n, 1:n) end f = eigfact(a) @test a ≈ f[:vectors] * Diagonal(f[:values]) / f[:vectors] diff --git a/test/linalg/generic.jl b/test/linalg/generic.jl index 70f0a2f8c3a91..1ce2337b0d043 100644 --- a/test/linalg/generic.jl +++ b/test/linalg/generic.jl @@ -57,10 +57,10 @@ let X = [3 9 5; 2 1 10] @test diff(X,1) == [4 -5 -3; -5 -3 8] @test diff(X,2) == [6 -4; -3 -2; -1 9] - @test diff(sub(X, 1:2, 1:2),1) == [4 -5] - @test diff(sub(X, 1:2, 1:2),2) == reshape([6; -3], (2,1)) - @test diff(sub(X, 2:3, 2:3),1) == [-3 8] - @test diff(sub(X, 2:3, 2:3),2) == reshape([-2; 9], (2,1)) + @test diff(view(X, 1:2, 1:2),1) == [4 -5] + @test diff(view(X, 1:2, 1:2),2) == reshape([6; -3], (2,1)) + @test diff(view(X, 2:3, 2:3),1) == [-3 8] + @test diff(view(X, 2:3, 2:3),2) == reshape([-2; 9], (2,1)) @test_throws ArgumentError diff(X,3) @test_throws ArgumentError diff(X,-1) end @@ -78,7 +78,7 @@ y = [3; 7; 10] x = 1:12 y = [5.5; 6.3; 7.6; 8.8; 10.9; 11.79; 13.48; 15.02; 17.77; 20.81; 22.0; 22.99] @test_approx_eq [linreg(x,y)...] [2.5559090909090867, 1.6960139860139862] -@test_approx_eq [linreg(sub(x,1:6),sub(y,1:6))...] [3.8366666666666642,1.3271428571428574] +@test_approx_eq [linreg(view(x,1:6),view(y,1:6))...] [3.8366666666666642,1.3271428571428574] # check (LinSpace, UnitRange) x = linspace(1.0, 12.0, 100) @@ -114,8 +114,8 @@ y4 = [6.58; 5.76; 7.71; 8.84; 8.47; 7.04; 5.25; 12.50; 5.56; 7.91; 6.89] # test diag let A = eye(4) @test diag(A) == ones(4) - @test diag(sub(A, 1:3, 1:3)) == ones(3) - @test diag(sub(A, 1:2, 1:2)) == ones(2) + @test diag(view(A, 1:3, 1:3)) == ones(3) + @test diag(view(A, 1:2, 1:2)) == ones(2) end # test generic axpy @@ -142,14 +142,14 @@ let aa = reshape([1.:6;], (2,3)) if atype == "Array" a = aa else - a = sub(aa, 1:2, 1:2) + a = view(aa, 1:2, 1:2) end # 2-argument version of scale! @test scale!(copy(a), 5.) == a*5 @test scale!(5., copy(a)) == a*5 b = randn(Base.LinAlg.SCAL_CUTOFF) # make sure we try BLAS path - subB = sub(b, :, :) + subB = view(b, :, :) @test scale!(copy(b), 5.) == b*5 @test scale!(copy(subB), 5.) == subB*5 @test scale!([1.; 2.], copy(a)) == a.*[1; 2] diff --git a/test/linalg/givens.jl b/test/linalg/givens.jl index 5d5a5dfd6c7f0..1078de8b839de 100644 --- a/test/linalg/givens.jl +++ b/test/linalg/givens.jl @@ -19,7 +19,7 @@ for elty in (Float32, Float64, Complex64, Complex128) if Atype == "Array" A = A else - A = sub(A, 1:10, 1:10) + A = view(A, 1:10, 1:10) end Ac = copy(A) R = Base.LinAlg.Rotation(Base.LinAlg.Givens{elty}[]) @@ -56,7 +56,7 @@ for elty in (Float32, Float64, Complex64, Complex128) if Atype == "Array" x = A[:, 1] else - x = sub(A, 1:10, 1) + x = view(A, 1:10, 1) end G, r = givens(x[2], x[4], 2, 4) @test (G*x)[2] ≈ r diff --git a/test/linalg/lq.jl b/test/linalg/lq.jl index c56692f68e1c7..e898dbcfdaed8 100644 --- a/test/linalg/lq.jl +++ b/test/linalg/lq.jl @@ -43,7 +43,7 @@ for eltya in (Float32, Float64, Complex64, Complex128) debug && println("\ntype of a: ", eltya, " type of b: ", eltyb, "\n") debug && println("LQ decomposition") for i = 1:2 - let a = i == 1 ? a : sub(a, 1:n - 1, 1:n - 1), b = i == 1 ? b : sub(b, 1:n - 1), n = i == 1 ? n : n - 1 + let a = i == 1 ? a : view(a, 1:n - 1, 1:n - 1), b = i == 1 ? b : view(b, 1:n - 1), n = i == 1 ? n : n - 1 lqa = lqfact(a) l,q = lqa[:L], lqa[:Q] qra = qrfact(a) diff --git a/test/linalg/lu.jl b/test/linalg/lu.jl index 1f1dc86bfd04b..5891fe63a7ea2 100644 --- a/test/linalg/lu.jl +++ b/test/linalg/lu.jl @@ -60,8 +60,8 @@ debug && println("(Automatic) Square LU decomposition") b = Bs c = Cs else - b = sub(Bs, 1:n, 1) - c = sub(Cs, 1:n) + b = view(Bs, 1:n, 1) + c = view(Cs, 1:n) end @test norm(a*(lua\b) - b, 1) < ε*κ*n*2 # Two because the right hand side has two columns @test norm(a'*(lua'\b) - b, 1) < ε*κ*n*2 # Two because the right hand side has two columns @@ -97,7 +97,7 @@ debug && println("Tridiagonal LU") if atype == "Array" b = Bs else - b = sub(Bs, 1:n, 1) + b = view(Bs, 1:n, 1) end @test norm(d*(lud\b) - b, 1) < ε*κd*n*2 # Two because the right hand side has two columns @@ -157,7 +157,7 @@ let Bs = b if atype == "Array" b = Bs else - b = sub(Bs, 1:n, 1) + b = view(Bs, 1:n, 1) end @test a*(lua\b) ≈ b end diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 803d44b1ff654..53d1368506e95 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -24,15 +24,15 @@ let AAi = AA+(0.5*im).*BB BBi = BB+(2.5*im).*AA[[2,1],[2,1]] for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:2, 1:2) - B = Btype == "Array" ? BB : sub(BB, 1:2, 1:2) + A = Atype == "Array" ? AA : view(AA, 1:2, 1:2) + B = Btype == "Array" ? BB : view(BB, 1:2, 1:2) @test A*B == [19 22; 43 50] @test At_mul_B(A, B) == [26 30; 38 44] @test A_mul_Bt(A, B) == [17 23; 39 53] @test At_mul_Bt(A, B) == [23 31; 34 46] - Ai = Atype == "Array" ? AAi : sub(AAi, 1:2, 1:2) - Bi = Btype == "Array" ? BBi : sub(BBi, 1:2, 1:2) + Ai = Atype == "Array" ? AAi : view(AAi, 1:2, 1:2) + Bi = Btype == "Array" ? BBi : view(BBi, 1:2, 1:2) @test Ai*Bi == [-21+53.5im -4.25+51.5im; -12+95.5im 13.75+85.5im] @test Ac_mul_B(Ai, Bi) == [68.5-12im 57.5-28im; 88-3im 76.5-25im] @test A_mul_Bc(Ai, Bi) == [64.5+5.5im 43+31.5im; 104-18.5im 80.5+31.5im] @@ -47,15 +47,15 @@ let AAi = AA+(0.5*im).*BB BBi = BB+(2.5*im).*AA[[2,1,3],[2,3,1]] for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:3, 1:3) - B = Btype == "Array" ? BB : sub(BB, 1:3, 1:3) + A = Atype == "Array" ? AA : view(AA, 1:3, 1:3) + B = Btype == "Array" ? BB : view(BB, 1:3, 1:3) @test A*B == [-26 38 -27; 1 -4 -6; 28 -46 15] @test Ac_mul_B(A, B) == [-6 2 -25; 3 -12 -18; 12 -26 -11] @test A_mul_Bc(A, B) == [-14 0 6; 4 -3 -3; 22 -6 -12] @test Ac_mul_Bc(A, B) == [6 -8 -6; 12 -9 -9; 18 -10 -12] - Ai = Atype == "Array" ? AAi : sub(AAi, 1:3, 1:3) - Bi = Btype == "Array" ? BBi : sub(BBi, 1:3, 1:3) + Ai = Atype == "Array" ? AAi : view(AAi, 1:3, 1:3) + Bi = Btype == "Array" ? BBi : view(BBi, 1:3, 1:3) @test Ai*Bi == [-44.75+13im 11.75-25im -38.25+30im; -47.75-16.5im -51.5+51.5im -56+6im; 16.75-4.5im -53.5+52im -15.5im] @test Ac_mul_B(Ai, Bi) == [-21+2im -1.75+49im -51.25+19.5im; 25.5+56.5im -7-35.5im 22+35.5im; -3+12im -32.25+43im -34.75-2.5im] @test A_mul_Bc(Ai, Bi) == [-20.25+15.5im -28.75-54.5im 22.25+68.5im; -12.25+13im -15.5+75im -23+27im; 18.25+im 1.5+94.5im -27-54.5im] @@ -83,25 +83,25 @@ let AA = [1 2 3; 4 5 6] .- 3 BB = [2 -2; 3 -5; -4 7] for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:2, 1:3) - B = Btype == "Array" ? BB : sub(BB, 1:3, 1:2) + A = Atype == "Array" ? AA : view(AA, 1:2, 1:3) + B = Btype == "Array" ? BB : view(BB, 1:3, 1:2) @test A*B == [-7 9; -4 9] @test At_mul_Bt(A, B) == [-6 -11 15; -6 -13 18; -6 -15 21] end AA = ones(Int, 2, 100) BB = ones(Int, 100, 3) for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:2, 1:100) - B = Btype == "Array" ? BB : sub(BB, 1:100, 1:3) + A = Atype == "Array" ? AA : view(AA, 1:2, 1:100) + B = Btype == "Array" ? BB : view(BB, 1:100, 1:3) @test A*B == [100 100 100; 100 100 100] end AA = rand(1:20, 5, 5) .- 10 BB = rand(1:20, 5, 5) .- 10 CC = Array{Int}(size(AA, 1), size(BB, 2)) for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"], Ctype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:5, 1:5) - B = Btype == "Array" ? BB : sub(BB, 1:5, 1:5) - C = Btype == "Array" ? CC : sub(CC, 1:5, 1:5) + A = Atype == "Array" ? AA : view(AA, 1:5, 1:5) + B = Btype == "Array" ? BB : view(BB, 1:5, 1:5) + C = Btype == "Array" ? CC : view(CC, 1:5, 1:5) @test At_mul_B(A, B) == A'*B @test A_mul_Bt(A, B) == A*B' # Preallocated @@ -118,8 +118,8 @@ let vv = [1,2] CC = Array{Int}(2, 2) for vtype = ["Array", "SubArray"], Ctype = ["Array", "SubArray"] - v = vtype == "Array" ? vv : sub(vv, 1:2) - C = Ctype == "Array" ? CC : sub(CC, 1:2, 1:2) + v = vtype == "Array" ? vv : view(vv, 1:2) + C = Ctype == "Array" ? CC : view(CC, 1:2, 1:2) @test @inferred(A_mul_Bc!(C, v, v)) == [1 2; 2 4] end end @@ -129,23 +129,23 @@ let AA = rand(5,5) BB = rand(5) for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:5, 1:5) - B = Btype == "Array" ? BB : sub(BB, 1:5) + A = Atype == "Array" ? AA : view(AA, 1:5, 1:5) + B = Btype == "Array" ? BB : view(BB, 1:5) @test_throws DimensionMismatch Base.LinAlg.generic_matvecmul!(zeros(6),'N',A,B) @test_throws DimensionMismatch Base.LinAlg.generic_matvecmul!(B,'N',A,zeros(6)) end vv = [1,2,3] CC = Array{Int}(3, 3) for vtype = ["Array", "SubArray"], Ctype = ["Array", "SubArray"] - v = vtype == "Array" ? vv : sub(vv, 1:3) - C = Ctype == "Array" ? CC : sub(CC, 1:3, 1:3) + v = vtype == "Array" ? vv : view(vv, 1:3) + C = Ctype == "Array" ? CC : view(CC, 1:3, 1:3) @test A_mul_Bt!(C, v, v) == v*v' end vvf = map(Float64,vv) CC = Array{Float64}(3, 3) for vtype = ["Array", "SubArray"], Ctype = ["Array", "SubArray"] - vf = vtype == "Array" ? vvf : sub(vvf, 1:3) - C = Ctype == "Array" ? CC : sub(CC, 1:3, 1:3) + vf = vtype == "Array" ? vvf : view(vvf, 1:3) + C = Ctype == "Array" ? CC : view(CC, 1:3, 1:3) @test A_mul_Bt!(C, vf, vf) == vf*vf' end end @@ -156,9 +156,9 @@ let BB = rand(Float64,6,6) CC = zeros(Float64,6,6) for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"], Ctype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:6, 1:6) - B = Btype == "Array" ? BB : sub(BB, 1:6, 1:6) - C = Ctype == "Array" ? CC : sub(CC, 1:6, 1:6) + A = Atype == "Array" ? AA : view(AA, 1:6, 1:6) + B = Btype == "Array" ? BB : view(BB, 1:6, 1:6) + C = Ctype == "Array" ? CC : view(CC, 1:6, 1:6) @test Base.LinAlg.At_mul_Bt!(C,A,B) == A.'*B.' @test Base.LinAlg.A_mul_Bc!(C,A,B) == A*B.' @test Base.LinAlg.Ac_mul_B!(C,A,B) == A.'*B @@ -169,26 +169,26 @@ end let A = reshape(map(Float64,1:20),5,4) Aref = A[1:2:end,1:2:end] - Asub = sub(A, 1:2:5, 1:2:4) + Asub = view(A, 1:2:5, 1:2:4) b = [1.2,-2.5] @test (Aref*b) == (Asub*b) @test At_mul_B(Asub, Asub) == At_mul_B(Aref, Aref) @test A_mul_Bt(Asub, Asub) == A_mul_Bt(Aref, Aref) Ai = A .+ im Aref = Ai[1:2:end,1:2:end] - Asub = sub(Ai, 1:2:5, 1:2:4) + Asub = view(Ai, 1:2:5, 1:2:4) @test Ac_mul_B(Asub, Asub) == Ac_mul_B(Aref, Aref) @test A_mul_Bc(Asub, Asub) == A_mul_Bc(Aref, Aref) end # issue #15286 -let A = reshape(map(Float64, 1:20), 5, 4), C = zeros(8, 8), sC = sub(C, 1:2:8, 1:2:8), B = reshape(map(Float64,-9:10),5,4) +let A = reshape(map(Float64, 1:20), 5, 4), C = zeros(8, 8), sC = view(C, 1:2:8, 1:2:8), B = reshape(map(Float64,-9:10),5,4) @test At_mul_B!(sC, A, A) == A'*A @test At_mul_B!(sC, A, B) == A'*B Aim = A .- im C = zeros(Complex128,8,8) - sC = sub(C, 1:2:8, 1:2:8) + sC = view(C, 1:2:8, 1:2:8) B = reshape(map(Float64,-9:10),5,4) .+ im @test Ac_mul_B!(sC, Aim, Aim) == Aim'*Aim @test Ac_mul_B!(sC, Aim, B) == Aim'*B @@ -199,17 +199,17 @@ let AA = reshape(1:1503, 501, 3).-750.0 res = Float64[135228751 9979252 -115270247; 9979252 10481254 10983256; -115270247 10983256 137236759] for Atype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:501, 1:3) + A = Atype == "Array" ? AA : view(AA, 1:501, 1:3) @test At_mul_B(A, A) == res @test A_mul_Bt(A',A') == res end cutoff = 501 A = reshape(1:6*cutoff,2*cutoff,3).-(6*cutoff)/2 - Asub = sub(A, 1:2:2*cutoff, 1:3) + Asub = view(A, 1:2:2*cutoff, 1:3) Aref = A[1:2:2*cutoff, 1:3] @test At_mul_B(Asub, Asub) == At_mul_B(Aref, Aref) Ai = A .- im - Asub = sub(Ai, 1:2:2*cutoff, 1:3) + Asub = view(Ai, 1:2:2*cutoff, 1:3) Aref = Ai[1:2:2*cutoff, 1:3] @test Ac_mul_B(Asub, Asub) == Ac_mul_B(Aref, Aref) @@ -221,7 +221,7 @@ end let AA = fill(complex(1,1), 10, 10) for Atype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:10, 1:10) + A = Atype == "Array" ? AA : view(AA, 1:10, 1:10) A2 = A^2 @test A2[1,1] == 20im end @@ -233,9 +233,9 @@ let CC = rand(5, 6) for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"] for Ctype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:5, 1:5) - B = Btype == "Array" ? BB : sub(BB, 1:5) - C = Ctype == "Array" ? CC : sub(CC, 1:5, 1:6) + A = Atype == "Array" ? AA : view(AA, 1:5, 1:5) + B = Btype == "Array" ? BB : view(BB, 1:5) + C = Ctype == "Array" ? CC : view(CC, 1:5, 1:6) @test_throws DimensionMismatch scale!(A, B, C) end @@ -258,8 +258,8 @@ end vecdot_(x,y) = invoke(vecdot, (Any,Any), x,y) # generic vecdot let AA = [1+2im 3+4im; 5+6im 7+8im], BB = [2+7im 4+1im; 3+8im 6+5im] for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"] - A = Atype == "Array" ? AA : sub(AA, 1:2, 1:2) - B = Btype == "Array" ? BB : sub(BB, 1:2, 1:2) + A = Atype == "Array" ? AA : view(AA, 1:2, 1:2) + B = Btype == "Array" ? BB : view(BB, 1:2, 1:2) @test vecdot(A,B) == dot(vec(A),vec(B)) == vecdot_(A,B) == vecdot(float(A),float(B)) @test vecdot(Int[], Int[]) == 0 == vecdot_(Int[], Int[]) @test_throws MethodError vecdot(Any[], Any[]) @@ -308,8 +308,8 @@ let aa = rand(3,3) bb = rand(3,3) for atype = ["Array", "SubArray"], btype = ["Array", "SubArray"] - a = atype == "Array" ? aa : sub(aa, 1:3, 1:3) - b = btype == "Array" ? bb : sub(bb, 1:3, 1:3) + a = atype == "Array" ? aa : view(aa, 1:3, 1:3) + b = btype == "Array" ? bb : view(bb, 1:3, 1:3) @test_throws ArgumentError A_mul_B!(a, a, b) @test_throws ArgumentError A_mul_B!(a, b, a) @test_throws ArgumentError A_mul_B!(a, a, a) diff --git a/test/linalg/qr.jl b/test/linalg/qr.jl index b024351d9d1b9..39829db9fdfc6 100644 --- a/test/linalg/qr.jl +++ b/test/linalg/qr.jl @@ -41,7 +41,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) debug && println("\ntype of a: ", eltya, " type of b: ", eltyb, "\n") debug && println("QR decomposition (without pivoting)") for i = 1:2 - let a = i == 1 ? a : sub(a, 1:n - 1, 1:n - 1), b = i == 1 ? b : sub(b, 1:n - 1), n = i == 1 ? n : n - 1 + let a = i == 1 ? a : view(a, 1:n - 1, 1:n - 1), b = i == 1 ? b : view(b, 1:n - 1), n = i == 1 ? n : n - 1 qra = @inferred qrfact(a) @inferred qr(a) q, r = qra[:Q], qra[:R] @@ -58,7 +58,7 @@ debug && println("QR decomposition (without pivoting)") if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) ac = copy(a) - @test qrfact!(a[:, 1:5])\b == qrfact!(sub(ac, :, 1:5))\b + @test qrfact!(a[:, 1:5])\b == qrfact!(view(ac, :, 1:5))\b end debug && println("Thin QR decomposition (without pivoting)") diff --git a/test/linalg/schur.jl b/test/linalg/schur.jl index 0ff240832d99d..d3dc91567ed2a 100644 --- a/test/linalg/schur.jl +++ b/test/linalg/schur.jl @@ -24,9 +24,9 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) if atype == "Array" a = a else - a = sub(a, 1:n, 1:n) - asym = sub(asym, 1:n, 1:n) - apd = sub(apd, 1:n, 1:n) + a = view(a, 1:n, 1:n) + asym = view(asym, 1:n, 1:n) + apd = view(apd, 1:n, 1:n) end ε = εa = eps(abs(float(one(eltya)))) @@ -61,8 +61,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) a1_sf = a[1:n1, 1:n1] a2_sf = a[n1+1:n2, n1+1:n2] else - a1_sf = sub(a, 1:n1, 1:n1) - a2_sf = sub(a, n1+1:n2, n1+1:n2) + a1_sf = view(a, 1:n1, 1:n1) + a2_sf = view(a, n1+1:n2, n1+1:n2) end f = schurfact(a1_sf, a2_sf) @test_approx_eq f[:Q]*f[:S]*f[:Z]' a1_sf diff --git a/test/linalg/svd.jl b/test/linalg/svd.jl index 7e44e83e6e566..d98a63c1d7625 100644 --- a/test/linalg/svd.jl +++ b/test/linalg/svd.jl @@ -28,8 +28,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) a = aa a2 = aa2 else - a = sub(aa, 1:n, 1:n) - a2 = sub(aa2, 1:n, 1:n) + a = view(aa, 1:n, 1:n) + a2 = view(aa2, 1:n, 1:n) end ε = εa = eps(abs(float(one(eltya)))) diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index 6f01fb1eff2d6..76a177ceb6f9c 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -123,7 +123,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test factorize(A1) == A1 # [c]transpose[!] (test views as well, see issue #14317) - let vrange = 1:n-1, viewA1 = t1(sub(A1.data, vrange, vrange)) + let vrange = 1:n-1, viewA1 = t1(view(A1.data, vrange, vrange)) # transpose @test full(A1.') == full(A1).' @test full(viewA1.') == full(viewA1).' @@ -132,10 +132,10 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test full(viewA1') == full(viewA1)' # transpose! @test transpose!(copy(A1)) == A1.' - @test transpose!(t1(sub(copy(A1).data, vrange, vrange))) == viewA1.' + @test transpose!(t1(view(copy(A1).data, vrange, vrange))) == viewA1.' # ctranspose! @test ctranspose!(copy(A1)) == A1' - @test ctranspose!(t1(sub(copy(A1).data, vrange, vrange))) == viewA1' + @test ctranspose!(t1(view(copy(A1).data, vrange, vrange))) == viewA1' end # diag @@ -150,7 +150,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test full(-A1) == -full(A1) # copy and copy! (test views as well, see issue #14317) - let vrange = 1:n-1, viewA1 = t1(sub(A1.data, vrange, vrange)) + let vrange = 1:n-1, viewA1 = t1(view(A1.data, vrange, vrange)) # copy @test copy(A1) == copy(full(A1)) @test copy(viewA1) == copy(full(viewA1)) @@ -493,4 +493,4 @@ end @test_throws ArgumentError UpperTriangular(LowerTriangular(randn(3,3))) # Issue 16196 -@test UpperTriangular(eye(3)) \ sub(ones(3), [1,2,3]) == ones(3) +@test UpperTriangular(eye(3)) \ view(ones(3), [1,2,3]) == ones(3) diff --git a/test/linalg/tridiag.jl b/test/linalg/tridiag.jl index 6b8881eaf1723..258569cb1fa9a 100644 --- a/test/linalg/tridiag.jl +++ b/test/linalg/tridiag.jl @@ -76,8 +76,8 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) BB = Bs vv = vs else - BB = sub(Bs, 1:n, 1) - vv = sub(vs, 1:n) + BB = view(Bs, 1:n, 1) + vv = view(vs, 1:n) end end @@ -113,7 +113,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) if atype == "Array" vv = vs else - vv = sub(vs, 1:n) + vv = view(vs, 1:n) end end invFsv = Fs\vv diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index b2ba92201c1e5..4d1c3e20d0ad8 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -43,8 +43,8 @@ let AA = randn(2, 2) A = AA S = SS else - A = sub(AA, 1:2, 1:2) - S = sub(SS, 1:3, 1:3) + A = view(AA, 1:2, 1:2) + S = view(SS, 1:3, 1:3) end @test A + I == A + eye(A) @@ -74,7 +74,7 @@ let AA = randn(2, 2) if atype == "Array" T = LowerTriangular(randn(3,3)) else - T = LowerTriangular(sub(randn(3,3), 1:3, 1:3)) + T = LowerTriangular(view(randn(3,3), 1:3, 1:3)) end @test T + J == full(T) + J @test J + T == J + full(T) @@ -85,7 +85,7 @@ let AA = randn(2, 2) if atype == "Array" T = LinAlg.UnitLowerTriangular(randn(3,3)) else - T = LinAlg.UnitLowerTriangular(sub(randn(3,3), 1:3, 1:3)) + T = LinAlg.UnitLowerTriangular(view(randn(3,3), 1:3, 1:3)) end @test T + J == full(T) + J @test J + T == J + full(T) @@ -96,7 +96,7 @@ let AA = randn(2, 2) if atype == "Array" T = UpperTriangular(randn(3,3)) else - T = UpperTriangular(sub(randn(3,3), 1:3, 1:3)) + T = UpperTriangular(view(randn(3,3), 1:3, 1:3)) end @test T + J == full(T) + J @test J + T == J + full(T) @@ -107,7 +107,7 @@ let AA = randn(2, 2) if atype == "Array" T = LinAlg.UnitUpperTriangular(randn(3,3)) else - T = LinAlg.UnitUpperTriangular(sub(randn(3,3), 1:3, 1:3)) + T = LinAlg.UnitUpperTriangular(view(randn(3,3), 1:3, 1:3)) end @test T + J == full(T) + J @test J + T == J + full(T) diff --git a/test/numbers.jl b/test/numbers.jl index 8e220dcaecf7a..217faa977d90a 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2424,7 +2424,7 @@ end # Fill a pre allocated 2x4 matrix let a = zeros(Int,(2,4)) for i in 0:3 - digits!(sub(a,:,i+1),i,2) + digits!(view(a,:,i+1),i,2) end @test a == [0 1 0 1; 0 0 1 1] diff --git a/test/offsetarray.jl b/test/offsetarray.jl index d7bee902c22ac..caf19f96b77f6 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -105,7 +105,7 @@ using OAs # Basics A0 = [1 3; 2 4] A = OffsetArray(A0, (-1,2)) # LinearFast -S = OffsetArray(slice(A0, 1:2, 1:2), (-1,2)) # LinearSlow +S = OffsetArray(view(A0, 1:2, 1:2), (-1,2)) # LinearSlow @test indices(A) == indices(S) == (0:1, 3:4) @test A[0,3] == A[1] == S[0,3] == S[1] == 1 @test A[1,3] == A[2] == S[1,3] == S[2] == 2 @@ -137,26 +137,26 @@ S = OffsetArray(slice(A0, 1:2, 1:2), (-1,2)) # LinearSlow @test eachindex(S) == CartesianRange((0:1,3:4)) # slice -S = slice(A, :, 3) +S = view(A, :, 3) @test S == OffsetArray([1,2], (A.offsets[1],)) @test S[0] == 1 @test S[1] == 2 @test_throws BoundsError S[2] -S = slice(A, 0, :) +S = view(A, 0, :) @test S == OffsetArray([1,3], (A.offsets[2],)) @test S[3] == 1 @test S[4] == 3 @test_throws BoundsError S[1] -S = slice(A, 0:0, 4) +S = view(A, 0:0, 4) @test S == [3] @test S[1] == 3 @test_throws BoundsError S[0] -S = slice(A, 1, 3:4) +S = view(A, 1, 3:4) @test S == [2,4] @test S[1] == 2 @test S[2] == 4 @test_throws BoundsError S[3] -S = slice(A, :, :) +S = view(A, :, :) @test S == A @test S[0,3] == S[1] == 1 @test S[1,3] == S[2] == 2 diff --git a/test/perf/array/indexing.jl b/test/perf/array/indexing.jl index 948e7800269d0..7f6a559f3a33b 100644 --- a/test/perf/array/indexing.jl +++ b/test/perf/array/indexing.jl @@ -167,7 +167,7 @@ function makearrays{T}(::Type{T}, sz) Astrd1 = ArrayStrides1(A) outersz = (sz[1]+1,sz[2]+2) B = reshape(convert(Vector{T}, [1:prod(outersz);]), outersz) - Asub = sub(B, 1:sz[1], 2:sz[2]+1) + Asub = view(B, 1:sz[1], 2:sz[2]+1) Bit = trues(sz) (A, AF, AS, ASS, Asub, Bit,) end diff --git a/test/perf/kernel/laplace.jl b/test/perf/kernel/laplace.jl index 35d5fe783905d..73e3c4ed66915 100644 --- a/test/perf/kernel/laplace.jl +++ b/test/perf/kernel/laplace.jl @@ -40,7 +40,7 @@ end function laplace_iter_vec_sub(u, dx2, dy2, Niter, N) for i = 1:Niter - u[2:N-1, 2:N-1] = ((sub(u, 1:N-2, 2:N-1) + sub(u,3:N, 2:N-1))*dy2 + (sub(u,2:N-1,1:N-2) + sub(u,2:N-1, 3:N))*dx2) * (1./ (2*(dx2+dy2))) + u[2:N-1, 2:N-1] = ((view(u, 1:N-2, 2:N-1) + view(u,3:N, 2:N-1))*dy2 + (view(u,2:N-1,1:N-2) + view(u,2:N-1, 3:N))*dx2) * (1./ (2*(dx2+dy2))) end return u end diff --git a/test/reduce.jl b/test/reduce.jl index 723a85c39037d..8db11f0a4014a 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -277,11 +277,11 @@ end A = reshape(map(UInt8, 101:109), (3,3)) @test @inferred(sum(A)) == 945 -@test @inferred(sum(sub(A, 1:3, 1:3))) == 945 +@test @inferred(sum(view(A, 1:3, 1:3))) == 945 A = reshape(map(UInt8, 1:100), (10,10)) @test @inferred(sum(A)) == 5050 -@test @inferred(sum(sub(A, 1:10, 1:10))) == 5050 +@test @inferred(sum(view(A, 1:10, 1:10))) == 5050 # issue #11618 @test sum([-0.0]) === -0.0 diff --git a/test/serialize.jl b/test/serialize.jl index d7a1fd996e3b8..c7c9e7e3d3c34 100644 --- a/test/serialize.jl +++ b/test/serialize.jl @@ -227,9 +227,9 @@ end # SubArray create_serialization_stream() do s # slices - slc1 = slice(ones(UInt8, 4), 2:3) + slc1 = view(ones(UInt8, 4), 2:3) serialize(s, slc1) - slc2 = slice(ones(UInt8, 4, 4) .+ [0x00, 0x01, 0x02, 0x03], 1, 2:4) + slc2 = view(ones(UInt8, 4, 4) .+ [0x00, 0x01, 0x02, 0x03], 1, 2:4) serialize(s, slc2) seek(s, 0) @@ -251,7 +251,7 @@ Base.getindex(A::ArrayWrapper, i::Real...) = getindex(A.data, i...) end let A = rand(3,4) - for B in (sub(A, :, 2:4), slice(A, 2, 1:3)) + for B in (view(A, :, 2:4), view(A, 2, 1:3)) C = ArrayWrappers.ArrayWrapper(B) io = IOBuffer() serialize(io, C) diff --git a/test/sorting.jl b/test/sorting.jl index 2ec0d37095512..1a40b9223ab51 100644 --- a/test/sorting.jl +++ b/test/sorting.jl @@ -8,12 +8,12 @@ using Base.Order: Forward @test sort(['a':'z';], rev=true) == ['z':-1:'a';] @test sortperm([2,3,1]) == [3,1,2] @test sortperm!([1,2,3], [2,3,1]) == [3,1,2] -let s = sub([1,2,3,4], 1:3) +let s = view([1,2,3,4], 1:3) r = sortperm!(s, [2,3,1]) @test r == [3,1,2] @test r === s end -@test_throws ArgumentError sortperm!(sub([1,2,3,4], 1:4), [2,3,1]) +@test_throws ArgumentError sortperm!(view([1,2,3,4], 1:4), [2,3,1]) @test !issorted([2,3,1]) @test issorted([1,2,3]) @test reverse([2,3,1]) == [1,3,2] @@ -125,7 +125,7 @@ for alg in [InsertionSort, MergeSort] @test issorted(b) @test a[ix] == b - sortperm!(sub(ix, 1:100), sub(a, 1:100), alg=alg) + sortperm!(view(ix, 1:100), view(a, 1:100), alg=alg) b = a[ix][1:100] @test issorted(b) @@ -141,7 +141,7 @@ for alg in [InsertionSort, MergeSort] @test issorted(b, rev=true) @test a[ix] == b - sortperm!(sub(ix, 1:100), sub(a, 1:100), alg=alg, rev=true) + sortperm!(view(ix, 1:100), view(a, 1:100), alg=alg, rev=true) b = a[ix][1:100] @test issorted(b, rev=true) @@ -157,7 +157,7 @@ for alg in [InsertionSort, MergeSort] @test issorted(b, by=x->1/x) @test a[ix] == b - sortperm!(sub(ix, 1:100), sub(a, 1:100), by=x->1/x) + sortperm!(view(ix, 1:100), view(a, 1:100), by=x->1/x) b = a[ix][1:100] @test issorted(b, by=x->1/x) diff --git a/test/sparsedir/cholmod.jl b/test/sparsedir/cholmod.jl index 268f979cfeccb..6f4ca625c1f05 100644 --- a/test/sparsedir/cholmod.jl +++ b/test/sparsedir/cholmod.jl @@ -636,8 +636,8 @@ Fnew = deserialize(b) # test \ for Factor and StridedVecOrMat let x = rand(5) A = cholfact(sparse(diagm(x.\1))) - @test_approx_eq A\sub(ones(10),1:2:10) x - @test_approx_eq A\slice(eye(5,5),:,:) diagm(x) + @test_approx_eq A\view(ones(10),1:2:10) x + @test_approx_eq A\view(eye(5,5),:,:) diagm(x) end # Real factorization and complex rhs diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index d4b941b0744fa..3dc15f2cec64a 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -289,7 +289,7 @@ cA = sprandn(5,5,0.2) + im*sprandn(5,5,0.2) @test full(conj(cA)) == conj(full(cA)) # transpose of SubArrays -A = sub(sprandn(10, 10, 0.3), 1:4, 1:4) +A = view(sprandn(10, 10, 0.3), 1:4, 1:4) @test transpose(full(A)) == full(transpose(A)) @test ctranspose(full(A)) == full(ctranspose(A)) diff --git a/test/staged.jl b/test/staged.jl index 0a635ee1a1ba4..8f064b56b5e3a 100644 --- a/test/staged.jl +++ b/test/staged.jl @@ -66,7 +66,7 @@ splat3(A, 1:2, 1:2, 1) splat3(A, 1:2, 1, 1:2) @test takebuf_string(stagediobuf) == "(UnitRange{$intstr},$intstr,UnitRange{$intstr})" -B = slice(A, 1:3, 2, 1:3); +B = view(A, 1:3, 2, 1:3); @generated function mygetindex(S::SubArray, indexes::Real...) T, N, A, I = S.parameters if N != length(indexes) diff --git a/test/subarray.jl b/test/subarray.jl index 9df96e9ccce8b..f52578efdc1fd 100644 --- a/test/subarray.jl +++ b/test/subarray.jl @@ -173,19 +173,13 @@ function runsubarraytests(A::Array, I...) # Direct test of linear indexing inference C = Agen_nodrop(A, I...) ld = min(single_stride_dim(C), dim_break_linindex(I)) - # sub - S = sub(A, I...) + S = view(A, I...) if Base.iscontiguous(S) @test S.stride1 == 1 end test_linear(S, C) test_cartesian(S, C) test_mixed(S, C) - # slice - S = slice(A, I...) - test_linear(S, C) - test_cartesian(S, C) - test_mixed(S, C) end function runsubarraytests(A::ANY, I...) @@ -216,7 +210,7 @@ function runsubarraytests(A::ANY, I...) # sub local S try - S = sub(A, I...) + S = view(A, I...) catch err @show typeof(A) @show A.indexes @@ -228,7 +222,7 @@ function runsubarraytests(A::ANY, I...) test_mixed(S, C) # slice try - S = slice(A, I...) + S = view(A, I...) catch err @show typeof(A) @show A.indexes @@ -276,9 +270,9 @@ runviews{T}(SB::AbstractArray{T,0}, indexN, indexNN, indexNNN) = nothing testfull = Bool(parse(Int,(get(ENV, "JULIA_TESTFULL", "0")))) ### Views from Arrays ### -index5 = (1, :, 2:5, [4,1,5], reshape([2]), sub(1:5,[2 3 4 1])) # all work with at least size 5 -index25 = (3, :, 2:11, [19,9,7], reshape([10]), sub(1:25,[19 15; 4 24])) -index125 = (113, :, 85:121, [99,14,103], reshape([72]), sub(1:125,reshape([25,4,102,67], 1, 2, 2))) +index5 = (1, :, 2:5, [4,1,5], reshape([2]), view(1:5,[2 3 4 1])) # all work with at least size 5 +index25 = (3, :, 2:11, [19,9,7], reshape([10]), view(1:25,[19 15; 4 24])) +index125 = (113, :, 85:121, [99,14,103], reshape([72]), view(1:125,reshape([25,4,102,67], 1, 2, 2))) if testfull let A = copy(reshape(1:5*7*11, 11, 7, 5)) @@ -295,10 +289,8 @@ oindex = (:, 6, 3:7, reshape([12]), [8,4,6,12,5,7], [3:7 1:5 2:6 4:8 5:9]) if testfull let B = copy(reshape(1:13^3, 13, 13, 13)) for o3 in oindex, o2 in oindex, o1 in oindex - sliceB = slice(B, o1, o2, o3) - runviews(sliceB, index5, index25, index125) - subB = sub(B, o1, o2, o3) - runviews(subB, index5, index25, index125) + viewB = view(B, o1, o2, o3) + runviews(viewB, index5, index25, index125) end end end @@ -317,30 +309,28 @@ if !testfull (13:-2:1,:,:), ([8,4,6,12,5,7],:,3:7), (6,6,[8,4,6,12,5,7]), - (1,:,sub(1:13,[9,12,4,13,1])), - (sub(1:13,[9,12,4,13,1]),2:6,4), + (1,:,view(1:13,[9,12,4,13,1])), + (view(1:13,[9,12,4,13,1]),2:6,4), ([1:5 2:6 3:7 4:8 5:9], :, 3), (:, [46:-1:42 88:-1:84 22:-1:18 49:-1:45 8:-1:4])) runsubarraytests(B, oind...) - sliceB = slice(B, oind...) - runviews(sliceB, index5, index25, index125) - subB = sub(B, oind...) - runviews(subB, index5, index25, index125) + viewB = view(B, oind...) + runviews(viewB, index5, index25, index125) end end end # issue #11289 x11289 = randn(5,5) -@test isempty(sub(x11289, Int[], :)) -@test isempty(sub(x11289, [2,5], Int[])) -@test isempty(sub(x11289, Int[], 2)) +@test isempty(view(x11289, Int[], :)) +@test isempty(view(x11289, [2,5], Int[])) +@test isempty(view(x11289, Int[], 2)) ####### "Classical" tests ####### # sub A = copy(reshape(1:120, 3, 5, 8)) -sA = sub(A, 2, 1:5, :) +sA = view(A, 2:2, 1:5, :) @test strides(sA) == (1, 3, 15) @test parent(sA) == A @test parentindexes(sA) == (Base.NoSlice(2), 1:5, :) @@ -354,7 +344,7 @@ sA[2:5:end] = -1 @test stride(sA,3) == 15 @test stride(sA,4) == 120 test_bounds(sA) -sA = sub(A, 1:3, 1:5, 5) +sA = view(A, 1:3, 1:5, 5) @test Base.parentdims(sA) == [1:2;] sA[1:3,1:5] = -2 @test all(A[:,:,5] .== -2) @@ -362,34 +352,34 @@ sA[:] = -3 @test all(A[:,:,5] .== -3) @test strides(sA) == (1,3) test_bounds(sA) -sA = sub(A, 1:3, 3, 2:5) +sA = view(A, 1:3, 3:3, 2:5) @test Base.parentdims(sA) == [1:3;] @test size(sA) == (3,1,4) @test sA == A[1:3,3:3,2:5] @test sA[:] == A[1:3,3,2:5][:] test_bounds(sA) -sA = sub(A, 1:2:3, 1:3:5, 1:2:8) +sA = view(A, 1:2:3, 1:3:5, 1:2:8) @test Base.parentdims(sA) == [1:3;] @test strides(sA) == (2,9,30) @test sA[:] == A[1:2:3, 1:3:5, 1:2:8][:] # issue #8807 -@test sub(sub([1:5;], 1:5), 1:5) == [1:5;] +@test view(view([1:5;], 1:5), 1:5) == [1:5;] # Test with mixed types @test sA[:, Int16[1,2], big(2)] == [31 40; 33 42] test_bounds(sA) # sub logical indexing #4763 -A = sub([1:10;], 5:8) +A = view([1:10;], 5:8) @test A[A.<7] == [5, 6] @test Base.unsafe_getindex(A, A.<7) == [5, 6] B = reshape(1:16, 4, 4) -sB = sub(B, 2:3, 2:3) +sB = view(B, 2:3, 2:3) @test sB[sB.>8] == [10, 11] @test Base.unsafe_getindex(sB, sB.>8) == [10, 11] # slice A = copy(reshape(1:120, 3, 5, 8)) -sA = slice(A, 2, :, 1:8) +sA = view(A, 2, :, 1:8) @test parent(sA) == A @test parentindexes(sA) == (2, :, 1:8) @test Base.parentdims(sA) == [2:3;] @@ -402,12 +392,12 @@ sA[2:5:end] = -1 @test all(sA[2:5:end] .== -1) @test all(A[5:15:120] .== -1) test_bounds(sA) -sA = slice(A, 1:3, 1:5, 5) +sA = view(A, 1:3, 1:5, 5) @test Base.parentdims(sA) == [1:2;] @test size(sA) == (3,5) @test strides(sA) == (1,3) test_bounds(sA) -sA = slice(A, 1:2:3, 3, 1:2:8) +sA = view(A, 1:2:3, 3, 1:2:8) @test Base.parentdims(sA) == [1,3] @test size(sA) == (2,4) @test strides(sA) == (2,30) @@ -422,56 +412,56 @@ a = [5:8;] A = rand(2, 2, 3) msk = ones(Bool, 2, 2) msk[2,1] = false -sA = sub(A, :, :, 1) +sA = view(A, :, :, 1) sA[msk] = 1.0 @test sA[msk] == ones(countnz(msk)) # bounds checking upon construction; see #4044, #10296 -@test_throws BoundsError sub(1:10, 8:11) +@test_throws BoundsError view(1:10, 8:11) A = reshape(1:20, 5, 4) -sA = sub(A, 1:2, 1:3) -@test_throws BoundsError sub(sA, 1:3, 1:3) -@test_throws BoundsError sub(sA, 1:2, 1:4) -sub(sA, 1:2, 1:2) -@test_throws BoundsError sub(A, 17:23) -sub(A, 17:20) +sA = view(A, 1:2, 1:3) +@test_throws BoundsError view(sA, 1:3, 1:3) +@test_throws BoundsError view(sA, 1:2, 1:4) +view(sA, 1:2, 1:2) +@test_throws BoundsError view(A, 17:23) +view(A, 17:20) # Linear indexing by one multidimensional array: A = reshape(1:120, 3, 5, 8) -sA = sub(A, :, :, :) +sA = view(A, :, :, :) @test sA[[72 17; 107 117]] == [72 17; 107 117] @test sA[[99 38 119 14 76 81]] == [99 38 119 14 76 81] @test sA[[ones(Int, 2, 2, 2); 2ones(Int, 2, 2, 2)]] == [ones(Int, 2, 2, 2); 2ones(Int, 2, 2, 2)] -sA = sub(A, 1:2, 2:3, 3:4) +sA = view(A, 1:2, 2:3, 3:4) @test sA[(1:8)'] == [34 35 37 38 49 50 52 53] @test sA[[1 2 4 4; 6 1 1 4]] == [34 35 38 38; 50 34 34 38] # issue #11871 let a = ones(Float64, (2,2)), - b = sub(a, 1:2, 1:2) + b = view(a, 1:2, 1:2) b[2] = 2 @test b[2] === 2.0 end # issue #15138 let a = [1,2,3], - b = sub(a, UInt(1):UInt(2)) - @test b == slice(a, UInt(1):UInt(2)) == slice(slice(a, :), UInt(1):UInt(2)) == [1,2] + b = view(a, UInt(1):UInt(2)) + @test b == view(a, UInt(1):UInt(2)) == view(view(a, :), UInt(1):UInt(2)) == [1,2] end let A = reshape(1:4, 2, 2) - B = sub(A, :, :) + B = view(A, :, :) @test parent(B) === A - @test parent(sub(B, 0x1, :)) === parent(slice(B, 0x1, :)) === A + @test parent(view(B, 0x1, :)) === parent(view(B, 0x1, :)) === A end # issue #15168 -let A = rand(10), sA = sub(copy(A), :) +let A = rand(10), sA = view(copy(A), :) @test sA[Int16(1)] === sA[Int32(1)] === sA[Int64(1)] === A[1] permute!(sA, collect(Int16, 1:10)) @test A == sA end # the following segfaults with LLVM 3.8 on Windows, ref #15417 -@test collect(sub(sub(reshape(1:13^3, 13, 13, 13), 3:7, 6, :), 1:2:5, :, 1:2:5)) == +@test collect(view(view(reshape(1:13^3, 13, 13, 13), 3:7, 6, :), 1:2:5, :, 1:2:5)) == cat(3,[68,70,72],[406,408,410],[744,746,748]) diff --git a/test/unicode/utf32.jl b/test/unicode/utf32.jl index 875c22330749d..d1ea8b4a690db 100644 --- a/test/unicode/utf32.jl +++ b/test/unicode/utf32.jl @@ -198,7 +198,7 @@ for (fun, S, T) in ((utf16, UInt16, UTF16String), (utf32, UInt32, UTF32String)) @test_throws UnicodeError map(islower, x) @test_throws ArgumentError map(islower, tst) # SubArray conversion - subarr = sub(cmp, 1:6) + subarr = view(cmp, 1:6) @test convert(T, subarr) == str[4:end] end From 0b8d3bebdae8030d31281781572ceb1787be6ed0 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Fri, 17 Jun 2016 16:34:20 +0100 Subject: [PATCH 0038/1117] remove NoSlice --- base/subarray.jl | 40 ++++++---------------------------------- base/sysimg.jl | 6 +++--- doc/stdlib/arrays.rst | 8 +------- test/subarray.jl | 4 ++-- 4 files changed, 12 insertions(+), 46 deletions(-) diff --git a/base/subarray.jl b/base/subarray.jl index ff155d3a49675..06027d4ad5d7d 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -32,40 +32,16 @@ end check_parent_index_match{T,N}(parent::AbstractArray{T,N}, indexes::NTuple{N}) = nothing check_parent_index_match(parent, indexes) = throw(ArgumentError("number of indices ($(length(indexes))) must match the parent dimensionality ($(ndims(parent)))")) -# The NoSlice one-element vector type keeps dimensions without losing performance -immutable NoSlice <: AbstractVector{Int} - i::Int -end -size(::NoSlice) = (1,) -length(::NoSlice) = 1 -linearindexing(::Type{NoSlice}) = LinearFast() -function getindex(N::NoSlice, i::Int) - @_inline_meta - @boundscheck i == 1 || throw_boundserror(N, i) - N.i -end -function getindex(N::NoSlice, i::NoSlice) - @_inline_meta - @boundscheck i.i == 1 || throw_boundserror(N, i) - N -end -getindex(N::NoSlice, ::Colon) = N -function getindex(N::NoSlice, r::Range{Int}) - @_inline_meta - @boundscheck checkbounds(N, r) - N -end - # This computes the linear indexing compatability for a given tuple of indices viewindexing() = LinearFast() # Leading scalar indexes simply increase the stride -viewindexing(I::Tuple{Union{Real, NoSlice}, Vararg{Any}}) = (@_inline_meta; viewindexing(tail(I))) +viewindexing(I::Tuple{Real, Vararg{Any}}) = (@_inline_meta; viewindexing(tail(I))) # Colons may begin a section which may be followed by any number of Colons viewindexing(I::Tuple{Colon, Colon, Vararg{Any}}) = (@_inline_meta; viewindexing(tail(I))) # A UnitRange can follow Colons, but only if all other indices are scalar -viewindexing(I::Tuple{Colon, UnitRange, Vararg{Union{Real, NoSlice}}}) = LinearFast() +viewindexing(I::Tuple{Colon, UnitRange, Vararg{Real}}) = LinearFast() # In general, ranges are only fast if all other indices are scalar -viewindexing(I::Tuple{Union{Range, Colon}, Vararg{Union{Real, NoSlice}}}) = LinearFast() +viewindexing(I::Tuple{Union{Range, Colon}, Vararg{Real}}) = LinearFast() # All other index combinations are slow viewindexing(I::Tuple{Vararg{Any}}) = LinearSlow() # Of course, all other array types are slow @@ -214,7 +190,6 @@ substrides(parent, I::Tuple) = substrides(1, parent, 1, I) substrides(s, parent, dim, ::Tuple{}) = () substrides(s, parent, dim, I::Tuple{Real, Vararg{Any}}) = (substrides(s*size(parent, dim), parent, dim+1, tail(I))...) substrides(s, parent, dim, I::Tuple{AbstractCartesianIndex, Vararg{Any}}) = substrides(s, parent, dim, (I[1].I..., tail(I)...)) -substrides(s, parent, dim, I::Tuple{NoSlice, Vararg{Any}}) = (s, substrides(s*size(parent, dim), parent, dim+1, tail(I))...) substrides(s, parent, dim, I::Tuple{Colon, Vararg{Any}}) = (s, substrides(s*size(parent, dim), parent, dim+1, tail(I))...) substrides(s, parent, dim, I::Tuple{Range, Vararg{Any}}) = (s*step(I[1]), substrides(s*size(parent, dim), parent, dim+1, tail(I))...) substrides(s, parent, dim, I::Tuple{Any, Vararg{Any}}) = throw(ArgumentError("strides is invalid for SubArrays with indices of type $(typeof(I[1]))")) @@ -223,7 +198,7 @@ stride(V::SubArray, d::Integer) = d <= ndims(V) ? strides(V)[d] : strides(V)[end compute_stride1(parent, I::Tuple) = compute_stride1(1, parent, 1, I) compute_stride1(s, parent, dim, I::Tuple{}) = s -compute_stride1(s, parent, dim, I::Tuple{Union{DroppedScalar, NoSlice}, Vararg{Any}}) = +compute_stride1(s, parent, dim, I::Tuple{DroppedScalar, Vararg{Any}}) = (@_inline_meta; compute_stride1(s*size(parent, dim), parent, dim+1, tail(I))) compute_stride1(s, parent, dim, I::Tuple{Range, Vararg{Any}}) = s*step(I[1]) compute_stride1(s, parent, dim, I::Tuple{Colon, Vararg{Any}}) = s @@ -263,8 +238,6 @@ compute_offset1(parent, stride1::Integer, dims, inds, I::Tuple) = compute_linind compute_linindex(parent, I) = compute_linindex(1, 1, parent, 1, I) compute_linindex(f, s, parent, dim, I::Tuple{Real, Vararg{Any}}) = (@_inline_meta; compute_linindex(f + (I[1]-first(indices(parent,dim)))*s, s*size(parent, dim), parent, dim+1, tail(I))) -compute_linindex(f, s, parent, dim, I::Tuple{NoSlice, Vararg{Any}}) = - (@_inline_meta; compute_linindex(f + (I[1].i-first(indices(parent,dim)))*s, s*size(parent, dim), parent, dim+1, tail(I))) # Just splat out the cartesian indices and continue compute_linindex(f, s, parent, dim, I::Tuple{AbstractCartesianIndex, Vararg{Any}}) = (@_inline_meta; compute_linindex(f, s, parent, dim, (I[1].I..., tail(I)...))) @@ -277,18 +250,17 @@ compute_linindex(f, s, parent, dim, I::Tuple{}) = f find_extended_dims(I) = (@_inline_meta; _find_extended_dims((), (), 1, I...)) _find_extended_dims(dims, inds, dim) = dims, inds _find_extended_dims(dims, inds, dim, ::Real, I...) = _find_extended_dims(dims, inds, dim+1, I...) -_find_extended_dims(dims, inds, dim, ::NoSlice, I...) = _find_extended_dims(dims, inds, dim+1, I...) _find_extended_dims(dims, inds, dim, i1::AbstractCartesianIndex, I...) = _find_extended_dims(dims, inds, dim, i1.I..., I...) _find_extended_dims(dims, inds, dim, i1, I...) = _find_extended_dims((dims..., dim), (inds..., i1), dim+1, I...) -unsafe_convert{T,N,P,I<:Tuple{Vararg{Union{RangeIndex, NoSlice}}}}(::Type{Ptr{T}}, V::SubArray{T,N,P,I}) = +unsafe_convert{T,N,P,I<:Tuple{Vararg{RangeIndex}}}(::Type{Ptr{T}}, V::SubArray{T,N,P,I}) = unsafe_convert(Ptr{T}, V.parent) + (first_index(V)-1)*sizeof(T) pointer(V::FastSubArray, i::Int) = pointer(V.parent, V.offset1 + V.stride1*i) pointer(V::FastContiguousSubArray, i::Int) = pointer(V.parent, V.offset1 + i) pointer(V::SubArray, i::Int) = pointer(V, smart_ind2sub(shape(V), i)) -function pointer{T,N,P<:Array,I<:Tuple{Vararg{Union{RangeIndex, NoSlice}}}}(V::SubArray{T,N,P,I}, is::Tuple{Vararg{Int}}) +function pointer{T,N,P<:Array,I<:Tuple{Vararg{RangeIndex}}}(V::SubArray{T,N,P,I}, is::Tuple{Vararg{Int}}) index = first_index(V) strds = strides(V) for d = 1:length(is) diff --git a/base/sysimg.jl b/base/sysimg.jl index dac5f9c4b1727..d54c548d93b86 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -117,9 +117,9 @@ include("iterator.jl") # Definition of StridedArray typealias StridedReshapedArray{T,N,A<:DenseArray} ReshapedArray{T,N,A} -typealias StridedArray{T,N,A<:Union{DenseArray,StridedReshapedArray},I<:Tuple{Vararg{Union{RangeIndex, NoSlice, AbstractCartesianIndex}}}} Union{DenseArray{T,N}, SubArray{T,N,A,I}, StridedReshapedArray{T,N}} -typealias StridedVector{T,A<:Union{DenseArray,StridedReshapedArray},I<:Tuple{Vararg{Union{RangeIndex, NoSlice, AbstractCartesianIndex}}}} Union{DenseArray{T,1}, SubArray{T,1,A,I}, StridedReshapedArray{T,1}} -typealias StridedMatrix{T,A<:Union{DenseArray,StridedReshapedArray},I<:Tuple{Vararg{Union{RangeIndex, NoSlice, AbstractCartesianIndex}}}} Union{DenseArray{T,2}, SubArray{T,2,A,I}, StridedReshapedArray{T,2}} +typealias StridedArray{T,N,A<:Union{DenseArray,StridedReshapedArray},I<:Tuple{Vararg{Union{RangeIndex, AbstractCartesianIndex}}}} Union{DenseArray{T,N}, SubArray{T,N,A,I}, StridedReshapedArray{T,N}} +typealias StridedVector{T,A<:Union{DenseArray,StridedReshapedArray},I<:Tuple{Vararg{Union{RangeIndex, AbstractCartesianIndex}}}} Union{DenseArray{T,1}, SubArray{T,1,A,I}, StridedReshapedArray{T,1}} +typealias StridedMatrix{T,A<:Union{DenseArray,StridedReshapedArray},I<:Tuple{Vararg{Union{RangeIndex, AbstractCartesianIndex}}}} Union{DenseArray{T,2}, SubArray{T,2,A,I}, StridedReshapedArray{T,2}} typealias StridedVecOrMat{T} Union{StridedVector{T}, StridedMatrix{T}} # For OS specific stuff diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index ccad51321734d..c258443fd465c 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -377,7 +377,7 @@ Indexing, Assignment, and Concatenation Returns a subset of array ``A`` as specified by ``inds``\ , where each ``ind`` may be an ``Int``\ , a ``Range``\ , or a ``Vector``\ . See the manual section on :ref:`array indexing ` for details. -.. function:: sub(A, inds...) +.. function:: view(A, inds...) .. Docstring generated from Julia source @@ -401,12 +401,6 @@ Indexing, Assignment, and Concatenation Return all the data of ``A`` where the index for dimension ``d`` equals ``i``\ . Equivalent to ``A[:,:,...,i,:,:,...]`` where ``i`` is in position ``d``\ . -.. function:: slice(A, inds...) - - .. Docstring generated from Julia source - - Returns a view of array ``A`` with the given indices like :func:`sub`\ , but drops all dimensions indexed with scalars. - .. function:: setindex!(A, X, inds...) .. Docstring generated from Julia source diff --git a/test/subarray.jl b/test/subarray.jl index f52578efdc1fd..14ebec68c3553 100644 --- a/test/subarray.jl +++ b/test/subarray.jl @@ -333,7 +333,7 @@ A = copy(reshape(1:120, 3, 5, 8)) sA = view(A, 2:2, 1:5, :) @test strides(sA) == (1, 3, 15) @test parent(sA) == A -@test parentindexes(sA) == (Base.NoSlice(2), 1:5, :) +@test parentindexes(sA) == (2:2, 1:5, :) @test Base.parentdims(sA) == [1:3;] @test size(sA) == (1, 5, 8) @test sA[1, 2, 1:8][:] == [5:15:120;] @@ -463,5 +463,5 @@ let A = rand(10), sA = view(copy(A), :) end # the following segfaults with LLVM 3.8 on Windows, ref #15417 -@test collect(view(view(reshape(1:13^3, 13, 13, 13), 3:7, 6, :), 1:2:5, :, 1:2:5)) == +@test collect(view(view(reshape(1:13^3, 13, 13, 13), 3:7, 6:6, :), 1:2:5, :, 1:2:5)) == cat(3,[68,70,72],[406,408,410],[744,746,748]) From 7c792a87b5c8fe20852ceab8400e6199f08bb9e4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 17 Jun 2016 15:30:01 -0400 Subject: [PATCH 0039/1117] update devdocs to mention :invoke Expr head --- doc/devdocs/ast.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/devdocs/ast.rst b/doc/devdocs/ast.rst index 50eda4da0dfe0..cc4eb6aeb7961 100644 --- a/doc/devdocs/ast.rst +++ b/doc/devdocs/ast.rst @@ -74,8 +74,12 @@ Expr types These symbols appear in the ``head`` field of ``Expr``\s in lowered form. ``call`` - function call. ``args[1]`` is the function to call, ``args[2:end]`` are the - arguments. + function call (dynamic dispatch). ``args[1]`` is the function to call, + ``args[2:end]`` are the arguments. + +``invoke`` + function call (static dispatch). ``args[1]`` is the LambdaInfo to call, + ``args[2:end]`` are the arguments (including the function that is being called, at ``args[2]``). ``static_parameter`` reference a static parameter by index. From a4ac62cdcb3432ba36ddfe62564f06e8ad230ce9 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 17 Jun 2016 17:06:59 -0400 Subject: [PATCH 0040/1117] fix #16340, static parameter duplicated in inner function --- src/julia-syntax.scm | 2 +- test/core.jl | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index c98074a4511e9..2a081e9e5a7bb 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2347,7 +2347,7 @@ (map (lambda (decl) (make-var-info (decl-var decl))) args) (map make-var-info locl))) - (capt-sp (filter (lambda (v) (and (memq v fv) (not (memq v glo)))) + (capt-sp (filter (lambda (v) (and (memq v fv) (not (memq v glo)) (not (memq v new-sp)))) sp)) ;; captured vars: vars from the environment that occur ;; in our set of free variables (fv). diff --git a/test/core.jl b/test/core.jl index 0fb29bf35ca10..7b2330e99c15b 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4303,3 +4303,14 @@ type C16767{T} end @test B16767.types[1].types[1].parameters[1].types === Base.svec(A16767{B16767}) @test C16767.types[1].types[1].parameters[1].types === Base.svec(A16767{C16767{:a}}) + +# issue #16340 +function f16340{T}(x::T) + function g{T}(y::T) + return (T,T) + end + return g +end +let g = f16340(1) + @test isa(typeof(g).name.mt.defs.tvars, TypeVar) +end From 0bcebccefd587d0b8f579f420f603199153e2f57 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Sat, 18 Jun 2016 00:35:32 -0500 Subject: [PATCH 0041/1117] Correction to Date adjuster API docstring --- base/dates/adjusters.jl | 12 ++++++------ doc/stdlib/dates.rst | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/base/dates/adjusters.jl b/base/dates/adjusters.jl index d45d1d493b1ce..cbe2da1f0ad13 100644 --- a/base/dates/adjusters.jl +++ b/base/dates/adjusters.jl @@ -160,14 +160,14 @@ end # Constructors using DateFunctions """ - Date(f::Function, y[, m]; step=Day(1), negate=false, limit=10000) -> Date + Date(f::Function, y[, m, d]; step=Day(1), negate=false, limit=10000) -> Date Create a `Date` through the adjuster API. The starting point will be constructed from the -provided `y, m` arguments, and will be adjusted until `f::Function` returns `true`. The step -size in adjusting can be provided manually through the `step` keyword. If `negate=true`, -then the adjusting will stop when `f::Function` returns `false` instead of `true`. `limit` -provides a limit to the max number of iterations the adjustment API will pursue before -throwing an error (given that `f::Function` is never satisfied). +provided `y, m, d` arguments, and will be adjusted until `f::Function` returns `true`. The +step size in adjusting can be provided manually through the `step` keyword. If +`negate=true`, then the adjusting will stop when `f::Function` returns `false` instead of +`true`. `limit` provides a limit to the max number of iterations the adjustment API will +pursue before throwing an error (given that `f::Function` is never satisfied). """ function Date(func::Function,y,m=1,d=1;step::Period=Day(1),negate::Bool=false,limit::Int=10000) return adjust(DateFunction(func,negate,Date(y,m,d)),Date(y,m,d),step,limit) diff --git a/doc/stdlib/dates.rst b/doc/stdlib/dates.rst index ea6ba2912d35e..dd9ef62218a81 100644 --- a/doc/stdlib/dates.rst +++ b/doc/stdlib/dates.rst @@ -181,11 +181,11 @@ Alternatively, you can write ``using Base.Dates`` to bring all exported function Construct a ``Date`` type by ``Period`` type parts. Arguments may be in any order. ``Date`` parts not provided will default to the value of ``Dates.default(period)``\ . -.. function:: Date(f::Function, y[, m]; step=Day(1), negate=false, limit=10000) -> Date +.. function:: Date(f::Function, y[, m, d]; step=Day(1), negate=false, limit=10000) -> Date .. Docstring generated from Julia source - Create a ``Date`` through the adjuster API. The starting point will be constructed from the provided ``y, m`` arguments, and will be adjusted until ``f::Function`` returns ``true``\ . The step size in adjusting can be provided manually through the ``step`` keyword. If ``negate=true``\ , then the adjusting will stop when ``f::Function`` returns ``false`` instead of ``true``\ . ``limit`` provides a limit to the max number of iterations the adjustment API will pursue before throwing an error (given that ``f::Function`` is never satisfied). + Create a ``Date`` through the adjuster API. The starting point will be constructed from the provided ``y, m, d`` arguments, and will be adjusted until ``f::Function`` returns ``true``\ . The step size in adjusting can be provided manually through the ``step`` keyword. If ``negate=true``\ , then the adjusting will stop when ``f::Function`` returns ``false`` instead of ``true``\ . ``limit`` provides a limit to the max number of iterations the adjustment API will pursue before throwing an error (given that ``f::Function`` is never satisfied). .. function:: Date(dt::DateTime) -> Date From daae6f307fb63f9a6deed120c8dc4f5d1a5b3bab Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 18 Jun 2016 22:02:58 +1000 Subject: [PATCH 0042/1117] Make popmeta! search further than the first meta expression With the push_loc mechanism for source file tracking, it's now common to have more than one meta expression per function body, at least in generated functions. This means that popmeta! needs to search further than the first meta expression to avoid ignoring metadata, including inline annotations. Fixes #16712. --- base/expr.jl | 38 ++++++++++++++++++++++---------------- test/meta.jl | 42 ++++++++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/base/expr.jl b/base/expr.jl index ee86e99d4e3f3..1ea47248df842 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -133,25 +133,31 @@ function popmeta!(body::Expr, sym::Symbol) end popmeta!(arg, sym) = (false, []) function popmeta!(body::Array{Any,1}, sym::Symbol) - idx, args = findmeta_block(body) + idx, blockargs = findmeta_block(body, args -> findmetaarg(args,sym)!=0) if idx == 0 return false, [] end - metaex = args[idx] - metaargs = metaex.args + metaargs = blockargs[idx].args + i = findmetaarg(blockargs[idx].args, sym) + if i == 0 + return false, [] + end + ret = isa(metaargs[i], Expr) ? (metaargs[i]::Expr).args : [] + deleteat!(metaargs, i) + isempty(metaargs) && deleteat!(blockargs, idx) + true, ret +end + +# Find index of `sym` in a meta expression argument list, or 0. +function findmetaarg(metaargs, sym) for i = 1:length(metaargs) - if isa(metaargs[i], Symbol) && (metaargs[i]::Symbol) == sym - deleteat!(metaargs, i) - isempty(metaargs) && deleteat!(args, idx) - return true, [] - elseif isa(metaargs[i], Expr) && (metaargs[i]::Expr).head == sym - ret = (metaargs[i]::Expr).args - deleteat!(metaargs, i) - isempty(metaargs) && deleteat!(args, idx) - return true, ret + arg = metaargs[i] + if (isa(arg, Symbol) && (arg::Symbol) == sym) || + (isa(arg, Expr) && (arg::Expr).head == sym) + return i end end - false, [] + return 0 end function findmeta(ex::Expr) @@ -165,14 +171,14 @@ end findmeta(ex::Array{Any,1}) = findmeta_block(ex) -function findmeta_block(exargs) +function findmeta_block(exargs, argsmatch=args->true) for i = 1:length(exargs) a = exargs[i] if isa(a, Expr) - if (a::Expr).head == :meta + if (a::Expr).head == :meta && argsmatch((a::Expr).args) return i, exargs elseif (a::Expr).head == :block - idx, exa = findmeta_block(a.args) + idx, exa = findmeta_block(a.args, argsmatch) if idx != 0 return idx, exa end diff --git a/test/meta.jl b/test/meta.jl index 427e5bd895e2b..1d7638e933cf1 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -67,28 +67,38 @@ body.args = Base.uncompressed_ast(ast) @test popmeta!(body, :test) == (true, [42]) @test popmeta!(body, :nonexistent) == (false, []) -metaex = Expr(:meta, :foo) +# Simple popmeta!() tests ex1 = quote - $metaex + $(Expr(:meta, :foo)) x*x+1 end -metaex = Expr(:meta, :foo) -ex2 = quote - y = x - $metaex - y*y+1 -end -metaex = Expr(:block, Expr(:meta, :foo)) -ex3 = quote +@test popmeta!(ex1, :foo)[1] +@test !popmeta!(ex1, :foo)[1] +@test !popmeta!(ex1, :bar)[1] +@test !(popmeta!(:(x*x+1), :foo)[1]) + +# Find and pop meta information from general ast locations +multi_meta = quote + $(Expr(:meta, :foo1)) y = x - $metaex + $(Expr(:meta, :foo2, :foo3)) + begin + $(Expr(:meta, :foo4, Expr(:foo5, 1, 2))) + end x*x+1 end - -@test popmeta!(ex1, :foo)[1] -@test popmeta!(ex2, :foo)[1] -@test popmeta!(ex3, :foo)[1] -@test !(popmeta!(:(x*x+1), :foo)[1]) +@test popmeta!(deepcopy(multi_meta), :foo1) == (true, []) +@test popmeta!(deepcopy(multi_meta), :foo2) == (true, []) +@test popmeta!(deepcopy(multi_meta), :foo3) == (true, []) +@test popmeta!(deepcopy(multi_meta), :foo4) == (true, []) +@test popmeta!(deepcopy(multi_meta), :foo5) == (true, [1,2]) +@test popmeta!(deepcopy(multi_meta), :bar) == (false, []) + +# Test that popmeta!() removes meta blocks entirely when they become empty. +for m in [:foo1, :foo2, :foo3, :foo4, :foo5] + @test popmeta!(multi_meta, m)[1] +end +@test Base.findmeta(multi_meta.args)[1] == 0 end From dba93e70592600825183cf68d61a283014d87877 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 18 Jun 2016 22:11:07 +1000 Subject: [PATCH 0043/1117] Clarify popmeta! search behaviour in devdocs --- doc/devdocs/meta.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/devdocs/meta.rst b/doc/devdocs/meta.rst index 1a950fc347012..9ed897362e43f 100644 --- a/doc/devdocs/meta.rst +++ b/doc/devdocs/meta.rst @@ -45,10 +45,10 @@ to specify additional information. To use the metadata, you have to parse these ``:meta`` expressions. If your implementation can be performed within Julia, ``Base.popmeta!`` is very handy: ``Base.popmeta!(body, :symbol)`` will scan a function *body* -expression (one without the function signature) for a ``:meta`` -expression, extract any arguments, and return a tuple ``(found::Bool, -args::Array{Any})``. If the metadata did not have any arguments, or -``:symbol`` was not found, the ``args`` array will be empty. +expression (one without the function signature) for the first ``:meta`` +expression containing ``:symbol``, extract any arguments, and return a tuple +``(found::Bool, args::Array{Any})``. If the metadata did not have any +arguments, or ``:symbol`` was not found, the ``args`` array will be empty. Not yet provided is a convenient infrastructure for parsing ``:meta`` expressions from C++. From ecfbe713266a1c31db4eb6fd7ac46b2d6d89410c Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 17 Jun 2016 17:19:23 -0400 Subject: [PATCH 0044/1117] Implement deepcopy for BigInt and BigFloat Workaround #16667 for `BigFloat` and `BigInt` --- base/gmp.jl | 10 ++++++++++ base/mpfr.jl | 14 ++++++++++++++ base/sysimg.jl | 2 ++ test/copy.jl | 26 ++++++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/base/gmp.jl b/base/gmp.jl index 3bcb2ade3e867..b933811775cd0 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -554,4 +554,14 @@ Base.checked_fld(a::BigInt, b::BigInt) = fld(a, b) Base.checked_mod(a::BigInt, b::BigInt) = mod(a, b) Base.checked_cld(a::BigInt, b::BigInt) = cld(a, b) +function Base.deepcopy_internal(x::BigInt, stackdict::ObjectIdDict) + if haskey(stackdict, x) + return stackdict[x] + end + y = BigInt() + ccall((:__gmpz_set,:libgmp), Void, (Ptr{BigInt}, Ptr{BigInt}), &y, &x) + stackdict[x] = y + return y +end + end # module diff --git a/base/mpfr.jl b/base/mpfr.jl index 1733bcb65485d..3b3da4f26b6f0 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -880,4 +880,18 @@ get_emin_max() = ccall((:mpfr_get_emin_max, :libmpfr), Clong, ()) set_emax!(x) = ccall((:mpfr_set_emax, :libmpfr), Void, (Clong,), x) set_emin!(x) = ccall((:mpfr_set_emin, :libmpfr), Void, (Clong,), x) +function Base.deepcopy_internal(x::BigFloat, stackdict::ObjectIdDict) + if haskey(stackdict, x) + return stackdict[x] + end + N = precision(x) + y = BigFloat(zero(Clong), zero(Cint), zero(Clong), C_NULL) + ccall((:mpfr_init2,:libmpfr), Void, (Ptr{BigFloat}, Clong), &y, N) + finalizer(y, Base.GMP._mpfr_clear_func) + ccall((:mpfr_set, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), + &y, &x, ROUNDING_MODE[end]) + stackdict[x] = y + return y +end + end #module diff --git a/base/sysimg.jl b/base/sysimg.jl index d54c548d93b86..05bca6889362e 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -214,6 +214,8 @@ importall .Sort # version include("version.jl") +function deepcopy_internal end + # BigInts and BigFloats include("gmp.jl") importall .GMP diff --git a/test/copy.jl b/test/copy.jl index 7be5a705d5f57..617c16f85d54f 100644 --- a/test/copy.jl +++ b/test/copy.jl @@ -82,3 +82,29 @@ let a1 = Base.svec(1, 2, 3, []), a2 = Base.svec(1, 2, 3) @test a3 !== b3 @test a3[1] === a3[2] end + +# issue #16667 +let x = BigInt[1:1000;], y = deepcopy(x), v + # Finalize the original values to make sure the deep copy is indeed + # independent + for v in x + finalize(v) + end + # Allocate some memory to make it more likely to trigger an error + # if `deepcopy` went wrong + x = BigInt[1:1000;] + @test y == x +end +let x = BigFloat[1:1000;], y, z, v + y, z = setprecision(2) do + deepcopy(x), BigFloat[1:1000;] + end + for v in x + finalize(v) + end + x = BigFloat[1:1000;] + # Make sure the difference in precision doesn't affect deep copy + @test y == x + # Check that the setprecision indeed does something + @test z != x +end From 400713c7d876acfbe6dd2db8cd05cf06f0fb58f4 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 18 Jun 2016 15:09:18 -0500 Subject: [PATCH 0045/1117] Support `append!(::Vector, iter)` --- base/collections.jl | 19 +++++++++++++++++++ test/arrayops.jl | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/base/collections.jl b/base/collections.jl index 11d3d23cc66cf..545112e25ba8b 100644 --- a/base/collections.jl +++ b/base/collections.jl @@ -17,6 +17,25 @@ export peek +# Some algorithms that can be defined only after infrastructure is in place +Base.append!(a::Vector, iter) = _append!(a, Base.iteratorsize(iter), iter) + +function _append!(a, ::Base.HasLength, iter) + n = length(a) + resize!(a, n+length(iter)) + @inbounds for (i,item) in zip(n+1:length(a), iter) + a[i] = item + end + a +end + +function _append!(a, ::Base.IteratorSize, iter) + for item in iter + push!(a, item) + end + a +end + # Heap operations on flat arrays # ------------------------------ diff --git a/test/arrayops.jl b/test/arrayops.jl index 6a061d8011cc1..74c0f8ae9f8a4 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1043,6 +1043,10 @@ A = [1,2] @test append!(A, A) == [1,2,1,2] @test prepend!(A, A) == [1,2,1,2,1,2,1,2] +A = [1,2] +s = Set([1,2,3]) +@test sort(append!(A, s)) == [1,1,2,2,3] + # cases where shared arrays can/can't be grown A = [1 3;2 4] B = reshape(A, 4) From ac0cc141bae41f206c31d697fe831bd18c926822 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Sat, 18 Jun 2016 18:43:54 -0400 Subject: [PATCH 0046/1117] deprecate WString and wstring (#16975) --- base/c.jl | 24 ++++++++++++++++++----- base/deprecated.jl | 10 ++++++++++ base/docs/helpdb/Base.jl | 9 --------- base/exports.jl | 2 -- base/unicode/utf16.jl | 3 +++ base/unicode/utf32.jl | 21 +++----------------- doc/manual/calling-c-and-fortran-code.rst | 14 +++++-------- doc/manual/strings.rst | 4 +--- doc/stdlib/strings.rst | 7 ------- test/strings/basic.jl | 10 +--------- test/unicode/utf32.jl | 6 ------ 11 files changed, 42 insertions(+), 68 deletions(-) diff --git a/base/c.jl b/base/c.jl index ee7eb1ab15618..53ed7e4f03f01 100644 --- a/base/c.jl +++ b/base/c.jl @@ -81,7 +81,12 @@ unsafe_string(s::Cstring) = unsafe_string(convert(Ptr{UInt8}, s)) # convert strings to String etc. to pass as pointers cconvert(::Type{Cstring}, s::AbstractString) = String(s) -cconvert(::Type{Cwstring}, s::AbstractString) = wstring(s) + +function cconvert(::Type{Cwstring}, s::AbstractString) + v = transcode(Cwchar_t, String(s).data) + !isempty(v) && v[end] == 0 || push!(v, 0) + return v +end containsnul(p::Ptr, len) = C_NULL != ccall(:memchr, Ptr{Cchar}, (Ptr{Cchar}, Cint, Csize_t), p, 0, len) @@ -90,17 +95,25 @@ containsnul(s::AbstractString) = '\0' in s function unsafe_convert(::Type{Cstring}, s::String) p = unsafe_convert(Ptr{Cchar}, s) - if containsnul(p, sizeof(s)) + containsnul(p, sizeof(s)) && throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))")) - end return Cstring(p) end +function unsafe_convert(::Type{Cwstring}, v::Vector{Cwchar_t}) + for i = 1:length(v)-1 + v[i] == 0 && + throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(v))")) + end + v[end] == 0 || + throw(ArgumentError("C string data must be NUL terminated: $(repr(v))")) + p = unsafe_convert(Ptr{Cwchar_t}, v) + return Cwstring(p) +end + # symbols are guaranteed not to contain embedded NUL convert(::Type{Cstring}, s::Symbol) = Cstring(unsafe_convert(Ptr{Cchar}, s)) -# in string.jl: unsafe_convert(::Type{Cwstring}, s::WString) - # FIXME: this should be handled by implicit conversion to Cwstring, but good luck with that if is_windows() function cwstring(s::AbstractString) @@ -113,6 +126,7 @@ end # transcoding between data in UTF-8 and UTF-16 for Windows APIs transcode{T<:Union{UInt8,UInt16}}(::Type{T}, src::Vector{T}) = src +transcode(::Type{Int32}, src::Vector{UInt32}) = reinterpret(Int32, src) function transcode(::Type{UInt16}, src::Vector{UInt8}) dst = UInt16[] diff --git a/base/deprecated.jl b/base/deprecated.jl index d938921f4b65f..a0b22e31af6c3 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -488,6 +488,16 @@ end end ) +if sizeof(Cwchar_t) == 2 + @deprecate_binding WString UTF16String + @deprecate_binding wstring utf16 + utf16(s::Cwstring) = utf16(convert(Ptr{Cwchar_t}, s)) +elseif sizeof(Cwchar_t) == 4 + @deprecate_binding WString UTF32String + @deprecate_binding wstring utf32 + utf32(s::Cwstring) = utf32(convert(Ptr{Cwchar_t}, s)) +end + @deprecate ==(x::Char, y::Integer) UInt32(x) == y @deprecate ==(x::Integer, y::Char) x == UInt32(y) @deprecate isless(x::Char, y::Integer) UInt32(x) < y diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 3d35be2c6cebf..ad8b9d701af67 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2775,15 +2775,6 @@ Equivalent to `writedlm` with `delim` set to comma. """ writecsv -""" - wstring(s) - -This is a synonym for either `utf32(s)` or `utf16(s)`, depending on whether `Cwchar_t` is 32 -or 16 bits, respectively. The synonym `WString` for `UTF32String` or `UTF16String` is also -provided. -""" -wstring - """ withenv(f::Function, kv::Pair...) diff --git a/base/exports.jl b/base/exports.jl index e1c8a74cdb79e..d5dc7801aa060 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -127,7 +127,6 @@ export VersionNumber, WeakKeyDict, WorkerConfig, - WString, Zip, # Ccall types @@ -881,7 +880,6 @@ export utf16, utf32, warn, - wstring, # random numbers AbstractRNG, diff --git a/base/unicode/utf16.jl b/base/unicode/utf16.jl index e1b2093e26d3a..21cb4059056e5 100644 --- a/base/unicode/utf16.jl +++ b/base/unicode/utf16.jl @@ -270,3 +270,6 @@ function map(fun, str::UTF16String) push!(buf, 0) UTF16String(buf) end + +cconvert(::Type{Cwstring}, v::Vector{UInt16}) = transcode(Cwchar_t, v) +cconvert(::Type{Cwstring}, s::UTF16String) = transcode(Cwchar_t, s.data) diff --git a/base/unicode/utf32.jl b/base/unicode/utf32.jl index 929bdcc22fe7d..9250e9310909e 100644 --- a/base/unicode/utf32.jl +++ b/base/unicode/utf32.jl @@ -153,6 +153,9 @@ function convert(T::Type{UTF32String}, bytes::AbstractArray{UInt8}) UTF32String(d) end +cconvert(::Type{Cwstring}, v::Vector{UInt32}) = transcode(Cwchar_t, v) +cconvert(::Type{Cwstring}, s::UTF32String) = transcode(Cwchar_t, s.data) + function isvalid(::Type{UTF32String}, str::Union{Vector{UInt32}, Vector{Char}}) for c in str @inbounds if !isvalid(Char, UInt32(c)) ; return false ; end @@ -186,24 +189,6 @@ function map(f, s::UTF32String) UTF32String(out) end -if sizeof(Cwchar_t) == 2 - const WString = UTF16String - const wstring = utf16 -elseif sizeof(Cwchar_t) == 4 - const WString = UTF32String - const wstring = utf32 -end -wstring(s::Cwstring) = wstring(convert(Ptr{Cwchar_t}, s)) - -# Cwstring is defined in c.jl, but conversion needs to be defined here -# to have WString -function unsafe_convert(::Type{Cwstring}, s::WString) - if containsnul(s) - throw(ArgumentError("embedded NUL chars are not allowed in C strings: $(repr(s))")) - end - return Cwstring(unsafe_convert(Ptr{Cwchar_t}, s)) -end - pointer(x::Union{UTF16String,UTF32String}) = pointer(x.data) pointer(x::Union{UTF16String,UTF32String}, i::Integer) = pointer(x)+(i-1)*sizeof(eltype(x.data)) pointer{T<:Union{UTF16String,UTF32String}}(x::SubString{T}) = pointer(x.string.data) + x.offset*sizeof(eltype(x.string.data)) diff --git a/doc/manual/calling-c-and-fortran-code.rst b/doc/manual/calling-c-and-fortran-code.rst index b16d4efc83447..84e3b35ae4010 100644 --- a/doc/manual/calling-c-and-fortran-code.rst +++ b/doc/manual/calling-c-and-fortran-code.rst @@ -486,15 +486,11 @@ C name Standard Julia Alias Julia Base Type .. note:: For ``wchar_t*`` arguments, the Julia type should be ``Cwstring`` (if the C - routine expects a NUL-terminated string) or ``Ptr{Cwchar_t}`` otherwise, and - data can be converted to/from ordinary Julia strings by the ``wstring(s)`` - function (equivalent to either ``utf16(s)`` or ``utf32(s)`` depending upon the - width of ``Cwchar_t``); this conversion will be called automatically for - ``Cwstring`` arguments. Note also that ASCII, UTF-8, UTF-16, and UTF-32 - string data in Julia is internally NUL-terminated, so it can be passed to C - functions expecting NUL-terminated data without making a copy (but using the - ``Cwstring`` type will cause an error to be thrown if the string itself - contains NUL characters). + routine expects a NUL-terminated string) or ``Ptr{Cwchar_t}`` otherwise. Note + also that ASCII, UTF-8, UTF-16, and UTF-32 string data in Julia is internally + NUL-terminated, so it can be passed to C functions expecting NUL-terminated + data without making a copy (but using the ``Cwstring`` type will cause an + error to be thrown if the string itself contains NUL characters). .. note:: diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 8bafefa5f3ba8..7c5c34a2e33e4 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -353,9 +353,7 @@ UTF-8 is not the only encoding that Julia supports, and adding support for new encodings is quite easy. In particular, Julia also provides :obj:`UTF16String` and :obj:`UTF32String` types, constructed by :func:`utf16` and :func:`utf32` respectively, for UTF-16 and -UTF-32 encodings. It also provides aliases :obj:`WString` and -:func:`wstring` for either UTF-16 or UTF-32 strings, depending on the -size of ``Cwchar_t``. Additional discussion of other encodings and how to +UTF-32 encodings. Additional discussion of other encodings and how to implement support for them is beyond the scope of this document for the time being. For further discussion of UTF-8 encoding issues, see the section below on `byte array literals <#Byte+Array+Literals>`_, diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index 80dcc8a5aafdc..b92a95c3d3040 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -500,10 +500,3 @@ .. Docstring generated from Julia source Create a string from the address of a NUL-terminated UTF-32 string. A copy is made; the pointer can be safely freed. If ``length`` is specified, the string does not have to be NUL-terminated. - -.. function:: wstring(s) - - .. Docstring generated from Julia source - - This is a synonym for either ``utf32(s)`` or ``utf16(s)``\ , depending on whether ``Cwchar_t`` is 32 or 16 bits, respectively. The synonym ``WString`` for ``UTF32String`` or ``UTF16String`` is also provided. - diff --git a/test/strings/basic.jl b/test/strings/basic.jl index b88c39f67dfa9..de45bf8405740 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -237,22 +237,14 @@ end let s = normalize_string("tést",:NFKC) @test unsafe_string(Base.unsafe_convert(Cstring, s)) == s @test unsafe_string(convert(Cstring, Symbol(s))) == s - @test wstring(Base.unsafe_convert(Cwstring, wstring(s))) == s -end -let s = "ba\0d" - @test_throws ArgumentError Base.unsafe_convert(Cstring, s) - @test_throws ArgumentError Base.unsafe_convert(Cwstring, wstring(s)) end +@test_throws ArgumentError Base.unsafe_convert(Cstring, "ba\0d") cstrdup(s) = @static is_windows() ? ccall(:_strdup, Cstring, (Cstring,), s) : ccall(:strdup, Cstring, (Cstring,), s) let p = cstrdup("hello") @test unsafe_string(p) == "hello" == unsafe_wrap(String, cstrdup(p), true) Libc.free(p) end -let p = @static is_windows() ? ccall(:_wcsdup, Cwstring, (Cwstring,), "tést") : ccall(:wcsdup, Cwstring, (Cwstring,), "tést") - @test wstring(p) == "tést" - Libc.free(p) -end # issue # 11389: Vector{UInt32} was copied with UTF32String, unlike Vector{Char} a = UInt32[48,0] diff --git a/test/unicode/utf32.jl b/test/unicode/utf32.jl index d1ea8b4a690db..f4455271be054 100644 --- a/test/unicode/utf32.jl +++ b/test/unicode/utf32.jl @@ -169,12 +169,6 @@ for T in (String, UTF16String, UTF32String) end end -# Wstring -u8 = "\U10ffff\U1d565\U1d7f6\U00066\U2008a" -w = wstring(u8) -@test length(w) == 5 && String(w) == u8 && collect(u8) == collect(w) -@test u8 == WString(w.data) - # 12268 for (fun, S, T) in ((utf16, UInt16, UTF16String), (utf32, UInt32, UTF32String)) # AbstractString From 740253669f65e3ae62e3d0a964a32cfe131fc215 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 18 Jun 2016 21:26:09 -0400 Subject: [PATCH 0047/1117] Correct type of `jl_lineno` Ref #16131 --- base/docs/Docs.jl | 2 +- base/docs/core.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index f72965ea3d41b..936a22a6e162a 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -448,7 +448,7 @@ function metadata(expr) push!(args, :($(Pair)(:source, $(quot(expr))))) # Filename and linenumber of the docstring. push!(args, :($(Pair)(:path, $(Base).@__FILE__))) - push!(args, :($(Pair)(:linenumber, $(unsafe_load(cglobal(:jl_lineno, Int)))))) + push!(args, :($(Pair)(:linenumber, $(unsafe_load(cglobal(:jl_lineno, Cint)))))) # Module in which the docstring is defined. push!(args, :($(Pair)(:module, $(current_module)()))) # Field docs for concrete types. diff --git a/base/docs/core.jl b/base/docs/core.jl index 1666333268337..f1350d57c8c5c 100644 --- a/base/docs/core.jl +++ b/base/docs/core.jl @@ -8,7 +8,7 @@ function doc!(str, ex) ptr = unsafe_load(Core.Intrinsics.cglobal(:jl_filename, Ptr{UInt8})) len = ccall(:strlen, Csize_t, (Ptr{UInt8},), ptr) file = ccall(:jl_symbol_n, Any, (Ptr{UInt8}, Int32), ptr, len) - line = unsafe_load(Core.Intrinsics.cglobal(:jl_lineno, Int)) + line = unsafe_load(Core.Intrinsics.cglobal(:jl_lineno, Int32)) # Cint push!(DOCS, (current_module(), ex, str, file, line)) end const DOCS = Array{Any, 1}() From e163594e732fa671bef08cc2f86a8f6ed4872699 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 18 Jun 2016 21:48:25 -0400 Subject: [PATCH 0048/1117] Lazy initialize gmp and mpfr finalizers --- base/gmp.jl | 7 +------ base/mpfr.jl | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index b933811775cd0..a71d54d56c8c9 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -44,14 +44,11 @@ type BigInt <: Integer function BigInt() b = new(zero(Cint), zero(Cint), C_NULL) ccall((:__gmpz_init,:libgmp), Void, (Ptr{BigInt},), &b) - finalizer(b, _gmp_clear_func) + finalizer(b, cglobal((:__gmpz_clear, :libgmp))) return b end end -_gmp_clear_func = C_NULL -_mpfr_clear_func = C_NULL - function __init__() try if gmp_version().major != GMP_VERSION.major || gmp_bits_per_limb() != GMP_BITS_PER_LIMB @@ -60,8 +57,6 @@ function __init__() "Please rebuild Julia.")) end - global _gmp_clear_func = cglobal((:__gmpz_clear, :libgmp)) - global _mpfr_clear_func = cglobal((:mpfr_clear, :libmpfr)) ccall((:__gmp_set_memory_functions, :libgmp), Void, (Ptr{Void},Ptr{Void},Ptr{Void}), cglobal(:jl_gc_counted_malloc), diff --git a/base/mpfr.jl b/base/mpfr.jl index 3b3da4f26b6f0..2ebdbe38a57fa 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -51,7 +51,7 @@ type BigFloat <: AbstractFloat N = precision(BigFloat) z = new(zero(Clong), zero(Cint), zero(Clong), C_NULL) ccall((:mpfr_init2,:libmpfr), Void, (Ptr{BigFloat}, Clong), &z, N) - finalizer(z, Base.GMP._mpfr_clear_func) + finalizer(z, cglobal((:mpfr_clear, :libmpfr))) return z end # Not recommended for general use @@ -887,7 +887,7 @@ function Base.deepcopy_internal(x::BigFloat, stackdict::ObjectIdDict) N = precision(x) y = BigFloat(zero(Clong), zero(Cint), zero(Clong), C_NULL) ccall((:mpfr_init2,:libmpfr), Void, (Ptr{BigFloat}, Clong), &y, N) - finalizer(y, Base.GMP._mpfr_clear_func) + finalizer(y, cglobal((:mpfr_clear, :libmpfr))) ccall((:mpfr_set, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &y, &x, ROUNDING_MODE[end]) stackdict[x] = y From a1b8c31fc77d422df56e36cded251f47095f928a Mon Sep 17 00:00:00 2001 From: kshyatt Date: Wed, 25 May 2016 20:28:38 -0500 Subject: [PATCH 0049/1117] Add logdet and logabsdet for Triangular types Somehow we didn't have `logdet`/`logabsdet` working for these. Added now with tests. --- base/linalg/generic.jl | 5 ++++- base/linalg/triangular.jl | 14 ++++++++++++++ test/linalg/triangular.jl | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index a97f25458b7a4..2e91295d1a3fe 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -563,8 +563,11 @@ function det{T}(A::AbstractMatrix{T}) end det(x::Number) = x -logdet(A::AbstractMatrix) = logdet(lufact(A)) logabsdet(A::AbstractMatrix) = logabsdet(lufact(A)) +function logdet(A::AbstractMatrix) + d,s = logabsdet(A) + return d + log(s) +end # isapprox: approximate equality of arrays [like isapprox(Number,Number)] function isapprox{T<:Number,S<:Number}(x::AbstractArray{T}, y::AbstractArray{S}; rtol::Real=Base.rtoldefault(T,S), atol::Real=0, norm::Function=vecnorm) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index ccce61d38edb1..18c0f551a7594 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -1964,8 +1964,22 @@ function eigvecs{T}(A::AbstractTriangular{T}) end det{T}(A::UnitUpperTriangular{T}) = one(T) det{T}(A::UnitLowerTriangular{T}) = one(T) +logdet{T}(A::UnitUpperTriangular{T}) = zero(T) +logdet{T}(A::UnitLowerTriangular{T}) = zero(T) +logabsdet{T}(A::UnitUpperTriangular{T}) = zero(T), one(T) +logabsdet{T}(A::UnitLowerTriangular{T}) = zero(T), one(T) det{T}(A::UpperTriangular{T}) = prod(diag(A.data)) det{T}(A::LowerTriangular{T}) = prod(diag(A.data)) +function logabsdet{T}(A::Union{UpperTriangular{T},LowerTriangular{T}}) + sgn = one(T) + abs_det = zero(real(T)) + @inbounds for i in 1:size(A,1) + diag_i = A.data[i,i] + sgn *= sign(diag_i) + abs_det += log(abs(diag_i)) + end + return abs_det, sgn +end eigfact(A::AbstractTriangular) = Eigen(eigvals(A), eigvecs(A)) diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index 76a177ceb6f9c..08d994433e3d4 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -225,6 +225,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa # Determinant @test_approx_eq_eps det(A1) det(lufact(full(A1))) sqrt(eps(real(float(one(elty1)))))*n*n + @test_approx_eq_eps logdet(A1) logdet(lufact(full(A1))) sqrt(eps(real(float(one(elty1)))))*n*n # Matrix square root @test sqrtm(A1) |> t -> t*t ≈ A1 From 9458a296c88a80a95fa5b2fc3e9865646124a71e Mon Sep 17 00:00:00 2001 From: Jim Garrison Date: Sun, 19 Jun 2016 00:14:48 -0700 Subject: [PATCH 0050/1117] Fix link in generator doc [av skip] --- doc/manual/arrays.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 67b95d890c14c..007483ae3c8db 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -224,7 +224,7 @@ Generator Expressions Comprehensions can also be written without the enclosing square brackets, producing an object known as a generator. This object can be iterated to produce values on demand, instead of allocating an array and storing them in advance -(see ). +(see :ref:`man-interfaces-iteration`). For example, the following expression sums a series without allocating memory: .. doctest:: From 5ce60379f3320e893eccefb645f72528be629585 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 19 Jun 2016 00:10:32 -0400 Subject: [PATCH 0051/1117] Fix #16280 Cleaned up docs for `trmv`, changed wording to be clearer. --- base/linalg/blas.jl | 23 ++++++++++++++--------- doc/stdlib/linalg.rst | 8 ++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/base/linalg/blas.jl b/base/linalg/blas.jl index 0c592b18c7d97..4e3b8422577a8 100644 --- a/base/linalg/blas.jl +++ b/base/linalg/blas.jl @@ -740,21 +740,26 @@ end ### trmv, Triangular matrix-vector multiplication """ - trmv(side, ul, tA, dA, alpha, A, b) + trmv(ul, tA, dA, A, b) -Returns `alpha*A*b` or one of the other three variants determined by `side` (`A` on left or -right) and `tA` (transpose `A`). Only the `ul` triangle of `A` is used. `dA` indicates if -`A` is unit-triangular (the diagonal is assumed to be all ones). +Returns `op(A)*b`, where `op` is determined by `tA` +(`N` for identity, `T` for transpose `A`, and `C` for conjugate +transpose `A`). Only the `ul` triangle (`U` for upper, `L` +for lower) of `A` is used. `dA` indicates if `A` is +unit-triangular (the diagonal is assumed to be all ones if `U`, +or non-unit if `N`). """ function trmv end """ - trmv!(side, ul, tA, dA, alpha, A, b) + trmv!(ul, tA, dA, A, b) -Update `b` as `alpha*A*b` or one of the other three variants determined by `side` (`A` on -left or right) and `tA` (transpose `A`). Only the `ul` triangle of `A` is used. `dA` -indicates if `A` is unit-triangular (the diagonal is assumed to be all ones). Returns the -updated `b`. +Returns `op(A)*b`, where `op` is determined by `tA` +(`N` for identity, `T` for transpose `A`, and `C` for conjugate +transpose `A`). Only the `ul` triangle (`U` for upper, `L` +for lower) of `A` is used. `dA` indicates if `A` is +unit-triangular (the diagonal is assumed to be all ones if `U`, +or non-unit if `N`). The multiplication occurs in-place on `b`. """ function trmv! end diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 00e05696f5b32..d8d23e343937d 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1630,17 +1630,17 @@ Usually a function has 4 methods defined, one each for ``Float64``, Returns the solution to ``A*X = alpha*B`` or one of the other three variants determined by ``side`` (``A`` on left or right of ``X``\ ) and ``tA`` (transpose ``A``\ ). Only the ``ul`` triangle of ``A`` is used. ``dA`` indicates if ``A`` is unit-triangular (the diagonal is assumed to be all ones). -.. function:: trmv!(side, ul, tA, dA, alpha, A, b) +.. function:: trmv!(ul, tA, dA, A, b) .. Docstring generated from Julia source - Update ``b`` as ``alpha*A*b`` or one of the other three variants determined by ``side`` (``A`` on left or right) and ``tA`` (transpose ``A``\ ). Only the ``ul`` triangle of ``A`` is used. ``dA`` indicates if ``A`` is unit-triangular (the diagonal is assumed to be all ones). Returns the updated ``b``\ . + Returns ``op(A)*b``\ , where ``op`` is determined by ``tA`` (``N`` for identity, ``T`` for transpose ``A``\ , and ``C`` for conjugate transpose ``A``\ ). Only the ``ul`` triangle (``U`` for upper, ``L`` for lower) of ``A`` is used. ``dA`` indicates if ``A`` is unit-triangular (the diagonal is assumed to be all ones if ``U``\ , or non-unit if ``N``\ ). The multiplication occurs in-place on ``b``\ . -.. function:: trmv(side, ul, tA, dA, alpha, A, b) +.. function:: trmv(ul, tA, dA, A, b) .. Docstring generated from Julia source - Returns ``alpha*A*b`` or one of the other three variants determined by ``side`` (``A`` on left or right) and ``tA`` (transpose ``A``\ ). Only the ``ul`` triangle of ``A`` is used. ``dA`` indicates if ``A`` is unit-triangular (the diagonal is assumed to be all ones). + Returns ``op(A)*b``\ , where ``op`` is determined by ``tA`` (``N`` for identity, ``T`` for transpose ``A``\ , and ``C`` for conjugate transpose ``A``\ ). Only the ``ul`` triangle (``U`` for upper, ``L`` for lower) of ``A`` is used. ``dA`` indicates if ``A`` is unit-triangular (the diagonal is assumed to be all ones if ``U``\ , or non-unit if ``N``\ ). .. function:: trsv!(ul, tA, dA, A, b) From 1629709e8cf1f1aa0922c6d2cc55d49a5050ccaf Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Tue, 24 May 2016 18:44:50 +0100 Subject: [PATCH 0052/1117] Create `@sub` macro for creating SubArrays via indexing. --- base/exports.jl | 1 + base/subarray.jl | 73 +++++++++++++++++++++++++++++++++++++++++++ doc/stdlib/arrays.rst | 6 ++++ test/subarray.jl | 26 +++++++++++++++ 4 files changed, 106 insertions(+) diff --git a/base/exports.jl b/base/exports.jl index d5dc7801aa060..cd76c3778e6a0 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1412,6 +1412,7 @@ export @enum, @label, @goto, + @view, # SparseArrays module re-exports SparseArrays, diff --git a/base/subarray.jl b/base/subarray.jl index 06027d4ad5d7d..adf7807b321ec 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -318,3 +318,76 @@ function parentdims(s::SubArray) end dimindex end + +""" + replace_ref_end!(ex) + +Recursively replace occurences of the symbol :end in a "ref" expression (i.e. A[...]) `ex` +with the appropriate function calls (`endof`, `size` or `trailingsize`). Replacement uses +the closest enclosing ref, so + + A[B[end]] + +should transform to + + A[B[endof(B)]] + +""" +function replace_ref_end!(ex,withex=nothing) + if isa(ex,Symbol) && ex == :end + withex == nothing && error("Invalid use of end") + return withex + elseif isa(ex,Expr) + if ex.head == :ref + S = ex.args[1] = replace_ref_end!(ex.args[1],withex) + # new :ref, so redefine withex + nargs = length(ex.args)-1 + if nargs == 0 + return ex + elseif nargs == 1 + # replace with endof(S) + ex.args[2] = replace_ref_end!(ex.args[2],:(Base.endof($S))) + else + n = 1 + J = endof(ex.args) + for j = 2:J-1 + exj = ex.args[j] = replace_ref_end!(ex.args[j],:(Base.size($S,$n))) + if isa(exj,Expr) && exj.head == :... + # splatted object + exjs = exj.args[1] + n = :($n + length($exjs)) + elseif isa(n, Expr) + # previous expression splatted + n = :($n + 1) + else + # an integer + n += 1 + end + end + ex.args[J] = replace_ref_end!(ex.args[J],:(Base.trailingsize($S,$n))) + end + else + # recursive search + for i = eachindex(ex.args) + ex.args[i] = replace_ref_end!(ex.args[i],withex) + end + end + end + ex +end + +""" + @view A[inds...] + +Creates a `SubArray` from an indexing expression. This can only be applied directly to a +reference expression (e.g. `@view A[1,2:end]`), and should *not* be used as the target of +an assignment (e.g. `@view(A[1,2:end]) = ...`). +""" +macro view(ex) + if isa(ex, Expr) && ex.head == :ref + ex = replace_ref_end!(ex) + Expr(:&&, true, esc(Expr(:call,:(Base.view),ex.args...))) + else + throw(ArgumentError("Invalid use of @view macro: argument must be a reference expression A[...].")) + end +end diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index c258443fd465c..2cc45514e1a82 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -383,6 +383,12 @@ Indexing, Assignment, and Concatenation Like :func:`getindex`\ , but returns a view into the parent array ``A`` with the given indices instead of making a copy. Calling :func:`getindex` or :func:`setindex!` on the returned :obj:`SubArray` computes the indices to the parent array on the fly without checking bounds. +.. function:: @view A[inds...] + + .. Docstring generated from Julia source + + Creates a ``SubArray`` from an indexing expression. This can only be applied directly to a reference expression (e.g. ``@view A[1,2:end]``\ ), and should *not* be used as the target of an assignment (e.g. ``@view(A[1,2:end]) = ...``\ ). + .. function:: parent(A) .. Docstring generated from Julia source diff --git a/test/subarray.jl b/test/subarray.jl index 14ebec68c3553..771e0c963f1e0 100644 --- a/test/subarray.jl +++ b/test/subarray.jl @@ -465,3 +465,29 @@ end # the following segfaults with LLVM 3.8 on Windows, ref #15417 @test collect(view(view(reshape(1:13^3, 13, 13, 13), 3:7, 6:6, :), 1:2:5, :, 1:2:5)) == cat(3,[68,70,72],[406,408,410],[744,746,748]) + + + +# tests @view (and replace_ref_end!) +X = reshape(1:24,2,3,4) +Y = 4:-1:1 + +@test isa(@view(X[1:3]), SubArray) + + +@test X[1:end] == @view X[1:end] +@test X[1:end-3] == @view X[1:end-3] +@test X[1:end,2,2] == @view X[1:end,2,2] +@test X[1,1:end-2] == @view X[1,1:end-2] +@test X[1,2,1:end-2] == @view X[1,2,1:end-2] +@test X[1,2,Y[2:end]] == @view X[1,2,Y[2:end]] +@test X[1:end,2,Y[2:end]] == @view X[1:end,2,Y[2:end]] + +u = (1,2:3) +@test X[u...,2:end] == @view X[u...,2:end] +@test X[(1,)...,(2,)...,2:end] == @view X[(1,)...,(2,)...,2:end] + +# test macro hygiene +let size=(x,y)-> error("should not happen") + @test X[1:end,2,2] == @view X[1:end,2,2] +end From 590167b2b2ad98bdf8d09484cd915df50f994cd2 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sun, 19 Jun 2016 06:47:37 -0700 Subject: [PATCH 0053/1117] Reverting a problematic libuv commit to hopefully fix win64 freeze closes #16556 if we're lucky --- .../libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/md5 | 1 - .../sha512 | 1 - .../libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 | 1 + .../sha512 | 1 + deps/libuv.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/md5 delete mode 100644 deps/checksums/libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/sha512 create mode 100644 deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 create mode 100644 deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 diff --git a/deps/checksums/libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/md5 b/deps/checksums/libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/md5 deleted file mode 100644 index 44760e79d0cef..0000000000000 --- a/deps/checksums/libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -f3d5f34ce0d8b45f2c3d927e06173bfe diff --git a/deps/checksums/libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/sha512 b/deps/checksums/libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/sha512 deleted file mode 100644 index 54e48ad472d7c..0000000000000 --- a/deps/checksums/libuv-a1d9166a440e4a0664c0e6de6ebe25350de56a42.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -18d822378afaa621ef5fd01da1b61a7772ff02964b4a1669f3b00032bdf5f6318921204a37cd1f16280d175ca061c0ac774c25b4820ab065baf06710847cde77 diff --git a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 b/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 new file mode 100644 index 0000000000000..6dfbbd2a475fb --- /dev/null +++ b/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 @@ -0,0 +1 @@ +7248e38acefd92761c54640622357b7b diff --git a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 b/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 new file mode 100644 index 0000000000000..e04d388782eba --- /dev/null +++ b/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 @@ -0,0 +1 @@ +2fad17bddc568125d50ce50b7f27aa2de4380400589043d74b180883c96070c2dcad93557f2f5f5416cc297de3a66b35ec8942bd33d8ef1bb22744dad60b760d diff --git a/deps/libuv.version b/deps/libuv.version index 7bef159edec3f..efe634777261b 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv1.9.0 -LIBUV_SHA1=a1d9166a440e4a0664c0e6de6ebe25350de56a42 +LIBUV_SHA1=ecbd6eddfac4940ab8db57c73166a7378563ebd3 From 6eab5bd9bedc35e9a05c412ed0d09fd827cd0e7f Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Sun, 19 Jun 2016 20:52:08 +0100 Subject: [PATCH 0054/1117] add #16972 to NEWS.md --- NEWS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index bae9cb041ccd8..d6a32e4271dd1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -219,6 +219,8 @@ Deprecated or removed * `xdump` is removed, and `dump` now simply shows the full representation of a value. `dump` should not be overloaded, since it is for examining concrete structure ([#4163]). + * `sub` and `slice` have been deprecated in favor of `view` ([#16972]) + [PkgDev]: https://github.com/JuliaLang/PkgDev.jl [#1090]: https://github.com/JuliaLang/julia/issues/1090 @@ -269,8 +271,9 @@ Deprecated or removed [#16058]: https://github.com/JuliaLang/julia/issues/16058 [#16107]: https://github.com/JuliaLang/julia/issues/16107 [#16219]: https://github.com/JuliaLang/julia/issues/16219 +[#16260]: https://github.com/JuliaLang/julia/issues/16260 [#16362]: https://github.com/JuliaLang/julia/issues/16362 [#16403]: https://github.com/JuliaLang/julia/issues/16403 [#16481]: https://github.com/JuliaLang/julia/issues/16481 [#16731]: https://github.com/JuliaLang/julia/issues/16731 -[#16260]: https://github.com/JuliaLang/julia/issues/16260 +[#16972]: https://github.com/JuliaLang/julia/issues/16972 From 0ef2a91c8e0dca13c3ef08009f1d6478bb19ead0 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 19 Jun 2016 13:58:21 -0400 Subject: [PATCH 0055/1117] Added type info to function sigs in Bidiagonal Also added examples of how to construct the type to illustrate the use of the various `uplo`/`isupper` flags. --- base/linalg/bidiag.jl | 32 +++++++++++++++++++++++++++++--- doc/stdlib/linalg.rst | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/base/linalg/bidiag.jl b/base/linalg/bidiag.jl index 9a0785e2e8f35..1b6452b2a4373 100644 --- a/base/linalg/bidiag.jl +++ b/base/linalg/bidiag.jl @@ -13,23 +13,41 @@ type Bidiagonal{T} <: AbstractMatrix{T} end end """ - Bidiagonal(dv, ev, isupper) + Bidiagonal(dv, ev, isupper::Bool) Constructs an upper (`isupper=true`) or lower (`isupper=false`) bidiagonal matrix using the given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` and provides efficient specialized linear solvers, but may be converted into a regular matrix with [`full`](:func:`full`). `ev`'s length must be one less than the length of `dv`. + +**Example** + +```julia +dv = rand(5) +ev = rand(4) +Bu = Bidiagonal(dv, ev, true) #e is on the first superdiagonal +Bl = Bidiagonal(dv, ev, false) #e is on the first subdiagonal +``` """ Bidiagonal{T}(dv::AbstractVector{T}, ev::AbstractVector{T}, isupper::Bool) = Bidiagonal{T}(collect(dv), collect(ev), isupper) Bidiagonal(dv::AbstractVector, ev::AbstractVector) = throw(ArgumentError("did you want an upper or lower Bidiagonal? Try again with an additional true (upper) or false (lower) argument.")) """ - Bidiagonal(dv, ev, uplo) + Bidiagonal(dv, ev, uplo::Char) Constructs an upper (`uplo='U'`) or lower (`uplo='L'`) bidiagonal matrix using the given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` and provides efficient specialized linear solvers, but may be converted into a regular matrix with [`full`](:func:`full`). `ev`'s length must be one less than the length of `dv`. + +**Example** + +```julia +dv = rand(5) +ev = rand(4) +Bu = Bidiagonal(dv, ev, 'U') #e is on the first superdiagonal +Bl = Bidiagonal(dv, ev, 'L') #e is on the first subdiagonal +``` """ #Convert from BLAS uplo flag to boolean internal Bidiagonal(dv::AbstractVector, ev::AbstractVector, uplo::Char) = begin @@ -48,10 +66,18 @@ function Bidiagonal{Td,Te}(dv::AbstractVector{Td}, ev::AbstractVector{Te}, isupp end """ - Bidiagonal(A, uplo) + Bidiagonal(A, isupper::Bool) Construct a `Bidiagonal` matrix from the main diagonal of `A` and its first super- (if `isupper=true`) or sub-diagonal (if `isupper=false`). + +**Example** + +```julia +A = rand(5,5) +Bu = Bidiagonal(A, true) #contains the main diagonal and first superdiagonal of A +Bl = Bidiagonal(A, false) #contains the main diagonal and first subdiagonal of A +``` """ Bidiagonal(A::AbstractMatrix, isupper::Bool)=Bidiagonal(diag(A), diag(A, isupper?1:-1), isupper) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index d8d23e343937d..bc103c1bd8fc2 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -71,24 +71,50 @@ Linear algebra functions in Julia are largely implemented by calling functions f Constructs a matrix with ``V`` as its diagonal. -.. function:: Bidiagonal(dv, ev, isupper) +.. function:: Bidiagonal(dv, ev, isupper::Bool) .. Docstring generated from Julia source Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . -.. function:: Bidiagonal(dv, ev, uplo) + **Example** + + .. code-block:: julia + + dv = rand(5) + ev = rand(4) + Bu = Bidiagonal(dv, ev, true) #e is on the first superdiagonal + Bl = Bidiagonal(dv, ev, false) #e is on the first subdiagonal + +.. function:: Bidiagonal(dv, ev, uplo::Char) .. Docstring generated from Julia source Constructs an upper (``uplo='U'``\ ) or lower (``uplo='L'``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . -.. function:: Bidiagonal(A, uplo) + **Example** + + .. code-block:: julia + + dv = rand(5) + ev = rand(4) + Bu = Bidiagonal(dv, ev, 'U') #e is on the first superdiagonal + Bl = Bidiagonal(dv, ev, 'L') #e is on the first subdiagonal + +.. function:: Bidiagonal(A, isupper::Bool) .. Docstring generated from Julia source Construct a ``Bidiagonal`` matrix from the main diagonal of ``A`` and its first super- (if ``isupper=true``\ ) or sub-diagonal (if ``isupper=false``\ ). + **Example** + + .. code-block:: julia + + A = rand(5,5) + Bu = Bidiagonal(A, true) #contains the main diagonal and first superdiagonal of A + Bl = Bidiagonal(A, false) #contains the main diagonal and first subdiagonal of A + .. function:: SymTridiagonal(dv, ev) .. Docstring generated from Julia source @@ -900,12 +926,6 @@ Linear algebra functions in Julia are largely implemented by calling functions f Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . -.. function:: Bidiagonal(dv, ev, isupper) - - .. Docstring generated from Julia source - - Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . - .. function:: rank(M) .. Docstring generated from Julia source From bf282edbf91f057ee02e41d3011bbfa02bab87e3 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 19 Jun 2016 22:30:23 -0400 Subject: [PATCH 0056/1117] Core.TopModule is a relic of before (core ...) was an option for lowering --- base/array.jl | 2 ++ base/boot.jl | 14 +------------- base/coreimg.jl | 2 +- base/deepcopy.jl | 2 +- base/essentials.jl | 6 +++--- base/inference.jl | 4 +++- base/reflection.jl | 6 +++--- base/serialize.jl | 3 ++- base/show.jl | 16 ++++++++-------- base/sysimg.jl | 2 +- test/copy.jl | 4 ++-- test/core.jl | 6 +++--- 12 files changed, 30 insertions(+), 37 deletions(-) diff --git a/base/array.jl b/base/array.jl index 56709861e0f54..d7881dab3eb49 100644 --- a/base/array.jl +++ b/base/array.jl @@ -12,6 +12,8 @@ typealias DenseVecOrMat{T} Union{DenseVector{T}, DenseMatrix{T}} ## Basic functions ## +import Core: arraysize, arrayset, arrayref + size(a::Array, d) = arraysize(a, d) size(a::Vector) = (arraysize(a,1),) size(a::Matrix) = (arraysize(a,1), arraysize(a,2)) diff --git a/base/boot.jl b/base/boot.jl index 96cd8d3c62245..4c93011525708 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -139,7 +139,7 @@ export fieldtype, getfield, setfield!, nfields, throw, tuple, is, ===, isdefined, eval, # sizeof # not exported, to avoid conflicting with Base.sizeof # type reflection - issubtype, typeof, isa, + issubtype, typeof, isa, typeassert, # method reflection applicable, invoke, # constants @@ -373,16 +373,4 @@ show(a::ANY) = show(STDOUT, a) print(a::ANY...) = print(STDOUT, a...) println(a::ANY...) = println(STDOUT, a...) - -module TopModule - # this defines the types that lowering expects to be defined in a (top) module - # that are usually inherited from Core, but could be defined custom for a module - using Core: Box, IntrinsicFunction, Builtin, - arrayref, arrayset, arraysize, - _expr, _apply, typeassert, apply_type, svec, kwfunc - export Box, IntrinsicFunction, Builtin, - arrayref, arrayset, arraysize, - _expr, _apply, typeassert, apply_type, svec, kwfunc -end -using .TopModule ccall(:jl_set_istopmod, Void, (Bool,), true) diff --git a/base/coreimg.jl b/base/coreimg.jl index 867a725490a04..4dfce3786e285 100644 --- a/base/coreimg.jl +++ b/base/coreimg.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license Main.Core.eval(Main.Core, :(baremodule Inference -using Core.TopModule, Core.Intrinsics +using Core.Intrinsics import Core: print, println, show, write, unsafe_write, STDOUT, STDERR if false # show that the IO system is already (relatively) operational print("HELLO") diff --git a/base/deepcopy.jl b/base/deepcopy.jl index 110921e64e9bf..c01182eddba5f 100644 --- a/base/deepcopy.jl +++ b/base/deepcopy.jl @@ -17,7 +17,7 @@ function deepcopy_internal(x::SimpleVector, stackdict::ObjectIdDict) if haskey(stackdict, x) return stackdict[x] end - y = svec(Any[deepcopy_internal(x[i], stackdict) for i = 1:length(x)]...) + y = Core.svec(Any[deepcopy_internal(x[i], stackdict) for i = 1:length(x)]...) stackdict[x] = y return y end diff --git a/base/essentials.jl b/base/essentials.jl index 3c2563abfd905..bdc95b133b7f4 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -100,7 +100,7 @@ function append_any(xs...) ccall(:jl_array_grow_end, Void, (Any, UInt), out, 16) l += 16 end - arrayset(out, y, i) + Core.arrayset(out, y, i) i += 1 end end @@ -109,7 +109,7 @@ function append_any(xs...) end # simple Array{Any} operations needed for bootstrap -setindex!(A::Array{Any}, x::ANY, i::Int) = arrayset(A, x, i) +setindex!(A::Array{Any}, x::ANY, i::Int) = Core.arrayset(A, x, i) function length_checked_equal(args...) n = length(args[1]) @@ -184,7 +184,7 @@ end map(f, v::SimpleVector) = Any[ f(v[i]) for i = 1:length(v) ] -getindex(v::SimpleVector, I::AbstractArray) = svec(Any[ v[i] for i in I ]...) +getindex(v::SimpleVector, I::AbstractArray) = Core.svec(Any[ v[i] for i in I ]...) function isassigned(v::SimpleVector, i::Int) 1 <= i <= length(v) || return false diff --git a/base/inference.jl b/base/inference.jl index eb228bc2fd4b9..5e166a79d6cf9 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +import Core: _apply, svec, apply_type, Builtin, IntrinsicFunction + #### parameters limiting potentially-infinite types #### const MAX_TYPEUNION_LEN = 3 const MAX_TYPE_DEPTH = 7 @@ -292,7 +294,7 @@ add_tfunc(Core.sizeof, 1, 1, x->Int) add_tfunc(nfields, 1, 1, x->(isa(x,Const) ? Const(nfields(x.val)) : isType(x) && isleaftype(x.parameters[1]) ? Const(nfields(x.parameters[1])) : Int)) -add_tfunc(_expr, 1, IInf, (args...)->Expr) +add_tfunc(Core._expr, 1, IInf, (args...)->Expr) add_tfunc(applicable, 1, IInf, (f, args...)->Bool) add_tfunc(Core.Intrinsics.arraylen, 1, 1, x->Int) add_tfunc(arraysize, 2, 2, (a,d)->Int) diff --git a/base/reflection.jl b/base/reflection.jl index 8f968ef6065cb..d422eeeb189dc 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -246,14 +246,14 @@ function MethodList(mt::MethodTable) end function methods(f::ANY, t::ANY) - if isa(f,Builtin) + if isa(f, Core.Builtin) throw(ArgumentError("argument is not a generic function")) end t = to_tuple_type(t) return MethodList(Method[m[3] for m in _methods(f,t,-1)], typeof(f).name.mt) end -methods(f::Builtin) = MethodList(Method[], typeof(f).name.mt) +methods(f::Core.Builtin) = MethodList(Method[], typeof(f).name.mt) function methods(f::ANY) # return all matches @@ -372,7 +372,7 @@ function return_types(f::ANY, types::ANY=Tuple) end function which(f::ANY, t::ANY) - if isa(f,Builtin) + if isa(f, Core.Builtin) throw(ArgumentError("argument is not a generic function")) end t = to_tuple_type(t) diff --git a/base/serialize.jl b/base/serialize.jl index e73d47172c203..f3b2589872a5a 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -2,7 +2,8 @@ module Serializer -import Base: GMP, Bottom, svec, unsafe_convert, uncompressed_ast +import Base: GMP, Bottom, unsafe_convert, uncompressed_ast +import Core: svec using Base: ViewIndex, index_lengths export serialize, deserialize diff --git a/base/show.jl b/base/show.jl index e48497ac20dc9..1f1f9fb0c0f4c 100644 --- a/base/show.jl +++ b/base/show.jl @@ -142,7 +142,7 @@ function show(io::IO, f::Function) ft = typeof(f) mt = ft.name.mt if get(io, :multiline, false) - if isa(f, Builtin) + if isa(f, Core.Builtin) print(io, mt.name, " (built-in function)") else name = mt.name @@ -163,8 +163,8 @@ function show(io::IO, f::Function) end end -function show(io::IO, x::IntrinsicFunction) - print(io, "(intrinsic function #", box(Int32,unbox(IntrinsicFunction,x)), ")") +function show(io::IO, x::Core.IntrinsicFunction) + print(io, "(intrinsic function #", box(Int32, unbox(Core.IntrinsicFunction, x)), ")") end function show(io::IO, x::Union) @@ -467,13 +467,13 @@ unquoted(ex::QuoteNode) = ex.value unquoted(ex::Expr) = ex.args[1] function is_intrinsic_expr(x::ANY) - isa(x, IntrinsicFunction) && return true + isa(x, Core.IntrinsicFunction) && return true if isa(x, GlobalRef) x = x::GlobalRef return (isdefined(x.mod, x.name) && - isa(getfield(x.mod, x.name), IntrinsicFunction)) + isa(getfield(x.mod, x.name), Core.IntrinsicFunction)) elseif isa(x, Expr) - return (x::Expr).typ === IntrinsicFunction + return (x::Expr).typ === Core.IntrinsicFunction end return false end @@ -487,10 +487,10 @@ const indent_width = 4 function show_expr_type(io::IO, ty, emph) if is(ty, Function) print(io, "::F") - elseif is(ty, IntrinsicFunction) + elseif is(ty, Core.IntrinsicFunction) print(io, "::I") else - if emph && (!isleaftype(ty) || ty == Box) + if emph && (!isleaftype(ty) || ty == Core.Box) emphasize(io, "::$ty") else print(io, "::$ty") diff --git a/base/sysimg.jl b/base/sysimg.jl index 05bca6889362e..f7af0b3606c39 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -2,7 +2,7 @@ baremodule Base -using Core.TopModule, Core.Intrinsics +using Core.Intrinsics ccall(:jl_set_istopmod, Void, (Bool,), true) include = Core.include include("coreio.jl") diff --git a/test/copy.jl b/test/copy.jl index 617c16f85d54f..59adeebf2ec10 100644 --- a/test/copy.jl +++ b/test/copy.jl @@ -69,8 +69,8 @@ end @test isnull(deepcopy(Nullable{Array}())) # issue #15250 -let a1 = Base.svec(1, 2, 3, []), a2 = Base.svec(1, 2, 3) - a3 = Base.svec(a1,a1) +let a1 = Core.svec(1, 2, 3, []), a2 = Core.svec(1, 2, 3) + a3 = Core.svec(a1,a1) b1 = deepcopy(a1) @test a1 == b1 @test a1 !== b1 diff --git a/test/core.jl b/test/core.jl index 7b2330e99c15b..a6bec9ebe0054 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3874,7 +3874,7 @@ end # issue #15370 @test isdefined(Core, :Box) -@test isdefined(Base, :Box) +@test !isdefined(Base, :Box) @test !isdefined(Main, :Box) # issue #1784 @@ -4301,8 +4301,8 @@ end type C16767{T} b::A16767{C16767{:a}} end -@test B16767.types[1].types[1].parameters[1].types === Base.svec(A16767{B16767}) -@test C16767.types[1].types[1].parameters[1].types === Base.svec(A16767{C16767{:a}}) +@test B16767.types[1].types[1].parameters[1].types === Core.svec(A16767{B16767}) +@test C16767.types[1].types[1].parameters[1].types === Core.svec(A16767{C16767{:a}}) # issue #16340 function f16340{T}(x::T) From 6997ad1f34d76c21d15135339314ae707d0d907d Mon Sep 17 00:00:00 2001 From: Rafael Fourquet Date: Mon, 20 Jun 2016 18:02:04 +0530 Subject: [PATCH 0057/1117] implement separately getindex(::Type, x [,y [,z]]) for efficiency --- base/array.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base/array.jl b/base/array.jl index d7881dab3eb49..0c568126ed17a 100644 --- a/base/array.jl +++ b/base/array.jl @@ -135,7 +135,11 @@ function getindex(T::Type, vals...) end return a end + getindex(T::Type) = Array{T}(0) +getindex(T::Type, x) = (a = Array{T}(1); @inbounds a[1] = x; a) +getindex(T::Type, x, y) = (a = Array{T}(2); @inbounds (a[1] = x; a[2] = y); a) +getindex(T::Type, x, y, z) = (a = Array{T}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a) function getindex(::Type{Any}, vals::ANY...) a = Array{Any}(length(vals)) From 0a62ca2c019a3222cbf92ee839ccc4bf54fdfb12 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Mon, 20 Jun 2016 12:21:23 -0400 Subject: [PATCH 0058/1117] More docs for Pkg Document `dependents`, `coverage` option for `test`, and `status(pkg)`. --- base/pkg/pkg.jl | 19 +++++++++++++++++-- doc/stdlib/pkg.rst | 20 ++++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/base/pkg/pkg.jl b/base/pkg/pkg.jl index 8aea6250a5dfe..d0cc1087739fe 100644 --- a/base/pkg/pkg.jl +++ b/base/pkg/pkg.jl @@ -134,6 +134,12 @@ installed(pkg::AbstractString) = cd(Entry.installed,pkg) Prints out a summary of what packages are installed and what version and state they're in. """ status(io::IO=STDOUT) = cd(Entry.status,io) + +""" + status(pkg) + +Prints out a summary of what version and state `pkg`, specifically, is in. +""" status(pkg::AbstractString, io::IO=STDOUT) = cd(Entry.status,io,pkg) """ @@ -227,23 +233,32 @@ on all installed or updated packages. build(pkgs::AbstractString...) = cd(Entry.build,[pkgs...]) """ - test() + test(; coverage=false) Run the tests for all installed packages ensuring that each package's test dependencies are installed for the duration of the test. A package is tested by running its `test/runtests.jl` file and test dependencies are specified in `test/REQUIRE`. +Coverage statistics for the packages may be generated by passing `coverage=true`. +The default behavior is not to run coverage. """ test(;coverage::Bool=false) = cd(Entry.test; coverage=coverage) """ - test(pkgs...) + test(pkgs...; coverage=false) Run the tests for each package in `pkgs` ensuring that each package's test dependencies are installed for the duration of the test. A package is tested by running its `test/runtests.jl` file and test dependencies are specified in `test/REQUIRE`. +Coverage statistics for the packages may be generated by passing `coverage=true`. +The default behavior is not to run coverage. """ test(pkgs::AbstractString...; coverage::Bool=false) = cd(Entry.test,AbstractString[pkgs...]; coverage=coverage) +""" + dependents(packagename) + +List the packages that have `packagename` as a dependency. +""" dependents(packagename::AbstractString) = Reqs.dependents(packagename) """ diff --git a/doc/stdlib/pkg.rst b/doc/stdlib/pkg.rst index 96fe1a2aa29e1..37cd095ba7fa7 100644 --- a/doc/stdlib/pkg.rst +++ b/doc/stdlib/pkg.rst @@ -99,6 +99,12 @@ Functions for package development (e.g. ``tag``, ``publish``, etc.) have been mo Prints out a summary of what packages are installed and what version and state they're in. +.. function:: status(pkg) + + .. Docstring generated from Julia source + + Prints out a summary of what version and state ``pkg``\ , specifically, is in. + .. function:: update() .. Docstring generated from Julia source @@ -143,15 +149,21 @@ Functions for package development (e.g. ``tag``, ``publish``, etc.) have been mo Run the build script in ``deps/build.jl`` for each package in ``pkgs`` and all of their dependencies in depth-first recursive order. This is called automatically by ``Pkg.resolve()`` on all installed or updated packages. -.. function:: test() +.. function:: test(; coverage=false) + + .. Docstring generated from Julia source + + Run the tests for all installed packages ensuring that each package's test dependencies are installed for the duration of the test. A package is tested by running its ``test/runtests.jl`` file and test dependencies are specified in ``test/REQUIRE``\ . Coverage statistics for the packages may be generated by passing ``coverage=true``\ . The default behavior is not to run coverage. + +.. function:: test(pkgs...; coverage=false) .. Docstring generated from Julia source - Run the tests for all installed packages ensuring that each package's test dependencies are installed for the duration of the test. A package is tested by running its ``test/runtests.jl`` file and test dependencies are specified in ``test/REQUIRE``\ . + Run the tests for each package in ``pkgs`` ensuring that each package's test dependencies are installed for the duration of the test. A package is tested by running its ``test/runtests.jl`` file and test dependencies are specified in ``test/REQUIRE``\ . Coverage statistics for the packages may be generated by passing ``coverage=true``\ . The default behavior is not to run coverage. -.. function:: test(pkgs...) +.. function:: dependents(packagename) .. Docstring generated from Julia source - Run the tests for each package in ``pkgs`` ensuring that each package's test dependencies are installed for the duration of the test. A package is tested by running its ``test/runtests.jl`` file and test dependencies are specified in ``test/REQUIRE``\ . + List the packages that have ``packagename`` as a dependency. From 443168c275d5f06afa7a6c1884d8cddac6860a7f Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Mon, 20 Jun 2016 09:27:18 -0700 Subject: [PATCH 0059/1117] Refactors `[c]transpose[!]` methods operating on `SparseMatrixCSC`s in anticipation of introducing related `permute[!]` methods and taking advantage of #16371 for better structuring. Also adds several tests for these methods. --- base/sparse/sparsematrix.jl | 186 +++++++++++++++++------------------- test/sparsedir/sparse.jl | 25 +++++ 2 files changed, 113 insertions(+), 98 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 7e3bac45a3594..97cdcc1c04d58 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -615,112 +615,102 @@ end ## Transposition methods -# qftranspose! is the parent method on which the others are built. """ - qftranspose!{Tv,Ti}(C::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, q::AbstractVector, f) - -Column-permute and transpose `A` (`(AQ)^T`), applying `f` to each element of `A` in the - process, and store the result in preallocated `C`. Permutation vector `q` defines the - column-permutation `Q`. The number of columns of `C` (`C.n`) must match the number of - rows of `A` (`A.m`). The number of rows of `C` (`C.m`) must match the number of columns - of `A` (`A.n`). The length of `C`'s internal row-index (`length(C.rowval)`) and - entry-value (`length(C.nzval)`) arrays must be at least the number of allocated entries - in `A` (`nnz(A)`). The length of the permutation vector `q` (`length(q)`) must match the - number of columns of `A` (`A.n`). - -This method implements the HALFPERM algorithm described in F. Gustavson, "Two fast - algorithms for sparse matrices: multiplication and permuted transposition," ACM TOMS - 4(3), 250-269 (1978). The algorithm runs in `O(A.m, A.n, nnz(A))` time and requires no - space beyond that passed in. + halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, + q::AbstractVector, f = identity) + +Column-permute and transpose `A`, simultaneously applying `f` to each entry of `A`, storing +the result `(f(A)Q)^T` (`map(f, transpose(A[:,q]))`) in `X`. + +`X`'s dimensions must match those of `transpose(A)` (`X.m == A.n` and `X.n == A.m`), and `X` +must have enough storage to accommodate all allocated entries in `A` (`length(X.rowval) >= nnz(A)` +and `length(X.nzval) >= nnz(A)`). Column-permutation `q`'s length must match `A`'s column +count (`length(q) == A.n`). + +This method is the parent of several methods performing transposition and permutation +operations on `SparseMatrixCSC`s. As this method performs no argument checking, prefer +the safer child methods (`[c]transpose[!]`, `permute[!]`) to direct use. + +This method implements the `HALFPERM` algorithm described in F. Gustavson, "Two fast +algorithms for sparse matrices: multiplication and permuted transposition," ACM TOMS 4(3), +250-269 (1978). The algorithm runs in `O(A.m, A.n, nnz(A))` time and requires no space +beyond that passed in. """ -function qftranspose!{Tv,Ti}(C::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, q::AbstractVector, f) - # Attach source matrix - Am, An = A.m, A.n - Acolptr = A.colptr - Arowval = A.rowval - Anzval = A.nzval - Annz = Acolptr[end]-1 - - # Attach destination matrix - Cm, Cn = C.m, C.n - Ccolptr = C.colptr - Crowval = C.rowval - Cnzval = C.nzval - - # Check compatibility of source and destination - if !(Cm == An) - throw(DimensionMismatch("the number of rows of the first argument, C.m = $(Cm)," - * "must match the number of columns of the second argument, A.n = $(An)") ) - elseif !(Cn == Am) - throw(DimensionMismatch("the number of columns of the first argument, C.n = $(Cn)," - * "must match the number of rows of the second argument, A.m = $(Am)") ) - elseif !(length(q) == A.n) - throw(DimensionMismatch("the length of the permtuation vector, length(q) = " - * "$(length(q)), must match the number of columns of the second argument," - * "A.n = $(An)") ) - elseif !(length(Crowval) >= Annz) - throw(ArgumentError("the first argument's row-index array's length," - * "length(C.rowval) = $(length(Crowval)), must be at least the number of" - * "allocated entries in the second argument, nnz(A) = $(Annz)") ) - elseif !(length(Cnzval) >= Annz) - throw(ArgumentError("the first argument's entry-value array's length," - * "length(C.nzval) = $(length(Cnzval)), must be at least the number of" - * "allocated entries in the second argument, nnz(A) = $(Annz)") ) - end - - # Compute the column counts of C and store them shifted forward by one in Ccolptr - Ccolptr[1:end] = 0 - @inbounds for k in 1:Annz - Ccolptr[Arowval[k]+1] += 1 - end - - # From these column counts, compute C's column pointers - # and store them shifted forward by one in Ccolptr +function halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, + q::AbstractVector, f = identity) + _computecolptrs_halfperm!(X, A) + _distributevals_halfperm!(X, A, q, f) + return X +end +""" +Helper method for `halfperm!`. Computes `transpose(A[:,q])`'s column pointers, storing them +shifted one position forward in `X.colptr`; `_distributevals_halfperm!` fixes this shift. +""" +function _computecolptrs_halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, + A::SparseMatrixCSC{Tv,Ti}) + # Compute `transpose(A[:,q])`'s column counts. Store shifted forward one position in X.colptr. + fill!(X.colptr, 0) + @inbounds for k in 1:nnz(A) + X.colptr[A.rowval[k] + 1] += 1 + end + # Compute `transpose(A[:,q])`'s column pointers. Store shifted forward one position in X.colptr. + X.colptr[1] = 1 countsum = 1 - @inbounds for k in 2:(Cn+1) - overwritten = Ccolptr[k] - Ccolptr[k] = countsum + @inbounds for k in 2:(A.m + 1) + overwritten = X.colptr[k] + X.colptr[k] = countsum countsum += overwritten end - - # Distribution-sort the row indices and nonzero values into Crowval and Cnzval, - # tracking write positions in Ccolptr - @inbounds for Aj in 1:An - qAj = q[Aj] - for Ak in Acolptr[qAj]:(Acolptr[qAj+1]-1) - Ai = Arowval[Ak] - Ck = Ccolptr[Ai+1] - Crowval[Ck] = qAj - Cnzval[Ck] = f(Anzval[Ak]) - Ccolptr[Ai+1] += 1 - end - end - - # Tracking write positions in Ccolptr as in the last block fixes the colptr shift, - # but the first colptr remains incorrect - Ccolptr[1] = 1 - - C end -transpose!{Tv,Ti}(C::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}) = qftranspose!(C, A, 1:A.n, identity) -ctranspose!{Tv,Ti}(C::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}) = qftranspose!(C, A, 1:A.n, conj) -"See `qftranspose!`" ftranspose!{Tv,Ti}(C::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, f) = qftranspose!(C, A, 1:A.n, f) - """ - qftranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, q::AbstractVector, f) - -Return-allocating version of `qftranspose!`. See `qftranspose!` for documentation. +Helper method for `halfperm!`. With `transpose(A[:,q])`'s column pointers shifted one +position forward in `X.colptr`, computes `map(f, transpose(A[:,q]))` by appropriately +distributing `A.rowval` and `f`-transformed `A.nzval` into `X.rowval` and `X.nzval` +respectively. Simultaneously fixes the one-position-forward shift in `X.colptr`. """ -function qftranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, q::AbstractVector, f) - Cm, Cn, Cnnz = A.n, A.m, nnz(A) - Ccolptr = zeros(Ti, Cn+1) - Crowval = Array{Ti}(Cnnz) - Cnzval = Array{Tv}(Cnnz) - qftranspose!(SparseMatrixCSC(Cm, Cn, Ccolptr, Crowval, Cnzval), A, q, f) -end -transpose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = qftranspose(A, 1:A.n, identity) -ctranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = qftranspose(A, 1:A.n, conj) -"See `qftranspose`" ftranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, f) = qftranspose(A, 1:A.n, f) +function _distributevals_halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, + A::SparseMatrixCSC{Tv,Ti}, q::AbstractVector, f) + @inbounds for Xi in 1:A.n + Aj = q[Xi] + for Ak in nzrange(A, Aj) + Ai = A.rowval[Ak] + Xk = X.colptr[Ai + 1] + X.rowval[Xk] = Xi + X.nzval[Xk] = f(A.nzval[Ak]) + X.colptr[Ai + 1] += 1 + end + end + return # kill potential type instability +end + +function ftranspose!{Tv,Ti}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, f) + # Check compatibility of source argument A and destination argument X + if X.n != A.m + throw(DimensionMismatch(string("destination argument `X`'s column count, ", + "`X.n (= $(X.n))`, must match source argument `A`'s row count, `A.m (= $(A.m))`"))) + elseif X.m != A.n + throw(DimensionMismatch(string("destination argument `X`'s row count, + `X.m (= $(X.m))`, must match source argument `A`'s column count, `A.n (= $(A.n))`"))) + elseif length(X.rowval) < nnz(A) + throw(ArgumentError(string("the length of destination argument `X`'s `rowval` ", + "array, `length(X.rowval) (= $(length(X.rowval)))`, must be greater than or ", + "equal to source argument `A`'s allocated entry count, `nnz(A) (= $(nnz(A)))`"))) + elseif length(X.nzval) < nnz(A) + throw(ArgumentError(string("the length of destination argument `X`'s `nzval` ", + "array, `length(X.nzval) (= $(length(X.nzval)))`, must be greater than or ", + "equal to source argument `A`'s allocated entry count, `nnz(A) (= $(nnz(A)))`"))) + end + halfperm!(X, A, 1:A.n, f) +end +transpose!{Tv,Ti}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}) = ftranspose!(X, A, identity) +ctranspose!{Tv,Ti}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}) = ftranspose!(X, A, conj) + +function ftranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, f) + X = SparseMatrixCSC(A.n, A.m, Vector{Ti}(A.m+1), Vector{Ti}(nnz(A)), Vector{Tv}(nnz(A))) + halfperm!(X, A, 1:A.n, f) +end +transpose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = ftranspose(A, identity) +ctranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = ftranspose(A, conj) ## fkeep! and children tril!, triu!, droptol!, dropzeros[!] diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 3dc15f2cec64a..c1278217873b6 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -288,6 +288,31 @@ end cA = sprandn(5,5,0.2) + im*sprandn(5,5,0.2) @test full(conj(cA)) == conj(full(cA)) +# Test SparseMatrixCSC [c]transpose[!] methods +let smalldim = 5, largedim = 10, nzprob = 0.4 + (m, n) = (smalldim, smalldim) + A = sprand(m, n, nzprob) + # Test common error checking of [c]transpose! methods (ftranspose!) + @test_throws DimensionMismatch transpose!(A[:, 1:(smalldim - 1)], A) + @test_throws DimensionMismatch transpose!(A[1:(smalldim - 1), 1], A) + @test_throws ArgumentError transpose!((B = similar(A); resize!(B.rowval, nnz(A) - 1); B), A) + @test_throws ArgumentError transpose!((B = similar(A); resize!(B.nzval, nnz(A) - 1); B), A) + # Test overall functionality of [c]transpose[!] + for (m, n) in ((smalldim, smalldim), (smalldim, largedim), (largedim, smalldim)) + A = sprand(m, n, nzprob) + At = transpose(A) + # transpose[!] + fullAt = transpose(full(A)) + @test transpose(A) == fullAt + @test transpose!(similar(At), A) == fullAt + # ctranspose[!] + C = A + im*A/2 + fullCh = ctranspose(full(C)) + @test ctranspose(C) == fullCh + @test ctranspose!(similar(sparse(fullCh)), C) == fullCh + end +end + # transpose of SubArrays A = view(sprandn(10, 10, 0.3), 1:4, 1:4) @test transpose(full(A)) == full(transpose(A)) From 48b799a150714b09eab89f0fadedf55a271ba9dc Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 20 Jun 2016 11:11:16 -0700 Subject: [PATCH 0060/1117] Remove CSparse functions. The functions and tests for these function have been moved to the package SuiteSparse.jl in the JuliaSparse organization. --- LICENSE.md | 1 - base/deprecated.jl | 19 ++++ base/sparse/csparse.jl | 178 -------------------------------- base/sparse/sparse.jl | 1 - contrib/add_license_to_files.jl | 1 - doc/stdlib/arrays.rst | 12 --- test/sparsedir/sparse.jl | 28 +---- 7 files changed, 21 insertions(+), 219 deletions(-) delete mode 100644 base/sparse/csparse.jl diff --git a/LICENSE.md b/LICENSE.md index e560a3e359284..fbd0efad2ef9c 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -66,7 +66,6 @@ their own licenses: The following components of Julia's standard library have separate licenses: - base/fftw.jl (see [FFTW](http://fftw.org/doc/License-and-Copyright.html)) -- base/sparse/csparse.jl (LGPL-2.1+) - base/linalg/umfpack.jl (see [SUITESPARSE](http://faculty.cse.tamu.edu/davis/suitesparse.html)) - base/linalg/cholmod.jl (see [SUITESPARSE](http://faculty.cse.tamu.edu/davis/suitesparse.html)) diff --git a/base/deprecated.jl b/base/deprecated.jl index a0b22e31af6c3..b22c1a3cc6523 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -769,6 +769,25 @@ end @deprecate slice view @deprecate sub view +# Point users to SuiteSparse +function ereach{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, k::Integer, parent::Vector{Ti}) + error(string("ereach(A, k, parent) now lives in package SuiteSparse.jl. Run", + "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) +end +function etree{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, postorder::Bool) + error(string("etree(A[, post]) now lives in package SuiteSparse.jl. Run", + "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) +end +etree(A::SparseMatrixCSC) = etree(A, false) +function csc_permute{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}, q::Vector{Ti}) + error(string("csc_permute(A, pinv, q) now lives in package SuiteSparse.jl. Run", + "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) +end +function symperm{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}) + error(string("symperm(A, pinv) now lives in package SuiteSparse.jl. Run,", + "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) +end + # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/sparse/csparse.jl b/base/sparse/csparse.jl deleted file mode 100644 index ab037530dbcbe..0000000000000 --- a/base/sparse/csparse.jl +++ /dev/null @@ -1,178 +0,0 @@ -# These functions are based on C functions in the CSparse library by Tim Davis. -# These are pure Julia implementations, and do not link to the CSparse library. -# CSparse can be downloaded from http://www.cise.ufl.edu/research/sparse/CSparse/CSparse.tar.gz -# CSparse is Copyright (c) 2006-2007, Timothy A. Davis and released under -# Lesser GNU Public License, version 2.1 or later. A copy of the license can be -# downloaded from http://www.gnu.org/licenses/lgpl-2.1.html - -# Because these functions are based on code covered by LGPL-2.1+ the same license -# must apply to the code in this file which is -# Copyright (c) 2013-2014 Viral Shah, Douglas Bates and other contributors - -# Based on Direct Methods for Sparse Linear Systems, T. A. Davis, SIAM, Philadelphia, Sept. 2006. -# Section 2.4: Triplet form -# http://www.cise.ufl.edu/research/sparse/CSparse/ - - -# Compute the elimination tree of A using triu(A) returning the parent vector. -# A root node is indicated by 0. This tree may actually be a forest in that -# there may be more than one root, indicating complete separability. -# A trivial example is speye(n, n) in which every node is a root. - -""" - etree(A[, post]) - -Compute the elimination tree of a symmetric sparse matrix `A` from `triu(A)` -and, optionally, its post-ordering permutation. -""" -function etree{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, postorder::Bool) - m,n = size(A) - Ap = A.colptr - Ai = A.rowval - parent = zeros(Ti, n) - ancestor = zeros(Ti, n) - for k in 1:n, p in Ap[k]:(Ap[k+1] - 1) - i = Ai[p] - while i != 0 && i < k - inext = ancestor[i] # inext = ancestor of i - ancestor[i] = k # path compression - if (inext == 0) parent[i] = k end # no anc., parent is k - i = inext - end - end - if !postorder return parent end - head = zeros(Ti,n) # empty linked lists - next = zeros(Ti,n) - for j in n:-1:1 # traverse in reverse order - if (parent[j] == 0); continue; end # j is a root - next[j] = head[parent[j]] # add j to list of its parent - head[parent[j]] = j - end - stack = Ti[] - sizehint!(stack, n) - post = zeros(Ti,n) - k = 1 - for j in 1:n - if (parent[j] != 0) continue end # skip j if it is not a root - push!(stack, j) # place j on the stack - while (!isempty(stack)) # while (stack is not empty) - p = stack[end] # p = top of stack - i = head[p] # i = youngest child of p - if (i == 0) - pop!(stack) - post[k] = p # node p is the kth postordered node - k += 1 - else - head[p] = next[i] # remove i from children of p - push!(stack, i) - end - end - end - parent, post -end - -etree(A::SparseMatrixCSC) = etree(A, false) - -# find nonzero pattern of Cholesky L[k,1:k-1] using etree and triu(A[:,k]) -# based on cs_ereach p. 43, "Direct Methods for Sparse Linear Systems" -function ereach{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, k::Integer, parent::Vector{Ti}) - m,n = size(A); Ap = A.colptr; Ai = A.rowval - s = Ti[]; sizehint!(s, n) # to be used as a stack - visited = falses(n) - visited[k] = true - for p in Ap[k]:(Ap[k+1] - 1) - i = Ai[p] # A[i,k] is nonzero - if i > k continue end # only use upper triangular part of A - while !visited[i] # traverse up etree - push!(s,i) # L[k,i] is nonzero - visited[i] = true - i = parent[i] - end - end - s -end - -# based on cs_permute p. 21, "Direct Methods for Sparse Linear Systems" -function csc_permute{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}, q::Vector{Ti}) - m, n = size(A) - Ap = A.colptr - Ai = A.rowval - Ax = A.nzval - lpinv = length(pinv) - if m != lpinv - throw(DimensionMismatch( - "the number of rows of sparse matrix A must equal the length of pinv, $m != $lpinv")) - end - lq = length(q) - if n != lq - throw(DimensionMismatch( - "the number of columns of sparse matrix A must equal the length of q, $n != $lq")) - end - if !isperm(pinv) || !isperm(q) - throw(ArgumentError("both pinv and q must be permutations")) - end - C = copy(A); Cp = C.colptr; Ci = C.rowval; Cx = C.nzval - nz = one(Ti) - for k in 1:n - Cp[k] = nz - j = q[k] - for t = Ap[j]:(Ap[j+1]-1) - Cx[nz] = Ax[t] - Ci[nz] = pinv[Ai[t]] - nz += one(Ti) - end - end - Cp[n + 1] = nz - (C.').' # double transpose to order the columns -end - - -# based on cs_symperm p. 21, "Direct Methods for Sparse Linear Systems" -# form A[p,p] for a symmetric A stored in the upper triangle -""" - symperm(A, p) - -Return the symmetric permutation of `A`, which is `A[p,p]`. `A` should be -symmetric, sparse, and only contain nonzeros in the upper triangular part of the -matrix is stored. This algorithm ignores the lower triangular part of the -matrix. Only the upper triangular part of the result is returned. -""" -function symperm{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}) - m, n = size(A) - if m != n - throw(DimensionMismatch("sparse matrix A must be square")) - end - Ap = A.colptr - Ai = A.rowval - Ax = A.nzval - if !isperm(pinv) - throw(ArgumentError("pinv must be a permutation")) - end - lpinv = length(pinv) - if n != lpinv - throw(DimensionMismatch( - "dimensions of sparse matrix A must equal the length of pinv, $((m,n)) != $lpinv")) - end - C = copy(A); Cp = C.colptr; Ci = C.rowval; Cx = C.nzval - w = zeros(Ti,n) - for j in 1:n # count entries in each column of C - j2 = pinv[j] - for p in Ap[j]:(Ap[j+1]-1) - (i = Ai[p]) > j || (w[max(pinv[i],j2)] += one(Ti)) - end - end - Cp[:] = cumsum(vcat(one(Ti),w)) - copy!(w,Cp[1:n]) # needed to be consistent with cs_cumsum - for j in 1:n - j2 = pinv[j] - for p = Ap[j]:(Ap[j+1]-1) - (i = Ai[p]) > j && continue - i2 = pinv[i] - ind = max(i2,j2) - Ci[q = w[ind]] = min(i2,j2) - w[ind] += 1 - Cx[q] = Ax[p] - end - end - (C.').' # double transpose to order the columns -end diff --git a/base/sparse/sparse.jl b/base/sparse/sparse.jl index 9e262e840a5da..93340cbb467fc 100644 --- a/base/sparse/sparse.jl +++ b/base/sparse/sparse.jl @@ -36,7 +36,6 @@ export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, include("abstractsparse.jl") include("sparsematrix.jl") include("sparsevector.jl") -include("csparse.jl") include("linalg.jl") if Base.USE_GPL_LIBS diff --git a/contrib/add_license_to_files.jl b/contrib/add_license_to_files.jl index 897f5d52ea1fb..3ee803b132035 100644 --- a/contrib/add_license_to_files.jl +++ b/contrib/add_license_to_files.jl @@ -34,7 +34,6 @@ const skipfiles = [ # files to check - already copyright # see: https://github.com/JuliaLang/julia/pull/11073#issuecomment-98099389 "../base/special/trig.jl", - "../base/sparse/csparse.jl", "../base/linalg/givens.jl", # "../src/abi_llvm.cpp", diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 2cc45514e1a82..adb38dfcc3b93 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -992,18 +992,6 @@ dense counterparts. The following functions are specific to sparse arrays. Create a random sparse vector of length ``m`` or sparse matrix of size ``m`` by ``n`` with the specified (independent) probability ``p`` of any entry being nonzero, where nonzero values are sampled from the normal distribution. The optional ``rng`` argument specifies a random number generator, see :ref:`Random Numbers `\ . -.. function:: etree(A[, post]) - - .. Docstring generated from Julia source - - Compute the elimination tree of a symmetric sparse matrix ``A`` from ``triu(A)`` and, optionally, its post-ordering permutation. - -.. function:: symperm(A, p) - - .. Docstring generated from Julia source - - Return the symmetric permutation of ``A``\ , which is ``A[p,p]``\ . ``A`` should be symmetric, sparse, and only contain nonzeros in the upper triangular part of the matrix is stored. This algorithm ignores the lower triangular part of the matrix. Only the upper triangular part of the result is returned. - .. function:: nonzeros(A) .. Docstring generated from Julia source diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index c1278217873b6..226989779188a 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -368,20 +368,6 @@ end @test full(spdiagm((ones(2), ones(2)), (0, -1), 3, 3)) == [1.0 0.0 0.0; 1.0 1.0 0.0; 0.0 1.0 0.0] -# elimination tree -## upper triangle of the pattern test matrix from Figure 4.2 of -## "Direct Methods for Sparse Linear Systems" by Tim Davis, SIAM, 2006 -rowval = Int32[1,2,2,3,4,5,1,4,6,1,7,2,5,8,6,9,3,4,6,8,10,3,5,7,8,10,11] -colval = Int32[1,2,3,3,4,5,6,6,6,7,7,8,8,8,9,9,10,10,10,10,10,11,11,11,11,11,11] -A = sparse(rowval, colval, ones(length(rowval))) -p = etree(A) -P,post = etree(A, true) -@test P == p -@test P == Int32[6,3,8,6,8,7,9,10,10,11,0] -@test post == Int32[2,3,5,8,1,4,6,7,9,10,11] -@test isperm(post) - - # issue #4986, reinterpret sfe22 = speye(Float64, 2) mfe22 = eye(Float64, 2) @@ -1068,13 +1054,9 @@ A = sparse(tril(rand(5,5))) @test istril(A) @test !istril(sparse(ones(5,5))) -# symperm -srand(1234321) -A = triu(sprand(10,10,0.2)) # symperm operates on upper triangle -perm = randperm(10) -@test symperm(A,perm).colptr == [1,1,2,4,5,5,6,8,8,9,10] - # droptol +srand(1234321) +A = triu(sprand(10,10,0.2)) @test Base.droptol!(A,0.01).colptr == [1,1,1,2,2,3,4,6,6,7,9] @test isequal(Base.droptol!(sparse([1], [1], [1]), 1), SparseMatrixCSC(1,1,Int[1,1],Int[],Int[])) @@ -1316,12 +1298,6 @@ if Base.USE_GPL_LIBS end @test_throws DimensionMismatch Base.SparseArrays.normestinv(sprand(3,5,.9)) -# csc_permute -A = sprand(10,10,0.2) -p = randperm(10) -q = randperm(10) -@test Base.SparseArrays.csc_permute(A, invperm(p), q) == full(A)[p, q] - # issue #13008 @test_throws ArgumentError sparse(collect(1:100), collect(1:100), fill(5,100), 5, 5) @test_throws ArgumentError sparse(Int[], collect(1:5), collect(1:5)) From 28b5e0ad5907a6f1f81a48fe069bc5ef9043270d Mon Sep 17 00:00:00 2001 From: kshyatt Date: Mon, 20 Jun 2016 14:58:09 -0400 Subject: [PATCH 0061/1117] Add documentation for LQ factorization --- base/linalg/lq.jl | 19 +++++++++++++++++++ doc/stdlib/linalg.rst | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/base/linalg/lq.jl b/base/linalg/lq.jl index 58abb7d6c7d84..476b2c195ed29 100644 --- a/base/linalg/lq.jl +++ b/base/linalg/lq.jl @@ -17,10 +17,29 @@ end LQ{T}(factors::AbstractMatrix{T}, τ::Vector{T}) = LQ{T,typeof(factors)}(factors, τ) LQPackedQ{T}(factors::AbstractMatrix{T}, τ::Vector{T}) = LQPackedQ{T,typeof(factors)}(factors, τ) +""" + lqfact!(A) -> LQ + +Compute the LQ factorization of `A`, using the input +matrix as a workspace. See also [`lq`](:func:`lq). +""" lqfact!{T<:BlasFloat}(A::StridedMatrix{T}) = LQ(LAPACK.gelqf!(A)...) +""" + lqfact(A) -> LQ + +Compute the LQ factorization of `A`. See also [`lq`](:func:`lq). +""" lqfact{T<:BlasFloat}(A::StridedMatrix{T}) = lqfact!(copy(A)) lqfact(x::Number) = lqfact(fill(x,1,1)) +""" + lq(A; [thin=true]) -> L, Q + +Perform an LQ factorization of `A` such that `A = L*Q`. The +default is to compute a thin factorization. The LQ factorization +is the QR factorization of `A.'`. `L` is not extended with +zeros if the full `Q` is requested. +""" function lq(A::Union{Number, AbstractMatrix}; thin::Bool=true) F = lqfact(A) F[:L], full(F[:Q], thin=thin) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index bc103c1bd8fc2..63ec01e493d7d 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -507,6 +507,24 @@ Linear algebra functions in Julia are largely implemented by calling functions f Optionally takes a ``thin`` Boolean argument, which if ``true`` omits the columns that span the rows of ``R`` in the QR factorization that are zero. The resulting matrix is the ``Q`` in a thin QR factorization (sometimes called the reduced QR factorization). If ``false``\ , returns a ``Q`` that spans all rows of ``R`` in its corresponding QR factorization. +.. function:: lqfact!(A) -> LQ + + .. Docstring generated from Julia source + + Compute the LQ factorization of ``A``\ , using the input matrix as a workspace. See also :func:`lq\ . + +.. function:: lqfact(A) -> LQ + + .. Docstring generated from Julia source + + Compute the LQ factorization of ``A``\ . See also :func:`lq\ . + +.. function:: lq(A; [thin=true]) -> L, Q + + .. Docstring generated from Julia source + + Perform an LQ factorization of ``A`` such that ``A = L*Q``\ . The default is to compute a thin factorization. The LQ factorization is the QR factorization of ``A.'``\ . ``L`` is not extended with zeros if the full ``Q`` is requested. + .. function:: bkfact(A) -> BunchKaufman .. Docstring generated from Julia source From 129673679f381a177829cbec58711e0f78f7ec72 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 18 Jun 2016 21:58:40 -0400 Subject: [PATCH 0062/1117] fix compile-all / anticodegen / no-inference modes and remove any type-inferred code that isn't being used also refactor a bit of compile=all and cache-method code to make it clearer how jl_is_cacheable_sig parallels it --- base/docs/Docs.jl | 2 +- base/docs/core.jl | 2 +- base/inference.jl | 53 ++++--- base/sysimg.jl | 18 ++- src/alloc.c | 12 -- src/anticodegen.c | 26 ++- src/codegen.cpp | 7 +- src/disasm.cpp | 2 +- src/dump.c | 2 + src/gf.c | 366 +++++++++++++++++++++++++++++-------------- src/julia_internal.h | 1 + src/stackwalk.c | 2 +- 12 files changed, 318 insertions(+), 175 deletions(-) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 936a22a6e162a..c238f1fc8bd20 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -59,7 +59,7 @@ include("bindings.jl") import Base.Markdown: @doc_str, MD import Base.Meta: quot, isexpr import Base: Callable -import Core.Inference.CoreDocs: lazy_iterpolate +import ..CoreDocs: lazy_iterpolate export doc diff --git a/base/docs/core.jl b/base/docs/core.jl index f1350d57c8c5c..4934f4e43aaa4 100644 --- a/base/docs/core.jl +++ b/base/docs/core.jl @@ -2,7 +2,7 @@ module CoreDocs -import Core.Inference: esc, push!, getindex, current_module, unsafe_load, Csize_t +import ..esc, ..push!, ..getindex, ..current_module, ..unsafe_load, ..Csize_t function doc!(str, ex) ptr = unsafe_load(Core.Intrinsics.cglobal(:jl_filename, Ptr{UInt8})) diff --git a/base/inference.jl b/base/inference.jl index 5e166a79d6cf9..cd0906d203505 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -73,9 +73,10 @@ type InferenceState typegotoredo::Bool inworkq::Bool optimize::Bool + needtree::Bool inferred::Bool - function InferenceState(linfo::LambdaInfo, optimize::Bool) + function InferenceState(linfo::LambdaInfo, optimize::Bool, needtree::Bool) @assert isa(linfo.code,Array{Any,1}) nslots = length(linfo.slotnames) nl = label_counter(linfo.code)+1 @@ -160,7 +161,7 @@ type InferenceState ssavalue_uses, ssavalue_init, ObjectIdDict(), #Dict{InferenceState, Vector{LineNum}}(), Vector{Tuple{InferenceState, Vector{LineNum}}}(), - false, false, false, optimize, false) + false, false, false, optimize, needtree, false) push!(active, frame) nactive[] += 1 return frame @@ -1475,6 +1476,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr # don't call staged functions on abstract types. # (see issues #8504, #10230) # we can't guarantee that their type behavior is monotonic. + # XXX: this test is wrong if Types (such as DataType) are present return (nothing, Any, false) end try @@ -1487,6 +1489,8 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr linfo = specialize_method(method, atypes, sparams, cached) end + # XXX: the following logic is likely subtly broken if code.code was nothing, + # although it seems unlikely something bad (infinite recursion) will happen as a result if linfo.inInference # inference on this signature may be in progress, # find the corresponding frame in the active list @@ -1500,10 +1504,13 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr end # TODO: this assertion seems iffy assert(frame !== nothing) + if needtree + frame.needtree = true + end else # inference not started yet, make a new frame for a new lambda linfo.inInference = true - frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), optimize) + frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), optimize, needtree) end frame = frame::InferenceState @@ -1543,8 +1550,8 @@ end function typeinf_ext(linfo::LambdaInfo) if isdefined(linfo, :def) # method lambda - infer this specialization via the method cache - (code, _t, _) = typeinf_edge(linfo.def, linfo.specTypes, linfo.sparam_vals, true, true, true, linfo) - if code.inferred && linfo !== code + (code, _t, inferred) = typeinf_edge(linfo.def, linfo.specTypes, linfo.sparam_vals, true, true, true, linfo) + if inferred && code.inferred && linfo !== code # This case occurs when the IR for a function has been deleted. # `code` will be a newly-created LambdaInfo, and we need to copy its # contents to the existing one to copy the info to the method cache. @@ -1564,7 +1571,7 @@ function typeinf_ext(linfo::LambdaInfo) else # toplevel lambda - infer directly linfo.inInference = true - frame = InferenceState(linfo, true) + frame = InferenceState(linfo, true, true) typeinf_loop(frame) @assert frame.inferred # TODO: deal with this better return linfo @@ -1877,7 +1884,8 @@ function isinlineable(linfo::LambdaInfo) end end if !inlineable - body = Expr(:block); body.args = linfo.code + body = Expr(:block) + body.args = linfo.code inlineable = inline_worthy(body, cost) end end @@ -1905,9 +1913,6 @@ function finish(me::InferenceState) end type_annotate!(me.linfo, me.stmt_types, me, me.nargs) - # make sure (meta pure) is stripped from full tree - ispure = popmeta!(me.linfo.code, :pure)[1] - # run optimization passes on fulltree if me.optimize # This pass is required for the AST to be valid in codegen @@ -1925,22 +1930,34 @@ function finish(me::InferenceState) reindex_labels!(me.linfo, me) end widen_all_consts!(me.linfo) - - # finalize and record the linfo result - me.inferred = true + # make sure (meta pure) is stripped from full tree + ispure = popmeta!(me.linfo.code, :pure)[1] + me.linfo.pure = ispure # determine and cache inlineability me.linfo.inlineable = isinlineable(me.linfo) - if isdefined(me.linfo, :def) - # compress code for non-toplevel thunks - compressedtree = ccall(:jl_compress_ast, Any, (Any,Any), me.linfo, me.linfo.code) - me.linfo.code = compressedtree + if !me.needtree + me.needtree = me.linfo.inlineable || ccall(:jl_is_cacheable_sig, Int32, (Any, Any, Any), + me.linfo.specTypes, me.linfo.def.sig, me.linfo.def) != 0 end - me.linfo.pure = ispure + + if me.needtree + if isdefined(me.linfo, :def) + # compress code for non-toplevel thunks + compressedtree = ccall(:jl_compress_ast, Any, (Any,Any), me.linfo, me.linfo.code) + me.linfo.code = compressedtree + end + else + ccall(:jl_set_lambda_code_null, Void, (Any,), me.linfo) + me.linfo.inlineable = false + end + ccall(:jl_set_lambda_rettype, Void, (Any, Any), me.linfo, me.bestguess) me.linfo.inferred = true me.linfo.inInference = false + # finalize and record the linfo result + me.inferred = true # lazy-delete the item from active for several reasons: # efficiency, correctness, and recursion-safety diff --git a/base/sysimg.jl b/base/sysimg.jl index f7af0b3606c39..e76daa36856b4 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -12,7 +12,10 @@ eval(m,x) = Core.eval(m,x) # init core docsystem import Core: @doc, @__doc__, @doc_str -Core.atdoc!(Core.Inference.CoreDocs.docm) +if isdefined(Core, :Inference) + import Core.Inference.CoreDocs + Core.atdoc!(CoreDocs.docm) +end include("exports.jl") @@ -61,6 +64,12 @@ include("refpointer.jl") include("checked.jl") importall .Checked +# Symbol constructors +if !isdefined(Core, :Inference) + Symbol(s::String) = Symbol(s.data) + Symbol(a::Array{UInt8,1}) = + ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Int32), a, length(a)) +end # vararg Symbol constructor Symbol(x...) = Symbol(string(x...)) @@ -129,6 +138,11 @@ include("osutils.jl") include("c.jl") include("sysinfo.jl") +if !isdefined(Core, :Inference) + include("docs/core.jl") + Core.atdoc!(CoreDocs.docm) +end + # Core I/O include("io.jl") include("iostream.jl") @@ -352,7 +366,7 @@ include("docs/basedocs.jl") include("markdown/Markdown.jl") include("docs/Docs.jl") using .Docs, .Markdown -Docs.loaddocs(Core.Inference.CoreDocs.DOCS) +isdefined(Core, :Inference) && Docs.loaddocs(Core.Inference.CoreDocs.DOCS) function __init__() # Base library init diff --git a/src/alloc.c b/src/alloc.c index 5c3541b71200f..79056b762c694 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -520,18 +520,6 @@ JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t new_linfo->specTypes = types; new_linfo->def = m; new_linfo->sparam_vals = sp; - - if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF || - jl_options.compile_enabled == JL_OPTIONS_COMPILE_MIN) { - // copy fptr from the template method definition - new_linfo->fptr = linfo->fptr; - new_linfo->jlcall_api = linfo->jlcall_api; - if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF && new_linfo->fptr == NULL) { - jl_printf(JL_STDERR,"code missing for "); - jl_static_show(JL_STDERR, (jl_value_t*)new_linfo); - jl_printf(JL_STDERR, " sysimg may not have been built with --compile=all\n"); - } - } } else { new_linfo = jl_instantiate_staged(m, types, sp); diff --git a/src/anticodegen.c b/src/anticodegen.c index b803fe1f757e8..8e19c2989323d 100644 --- a/src/anticodegen.c +++ b/src/anticodegen.c @@ -9,13 +9,13 @@ int globalUnique = 0; #define UNAVAILABLE { jl_errorf("%s: not available in this build of Julia", __func__); } -void jl_dump_bitcode(char *fname, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE +void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE void jl_dump_objfile(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE int32_t jl_get_llvm_gv(jl_value_t *p) UNAVAILABLE void jl_write_malloc_log(void) UNAVAILABLE void jl_write_coverage_data(void) UNAVAILABLE void jl_generate_fptr(jl_lambda_info_t *li) UNAVAILABLE -void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx) UNAVAILABLE +void jl_compile_linfo(jl_lambda_info_t *li) UNAVAILABLE JL_DLLEXPORT void jl_clear_malloc_data(void) UNAVAILABLE JL_DLLEXPORT void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE @@ -23,8 +23,8 @@ JL_DLLEXPORT void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t JL_DLLEXPORT const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) UNAVAILABLE JL_DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, uint8_t strip_ir_metadata, uint8_t dump_module) UNAVAILABLE -JL_DLLEXPORT void *jl_LLVMCreateDisasm(const char *, void *, int, void*, void*) UNAVAILABLE -JL_DLLEXPORT size_t jl_LLVMDisasmInstruction(void *DC, uint8_t *, uint64_t, uint64_t PC, char *, size_t) UNAVAILABLE +JL_DLLEXPORT void *jl_LLVMCreateDisasm(const char *TripleName, void *DisInfo, int TagType, void *GetOpInfo, void *SymbolLookUp) UNAVAILABLE +JL_DLLEXPORT size_t jl_LLVMDisasmInstruction(void *DC, uint8_t *Bytes, uint64_t BytesSize, uint64_t PC, char *OutString, size_t OutStringSize) UNAVAILABLE void jl_init_codegen(void) { } void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig) @@ -32,17 +32,10 @@ void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig) if (!specsig) lam->fptr = fptr; } -void jl_getFunctionInfo(char **name, char **filename, size_t *line, - char **inlinedat_file, size_t *inlinedat_line, jl_lambda_info_t **outer_linfo, - size_t pointer, int *fromC, int skipC, int skipInline) + +int jl_getFunctionInfo(jl_frame_t **frames, uintptr_t pointer, int skipC, int noInline) { - *name = NULL; - *line = -1; - *filename = NULL; - *inlinedat_file = NULL; - *inlinedat_line = -1; - *outer_linfo = NULL; - *fromC = 0; + return 0; } jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, @@ -50,3 +43,8 @@ jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, { return NULL; } + +void jl_register_fptrs(uint64_t sysimage_base, void **fptrs, jl_lambda_info_t **linfos, size_t n) +{ + (void)sysimage_base; (void)fptrs; (void)linfos; (void)n; +} diff --git a/src/codegen.cpp b/src/codegen.cpp index e09a8db619bac..e33d370bca767 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -870,11 +870,8 @@ static void to_function(jl_lambda_info_t *li) // if not inlineable, code won't be needed again if (JL_DELETE_NON_INLINEABLE && li->def && li->inferred && !li->inlineable && - li != li->def->lambda_template && !jl_options.outputji) { - li->code = jl_nothing; - li->ssavaluetypes = jl_box_long(jl_array_len(li->ssavaluetypes)); jl_gc_wb(li, li->ssavaluetypes); - li->slotflags = NULL; - li->slotnames = NULL; + li != li->def->lambda_template && !imaging_mode) { + jl_set_lambda_code_null(li); } // done compiling: restore global state diff --git a/src/disasm.cpp b/src/disasm.cpp index e682d65576dd8..063923c5f7b42 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -233,7 +233,7 @@ static int OpInfoLookup(void *DisInfo, uint64_t PC, uint64_t Offset, uint64_t Si default: return 0; // Cannot handle input address size } int skipC = 0; - jl_frame_t *frame; + jl_frame_t *frame = NULL; jl_getFunctionInfo(&frame, pointer, skipC, 1); char *name = frame->func_name; free(frame->file_name); diff --git a/src/dump.c b/src/dump.c index 4b26f2ad40947..176697890ae84 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1267,6 +1267,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) // builtin types are not serialized, so their caches aren't // explicitly saved. so we reconstruct the caches of builtin // parametric types here. + // XXX: need to make sure to serialize all leaftypes in this cache + // that were referenced from compiled code jl_array_ptr_1d_push(datatype_list, (jl_value_t*)dt); } } diff --git a/src/gf.c b/src/gf.c index 98a50b4694296..688009ac1fa38 100644 --- a/src/gf.c +++ b/src/gf.c @@ -251,6 +251,15 @@ JL_DLLEXPORT void jl_set_lambda_rettype(jl_lambda_info_t *li, jl_value_t *rettyp li->jlcall_api = 0; } +JL_DLLEXPORT void jl_set_lambda_code_null(jl_lambda_info_t *li) +{ + li->code = jl_nothing; + li->ssavaluetypes = jl_box_long(jl_array_len(li->ssavaluetypes)); + jl_gc_wb(li, li->ssavaluetypes); + li->slotflags = NULL; + li->slotnames = NULL; +} + static int get_spec_unspec_list(jl_typemap_entry_t *l, void *closure) { if (jl_is_lambda_info(l->func.value) && !l->func.linfo->inferred) @@ -339,7 +348,6 @@ static jl_tupletype_t *join_tsig(jl_tupletype_t *tt, jl_tupletype_t *sig) { jl_svec_t *newparams = NULL; JL_GC_PUSH1(&newparams); - int changed = 0; size_t i, np; for (i = 0, np = jl_nparams(tt); i < np; i++) { jl_value_t *elt = jl_tparam(tt, i); @@ -369,14 +377,11 @@ static jl_tupletype_t *join_tsig(jl_tupletype_t *tt, jl_tupletype_t *sig) } // prepare to build a new type with the replacement above if (newelt) { - if (!changed) { - newparams = jl_svec_copy(tt->parameters); - changed = 1; - } + if (!newparams) newparams = jl_svec_copy(tt->parameters); jl_svecset(newparams, i, newelt); } } - if (changed) + if (newparams) tt = jl_apply_tuple_type(newparams); JL_GC_POP(); return tt; @@ -385,36 +390,25 @@ static jl_tupletype_t *join_tsig(jl_tupletype_t *tt, jl_tupletype_t *sig) static jl_value_t *ml_matches(union jl_typemap_t ml, int offs, jl_tupletype_t *type, int lim, int include_ambiguous); -static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *cache, jl_value_t *parent, - jl_tupletype_t *type, // the specialized type signature for type lambda - jl_tupletype_t *tt, // the original tupletype of the signature - jl_typemap_entry_t *m, jl_svec_t *sparams) +static void jl_cacheable_sig( + jl_tupletype_t *const type, // the specialized type signature for type lambda + jl_tupletype_t *const tt, // the original tupletype of the signature + jl_tupletype_t *decl, + jl_method_t *definition, + + jl_svec_t **const newparams, + int *const need_guard_entries, + int *const makesimplesig) { - JL_LOCK(&codegen_lock); // Might GC - size_t i; - jl_tupletype_t *decl = m->sig; - jl_method_t *definition = m->func.method; int8_t isstaged = definition->isstaged; - int need_guard_entries = 0; - int hasnewparams = 0; - int makesimplesig = 0; - jl_value_t *temp=NULL; - jl_value_t *temp2=NULL; - jl_value_t *temp3=NULL; - jl_lambda_info_t *newmeth=NULL; - jl_svec_t *newparams=NULL; - jl_svec_t *limited=NULL; - JL_GC_PUSH5(&temp, &temp2, &temp3, &newmeth, &newparams); - size_t np = jl_nparams(type); - newparams = jl_svec_copy(type->parameters); - - for (i=0; i < np; i++) { - jl_value_t *elt = jl_tparam(type,i); - jl_value_t *decl_i = jl_nth_slot_type(decl,i); + size_t i, np = jl_nparams(type); + for (i = 0; i < np; i++) { + jl_value_t *elt = jl_tparam(type, i); + jl_value_t *decl_i = jl_nth_slot_type(decl, i); if ((tt != type && elt != jl_tparam(tt, i)) || // if join_tsig made a swap is_kind(elt)) { // might see a kind if called at compile-time // kind slots always need guard entries (checking for subtypes of Type) - need_guard_entries = 1; + *need_guard_entries = 1; continue; } @@ -428,18 +422,18 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca if (jl_is_type_type(elt) && jl_is_tuple_type(jl_tparam0(elt)) && (!jl_subtype(decl_i, (jl_value_t*)jl_type_type, 0) || is_kind(decl_i))) { // Type{Tuple{...}} elt = (jl_value_t*)jl_anytuple_type_type; // Type{T<:Tuple} - jl_svecset(newparams, i, elt); - hasnewparams = 1; - need_guard_entries = 1; + if (!*newparams) *newparams = jl_svec_copy(type->parameters); + jl_svecset(*newparams, i, elt); + *need_guard_entries = 1; } - int notcalled_func = (i>0 && i<=8 && !(definition->called&(1<<(i-1))) && - jl_subtype(elt,(jl_value_t*)jl_function_type,0)); + int notcalled_func = (i > 0 && i <= 8 && !(definition->called & (1 << (i - 1))) && + jl_subtype(elt, (jl_value_t*)jl_function_type, 0)); if (decl_i == jl_ANY_flag) { // don't specialize on slots marked ANY - jl_svecset(newparams, i, (jl_value_t*)jl_any_type); - hasnewparams = 1; - need_guard_entries = 1; + if (!*newparams) *newparams = jl_svec_copy(type->parameters); + jl_svecset(*newparams, i, (jl_value_t*)jl_any_type); + *need_guard_entries = 1; } else if (notcalled_func && (decl_i == (jl_value_t*)jl_any_type || decl_i == (jl_value_t*)jl_function_type || @@ -448,15 +442,14 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_subtype((jl_value_t*)jl_datatype_type, decl_i, 0)))) { // and attempt to despecialize types marked Function, Callable, or Any // when called with a subtype of Function but is not called - jl_svecset(newparams, i, (jl_value_t*)jl_function_type); - makesimplesig = 1; - hasnewparams = 1; - need_guard_entries = 1; + if (!*newparams) *newparams = jl_svec_copy(type->parameters); + jl_svecset(*newparams, i, (jl_value_t*)jl_function_type); + *makesimplesig = 1; + *need_guard_entries = 1; } else if (jl_is_type_type(elt) && jl_is_type_type(jl_tparam0(elt)) && // give up on specializing static parameters for Type{Type{Type{...}}} - (jl_is_type_type(jl_tparam0(jl_tparam0(elt))) || - decl_i==NULL || !jl_has_typevars(decl_i))) { + (jl_is_type_type(jl_tparam0(jl_tparam0(elt))) || !jl_has_typevars(decl_i))) { /* actual argument was Type{...}, we computed its type as Type{Type{...}}. we must avoid unbounded nesting here, so @@ -464,8 +457,9 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca specific like Type{Type{Int32}} was actually declared. this can be determined using a type intersection. */ + if (!*newparams) *newparams = jl_svec_copy(type->parameters); if (i < jl_nparams(decl)) { - jl_value_t *declt = jl_tparam(decl,i); + jl_value_t *declt = jl_tparam(decl, i); // for T..., intersect with T if (jl_is_vararg_type(declt)) declt = jl_tparam0(declt); @@ -473,17 +467,132 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca assert(di != (jl_value_t*)jl_bottom_type); if (is_kind(di)) // issue #11355: DataType has a UID and so takes precedence in the cache - jl_svecset(newparams, i, (jl_value_t*)jl_typetype_type); + jl_svecset(*newparams, i, (jl_value_t*)jl_typetype_type); else - jl_svecset(newparams, i, di); + jl_svecset(*newparams, i, di); // TODO: recompute static parameter values, so in extreme cases we - // can give `T=Type` instead of `T=Type{Type{Type{...`. + // can give `T=Type` instead of `T=Type{Type{Type{...`. /* make editors happy:}}} */ + } + else { + jl_svecset(*newparams, i, (jl_value_t*)jl_typetype_type); + } + *need_guard_entries = 1; + } + else if (jl_is_type_type(elt) && very_general_type(decl_i) && + !jl_has_typevars(decl_i)) { + /* + here's a fairly simple heuristic: if this argument slot's + declared type is general (Type, Any, or ANY), + then don't specialize for every Type that got passed. + + Since every type x has its own type Type{x}, this would be + excessive specialization for an Any slot. + + This may require guard entries due to other potential matches. + In particular, TypeConstructors are problematic because they can + be alternate representations of any type. Extensionally, TC == TC.body, + but typeof(TC) != typeof(TC.body). This creates an ambiguity: + Type{TC} is type-equal to Type{TC.body}, yet a slot + x::TypeConstructor matches the first but not the second, while + also matching all other TypeConstructors. This means neither + Type{TC} nor TypeConstructor is more specific. + */ + if (!*newparams) *newparams = jl_svec_copy(type->parameters); + jl_svecset(*newparams, i, jl_typetype_type); + *need_guard_entries = 1; + } + } +} + +JL_DLLEXPORT int jl_is_cacheable_sig( + jl_tupletype_t *type, + jl_tupletype_t *decl, + jl_method_t *definition) +{ + // compute whether this type signature is a possible return value from jl_cacheable_sig + //return jl_cacheable_sig(type, NULL, definition->sig, definition, NULL, NULL); + + if (definition->isstaged) + // staged functions can't be optimized + // so assume the caller was intelligent about calling us + return 1; + + size_t i, np = jl_nparams(type); + for (i = 0; i < np; i++) { + jl_value_t *elt = jl_tparam(type, i); + jl_value_t *decl_i = jl_nth_slot_type(decl, i); + + if (jl_is_vararg_type(elt)) // varargs are always considered compilable + continue; + if (is_kind(elt)) // kind slots always need guard entries (checking for subtypes of Type) + continue; + if (decl_i == jl_ANY_flag) { + // don't specialize on slots marked ANY + if (elt != (jl_value_t*)jl_any_type) + return 0; + continue; + } + if (jl_is_type_type(elt)) { // if join_tsig would make a swap + // if the declared type was not Any or Union{Type, ...}, + // then the match must been with TypeConstructor or DataType + // and the result of matching the type signature + // needs to be corrected to the leaf type 'kind' + jl_value_t *kind = jl_typeof(jl_tparam0(elt)); + if (kind != (jl_value_t*)jl_tvar_type && jl_subtype(kind, decl_i, 0)) { + if (!jl_subtype((jl_value_t*)jl_type_type, decl_i, 0)) { + return 0; + } + } + } + // avoid specializing on an argument of type Tuple + // unless matching a declared type of `::Type` + if (jl_is_type_type(elt) && jl_is_tuple_type(jl_tparam0(elt)) && + (!jl_subtype(decl_i, (jl_value_t*)jl_type_type, 0) || is_kind(decl_i))) { // Type{Tuple{...}} + if (elt != (jl_value_t*)jl_anytuple_type_type) + return 0; + continue; + } + + int notcalled_func = (i > 0 && i <= 8 && !(definition->called & (1 << (i - 1))) && + jl_subtype(elt, (jl_value_t*)jl_function_type, 0)); + if (notcalled_func && (decl_i == (jl_value_t*)jl_any_type || + decl_i == (jl_value_t*)jl_function_type || + (jl_is_uniontype(decl_i) && jl_svec_len(((jl_uniontype_t*)decl_i)->types)==2 && + jl_subtype((jl_value_t*)jl_function_type, decl_i, 0) && + jl_subtype((jl_value_t*)jl_datatype_type, decl_i, 0)))) { + // and attempt to despecialize types marked Function, Callable, or Any + // when called with a subtype of Function but is not called + if (elt != (jl_value_t*)jl_function_type) + return 0; + continue; + } + else if (jl_is_type_type(elt) && jl_is_type_type(jl_tparam0(elt)) && + // give up on specializing static parameters for Type{Type{Type{...}}} + (jl_is_type_type(jl_tparam0(jl_tparam0(elt))) || !jl_has_typevars(decl_i))) { + /* + actual argument was Type{...}, we computed its type as + Type{Type{...}}. we must avoid unbounded nesting here, so + cache the signature as Type{T}, unless something more + specific like Type{Type{Int32}} was actually declared. + this can be determined using a type intersection. + */ + + if (i < jl_nparams(decl)) { + jl_value_t *declt = jl_tparam(decl, i); + // for T..., intersect with T + if (jl_is_vararg_type(declt)) + declt = jl_tparam0(declt); + jl_value_t *di = jl_type_intersection(declt, (jl_value_t*)jl_typetype_type); + assert(di != (jl_value_t*)jl_bottom_type); + if (is_kind(di)) + return 0; + else if (!jl_subtype(di, elt, 0) || !jl_subtype(elt, di, 0)) + return 0; } else { - jl_svecset(newparams, i, (jl_value_t*)jl_typetype_type); + return 0; } - need_guard_entries = 1; - hasnewparams = 1; + continue; } else if (jl_is_type_type(elt) && very_general_type(decl_i) && !jl_has_typevars(decl_i)) { @@ -504,25 +613,52 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca also matching all other TypeConstructors. This means neither Type{TC} nor TypeConstructor is more specific. */ - jl_svecset(newparams, i, jl_typetype_type); - need_guard_entries = 1; - hasnewparams = 1; + if (elt != (jl_value_t*)jl_typetype_type) + return 0; + continue; + } + else if (!jl_is_leaf_type(elt)) { + return 0; } } + return 1; +} + +static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *cache, jl_value_t *parent, + jl_tupletype_t *type, // the specialized type signature for type lambda + jl_tupletype_t *tt, // the original tupletype of the signature + jl_typemap_entry_t *m, + jl_svec_t *sparams) +{ + JL_LOCK(&codegen_lock); // Might GC + jl_method_t *definition = m->func.method; + jl_tupletype_t *decl = m->sig; + jl_value_t *temp = NULL; + jl_value_t *temp2 = NULL; + jl_value_t *temp3 = NULL; + jl_lambda_info_t *newmeth = NULL; + jl_svec_t *newparams = NULL; + JL_GC_PUSH5(&temp, &temp2, &temp3, &newmeth, &newparams); + + int need_guard_entries = 0; + int makesimplesig = 0; + jl_cacheable_sig(type, tt, decl, definition, + (jl_svec_t**)&newparams, &need_guard_entries, &makesimplesig); // for varargs methods, only specialize up to max_args. // in general, here we want to find the biggest type that's not a // supertype of any other method signatures. so far we are conservative // and the types we find should be bigger. - if (!isstaged && jl_nparams(type) > mt->max_args + if (!definition->isstaged && jl_nparams(type) > mt->max_args && jl_va_tuple_kind(decl) == JL_VARARG_UNBOUND) { - size_t nspec = mt->max_args + 2; - limited = jl_alloc_svec(nspec); - temp3 = (jl_value_t*)limited; - for(i=0; i < nspec-1; i++) { + size_t i, nspec = mt->max_args + 2; + jl_svec_t *limited = jl_alloc_svec(nspec); + temp = (jl_value_t*)limited; + if (!newparams) newparams = type->parameters; + for (i = 0; i < nspec - 1; i++) { jl_svecset(limited, i, jl_svecref(newparams, i)); } - jl_value_t *lasttype = jl_svecref(newparams,i-1); + jl_value_t *lasttype = jl_svecref(newparams, i - 1); // if all subsequent arguments are subtypes of lasttype, specialize // on that instead of decl. for example, if decl is // (Any...) @@ -536,9 +672,9 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca // leaftype signatures TypeConstructor and DataType // (assuming those made an unlikely appearance in Varargs position) size_t j = i; - int all_are_subtypes=1; - for(; j < jl_svec_len(newparams); j++) { - if (!jl_subtype(jl_svecref(newparams,j), lasttype, 0)) { + int all_are_subtypes = 1; + for (; j < jl_svec_len(newparams); j++) { + if (!jl_subtype(jl_svecref(newparams, j), lasttype, 0)) { all_are_subtypes = 0; break; } @@ -550,24 +686,24 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_svecset(limited, i, jl_wrap_vararg(lasttype, (jl_value_t*)NULL)); } else { - jl_value_t *lastdeclt = jl_tparam(decl,jl_nparams(decl)-1); + jl_value_t *lastdeclt = jl_tparam(decl, jl_nparams(decl) - 1); int nsp = jl_svec_len(sparams); if (nsp > 0) { - temp2 = (jl_value_t*)jl_alloc_svec_uninit(2*nsp); - for(j=0; j < nsp; j++) { - if (j==0 && jl_is_typevar(m->tvars)) - jl_svecset(temp2, 0, m->tvars); + jl_svec_t *env = jl_alloc_svec_uninit(2 * nsp); + temp2 = (jl_value_t*)env; + for (j = 0; j < nsp; j++) { + if (j == 0 && jl_is_typevar(m->tvars)) + jl_svecset(env, 0, m->tvars); else - jl_svecset(temp2, j*2, jl_svecref(m->tvars, j)); - jl_svecset(temp2, j*2+1, jl_svecref(sparams,j)); + jl_svecset(env, j * 2, jl_svecref(m->tvars, j)); + jl_svecset(env, j * 2 + 1, jl_svecref(sparams, j)); } lastdeclt = (jl_value_t*)jl_instantiate_type_with((jl_value_t*)lastdeclt, - jl_svec_data(temp2), nsp); + jl_svec_data(env), nsp); } jl_svecset(limited, i, lastdeclt); } newparams = limited; - hasnewparams = 1; // now there is a problem: the widened signature is more // general than just the given arguments, so it might conflict // with another definition that doesn't have cache instances yet. @@ -581,7 +717,7 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca int cache_with_orig = 0; jl_svec_t* guardsigs = jl_emptysvec; jl_tupletype_t *origtype = type; // backup the prior value of `type` - if (hasnewparams) { + if (newparams) { type = jl_apply_tuple_type(newparams); temp2 = (jl_value_t*)type; } @@ -593,7 +729,8 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca } else { int unmatched_tvars = 0; - for (i = 0; i < jl_array_len(temp); i++) { + size_t i, l = jl_array_len(temp); + for (i = 0; i < l; i++) { jl_value_t *m = jl_array_ptr_ref(temp, i); jl_value_t *env = jl_svecref(m, 1); int k, l; @@ -663,7 +800,7 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca // reduce the complexity of rejecting this entry in the cache // by replacing non-simple types with jl_any_type to build a new `type` // (the only case this applies to currently due to the above logic is jl_function_type) - size_t np = jl_nparams(origtype); + size_t i, np = jl_nparams(origtype); newparams = jl_svec_copy(origtype->parameters); for (i = 0; i < np; i++) { jl_value_t *elt = jl_svecref(newparams, i); @@ -1053,6 +1190,22 @@ JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) { + if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF || + jl_options.compile_enabled == JL_OPTIONS_COMPILE_MIN) { + // copy fptr from the template method definition + jl_method_t *def = li->def; + if (def && !def->isstaged) { + li->fptr = def->lambda_template->fptr; + li->jlcall_api = def->lambda_template->jlcall_api; + if (li->fptr != NULL) + return li; + } + if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF) { + jl_printf(JL_STDERR, "code missing for "); + jl_static_show(JL_STDERR, (jl_value_t*)li); + jl_printf(JL_STDERR, " sysimg may not have been built with --compile=all\n"); + } + } if (li->functionObjectsDecls.functionObject == NULL) { if (li->inInference || li->inCompile) { // if inference is running on this function, get a copy @@ -1351,27 +1504,17 @@ static void _compile_all_deq(jl_array_t *found) if (found_i % (1 + found_l / 300) == 0 || found_i == found_l - 1) // show 300 progress steps, to show progress without overwhelming log files jl_printf(JL_STDERR, " %d / %d\r", found_i + 1, found_l); jl_typemap_entry_t *ml = (jl_typemap_entry_t*)jl_array_ptr_ref(found, found_i); - if (ml->func.value == NULL) - continue; // XXX: how does this happen - - jl_lambda_info_t *templ = NULL; - if (jl_is_method(ml->func.value)) { - // type infer a copy of the template, to avoid modifying the template itself - templ = ml->func.method->lambda_template; - linfo = jl_specializations_get_linfo(ml->func.method, ml->sig, jl_emptysvec); - } - else if (jl_is_lambda_info(ml->func.value)) { - templ = ml->func.linfo; - linfo = ml->func.linfo; - } - else { - continue; // this should be unreachable - } + jl_method_t *m = ml->func.method; + jl_lambda_info_t *templ = m->lambda_template; + // type infer a copy of the template, to avoid modifying the template code itself + if (m->isstaged) + linfo = templ; + else + linfo = jl_specializations_get_linfo(m, ml->sig, jl_emptysvec); - if (!linfo->inferred || linfo->code == jl_nothing) { - // force this function to be recompiled + // infer this function now, if necessary + if (!linfo->inferred || linfo->code == jl_nothing) jl_type_infer(linfo, 1); - } // keep track of whether all possible signatures have been cached (and thus whether it can skip trying to compile the template function) // this is necessary because many intrinsics try to call static_eval and thus are not compilable unspecialized @@ -1386,8 +1529,7 @@ static void _compile_all_deq(jl_array_t *found) assert(linfo->functionID > 0); if (linfo != templ) { // copy the function pointer back to the lambda_template - templ->functionObjectsDecls.functionObject = linfo->functionObjectsDecls.functionObject; - templ->functionObjectsDecls.specFunctionObject = linfo->functionObjectsDecls.specFunctionObject; + templ->functionObjectsDecls = linfo->functionObjectsDecls; templ->functionID = linfo->functionID; templ->specFunctionID = linfo->specFunctionID; templ->jlcall_api = linfo->jlcall_api; @@ -1402,24 +1544,9 @@ static void _compile_all_deq(jl_array_t *found) static int _compile_all_enq(jl_typemap_entry_t *ml, void *env) { jl_array_t *found = (jl_array_t*)env; - jl_lambda_info_t *linfo = NULL; - if (ml->func.value == NULL) - return 1; - - if (jl_is_method(ml->func.value)) { - // method definition -- compile via template field - jl_method_t *m = ml->func.method; - if (m->invokes.unknown != NULL) - jl_typemap_visitor(m->invokes, _compile_all_enq, env); - if (!m->isstaged) - linfo = m->lambda_template; - } - else if (jl_is_lambda_info(ml->func.value)) { - linfo = ml->func.linfo; - if (linfo->fptr != NULL) - return 1; // builtin function - } - if (linfo && !linfo->functionID) { + // method definition -- compile template field + jl_method_t *m = ml->func.method; + if (!m->lambda_template->functionID) { // found a lambda that still needs to be compiled jl_array_ptr_1d_push(found, (jl_value_t*)ml); } @@ -1442,7 +1569,6 @@ static void _compile_all_enq_module(jl_module_t *m, jl_array_t *found) jl_methtable_t *mt = tn->mt; if (mt != NULL && (jl_value_t*)mt != jl_nothing) { jl_typemap_visitor(mt->defs, _compile_all_enq, (void*)found); - jl_typemap_visitor(mt->cache, _compile_all_enq, (void*)found); } } } @@ -1458,7 +1584,7 @@ static void _compile_all_enq_module(jl_module_t *m, jl_array_t *found) } } -static void jl_compile_all(void) +static void jl_compile_all_defs(void) { // this "found" array will contain // TypeMapEntries for Methods and LambdaInfos that need to be compiled @@ -1478,7 +1604,7 @@ static void jl_compile_all(void) static int _precompile_enq_tfunc(jl_typemap_entry_t *l, void *closure) { if (jl_is_lambda_info(l->func.value) && !l->func.linfo->functionID) - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)l->func.linfo->specTypes); + jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)l->sig); return 1; } @@ -1533,9 +1659,9 @@ static void jl_compile_specializations(void) } void jl_precompile(int all) { - jl_compile_specializations(); if (all) - jl_compile_all(); + jl_compile_all_defs(); + jl_compile_specializations(); } // diff --git a/src/julia_internal.h b/src/julia_internal.h index 91093cca95902..ec6ee07329704 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -74,6 +74,7 @@ void jl_generate_fptr(jl_lambda_info_t *li); void jl_compile_linfo(jl_lambda_info_t *li); JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types); jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li); +JL_DLLEXPORT void jl_set_lambda_code_null(jl_lambda_info_t *li); // invoke (compiling if necessary) the jlcall function pointer for a method jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method); diff --git a/src/stackwalk.c b/src/stackwalk.c index bed4b5197a5d5..1cea12c1f1c3a 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -387,7 +387,7 @@ JL_DLLEXPORT void jl_gdblookup(uintptr_t ip) // This function is not allowed to reference any TLS variables since // it can be called from an unmanaged thread on OSX. // it means calling getFunctionInfo with noInline = 1 - jl_frame_t *frames; + jl_frame_t *frames = NULL; int n = jl_getFunctionInfo(&frames, ip, 0, 1); int i; From ec905107eb16cc74ad2e08faabef0cd877645097 Mon Sep 17 00:00:00 2001 From: Sam Morrison Date: Mon, 4 Apr 2016 16:03:21 -0500 Subject: [PATCH 0063/1117] Add test_broken and test_skip macros --- base/test.jl | 159 ++++++++++++++++++++++++++++++++++---------- doc/stdlib/test.rst | 27 +++++++- test/test.jl | 37 ++++++++--- 3 files changed, 176 insertions(+), 47 deletions(-) diff --git a/base/test.jl b/base/test.jl index 5b5ee1fb3b5af..ae95963141446 100644 --- a/base/test.jl +++ b/base/test.jl @@ -13,7 +13,7 @@ and summarize them at the end of the test set with `@testset`. """ module Test -export @test, @test_throws +export @test, @test_throws, @test_broken, @test_skip export @testset # Legacy approximate testing functions, yet to be included export @test_approx_eq, @test_approx_eq_eps, @inferred @@ -90,6 +90,8 @@ end The test condition couldn't be evaluated due to an exception, or it evaluated to something other than a `Bool`. +In the case of `@test_broken` it is used to alert for an +unexpected `Pass` `Result`. """ type Error <: Result test_type::Symbol @@ -110,6 +112,11 @@ function Base.show(io::IO, t::Error) errmsg = sprint(showerror, t.value, t.backtrace) print(io, join(map(line->string(" ",line), split(errmsg, "\n")), "\n")) + elseif t.test_type == :test_unbroken + # A test that was expected to fail did not + println(io, " Unexpected Pass") + println(io, " Expression: ", t.orig_expr) + println(io, " Got correct result, please change to @test if no longer broken.") elseif t.test_type == :nontest_error # we had an error outside of a @test println(io, " Got an exception of type $(typeof(t.value)) outside of a @test") @@ -120,6 +127,24 @@ function Base.show(io::IO, t::Error) end end +""" + Broken + +The test condition either has been explicitly skipped or is the expected +(failed) result of a broken test. +""" +type Broken <: Result + test_type::Symbol + orig_expr +end +function Base.show(io::IO, t::Broken) + print_with_color(:yellow, io, "Test Broken\n") + if t.test_type == :skipped + println(io, " Skipped: ", t.orig_expr) + else + println(io, "Expression: ", t.orig_expr) + end +end #----------------------------------------------------------------------- @@ -153,9 +178,6 @@ end const comparison_prec = Base.operator_precedence(:(==)) # @test - check if the expression evaluates to true -# In the special case of a comparison, e.g. x == 5, generate code to -# evaluate each term in the comparison individually so the results -# can be displayed nicely. """ @test ex @@ -164,6 +186,46 @@ Returns a `Pass` `Result` if it does, a `Fail` `Result` if it is `false`, and an `Error` `Result` if it could not be evaluated. """ macro test(ex) + orig_ex = Expr(:inert,ex) + result = get_test_result(ex) + :(do_test($result, $orig_ex)) +end + +""" + @test_broken ex + +For use to indicate a test that should pass but currently consistently +fails. +Tests that the expression `ex` evaluates to `false` or causes an exception. +Returns a `Broken` `Result` if it does, and an `Error` `Result` if it is +`true`. +""" +macro test_broken(ex) + orig_ex = Expr(:quote,ex) + result = get_test_result(ex) + # code to call do_test with execution result and original expr + :(do_broken_test($result, $orig_ex)) +end + +""" + @test_skip ex + +For use to indicate a test that should pass but currently intermittently +fails. +Does not evaluate the expression. +""" +macro test_skip(ex) + orig_ex = Expr(:quote, ex) + testres = :(Broken(:skipped, $orig_ex)) + :(record(get_testset(), $testres)) +end + +# An internal function, called by the code generated by the @test +# macro to get results of the test expression. +# In the special case of a comparison, e.g. x == 5, generate code to +# evaluate each term in the comparison individually so the results +# can be displayed nicely. +function get_test_result(ex) orig_ex = Expr(:inert,ex) # Normalize comparison operator calls to :comparison expressions if isa(ex, Expr) && ex.head == :call && length(ex.args)==3 && @@ -188,8 +250,7 @@ macro test(ex) end end Base.remove_linenums!(result) - # code to call do_test with execution result and original expr - :(do_test($result, $orig_ex)) + result end # An internal function, called by the code generated by the @test @@ -205,6 +266,7 @@ function do_test(result::ExecutionResult, orig_expr) # Ideally it is true, but it may be false or non-Boolean. value = result.value testres = if isa(value, Bool) + # a true value Passes value ? Pass(:test, orig_expr, result.data, value) : Fail(:test, orig_expr, result.data, value) else @@ -220,6 +282,18 @@ function do_test(result::ExecutionResult, orig_expr) record(get_testset(), testres) end +function do_broken_test(result::ExecutionResult, orig_expr) + testres = Broken(:test, orig_expr) + # Assume the test is broken and only change if the result is true + if isa(result, Returned) + value = result.value + if isa(value, Bool) && value + testres = Error(:test_unbroken, orig_expr, value, nothing) + end + end + record(get_testset(), testres) +end + #----------------------------------------------------------------------- """ @@ -294,13 +368,15 @@ type TestSetException <: Exception pass::Int fail::Int error::Int + broken::Int end function Base.show(io::IO, ex::TestSetException) print(io, "Some tests did not pass: ") print(io, ex.pass, " passed, ") print(io, ex.fail, " failed, ") - print(io, ex.error, " errored.") + print(io, ex.error, " errored, ") + print(io, ex.broken, " broken.") end #----------------------------------------------------------------------- @@ -317,6 +393,7 @@ fallback_testset = FallbackTestSet() # Records nothing, and throws an error immediately whenever a Fail or # Error occurs. Takes no action in the event of a Pass result record(ts::FallbackTestSet, t::Pass) = t +record(ts::FallbackTestSet, t::Broken) = t function record(ts::FallbackTestSet, t::Union{Fail,Error}) println(t) error("There was an error during testing") @@ -342,9 +419,11 @@ DefaultTestSet(desc) = DefaultTestSet(desc, [], false) # For a passing result, simply store the result record(ts::DefaultTestSet, t::Pass) = (push!(ts.results, t); t) +record(ts::DefaultTestSet, t::Broken) = (push!(ts.results, t); t) + # For the other result types, immediately print the error message # but do not terminate. Print a backtrace. -function record(ts::DefaultTestSet, t::Union{Fail,Error}) +function record(ts::DefaultTestSet, t::Union{Fail, Error}) print_with_color(:white, ts.description, ": ") print(t) # don't print the backtrace for Errors because it gets printed in the show @@ -373,48 +452,50 @@ function finish(ts::DefaultTestSet) end # Calculate the overall number for each type so each of # the test result types are aligned - passes, fails, errors, c_passes, c_fails, c_errors = get_test_counts(ts) + passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = get_test_counts(ts) total_pass = passes + c_passes total_fail = fails + c_fails total_error = errors + c_errors + total_broken = broken + c_broken dig_pass = total_pass > 0 ? ndigits(total_pass) : 0 dig_fail = total_fail > 0 ? ndigits(total_fail) : 0 dig_error = total_error > 0 ? ndigits(total_error) : 0 - total = total_pass + total_fail + total_error + dig_broken = total_broken > 0 ? ndigits(total_broken) : 0 + total = total_pass + total_fail + total_error + total_broken dig_total = total > 0 ? ndigits(total) : 0 # For each category, take max of digits and header width if there are # tests of that type pass_width = dig_pass > 0 ? max(length("Pass"), dig_pass) : 0 fail_width = dig_fail > 0 ? max(length("Fail"), dig_fail) : 0 error_width = dig_error > 0 ? max(length("Error"), dig_error) : 0 + broken_width = dig_broken > 0 ? max(length("Broken"), dig_broken) : 0 total_width = dig_total > 0 ? max(length("Total"), dig_total) : 0 # Calculate the alignment of the test result counts by # recursively walking the tree of test sets align = max(get_alignment(ts, 0), length("Test Summary:")) # Print the outer test set header once - print_with_color(:white, rpad("Test Summary:",align," ")) - print(" | ") + print_with_color(:white, rpad("Test Summary:",align," ")" | ") if pass_width > 0 - print_with_color(:green, lpad("Pass",pass_width," ")) - print(" ") + print_with_color(:green, lpad("Pass",pass_width," "), " ") end if fail_width > 0 - print_with_color(:red, lpad("Fail",fail_width," ")) - print(" ") + print_with_color(:red, lpad("Fail",fail_width," ")," ") end if error_width > 0 - print_with_color(:red, lpad("Error",error_width," ")) - print(" ") + print_with_color(:red, lpad("Error",error_width," ")," ") + end + if broken_width > 0 + print_with_color(:yellow, lpad("Broken",broken_width," ")," ") end if total_width > 0 print_with_color(:blue, lpad("Total",total_width," ")) end println() # Recursively print a summary at every level - print_counts(ts, 0, align, pass_width, fail_width, error_width, total_width) + print_counts(ts, 0, align, pass_width, fail_width, error_width, broken_width, total_width) # Finally throw an error as we are the outermost test set if total != total_pass - throw(TestSetException(total_pass,total_fail,total_error)) + throw(TestSetException(total_pass,total_fail,total_error, total_broken)) end # return the testset so it is returned from the @testset macro @@ -442,42 +523,44 @@ get_alignment(ts, depth::Int) = 0 # Recursive function that counts the number of test results of each # type directly in the testset, and totals across the child testsets function get_test_counts(ts::DefaultTestSet) - passes, fails, errors = 0, 0, 0 - c_passes, c_fails, c_errors = 0, 0, 0 + passes, fails, errors, broken = 0, 0, 0, 0 + c_passes, c_fails, c_errors, c_broken = 0, 0, 0, 0 for t in ts.results isa(t, Pass) && (passes += 1) isa(t, Fail) && (fails += 1) isa(t, Error) && (errors += 1) + isa(t, Broken) && (broken += 1) if isa(t, DefaultTestSet) - np, nf, ne, ncp, ncf, nce = get_test_counts(t) + np, nf, ne, nb, ncp, ncf, nce , ncb = get_test_counts(t) c_passes += np + ncp c_fails += nf + ncf c_errors += ne + nce + c_broken += nb + ncb end end ts.anynonpass = (fails + errors + c_fails + c_errors > 0) - return passes, fails, errors, c_passes, c_fails, c_errors + return passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken end # Recursive function that prints out the results at each level of # the tree of test sets function print_counts(ts::DefaultTestSet, depth, align, - pass_width, fail_width, error_width, total_width) + pass_width, fail_width, error_width, broken_width, total_width) # Count results by each type at this level, and recursively # through and child test sets - passes, fails, errors, c_passes, c_fails, c_errors = get_test_counts(ts) - subtotal = passes + fails + errors + c_passes + c_fails + c_errors + passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = get_test_counts(ts) + subtotal = passes + fails + errors + broken + c_passes + c_fails + c_errors + c_broken # Print test set header, with an alignment that ensures all # the test results appear above each other - print(rpad(string(" "^depth, ts.description), align, " "), " | ") + print(rpad(string(lpad(" ",depth), ts.description), align, " "), " | ") np = passes + c_passes if np > 0 print_with_color(:green, lpad(string(np), pass_width, " "), " ") elseif pass_width > 0 # No passes at this level, but some at another level - print(" "^pass_width, " ") + print(lpad(" ", pass_width), " ") end nf = fails + c_fails @@ -485,7 +568,7 @@ function print_counts(ts::DefaultTestSet, depth, align, print_with_color(:red, lpad(string(nf), fail_width, " "), " ") elseif fail_width > 0 # No fails at this level, but some at another level - print(" "^fail_width, " ") + print(lpad(" ", fail_width), " ") end ne = errors + c_errors @@ -493,10 +576,18 @@ function print_counts(ts::DefaultTestSet, depth, align, print_with_color(:red, lpad(string(ne), error_width, " "), " ") elseif error_width > 0 # No errors at this level, but some at another level - print(" "^error_width, " ") + print(lpad(" ", error_width), " ") + end + + nb = broken + c_broken + if nb > 0 + print_with_color(:yellow, lpad(string(nb), broken_width, " "), " ") + elseif broken_width > 0 + # No errors at this level, but some at another level + print(lpad(" ", broken_width), " ") end - if np == 0 && nf == 0 && ne == 0 + if np == 0 && nf == 0 && ne == 0 && nb == 0 print_with_color(:blue, "No tests") else print_with_color(:blue, lpad(string(subtotal), total_width, " ")) @@ -504,11 +595,11 @@ function print_counts(ts::DefaultTestSet, depth, align, println() # Only print results at lower levels if we had failures - if np != subtotal + if np + nb != subtotal for t in ts.results if isa(t, DefaultTestSet) print_counts(t, depth + 1, align, - pass_width, fail_width, error_width, total_width) + pass_width, fail_width, error_width, broken_width, total_width) end end end diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index c4812b96608ee..0006659bd7dd4 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -29,8 +29,8 @@ the results are what you expect. It can be helpful to ensure your code still works after you make changes, and can be used when developing as a way of specifying the behaviors your code should have when complete. -Simple unit testing can be performed with the :func:`@test` and -:func:`@test_throws` macros: +Simple unit testing can be performed with the :func:`@test` and :func:`@test_throws` +macros: .. function:: @test ex @@ -285,6 +285,29 @@ writing new tests. julia> @inferred max(1,2) 2 +Broken Tests +------------ + +If a test fails consistently it can be changed to use the :func:`@test_broken` +macro. This will denote the test as ``Broken`` if the test continues to fail +and alerts the user via an ``Error`` if the test succeeds. + +.. function:: @test_broken ex + + .. Docstring generated from Julia source + + For use to indicate a test that should pass but currently consistently fails. Tests that the expression ``ex`` evaluates to ``false`` or causes an exception. Returns a ``Broken`` ``Result`` if it does, and an ``Error`` ``Result`` if it is ``true``\ . + +:func:`@test_skip` is also available for use to skip a test without +evaluation in the case of intermittent failures. This test will not run but +gives a `Broken` `Result`. + +.. function:: @test_skip ex + + .. Docstring generated from Julia source + + For use to indicate a test that should pass but currently intermittently fails. Does not evaluate the expression. + Creating Custom ``AbstractTestSet`` Types ----------------------------------------- diff --git a/test/test.jl b/test/test.jl index a2f3a23332038..0ef54e16b2f26 100644 --- a/test/test.jl +++ b/test/test.jl @@ -1,9 +1,5 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -using Base.Test - -# test file to test testing - # Test @test @test true @test 1 == 1 @@ -11,6 +7,22 @@ using Base.Test @test strip("\t hi \n") == "hi" @test strip("\t this should fail \n") != "hi" +# Test @test_broken with fail +@test_broken false +@test_broken 1 == 2 +@test_broken 1 != 1 +@test_broken strip("\t hi \n") != "hi" +@test_broken strip("\t this should fail \n") == "hi" +# Test @test_broken with errors +@test_broken error() +@test_broken absolute_nonsense + +#Test @test_skip +@test_skip error() +@test_skip true +@test_skip false +@test_skip gobbeldygook + a = Array(Float64, 2, 2, 2, 2, 2) a[1,1,1,1,1] = 10 @test a[1,1,1,1,1] == 10 @@ -42,6 +54,8 @@ fails = @testset NoThrowTestSet begin @test_throws OverflowError 1 + 1 # Fail - comparison @test 1+1 == 2+2 + # Error - unexpected pass + @test_broken true end for i in 1:3 @test isa(fails[i], Base.Test.Fail) @@ -49,12 +63,14 @@ end @test contains(sprint(show, fails[1]), "Thrown: ErrorException") @test contains(sprint(show, fails[2]), "No exception thrown") @test contains(sprint(show, fails[3]), "Evaluated: 2 == 4") +@test contains(sprint(show, fails[4]), "Unexpected Pass") # Test printing of a TestSetException -tse_str = sprint(show, Test.TestSetException(1,2,3)) +tse_str = sprint(show, Test.TestSetException(1,2,3,4)) @test contains(tse_str, "1 passed") @test contains(tse_str, "2 failed") @test contains(tse_str, "3 errored") +@test contains(tse_str, "4 broken") @test Test.finish(Test.FallbackTestSet()) !== nothing @@ -76,9 +92,7 @@ end end end -try - -@testset "outer" begin +try @testset "outer" begin @testset "inner1" begin @test true @test false @@ -139,9 +153,10 @@ redirect_stdout(OLD_STDOUT) error("No exception was thrown!") catch ex @test isa(ex, Test.TestSetException) - @test ex.pass == 24 - @test ex.fail == 6 - @test ex.error == 6 + @test ex.pass == 24 + @test ex.fail == 6 + @test ex.error == 6 + @test ex.broken == 0 end # Test @test_approx_eq From 09a963eff05e38bf427b3fb8c1595e1c65312eef Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 20 Jun 2016 14:31:45 -0400 Subject: [PATCH 0064/1117] only add comdat when using MSVC to work around linker limitation --- src/jitlayers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 35c920878e84f..3476b821e7ee5 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -539,7 +539,7 @@ ExecutionEngine *jl_ExecutionEngine; template // for GlobalObject's static T *addComdat(T *G) { -#if defined(_OS_WINDOWS_) +#if defined(_OS_WINDOWS_) && defined(_COMPILER_MICROSOFT_) if (imaging_mode && !G->isDeclaration()) { #ifdef LLVM35 // Add comdat information to make MSVC link.exe happy From 93f6d2811996dc08ec04586afe98394fefdbdabd Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Mon, 20 Jun 2016 10:25:03 -0700 Subject: [PATCH 0065/1117] Introduces `permute[!]` methods operating on `SparseMatrixCSC`s, associated documentation, and tests. Exports `permute`. --- base/exports.jl | 1 + base/sparse/sparse.jl | 4 +- base/sparse/sparsematrix.jl | 276 ++++++++++++++++++++++++++++++++++-- doc/stdlib/arrays.rst | 29 ++++ test/sparsedir/sparse.jl | 38 ++++- 5 files changed, 333 insertions(+), 15 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index cd76c3778e6a0..838410226e02c 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -558,6 +558,7 @@ export ones, parent, parentindexes, + permute, permute!, permutedims, permutedims!, diff --git a/base/sparse/sparse.jl b/base/sparse/sparse.jl index 93340cbb467fc..b09d036e6d8b1 100644 --- a/base/sparse/sparse.jl +++ b/base/sparse/sparse.jl @@ -24,14 +24,14 @@ import Base: @get!, acos, acosd, acot, acotd, acsch, asech, asin, asind, asinh, hcat, hvcat, imag, indmax, ishermitian, kron, length, log, log1p, max, min, maximum, minimum, norm, one, promote_eltype, real, reinterpret, reshape, rot180, rotl90, rotr90, round, scale!, setindex!, similar, size, transpose, tril, - triu, vcat, vec + triu, vcat, vec, permute! import Base.Broadcast: eltype_plus, broadcast_shape export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, SparseMatrixCSC, SparseVector, blkdiag, dense, droptol!, dropzeros!, dropzeros, etree, issparse, nonzeros, nzrange, rowvals, sparse, sparsevec, spdiagm, speye, spones, - sprand, sprandn, spzeros, symperm, nnz + sprand, sprandn, spzeros, symperm, nnz, permute include("abstractsparse.jl") include("sparsematrix.jl") diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 97cdcc1c04d58..489c290c543c8 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -613,11 +613,11 @@ function sparse(B::Bidiagonal) return sparse([1:m;1:m-1],[1:m;2:m],[B.dv;B.ev], Int(m), Int(m)) # upper bidiagonal end -## Transposition methods +## Transposition and permutation methods """ - halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, - q::AbstractVector, f = identity) + halfperm!{Tv,Ti,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, + q::AbstractVector{Tq}, f::Function = identity) Column-permute and transpose `A`, simultaneously applying `f` to each entry of `A`, storing the result `(f(A)Q)^T` (`map(f, transpose(A[:,q]))`) in `X`. @@ -636,8 +636,8 @@ algorithms for sparse matrices: multiplication and permuted transposition," ACM 250-269 (1978). The algorithm runs in `O(A.m, A.n, nnz(A))` time and requires no space beyond that passed in. """ -function halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, - q::AbstractVector, f = identity) +function halfperm!{Tv,Ti,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, + q::AbstractVector{Tq}, f::Function = identity) _computecolptrs_halfperm!(X, A) _distributevals_halfperm!(X, A, q, f) return X @@ -646,8 +646,7 @@ end Helper method for `halfperm!`. Computes `transpose(A[:,q])`'s column pointers, storing them shifted one position forward in `X.colptr`; `_distributevals_halfperm!` fixes this shift. """ -function _computecolptrs_halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, - A::SparseMatrixCSC{Tv,Ti}) +function _computecolptrs_halfperm!{Tv,Ti}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}) # Compute `transpose(A[:,q])`'s column counts. Store shifted forward one position in X.colptr. fill!(X.colptr, 0) @inbounds for k in 1:nnz(A) @@ -668,8 +667,8 @@ position forward in `X.colptr`, computes `map(f, transpose(A[:,q]))` by appropri distributing `A.rowval` and `f`-transformed `A.nzval` into `X.rowval` and `X.nzval` respectively. Simultaneously fixes the one-position-forward shift in `X.colptr`. """ -function _distributevals_halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, - A::SparseMatrixCSC{Tv,Ti}, q::AbstractVector, f) +function _distributevals_halfperm!{Tv,Ti,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, + A::SparseMatrixCSC{Tv,Ti}, q::AbstractVector{Tq}, f::Function) @inbounds for Xi in 1:A.n Aj = q[Xi] for Ak in nzrange(A, Aj) @@ -683,7 +682,7 @@ function _distributevals_halfperm!{Tv,Ti<:Integer}(X::SparseMatrixCSC{Tv,Ti}, return # kill potential type instability end -function ftranspose!{Tv,Ti}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, f) +function ftranspose!{Tv,Ti}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, f::Function) # Check compatibility of source argument A and destination argument X if X.n != A.m throw(DimensionMismatch(string("destination argument `X`'s column count, ", @@ -705,13 +704,268 @@ end transpose!{Tv,Ti}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}) = ftranspose!(X, A, identity) ctranspose!{Tv,Ti}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}) = ftranspose!(X, A, conj) -function ftranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, f) +function ftranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, f::Function) X = SparseMatrixCSC(A.n, A.m, Vector{Ti}(A.m+1), Vector{Ti}(nnz(A)), Vector{Tv}(nnz(A))) halfperm!(X, A, 1:A.n, f) end transpose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = ftranspose(A, identity) ctranspose{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = ftranspose(A, conj) +""" + unchecked_noalias_permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, + A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, q::AbstractVector{Tq}, + C::SparseMatrixCSC{Tv,Ti}) + +See [`permute!`](:func:`Base.SparseArrays.permute!`) for basic usage. Parent of `permute[!]` +methods operating on `SparseMatrixCSC`s that assume none of `X`, `A`, and `C` alias each +other. As this method performs no argument checking, prefer the safer child methods +(`permute[!]`) to direct use. + +This method consists of two major steps: (1) Column-permute (`Q`,`I[:,q]`) and transpose `A` +to generate intermediate result `(AQ)^T` (`transpose(A[:,q])`) in `C`. (2) Column-permute +(`P^T`, I[:,p]) and transpose intermediate result `(AQ)^T` to generate result +`((AQ)^T P^T)^T = PAQ` (`A[p,q]`) in `X`. + +The first step is a call to `halfperm!`, and the second is a variant on `halfperm!` that +avoids an unnecessary length-`nnz(A)` array-sweep and associated recomputation of column +pointers. See [`halfperm!`](:func:Base.SparseArrays.halfperm!) for additional algorithmic +information. + +See also: [`unchecked_aliasing_permute!`](:func:`Base.SparseArrays.unchecked_aliasing_permute!`) +""" +function unchecked_noalias_permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, + A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, q::AbstractVector{Tq}, + C::SparseMatrixCSC{Tv,Ti}) + halfperm!(C, A, q) + _computecolptrs_permute!(X, A, q, X.colptr) + _distributevals_halfperm!(X, C, p, identity) + return X +end +""" + unchecked_aliasing_permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}( + A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, q::AbstractVector{Tq}, + C::SparseMatrixCSC{Tv,Ti}, workcolptr::Vector{Ti}) + +See [`permute!`](:func:`Base.SparseArrays.permute!`) for basic usage. Parent of `permute!` +methods operating on `SparseMatrixCSC`s where the source and destination matrices are the +same. See [`unchecked_noalias_permute!`](:func:`Base.SparseArrays.unchecked_noalias_permute`) +for additional information; these methods are identical but for this method's requirement of +the additional `workcolptr`, `length(workcolptr) >= A.n + 1`, which enables efficient +handling of the source-destination aliasing. +""" +function unchecked_aliasing_permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}( + A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, q::AbstractVector{Tq}, + C::SparseMatrixCSC{Tv,Ti}, workcolptr::Vector{Ti}) + halfperm!(C, A, q) + _computecolptrs_permute!(A, A, q, workcolptr) + _distributevals_halfperm!(A, C, p, identity) + return A +end +""" +Helper method for `unchecked_noalias_permute!` and `unchecked_aliasing_permute!`. +Computes `PAQ`'s column pointers, storing them shifted one position forward in `X.colptr`; +`_distributevals_halfperm!` fixes this shift. Saves some work relative to +`_computecolptrs_halfperm!` as described in `uncheckednoalias_permute!`'s documentation. +""" +function _computecolptrs_permute!{Tv,Ti,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, + A::SparseMatrixCSC{Tv,Ti}, q::AbstractVector{Tq}, workcolptr::Vector{Ti}) + # Compute `A[p,q]`'s column counts. Store shifted forward one position in workcolptr. + @inbounds for k in 1:A.n + workcolptr[k+1] = A.colptr[q[k] + 1] - A.colptr[q[k]] + end + # Compute `A[p,q]`'s column pointers. Store shifted forward one position in X.colptr. + X.colptr[1] = 1 + countsum = 1 + @inbounds for k in 2:(X.n + 1) + overwritten = workcolptr[k] + X.colptr[k] = countsum + countsum += overwritten + end +end + +""" +Helper method for `permute` and `permute!` methods operating on `SparseMatrixCSC`s. +Checks compatibility of source argument `A`, row-permutation argument `p`, and +column-permutation argument `q`. +""" +function _checkargs_sourcecompatperms_permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}( + A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, q::AbstractVector{Tq}) + if length(q) != A.n + throw(DimensionMismatch(string("the length of column-permutation argument `q`, ", + "`length(q) (= $(length(q)))`, must match source argument `A`'s column ", + "count, `A.n (= $(A.n))`"))) + elseif length(p) != A.m + throw(DimensionMismatch(string("the length of row-permutation argument `p`, ", + "`length(p) (= $(length(p)))`, must match source argument `A`'s row count, ", + "`A.m (= $(A.m))`"))) + end +end +""" +Helper method for `permute` and `permute!` methods operating on `SparseMatrixCSC`s. +Checks whether row- and column- permutation arguments `p` and `q` are valid permutations. +""" +function _checkargs_permutationsvalid_permute!{Ti<:Integer,Tp<:Integer,Tq<:Integer}( + p::AbstractVector{Tp}, pcheckspace::Vector{Ti}, + q::AbstractVector{Tq}, qcheckspace::Vector{Ti}) + if !_ispermutationvalid_permute!(p, pcheckspace) + throw(ArgumentError("row-permutation argument `p` must be a valid permutation")) + elseif !_ispermutationvalid_permute!(q, qcheckspace) + throw(ArgumentError("column-permutation argument `q` must be a valid permutation")) + end +end +function _ispermutationvalid_permute!{Ti<:Integer,Tp<:Integer}(perm::AbstractVector{Tp}, + checkspace::Vector{Ti}) + n = length(perm) + checkspace[1:n] = 0 + for k in perm + (0 < k <= n) && ((checkspace[k] $= 1) == 1) || return false + end + return true +end +""" +Helper method for `permute` and `permute!` methods operating on `SparseMatrixCSC`s. +Checks compatibility of source argument `A` and destination argument `X`. +""" +function _checkargs_sourcecompatdest_permute!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, + X::SparseMatrixCSC{Tv,Ti}) + if X.m != A.m + throw(DimensionMismatch(string("destination argument `X`'s row count, ", + "`X.m (= $(X.m))`, must match source argument `A`'s row count, `A.m (= $(A.m))`"))) + elseif X.n != A.n + throw(DimensionMismatch(string("destination argument `X`'s column count, ", + "`X.n (= $(X.n))`, must match source argument `A`'s column count, `A.n (= $(A.n))`"))) + elseif length(X.rowval) < nnz(A) + throw(ArgumentError(string("the length of destination argument `X`'s `rowval` ", + "array, `length(X.rowval) (= $(length(X.rowval)))`, must be greater than or ", + "equal to source argument `A`'s allocated entry count, `nnz(A) (= $(nnz(A)))`"))) + elseif length(X.nzval) < nnz(A) + throw(ArgumentError(string("the length of destination argument `X`'s `nzval` ", + "array, `length(X.nzval) (= $(length(X.nzval)))`, must be greater than or ", + "equal to source argument `A`'s allocated entry count, `nnz(A) (= $(nnz(A)))`"))) + end +end +""" +Helper method for `permute` and `permute!` methods operating on `SparseMatrixCSC`s. +Checks compatibility of source argument `A` and intermediate result argument `C`. +""" +function _checkargs_sourcecompatworkmat_permute!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, + C::SparseMatrixCSC{Tv,Ti}) + if C.n != A.m + throw(DimensionMismatch(string("intermediate result argument `C`'s column count, ", + "`C.n (= $(C.n))`, must match source argument `A`'s row count, `A.m (= $(A.m))`"))) + elseif C.m != A.n + throw(DimensionMismatch(string("intermediate result argument `C`'s row count, ", + "`C.m (= $(C.m))`, must match source argument `A`'s column count, `A.n (= $(A.n))`"))) + elseif length(C.rowval) < nnz(A) + throw(ArgumentError(string("the length of intermediate result argument `C`'s ", + "`rowval` array, `length(C.rowval) (= $(length(C.rowval)))`, must be greater than ", + "or equal to source argument `A`'s allocated entry count, `nnz(A) (= $(nnz(A)))`"))) + elseif length(C.nzval) < nnz(A) + throw(ArgumentError(string("the length of intermediate result argument `C`'s ", + "`rowval` array, `length(C.nzval) (= $(length(C.nzval)))`, must be greater than ", + "or equal to source argument `A`'s allocated entry count, `nnz(A)` (= $(nnz(A)))"))) + end +end +""" +Helper method for `permute` and `permute!` methods operating on `SparseMatrixCSC`s. +Checks compatibility of source argument `A` and workspace argument `workcolptr`. +""" +function _checkargs_sourcecompatworkcolptr_permute!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, + workcolptr::Vector{Ti}) + if length(workcolptr) <= A.n + throw(DimensionMismatch(string("argument `workcolptr`'s length, ", + "`length(workcolptr) (= $(length(workcolptr)))`, must exceed source argument ", + "`A`'s column count, `A.n (= $(A.n))`"))) + end +end +""" + permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, + p::AbstractVector{Tp}, q::AbstractVector{Tq}[, C::SparseMatrixCSC{Tv,Ti}]) + +Bilaterally permute `A`, storing result `PAQ` (`A[p,q]`) in `X`. Stores intermediate result +`(AQ)^T` (`transpose(A[:,q])`) in optional argument `C` if present. Requires that none of +`X`, `A`, and, if present, `C` alias each other; to store result `PAQ` back into `A`, use +the following method lacking `X`: + + permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, + q::AbstractVector{Tq}[, C::SparseMatrixCSC{Tv,Ti}[, workcolptr::Vector{Ti}]]) + +`X`'s dimensions must match those of `A` (`X.m == A.m` and `X.n == A.n`), and `X` must +have enough storage to accommodate all allocated entries in `A` (`length(X.rowval) >= nnz(A)` +and `length(X.nzval) >= nnz(A)`). Column-permutation `q`'s length must match `A`'s column +count (`length(q) == A.n`). Row-permutation `p`'s length must match `A`'s row count +(`length(p) == A.m`). + +`C`'s dimensions must match those of `transpose(A)` (`C.m == A.n` and `C.n == A.m`), and `C` +must have enough storage to accommodate all allocated entries in `A` (`length(C.rowval)` >= nnz(A)` +and `length(C.nzval) >= nnz(A)`). + +For additional (algorithmic) information, and for versions of these methods that forgo +argument checking, see (unexported) parent methods [`unchecked_noalias_permute!`](:func:`Base.SparseArrays.unchecked_noalias_permute!`) +and [`unchecked_aliasing_permute!`](:func:`Base.SparseArrays.unchecked_aliasing_permute!`). + +See also: [`permute`](:func:`Base.SparseArrays.permute`) +""" +function permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, + A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, q::AbstractVector{Tq}) + _checkargs_sourcecompatdest_permute!(A, X) + _checkargs_sourcecompatperms_permute!(A, p, q) + C = SparseMatrixCSC(A.n, A.m, Vector{Ti}(A.m + 1), Vector{Ti}(nnz(A)), Vector{Tv}(nnz(A))) + _checkargs_permutationsvalid_permute!(p, C.colptr, q, X.colptr) + unchecked_noalias_permute!(X, A, p, q, C) +end +function permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, + A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, q::AbstractVector{Tq}, + C::SparseMatrixCSC{Tv,Ti}) + _checkargs_sourcecompatdest_permute!(A, X) + _checkargs_sourcecompatperms_permute!(A, p, q) + _checkargs_sourcecompatworkmat_permute!(A, C) + _checkargs_permutationsvalid_permute!(p, C.colptr, q, X.colptr) + unchecked_noalias_permute!(X, A, p, q, C) +end +function permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, + p::AbstractVector{Tp}, q::AbstractVector{Tq}) + _checkargs_sourcecompatperms_permute!(A, p, q) + C = SparseMatrixCSC(A.n, A.m, Vector{Ti}(A.m + 1), Vector{Ti}(nnz(A)), Vector{Tv}(nnz(A))) + workcolptr = Vector{Ti}(A.n + 1) + _checkargs_permutationsvalid_permute!(p, C.colptr, q, workcolptr) + unchecked_aliasing_permute!(A, p, q, C, workcolptr) +end +function permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, + p::AbstractVector{Tp}, q::AbstractVector{Tq}, C::SparseMatrixCSC{Tv,Ti}) + _checkargs_sourcecompatperms_permute!(A, p, q) + _checkargs_sourcecompatworkmat_permute!(A, C) + workcolptr = Vector{Ti}(A.n + 1) + _checkargs_permutationsvalid_permute!(p, C.colptr, q, workcolptr) + unchecked_aliasing_permute!(A, p, q, C, workcolptr) +end +function permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, + p::AbstractVector{Tp}, q::AbstractVector{Tq}, C::SparseMatrixCSC{Tv,Ti}, + workcolptr::Vector{Ti}) + _checkargs_sourcecompatperms_permute!(A, p, q) + _checkargs_sourcecompatworkmat_permute!(A, C) + _checkargs_sourcecompatworkcolptr_permute!(A, workcolptr) + _checkargs_permutationsvalid_permute!(p, C.colptr, q, workcolptr) + unchecked_aliasing_permute!(A, p, q, C, workcolptr) +end +""" + permute{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, + q::AbstractVector{Tq}) + +Bilaterally permute `A`, returning `PAQ` (`A[p,q]`). Column-permutation `q`'s length must +match `A`'s column count (`length(q) == A.n`). Row-permutation `p`'s length must match `A`'s +row count (`length(p) == A.m`). + +For expert drivers and additional information, see [`permute!`](:func:`Base.SparseArrays.permute!`). +""" +function permute{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, + p::AbstractVector{Tp}, q::AbstractVector{Tq}) + _checkargs_sourcecompatperms_permute!(A, p, q) + X = SparseMatrixCSC(A.m, A.n, Vector{Ti}(A.n + 1), Vector{Ti}(nnz(A)), Vector{Tv}(nnz(A))) + C = SparseMatrixCSC(A.n, A.m, Vector{Ti}(A.m + 1), Vector{Ti}(nnz(A)), Vector{Tv}(nnz(A))) + _checkargs_permutationsvalid_permute!(p, C.colptr, q, X.colptr) + unchecked_noalias_permute!(X, A, p, q, C) +end ## fkeep! and children tril!, triu!, droptol!, dropzeros[!] diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index adb38dfcc3b93..f2deb6bbf98be 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1056,3 +1056,32 @@ dense counterparts. The following functions are specific to sparse arrays. For an in-place version and algorithmic information, see :func:`Base.SparseArrays.dropzeros!`\ . +.. function:: permute{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, +.............. q::AbstractVector{Tq}) + + .. Docstring generated from Julia source + + Bilaterally permute ``A``\ , returning ``PAQ`` (``A[p,q]``\ ). Column-permutation ``q``\ 's length must match ``A``\ 's column count (``length(q) == A.n``\ ). Row-permutation ``p``\ 's length must match ``A``\ 's row count (``length(p) == A.m``\ ). + + For expert drivers and additional information, see :func:`Base.SparseArrays.permute!`\ . + +.. function:: permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, +.............. p::AbstractVector{Tp}, q::AbstractVector{Tq}[, C::SparseMatrixCSC{Tv,Ti}]) + + .. Docstring generated from Julia source + + Bilaterally permute ``A``\ , storing result ``PAQ`` (``A[p,q]``\ ) in ``X``\ . Stores intermediate result ``(AQ)^T`` (``transpose(A[:,q])``\ ) in optional argument ``C`` if present. Requires that none of ``X``\ , ``A``\ , and, if present, ``C`` alias each other; to store result ``PAQ`` back into ``A``\ , use the following method lacking ``X``\ : + + .. code-block:: julia + + permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, + q::AbstractVector{Tq}[, C::SparseMatrixCSC{Tv,Ti}[, workcolptr::Vector{Ti}]]) + + ``X``\ 's dimensions must match those of ``A`` (``X.m == A.m`` and ``X.n == A.n``\ ), and ``X`` must have enough storage to accommodate all allocated entries in ``A`` (``length(X.rowval) >= nnz(A)`` and ``length(X.nzval) >= nnz(A)``\ ). Column-permutation ``q``\ 's length must match ``A``\ 's column count (``length(q) == A.n``\ ). Row-permutation ``p``\ 's length must match ``A``\ 's row count (``length(p) == A.m``\ ). + + ``C``\ 's dimensions must match those of ``transpose(A)`` (``C.m == A.n`` and ``C.n == A.m``\ ), and ``C`` must have enough storage to accommodate all allocated entries in ``A`` (``length(C.rowval)`` >= nnz(A)``and``\ length(C.nzval) >= nnz(A)`). + + For additional (algorithmic) information, and for versions of these methods that forgo argument checking, see (unexported) parent methods :func:`Base.SparseArrays.unchecked_noalias_permute!` and :func:`Base.SparseArrays.unchecked_aliasing_permute!`\ . + + See also: :func:`Base.SparseArrays.permute` + diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 226989779188a..eb6b14be8d568 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -288,16 +288,40 @@ end cA = sprandn(5,5,0.2) + im*sprandn(5,5,0.2) @test full(conj(cA)) == conj(full(cA)) -# Test SparseMatrixCSC [c]transpose[!] methods +# Test SparseMatrixCSC [c]transpose[!] and permute[!] methods let smalldim = 5, largedim = 10, nzprob = 0.4 (m, n) = (smalldim, smalldim) A = sprand(m, n, nzprob) + X = similar(A) + C = transpose(A) + p = randperm(m) + q = randperm(n) # Test common error checking of [c]transpose! methods (ftranspose!) @test_throws DimensionMismatch transpose!(A[:, 1:(smalldim - 1)], A) @test_throws DimensionMismatch transpose!(A[1:(smalldim - 1), 1], A) @test_throws ArgumentError transpose!((B = similar(A); resize!(B.rowval, nnz(A) - 1); B), A) @test_throws ArgumentError transpose!((B = similar(A); resize!(B.nzval, nnz(A) - 1); B), A) - # Test overall functionality of [c]transpose[!] + # Test common error checking of permute[!] methods / source-perm compat + @test_throws DimensionMismatch permute(A, p[1:(end - 1)], q) + @test_throws DimensionMismatch permute(A, p, q[1:(end - 1)]) + # Test common error checking of permute[!] methods / source-dest compat + @test_throws DimensionMismatch permute!(A[1:(m - 1), :], A, p, q) + @test_throws DimensionMismatch permute!(A[:, 1:(m - 1)], A, p, q) + @test_throws ArgumentError permute!((Y = copy(X); resize!(Y.rowval, nnz(A) - 1); Y), A, p, q) + @test_throws ArgumentError permute!((Y = copy(X); resize!(Y.nzval, nnz(A) - 1); Y), A, p, q) + # Test common error checking of permute[!] methods / source-workmat compat + @test_throws DimensionMismatch permute!(X, A, p, q, C[1:(m - 1), :]) + @test_throws DimensionMismatch permute!(X, A, p, q, C[:, 1:(m - 1)]) + @test_throws ArgumentError permute!(X, A, p, q, (D = copy(C); resize!(D.rowval, nnz(A) - 1); D)) + @test_throws ArgumentError permute!(X, A, p, q, (D = copy(C); resize!(D.nzval, nnz(A) - 1); D)) + # Test common error checking of permute[!] methods / source-workcolptr compat + @test_throws DimensionMismatch permute!(A, p, q, C, Vector{eltype(A.rowval)}(length(A.colptr) - 1)) + # Test common error checking of permute[!] methods / permutation validity + @test_throws ArgumentError permute!(A, (r = copy(p); r[2] = r[1]; r), q) + @test_throws ArgumentError permute!(A, (r = copy(p); r[2] = m + 1; r), q) + @test_throws ArgumentError permute!(A, p, (r = copy(q); r[2] = r[1]; r)) + @test_throws ArgumentError permute!(A, p, (r = copy(q); r[2] = n + 1; r)) + # Test overall functionality of [c]transpose[!] and permute[!] for (m, n) in ((smalldim, smalldim), (smalldim, largedim), (largedim, smalldim)) A = sprand(m, n, nzprob) At = transpose(A) @@ -310,6 +334,16 @@ let smalldim = 5, largedim = 10, nzprob = 0.4 fullCh = ctranspose(full(C)) @test ctranspose(C) == fullCh @test ctranspose!(similar(sparse(fullCh)), C) == fullCh + # permute[!] + p = randperm(m) + q = randperm(n) + fullPAQ = full(A)[p,q] + @test permute(A, p, q) == sparse(full(A[p,q])) + @test permute!(similar(A), A, p, q) == fullPAQ + @test permute!(similar(A), A, p, q, similar(At)) == fullPAQ + @test permute!(copy(A), p, q) == fullPAQ + @test permute!(copy(A), p, q, similar(At)) == fullPAQ + @test permute!(copy(A), p, q, similar(At), similar(A.colptr)) == fullPAQ end end From aa8184681006b31439a4a973f01b9d0e0d898db7 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 20 Jun 2016 17:50:27 -0400 Subject: [PATCH 0066/1117] fix #17016, method sort order of `map(f, ::Tuple, ::Tuple)` fix a `_setindex!` method signature that was getting luckily hidden by incorrect method sorting --- base/abstractarray.jl | 3 ++- src/jltypes.c | 13 ++++++++++++- test/core.jl | 6 ++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index e937e9ca4208a..52f55c6b30493 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -789,6 +789,7 @@ end _setindex!(::LinearIndexing, A::AbstractArray, v, I...) = error("indexing $(typeof(A)) with types $(typeof(I)) is not supported") ## LinearFast Scalar indexing +_setindex!(::LinearFast, A::AbstractVector, v, ::Int) = error("indexed assignment not defined for ", typeof(A)) _setindex!(::LinearFast, A::AbstractArray, v, ::Int) = error("indexed assignment not defined for ", typeof(A)) _setindex!{T}(::LinearFast, A::AbstractArray{T,0}, v) = (@_propagate_inbounds_meta; setindex!(A, v, 1)) _setindex!(::LinearFast, A::AbstractArray, v, i::Real) = (@_propagate_inbounds_meta; setindex!(A, v, to_index(i))) @@ -800,7 +801,7 @@ function _setindex!{T,N}(::LinearFast, A::AbstractArray{T,N}, v, I::Vararg{Real, @inbounds r = setindex!(A, v, sub2ind(A, J...)) r end -function _setindex!(::LinearFast, A::AbstractArray, v, I1::Real, I::Real...) +function _setindex!(::LinearFast, A::AbstractVector, v, I1::Real, I::Real...) @_inline_meta J = to_indexes(I1, I...) @boundscheck checkbounds(A, J...) diff --git a/src/jltypes.c b/src/jltypes.c index 3b889ee6665ac..a02635d78c77b 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2823,6 +2823,11 @@ static int jl_tuple_morespecific(jl_datatype_t *cdt, jl_datatype_t *pdt, int inv return 0; } } + else if (ci==clenr-1 && pi==plenr-1 && clenr == plenr && !cseq && pseq) { + // make Vararg{X, 1} more specific than Vararg{X, N} + if (jl_is_vararg_type(child[ci]) && type_eqv_with_ANY(ce,pe)) + return 1; + } if (some_morespecific && cseq && !pseq) return 1; @@ -2837,7 +2842,13 @@ static int jl_tuple_morespecific(jl_datatype_t *cdt, jl_datatype_t *pdt, int inv // argument count (i.e. ...) } - if (cseq && pseq) return 1; + /* + Ideally this would test `clenr > plenr`, but that causes something + bad to happen with these two definitions: + sub2ind( inds::Indices{1}, I::Integer...) + sub2ind{N,T<:Integer}(inds::Union{Dims{N},Indices{N}}, I::AbstractVector{T}...) + */ + if (cseq && pseq) return clenr >= plenr || some_morespecific; ci++; pi++; } diff --git a/test/core.jl b/test/core.jl index a6bec9ebe0054..f89d31fa8fa8d 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2124,6 +2124,12 @@ c99991{T}(::Type{UnitRange{T}},x::Range{T}) = 2 @test c99991(UnitRange{Float64}, 1.0:2.0) == 1 @test c99991(UnitRange{Int}, 1:2) == 2 +# issue #17016, method specificity involving vararg tuples +typealias T_17016{N} Tuple{Any,Any,Vararg{Any,N}} +f17016(f, t::T_17016) = 0 +f17016(f, t1::Tuple) = 1 +@test f17016(0, (1,2,3)) == 0 + # issue #8798 let const npy_typestrs = Dict("b1"=>Bool, From 5408a15337b29d90c12c028d79bfb3a1d5d7b471 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Tue, 21 Jun 2016 08:28:37 -0400 Subject: [PATCH 0067/1117] fix various path-related functions on non-String strings, add tests (#17040) fixes #17024; discovered some buggy corner cases for splitext --- base/path.jl | 40 ++++++++++++++----------- test/path.jl | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/base/path.jl b/base/path.jl index 85aec28aadb07..494250dc29683 100644 --- a/base/path.jl +++ b/base/path.jl @@ -24,7 +24,7 @@ if is_unix() const path_dir_splitter = r"^(.*?)(/+)([^/]*)$" const path_ext_splitter = r"^((?:.*/)?(?:\.|[^/\.])[^/]*?)(\.[^/\.]*|)$" - splitdrive(path::AbstractString) = ("",path) + splitdrive(path::String) = ("",path) homedir() = ENV["HOME"] elseif is_windows() const path_separator = "\\" @@ -34,7 +34,7 @@ elseif is_windows() const path_dir_splitter = r"^(.*?)([/\\]+)([^/\\]*)$" const path_ext_splitter = r"^((?:.*[/\\])?(?:\.|[^/\\\.])[^/\\]*?)(\.[^/\\\.]*|)$" - function splitdrive(path::AbstractString) + function splitdrive(path::String) m = match(r"^(\w+:|\\\\\w+\\\w+|\\\\\?\\UNC\\\w+\\\w+|\\\\\?\\\w+:|)(.*)$", path) String(m.captures[1]), String(m.captures[2]) end @@ -43,8 +43,8 @@ else error("path primitives for this OS need to be defined") end -isabspath(path::AbstractString) = ismatch(path_absolute_re, path) -isdirpath(path::AbstractString) = ismatch(path_directory_re, splitdrive(path)[2]) +isabspath(path::String) = ismatch(path_absolute_re, path) +isdirpath(path::String) = ismatch(path_directory_re, splitdrive(path)[2]) function splitdir(path::String) a, b = splitdrive(path) @@ -53,12 +53,11 @@ function splitdir(path::String) a = string(a, isempty(m.captures[1]) ? m.captures[2][1] : m.captures[1]) a, String(m.captures[3]) end -splitdir(path::AbstractString) = splitdir(String(path)) dirname(path::AbstractString) = splitdir(path)[1] basename(path::AbstractString) = splitdir(path)[2] -function splitext(path::AbstractString) +function splitext(path::String) a, b = splitdrive(path) m = match(path_ext_splitter, b) m === nothing && return (path,"") @@ -67,7 +66,7 @@ end function pathsep(paths::AbstractString...) for path in paths - m = match(path_separator_re, path) + m = match(path_separator_re, String(path)) m !== nothing && return m.match[1:1] end return path_separator @@ -76,7 +75,7 @@ end joinpath(a::AbstractString) = a joinpath(a::AbstractString, b::AbstractString, c::AbstractString...) = joinpath(joinpath(a,b), c...) -function joinpath(a::AbstractString, b::AbstractString) +function joinpath(a::String, b::String) isabspath(b) && return b A, a = splitdrive(a) B, b = splitdrive(b) @@ -86,8 +85,9 @@ function joinpath(a::AbstractString, b::AbstractString) ismatch(path_separator_re, a[end:end]) ? string(C,a,b) : string(C,a,pathsep(a,b),b) end +joinpath(a::AbstractString, b::AbstractString) = joinpath(String(a), String(b)) -function normpath(path::AbstractString) +function normpath(path::String) isabs = isabspath(path) isdir = isdirpath(path) drive, path = splitdrive(path) @@ -122,17 +122,17 @@ function normpath(path::AbstractString) end normpath(a::AbstractString, b::AbstractString...) = normpath(joinpath(a,b...)) -abspath(a::AbstractString) = normpath(isabspath(a) ? a : joinpath(pwd(),a)) +abspath(a::String) = normpath(isabspath(a) ? a : joinpath(pwd(),a)) abspath(a::AbstractString, b::AbstractString...) = abspath(joinpath(a,b...)) if is_windows() function realpath(path::AbstractString) - path = cwstring(path) - buf = zeros(UInt16, length(path)) + p = cwstring(path) + buf = zeros(UInt16, length(p)) while true n = ccall((:GetFullPathNameW, "kernel32"), stdcall, UInt32, (Ptr{UInt16}, UInt32, Ptr{UInt16}, Ptr{Void}), - path, length(buf), buf, C_NULL) + p, length(buf), buf, C_NULL) systemerror(:realpath, n == 0) x = n < length(buf) # is the buffer big enough? resize!(buf, n) # shrink if x, grow if !x @@ -141,12 +141,12 @@ function realpath(path::AbstractString) end function longpath(path::AbstractString) - path = cwstring(path) - buf = zeros(UInt16, length(path)) + p = cwstring(path) + buf = zeros(UInt16, length(p)) while true n = ccall((:GetLongPathNameW, "kernel32"), stdcall, UInt32, (Ptr{UInt16}, Ptr{UInt16}, UInt32), - path, buf, length(buf)) + p, buf, length(buf)) systemerror(:longpath, n == 0) x = n < length(buf) # is the buffer big enough? resize!(buf, n) # shrink if x, grow if !x @@ -176,7 +176,7 @@ function expanduser(path::AbstractString) end end -function relpath(path::AbstractString, startpath::AbstractString = ".") +function relpath(path::String, startpath::String = ".") isempty(path) && throw(ArgumentError("`path` must be specified")) isempty(startpath) && throw(ArgumentError("`startpath` must be specified")) curdir = "." @@ -204,3 +204,9 @@ function relpath(path::AbstractString, startpath::AbstractString = ".") end return isempty(relpath_) ? curdir : relpath_ end +relpath(path::AbstractString, startpath::AbstractString) = + relpath(String(path), String(startpath)) + +for f in (:isabspath, :isdirpath, :splitdir, :splitdrive, :splitext, :normpath, :abspath) + @eval $f(path::AbstractString) = $f(String(path)) +end diff --git a/test/path.jl b/test/path.jl index ecce1aba7dee7..6c9aa764f3098 100644 --- a/test/path.jl +++ b/test/path.jl @@ -1,5 +1,89 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +for S in (String, GenericString) + dir = pwd() + sep = Base.Filesystem.path_separator + + @test abspath(S("foo")) == "$dir$(sep)foo" + @test abspath(S("foo"), S("bar")) == "$dir$(sep)foo$(sep)bar" + + @test basename(S("foo$(sep)bar")) == "bar" + @test dirname(S("foo$(sep)bar")) == "foo" + + @test expanduser(S("x")) == "x" + @test expanduser(S("~")) == (is_windows() ? "~" : homedir()) + + @test isabspath(S(homedir())) + @test !isabspath(S("foo")) + + @test !isdirpath(S("foo")) + @test isdirpath(S("foo$sep")) + @test isdirpath(S("")) + @test isdirpath(S(".")) + @test isdirpath(S("..")) + + @test joinpath(S("foo")) == "foo" + @test joinpath(S("foo"), S("bar")) == "foo$(sep)bar" + @test joinpath(S("foo"), S(homedir())) == homedir() + @test joinpath(S(abspath("foo")), S(homedir())) == homedir() + + @test normpath(S(joinpath("."))) == "." + @test normpath(S(joinpath(".."))) == ".." + @test normpath(S(joinpath("..","."))) == ".." + @test normpath(S(joinpath(".",".."))) == ".." + @test normpath(S(joinpath("..",".."))) == "..$(sep).." + @test normpath(S(joinpath(".","..",".."))) == "..$(sep).." + @test normpath(S(joinpath("..",".",".."))) == "..$(sep).." + @test normpath(S(joinpath("..","..","."))) == "..$(sep).." + + @test normpath(S(joinpath("foo","."))) == "foo$sep" + @test normpath(S(joinpath("foo",".."))) == "." + @test normpath(S(joinpath("foo","..","."))) == "." + @test normpath(S(joinpath("foo",".",".."))) == "." + @test normpath(S(joinpath("foo","..",".."))) == ".." + @test normpath(S(joinpath("foo",".","..",".."))) == ".." + @test normpath(S(joinpath("foo","..",".",".."))) == ".." + @test normpath(S(joinpath("foo","..","..","."))) == ".." + + @test normpath(S(joinpath(".","bar"))) == "bar" + @test normpath(S(joinpath("..","bar"))) == "..$(sep)bar" + @test normpath(S(joinpath("..",".","bar"))) == "..$(sep)bar" + @test normpath(S(joinpath(".","..","bar"))) == "..$(sep)bar" + @test normpath(S(joinpath("..","..","bar"))) == "..$(sep)..$(sep)bar" + @test normpath(S(joinpath(".","..","..","bar"))) == "..$(sep)..$(sep)bar" + @test normpath(S(joinpath("..",".","..","bar"))) == "..$(sep)..$(sep)bar" + @test normpath(S(joinpath("..","..",".","bar"))) == "..$(sep)..$(sep)bar" + + @test normpath(S(joinpath("foo",".","bar"))) == "foo$(sep)bar" + @test normpath(S(joinpath("foo","..","bar"))) == "bar" + @test normpath(S(joinpath("foo","..",".","bar"))) == "bar" + @test normpath(S(joinpath("foo",".","..","bar"))) == "bar" + @test normpath(S(joinpath("foo","..","..","bar"))) == "..$(sep)bar" + @test normpath(S(joinpath("foo",".","..","..","bar"))) == "..$(sep)bar" + @test normpath(S(joinpath("foo","..",".","..","bar"))) == "..$(sep)bar" + @test normpath(S(joinpath("foo","..","..",".","bar"))) == "..$(sep)bar" + + @test relpath(S(joinpath("foo","bar")), S("foo")) == "bar" + + @test joinpath(splitdir(S(homedir()))...) == homedir() + @test string(splitdrive(S(homedir()))...) == homedir() + + @test splitext(S("")) == ("", "") + @test splitext(S(".")) == (".", "") + # FIXME: @test splitext(S("..")) == ("..", "") + # FIXME: @test splitext(S("...")) == ("...", "") + @test splitext(S("foo")) == ("foo", "") + @test splitext(S("foo.")) == ("foo", ".") + # FIXME: @test splitext(S("foo..")) == ("foo", "..") + # FIXME: @test splitext(S("foo...")) == ("foo", "...") + @test splitext(S("foo.bar")) == ("foo", ".bar") + @test splitext(S(".foo")) == (".foo", "") + @test splitext(S(".foo.")) == (".foo", ".") + # FIXME: @test splitext(S(".foo..")) == (".foo", "..") + # FIXME: @test splitext(S(".foo...")) == (".foo", "...") + @test splitext(S(".foo.bar")) == (".foo", ".bar") +end + @test isabspath("~") == false @test isabspath("/") == true # on windows, this is relatively absolute @test isabspath("A:/") == is_windows() From 3f8b43c2b3d593c38b964d17465e7df40bbb6f41 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Tue, 21 Jun 2016 09:28:35 -0400 Subject: [PATCH 0068/1117] base/test.jl: fix some doc verbiage --- base/test.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/base/test.jl b/base/test.jl index ae95963141446..0dea0230b9f37 100644 --- a/base/test.jl +++ b/base/test.jl @@ -130,8 +130,9 @@ end """ Broken -The test condition either has been explicitly skipped or is the expected -(failed) result of a broken test. +The test condition is the expected (failed) result of a broken test. +This will result in a error if the test succeeds, forcing the tester +to update the test. """ type Broken <: Result test_type::Symbol @@ -195,10 +196,9 @@ end @test_broken ex For use to indicate a test that should pass but currently consistently -fails. -Tests that the expression `ex` evaluates to `false` or causes an exception. -Returns a `Broken` `Result` if it does, and an `Error` `Result` if it is -`true`. +fails. Tests that the expression `ex` evaluates to `false` or causes an +exception. Returns a `Broken` `Result` if it does, and an `Error` `Result` +if it is `true`. """ macro test_broken(ex) orig_ex = Expr(:quote,ex) @@ -211,8 +211,8 @@ end @test_skip ex For use to indicate a test that should pass but currently intermittently -fails. -Does not evaluate the expression. +fails. Does not evaluate the expression, which makes it useful for tests +of not-yet-implemented functionality. """ macro test_skip(ex) orig_ex = Expr(:quote, ex) From ca84cda101154e6c9737739209ead4d49b3b6939 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 21 Jun 2016 01:54:09 -0400 Subject: [PATCH 0069/1117] calculate ismonday with 100% less eval --- base/dates/adjusters.jl | 131 +++++++++++++++++++--------------------- test/dates/adjusters.jl | 4 +- 2 files changed, 65 insertions(+), 70 deletions(-) diff --git a/base/dates/adjusters.jl b/base/dates/adjusters.jl index cbe2da1f0ad13..4836bc4dafa40 100644 --- a/base/dates/adjusters.jl +++ b/base/dates/adjusters.jl @@ -1,17 +1,17 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license ### truncation -Base.trunc(dt::Date,p::Type{Year}) = Date(UTD(totaldays(year(dt),1,1))) -Base.trunc(dt::Date,p::Type{Month}) = firstdayofmonth(dt) -Base.trunc(dt::Date,p::Type{Day}) = dt +Base.trunc(dt::Date, p::Type{Year}) = Date(UTD(totaldays(year(dt), 1, 1))) +Base.trunc(dt::Date, p::Type{Month}) = firstdayofmonth(dt) +Base.trunc(dt::Date, p::Type{Day}) = dt -Base.trunc(dt::DateTime,p::Type{Year}) = DateTime(trunc(Date(dt),Year)) -Base.trunc(dt::DateTime,p::Type{Month}) = DateTime(trunc(Date(dt),Month)) -Base.trunc(dt::DateTime,p::Type{Day}) = DateTime(Date(dt)) -Base.trunc(dt::DateTime,p::Type{Hour}) = dt - Minute(dt) - Second(dt) - Millisecond(dt) -Base.trunc(dt::DateTime,p::Type{Minute}) = dt - Second(dt) - Millisecond(dt) -Base.trunc(dt::DateTime,p::Type{Second}) = dt - Millisecond(dt) -Base.trunc(dt::DateTime,p::Type{Millisecond}) = dt +Base.trunc(dt::DateTime, p::Type{Year}) = DateTime(trunc(Date(dt), Year)) +Base.trunc(dt::DateTime, p::Type{Month}) = DateTime(trunc(Date(dt), Month)) +Base.trunc(dt::DateTime, p::Type{Day}) = DateTime(Date(dt)) +Base.trunc(dt::DateTime, p::Type{Hour}) = dt - Minute(dt) - Second(dt) - Millisecond(dt) +Base.trunc(dt::DateTime, p::Type{Minute}) = dt - Second(dt) - Millisecond(dt) +Base.trunc(dt::DateTime, p::Type{Second}) = dt - Millisecond(dt) +Base.trunc(dt::DateTime, p::Type{Millisecond}) = dt """ trunc(dt::TimeType, ::Type{Period}) -> TimeType @@ -39,7 +39,7 @@ Adjusts `dt` to the Sunday of its week. """ function lastdayofweek end -lastdayofweek(dt::Date) = Date(UTD(value(dt) + (7-dayofweek(dt)))) +lastdayofweek(dt::Date) = Date(UTD(value(dt) + (7 - dayofweek(dt)))) lastdayofweek(dt::DateTime) = DateTime(lastdayofweek(Date(dt))) @vectorize_1arg TimeType firstdayofweek @@ -52,7 +52,7 @@ Adjusts `dt` to the first day of its month. """ function firstdayofmonth end -firstdayofmonth(dt::Date) = Date(UTD(value(dt)-day(dt)+1)) +firstdayofmonth(dt::Date) = Date(UTD(value(dt) - day(dt) + 1)) firstdayofmonth(dt::DateTime) = DateTime(firstdayofmonth(Date(dt))) """ @@ -63,8 +63,8 @@ Adjusts `dt` to the last day of its month. function lastdayofmonth end function lastdayofmonth(dt::Date) - y,m,d = yearmonthday(dt) - return Date(UTD(value(dt)+daysinmonth(y,m)-d)) + y, m, d = yearmonthday(dt) + return Date(UTD(value(dt) + daysinmonth(y, m) - d)) end lastdayofmonth(dt::DateTime) = DateTime(lastdayofmonth(Date(dt))) @@ -78,7 +78,7 @@ Adjusts `dt` to the first day of its year. """ function firstdayofyear end -firstdayofyear(dt::Date) = Date(UTD(value(dt)-dayofyear(dt)+1)) +firstdayofyear(dt::Date) = Date(UTD(value(dt) - dayofyear(dt) + 1)) firstdayofyear(dt::DateTime) = DateTime(firstdayofyear(Date(dt))) """ @@ -89,8 +89,8 @@ Adjusts `dt` to the last day of its year. function lastdayofyear end function lastdayofyear(dt::Date) - y,m,d = yearmonthday(dt) - return Date(UTD(value(dt)+daysinyear(y)-dayofyear(y,m,d))) + y, m, d = yearmonthday(dt) + return Date(UTD(value(dt) + daysinyear(y) - dayofyear(y, m, d))) end lastdayofyear(dt::DateTime) = DateTime(lastdayofyear(Date(dt))) @@ -107,7 +107,7 @@ function firstdayofquarter end function firstdayofquarter(dt::Date) y,m = yearmonth(dt) mm = m < 4 ? 1 : m < 7 ? 4 : m < 10 ? 7 : 10 - return Date(y,mm,1) + return Date(y, mm, 1) end firstdayofquarter(dt::DateTime) = DateTime(firstdayofquarter(Date(dt))) @@ -120,8 +120,8 @@ function lastdayofquarter end function lastdayofquarter(dt::Date) y,m = yearmonth(dt) - mm,d = m < 4 ? (3,31) : m < 7 ? (6,30) : m < 10 ? (9,30) : (12,31) - return Date(y,mm,d) + mm, d = m < 4 ? (3, 31) : m < 7 ? (6, 30) : m < 10 ? (9, 30) : (12, 31) + return Date(y, mm, d) end lastdayofquarter(dt::DateTime) = DateTime(lastdayofquarter(Date(dt))) @@ -132,20 +132,15 @@ lastdayofquarter(dt::DateTime) = DateTime(lastdayofquarter(Date(dt))) immutable DateFunction f::Function # validate boolean, single-arg inner constructor - function DateFunction(f::ANY,negate::Bool,dt::TimeType) - try - f(dt) in (true,false) || throw(ArgumentError("Provided function must take a single TimeType argument and return true or false")) - catch e - throw(ArgumentError("Provided function must take a single TimeType argument")) - end - n = negate ? (!) : identity - return new(@eval x->$n($f(x))) + function DateFunction(f::ANY, negate::Bool, dt::TimeType) + isa(f(dt), Bool) || throw(ArgumentError("Provided function must take a single TimeType argument and return true or false")) + return new(negate ? x -> !f(x)::Bool : f) end end -Base.show(io::IO,df::DateFunction) = println(io, df.f) +Base.show(io::IO, df::DateFunction) = println(io, df.f) # Core adjuster -function adjust(df::DateFunction,start,step,limit) +function adjust(df::DateFunction, start, step, limit) for i = 1:limit df.f(start) && return start start += step @@ -153,8 +148,8 @@ function adjust(df::DateFunction,start,step,limit) throw(ArgumentError("Adjustment limit reached: $limit iterations")) end -function adjust(func::Function,start;step::Period=Day(1),negate::Bool=false,limit::Int=10000) - return adjust(DateFunction(func,negate,start),start,step,limit) +function adjust(func::Function, start; step::Period=Day(1), negate::Bool=false, limit::Int=10000) + return adjust(DateFunction(func, negate, start), start, step, limit) end # Constructors using DateFunctions @@ -169,8 +164,8 @@ step size in adjusting can be provided manually through the `step` keyword. If `true`. `limit` provides a limit to the max number of iterations the adjustment API will pursue before throwing an error (given that `f::Function` is never satisfied). """ -function Date(func::Function,y,m=1,d=1;step::Period=Day(1),negate::Bool=false,limit::Int=10000) - return adjust(DateFunction(func,negate,Date(y,m,d)),Date(y,m,d),step,limit) +function Date(func::Function, y, m=1, d=1;step::Period=Day(1), negate::Bool=false, limit::Int=10000) + return adjust(DateFunction(func, negate, Date(y, m, d)), Date(y, m, d), step, limit) end """ @@ -185,30 +180,30 @@ pursue before throwing an error (in the case that `f::Function` is never satisfi """ DateTime(::Function, args...) -function DateTime(func::Function,y,m=1;step::Period=Day(1),negate::Bool=false,limit::Int=10000) - return adjust(DateFunction(func,negate,DateTime(y,m)),DateTime(y,m),step,limit) +function DateTime(func::Function, y, m=1; step::Period=Day(1), negate::Bool=false, limit::Int=10000) + return adjust(DateFunction(func, negate, DateTime(y, m)), DateTime(y, m), step, limit) end -function DateTime(func::Function,y,m,d;step::Period=Hour(1),negate::Bool=false,limit::Int=10000) - return adjust(DateFunction(func,negate,DateTime(y)),DateTime(y,m,d),step,limit) +function DateTime(func::Function, y, m, d; step::Period=Hour(1), negate::Bool=false, limit::Int=10000) + return adjust(DateFunction(func, negate, DateTime(y)), DateTime(y, m, d), step, limit) end -function DateTime(func::Function,y,m,d,h;step::Period=Minute(1),negate::Bool=false,limit::Int=10000) - return adjust(DateFunction(func,negate,DateTime(y)),DateTime(y,m,d,h),step,limit) +function DateTime(func::Function, y, m, d, h; step::Period=Minute(1), negate::Bool=false, limit::Int=10000) + return adjust(DateFunction(func, negate, DateTime(y)), DateTime(y, m, d, h), step, limit) end -function DateTime(func::Function,y,m,d,h,mi;step::Period=Second(1),negate::Bool=false,limit::Int=10000) - return adjust(DateFunction(func,negate,DateTime(y)),DateTime(y,m,d,h,mi),step,limit) +function DateTime(func::Function, y, m, d, h, mi; step::Period=Second(1), negate::Bool=false, limit::Int=10000) + return adjust(DateFunction(func, negate, DateTime(y)), DateTime(y, m, d, h, mi), step, limit) end -function DateTime(func::Function,y,m,d,h,mi,s;step::Period=Millisecond(1),negate::Bool=false,limit::Int=10000) - return adjust(DateFunction(func,negate,DateTime(y)),DateTime(y,m,d,h,mi,s),step,limit) +function DateTime(func::Function, y, m, d, h, mi, s; step::Period=Millisecond(1), negate::Bool=false, limit::Int=10000) + return adjust(DateFunction(func, negate, DateTime(y)), DateTime(y, m, d, h, mi, s), step, limit) end # Return the next TimeType that falls on dow -ISDAYOFWEEK = Dict(Mon=>DateFunction(ismonday,false,Date(0)), - Tue=>DateFunction(istuesday,false,Date(0)), - Wed=>DateFunction(iswednesday,false,Date(0)), - Thu=>DateFunction(isthursday,false,Date(0)), - Fri=>DateFunction(isfriday,false,Date(0)), - Sat=>DateFunction(issaturday,false,Date(0)), - Sun=>DateFunction(issunday,false,Date(0))) +ISDAYOFWEEK = Dict(Mon => DateFunction(ismonday, false, Date(0)), + Tue => DateFunction(istuesday, false, Date(0)), + Wed => DateFunction(iswednesday, false, Date(0)), + Thu => DateFunction(isthursday, false, Date(0)), + Fri => DateFunction(isfriday, false, Date(0)), + Sat => DateFunction(issaturday, false, Date(0)), + Sun => DateFunction(issunday, false, Date(0))) # "same" indicates whether the current date can be considered or not """ @@ -218,7 +213,7 @@ Adjusts `dt` to the next day of week corresponding to `dow` with `1 = Monday, 2 etc`. Setting `same=true` allows the current `dt` to be considered as the next `dow`, allowing for no adjustment to occur. """ -tonext(dt::TimeType,dow::Int;same::Bool=false) = adjust(ISDAYOFWEEK[dow],same ? dt : dt+Day(1),Day(1),7) +tonext(dt::TimeType, dow::Int; same::Bool=false) = adjust(ISDAYOFWEEK[dow], same ? dt : dt+Day(1), Day(1), 7) # Return the next TimeType where func evals true using step in incrementing """ @@ -229,8 +224,8 @@ returns `true`. `func` must take a single `TimeType` argument and return a `Bool allows `dt` to be considered in satisfying `func`. `negate` will make the adjustment process terminate when `func` returns `false` instead of `true`. """ -function tonext(func::Function,dt::TimeType;step::Period=Day(1),negate::Bool=false,limit::Int=10000,same::Bool=false) - return adjust(DateFunction(func,negate,dt),same ? dt : dt+step,step,limit) +function tonext(func::Function, dt::TimeType;step::Period=Day(1), negate::Bool=false, limit::Int=10000, same::Bool=false) + return adjust(DateFunction(func, negate, dt), same ? dt : dt+step, step, limit) end """ @@ -240,7 +235,7 @@ Adjusts `dt` to the previous day of week corresponding to `dow` with `1 = Monday Tuesday, etc`. Setting `same=true` allows the current `dt` to be considered as the previous `dow`, allowing for no adjustment to occur. """ -toprev(dt::TimeType,dow::Int;same::Bool=false) = adjust(ISDAYOFWEEK[dow],same ? dt : dt+Day(-1),Day(-1),7) +toprev(dt::TimeType, dow::Int; same::Bool=false) = adjust(ISDAYOFWEEK[dow], same ? dt : dt+Day(-1), Day(-1), 7) """ toprev(func::Function,dt::TimeType;step=Day(-1),negate=false,limit=10000,same=false) -> TimeType @@ -250,8 +245,8 @@ returns `true`. `func` must take a single `TimeType` argument and return a `Bool allows `dt` to be considered in satisfying `func`. `negate` will make the adjustment process terminate when `func` returns `false` instead of `true`. """ -function toprev(func::Function,dt::TimeType;step::Period=Day(-1),negate::Bool=false,limit::Int=10000,same::Bool=false) - return adjust(DateFunction(func,negate,dt),same ? dt : dt+step,step,limit) +function toprev(func::Function, dt::TimeType; step::Period=Day(-1), negate::Bool=false, limit::Int=10000, same::Bool=false) + return adjust(DateFunction(func, negate, dt), same ? dt : dt+step, step, limit) end # Return the first TimeType that falls on dow in the Month or Year @@ -261,9 +256,9 @@ end Adjusts `dt` to the first `dow` of its month. Alternatively, `of=Year` will adjust to the first `dow` of the year. """ -function tofirst(dt::TimeType,dow::Int;of::Union{Type{Year},Type{Month}}=Month) +function tofirst(dt::TimeType, dow::Int; of::Union{Type{Year}, Type{Month}}=Month) dt = of <: Month ? firstdayofmonth(dt) : firstdayofyear(dt) - return adjust(ISDAYOFWEEK[dow],dt,Day(1),366) + return adjust(ISDAYOFWEEK[dow], dt, Day(1), 366) end # Return the last TimeType that falls on dow in the Month or Year @@ -273,20 +268,20 @@ end Adjusts `dt` to the last `dow` of its month. Alternatively, `of=Year` will adjust to the last `dow` of the year. """ -function tolast(dt::TimeType,dow::Int;of::Union{Type{Year},Type{Month}}=Month) +function tolast(dt::TimeType, dow::Int; of::Union{Type{Year}, Type{Month}}=Month) dt = of <: Month ? lastdayofmonth(dt) : lastdayofyear(dt) - return adjust(ISDAYOFWEEK[dow],dt,Day(-1),366) + return adjust(ISDAYOFWEEK[dow], dt, Day(-1), 366) end -function recur{T<:TimeType}(fun::Function,start::T,stop::T;step::Period=Day(1),negate::Bool=false,limit::Int=10000) +function recur{T<:TimeType}(fun::Function, start::T, stop::T; step::Period=Day(1), negate::Bool=false, limit::Int=10000) ((start != stop) & ((step > zero(step)) != (stop > start))) && return T[] a = T[] check = start <= stop ? 1 : -1 - df = Dates.DateFunction(fun,negate,start) + df = Dates.DateFunction(fun, negate, start) while true - next = Dates.adjust(df,start,step,limit) - cmp(next,stop) == check && break - push!(a,next) + next = Dates.adjust(df, start, step, limit) + cmp(next, stop) == check && break + push!(a, next) start = next + step end return a @@ -300,6 +295,6 @@ should be "included" in the final set. `recur` applies `func` over each element of `dr`, including those elements for which `func` returns `true` in the resulting Array, unless `negate=true`, then only elements where `func` returns `false` are included. """ -function recur{T<:TimeType}(fun::Function,dr::StepRange{T};negate::Bool=false,limit::Int=10000) - return recur(fun,first(dr),last(dr);step=step(dr),negate=negate,limit=limit) +function recur{T<:TimeType}(fun::Function, dr::StepRange{T};negate::Bool=false, limit::Int=10000) + return recur(fun, first(dr), last(dr); step=step(dr), negate=negate, limit=limit) end diff --git a/test/dates/adjusters.jl b/test/dates/adjusters.jl index e4724c8e3039c..c19c16fd4fe42 100644 --- a/test/dates/adjusters.jl +++ b/test/dates/adjusters.jl @@ -316,8 +316,8 @@ januarymondays2014 = [Dates.Date(2014,1,6),Dates.Date(2014,1,13),Dates.Date(2014 @test Dates.recur(Dates.ismonday,startdate:stopdate) == januarymondays2014 @test Dates.recur(x->!Dates.ismonday(x),startdate,stopdate;negate=true) == januarymondays2014 -@test_throws ArgumentError Dates.recur((x,y)->x+y,Dates.Date(2013):Dates.Date(2014)) -@test_throws ArgumentError Dates.DateFunction((x,y)->x+y, false, Date(0)) +@test_throws MethodError Dates.recur((x,y)->x+y,Dates.Date(2013):Dates.Date(2014)) +@test_throws MethodError Dates.DateFunction((x,y)->x+y, false, Date(0)) @test_throws ArgumentError Dates.DateFunction((dt)->2, false, Date(0)) @test length(Dates.recur(x->true,Dates.Date(2013):Dates.Date(2013,2))) == 32 @test length(Dates.recur(x->true,Dates.Date(2013):Dates.Date(2013,1,1))) == 1 From 709063d326cc57de73f99b27d0d244c4ca949dc6 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Tue, 21 Jun 2016 11:53:23 -0400 Subject: [PATCH 0070/1117] use new test_broken macro for broken splitext tests (#17048) --- test/path.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/path.jl b/test/path.jl index 6c9aa764f3098..e983eda1bbd4d 100644 --- a/test/path.jl +++ b/test/path.jl @@ -70,17 +70,17 @@ for S in (String, GenericString) @test splitext(S("")) == ("", "") @test splitext(S(".")) == (".", "") - # FIXME: @test splitext(S("..")) == ("..", "") - # FIXME: @test splitext(S("...")) == ("...", "") + @test_broken splitext(S("..")) == ("..", "") + @test_broken splitext(S("...")) == ("...", "") @test splitext(S("foo")) == ("foo", "") @test splitext(S("foo.")) == ("foo", ".") - # FIXME: @test splitext(S("foo..")) == ("foo", "..") - # FIXME: @test splitext(S("foo...")) == ("foo", "...") + @test_broken splitext(S("foo..")) == ("foo", "..") + @test_broken splitext(S("foo...")) == ("foo", "...") @test splitext(S("foo.bar")) == ("foo", ".bar") @test splitext(S(".foo")) == (".foo", "") @test splitext(S(".foo.")) == (".foo", ".") - # FIXME: @test splitext(S(".foo..")) == (".foo", "..") - # FIXME: @test splitext(S(".foo...")) == (".foo", "...") + @test_broken splitext(S(".foo..")) == (".foo", "..") + @test_broken splitext(S(".foo...")) == (".foo", "...") @test splitext(S(".foo.bar")) == (".foo", ".bar") end From 0c2bcea6b1d66caf7b40129f3a7dbc200f509553 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 18 Jun 2016 17:50:43 -0500 Subject: [PATCH 0071/1117] Fix MethodError printing in case of ambiguity --- base/reflection.jl | 12 +++++++----- test/ambiguous.jl | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index d422eeeb189dc..a2f5889b42bca 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -181,11 +181,6 @@ function _methods(f::ANY,t::ANY,lim) tt = isa(t,Type) ? Tuple{ft, t.parameters...} : Tuple{ft, t...} return _methods_by_ftype(tt, lim) end -function methods_including_ambiguous(f::ANY, t::ANY) - ft = isa(f,Type) ? Type{f} : typeof(f) - tt = isa(t,Type) ? Tuple{ft, t.parameters...} : Tuple{ft, t...} - return ccall(:jl_matching_methods, Any, (Any,Cint,Cint), tt, -1, 1) -end function _methods_by_ftype(t::ANY, lim) tp = t.parameters::SimpleVector nu = 1 @@ -255,6 +250,13 @@ end methods(f::Core.Builtin) = MethodList(Method[], typeof(f).name.mt) +function methods_including_ambiguous(f::ANY, t::ANY) + ft = isa(f,Type) ? Type{f} : typeof(f) + tt = isa(t,Type) ? Tuple{ft, t.parameters...} : Tuple{ft, t...} + ms = ccall(:jl_matching_methods, Any, (Any,Cint,Cint), tt, -1, 1)::Array{Any,1} + return MethodList(Method[m[3] for m in ms], typeof(f).name.mt) +end + function methods(f::ANY) # return all matches return methods(f, Tuple{Vararg{Any}}) diff --git a/test/ambiguous.jl b/test/ambiguous.jl index 4c13725763ca6..fa2bc54c269e8 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -43,6 +43,21 @@ end callambig(x, y) = ambig(x, y) @test_throws MethodError callambig(0x03, 4) +# Printing ambiguity errors +let err = try + ambig(0x03, 4) + catch _e_ + _e_ + end + io = IOBuffer() + Base.showerror(io, err) + lines = split(takebuf_string(io), '\n') + ambig_checkline(str) = startswith(str, " ambig(x, y::Integer) at") || + startswith(str, " ambig(x::Integer, y) at") + @test ambig_checkline(lines[2]) + @test ambig_checkline(lines[3]) +end + ## Other ways of accessing functions # Test that non-ambiguous cases work io = IOBuffer() From 2b527adce11c484ce004ef318796aabf5cd90298 Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Tue, 21 Jun 2016 11:34:22 -0700 Subject: [PATCH 0072/1117] Remove qualifications from dropzeros[!] calls in sparse tests. --- test/sparsedir/sparse.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index eb6b14be8d568..64cc456147452 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -1109,28 +1109,28 @@ let smalldim = 5, largedim = 10, nzprob = 0.4, targetnumposzeros = 5, targetnumn map!(x -> x == 2 ? 0.0 : x == -2 ? -0.0 : x, Abothsigns.nzval) for Awithzeros in (Aposzeros, Anegzeros, Abothsigns) # Basic functionality / dropzeros! - @test Base.dropzeros!(copy(Awithzeros)) == A - @test Base.dropzeros!(copy(Awithzeros), false) == A + @test dropzeros!(copy(Awithzeros)) == A + @test dropzeros!(copy(Awithzeros), false) == A # Basic functionality / dropzeros - @test Base.SparseArrays.dropzeros(Awithzeros) == A - @test Base.SparseArrays.dropzeros(Awithzeros, false) == A + @test dropzeros(Awithzeros) == A + @test dropzeros(Awithzeros, false) == A # Check trimming works as expected - @test length(Base.dropzeros!(copy(Awithzeros)).nzval) == length(A.nzval) - @test length(Base.dropzeros!(copy(Awithzeros)).rowval) == length(A.rowval) - @test length(Base.dropzeros!(copy(Awithzeros), false).nzval) == length(Awithzeros.nzval) - @test length(Base.dropzeros!(copy(Awithzeros), false).rowval) == length(Awithzeros.rowval) + @test length(dropzeros!(copy(Awithzeros)).nzval) == length(A.nzval) + @test length(dropzeros!(copy(Awithzeros)).rowval) == length(A.rowval) + @test length(dropzeros!(copy(Awithzeros), false).nzval) == length(Awithzeros.nzval) + @test length(dropzeros!(copy(Awithzeros), false).rowval) == length(Awithzeros.rowval) end end # original lone dropzeros test A = sparse([1 2 3; 4 5 6; 7 8 9]) A.nzval[2] = A.nzval[6] = A.nzval[7] = 0 - @test Base.dropzeros!(A).colptr == [1, 3, 5, 7] + @test dropzeros!(A).colptr == [1, 3, 5, 7] # test for issue #5169, modified for new behavior following #15242/#14798 @test nnz(sparse([1, 1], [1, 2], [0.0, -0.0])) == 2 - @test nnz(Base.SparseArrays.dropzeros!(sparse([1, 1], [1, 2], [0.0, -0.0]))) == 0 + @test nnz(dropzeros!(sparse([1, 1], [1, 2], [0.0, -0.0]))) == 0 # test for issue #5437, modified for new behavior following #15242/#14798 @test nnz(sparse([1, 2, 3], [1, 2, 3], [0.0, 1.0, 2.0])) == 3 - @test nnz(Base.SparseArrays.dropzeros!(sparse([1, 2, 3],[1, 2, 3],[0.0, 1.0, 2.0]))) == 2 + @test nnz(dropzeros!(sparse([1, 2, 3],[1, 2, 3],[0.0, 1.0, 2.0]))) == 2 end #trace From 2123acab8aea2df12a019ce50bdb37ae3eb0ce55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Tue, 21 Jun 2016 21:30:25 +0200 Subject: [PATCH 0073/1117] Remove documentation of factorial(n,k) (#17055) This method for `factorial' has been moved to Combinatorics packages. --- base/docs/helpdb/Base.jl | 7 ------- doc/stdlib/math.rst | 6 ------ 2 files changed, 13 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index ad8b9d701af67..41be192c95955 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -6993,13 +6993,6 @@ If `n` is not an `Integer`, `factorial(n)` is equivalent to [`gamma(n+1)`](:func """ factorial(n) -""" - factorial(n,k) - -Compute `factorial(n)/factorial(k)`. -""" -factorial(n,k) - """ bitrand([rng], [dims...]) diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 6afa6e018046a..970f1e8e75b58 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1347,12 +1347,6 @@ Mathematical Functions Factorial of ``n``\ . If ``n`` is an :obj:`Integer`\ , the factorial is computed as an integer (promoted to at least 64 bits). Note that this may overflow if ``n`` is not small, but you can use ``factorial(big(n))`` to compute the result exactly in arbitrary precision. If ``n`` is not an ``Integer``\ , ``factorial(n)`` is equivalent to :func:`gamma(n+1) `\ . -.. function:: factorial(n,k) - - .. Docstring generated from Julia source - - Compute ``factorial(n)/factorial(k)``\ . - .. function:: gcd(x,y) .. Docstring generated from Julia source From fb753a624e3245192294bbff459a488fc643a300 Mon Sep 17 00:00:00 2001 From: Graham Inggs Date: Wed, 22 Jun 2016 14:51:22 +0200 Subject: [PATCH 0074/1117] Fix inconsistent use of GNU_SOURCE Julia is built with GNU_SOURCE, defined https://github.com/JuliaLang/julia/blob/master/src/Makefile#L21 but the embedded copy of libuv is not. See https://bugs.debian.org/748573 --- deps/libuv.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libuv.mk b/deps/libuv.mk index bde45ba580399..77672e142a9af 100644 --- a/deps/libuv.mk +++ b/deps/libuv.mk @@ -6,7 +6,7 @@ $(eval $(call git-external,libuv,LIBUV,configure,.libs/libuv.la,$(SRCDIR)/srccac UV_SRC_TARGET := $(BUILDDIR)/$(LIBUV_SRC_DIR)/.libs/libuv.la UV_OBJ_TARGET := $(build_libdir)/libuv.la -UV_CFLAGS := +UV_CFLAGS := -D_GNU_SOURCE ifeq ($(USEMSVC), 1) UV_CFLAGS += -DBUILDING_UV_SHARED endif From bfe998a41fa319c9f72472200fc057199a6f867f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 Jun 2016 10:43:52 -0400 Subject: [PATCH 0075/1117] Move non-standard string literal conventions --- doc/manual/strings.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 7c5c34a2e33e4..b4f0984b2a293 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -593,12 +593,16 @@ quite what is needed. For these kinds of situations, Julia provides :ref:`non-standard string literals `. A non-standard string literal looks like a regular double-quoted string literal, but is immediately prefixed by -an identifier, and doesn't behave quite like a normal string literal. Regular +an identifier, and doesn't behave quite like a normal string literal. The +convention is that non-standard literals with uppercase prefixes produce +actual string objects, while those with lowercase prefixes produce +non-string objects like byte arrays or compiled regular expressions. Regular expressions, byte array literals and version number literals, as described below, are some examples of non-standard string literals. Other examples are given in the :ref:`metaprogramming ` section. + Regular Expressions ------------------- @@ -828,11 +832,8 @@ Byte Array Literals Another useful non-standard string literal is the byte-array string literal: ``b"..."``. This form lets you use string notation to express -literal byte arrays — i.e. arrays of ``UInt8`` values. The convention is -that non-standard literals with uppercase prefixes produce actual string -objects, while those with lowercase prefixes produce non-string objects -like byte arrays or compiled regular expressions. The rules for byte -array literals are the following: +literal byte arrays — i.e. arrays of ``UInt8`` values. The rules for +byte array literals are the following: - ASCII characters and ASCII escapes produce a single byte. - ``\x`` and octal escape sequences produce the *byte* corresponding to From c4ea930bb750b617ffa9224dbcf68bc201f611b4 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 Jun 2016 10:48:20 -0400 Subject: [PATCH 0076/1117] Fix invalid version literal --- doc/manual/strings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index b4f0984b2a293..36180463ba46a 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -948,7 +948,7 @@ would only run with stable ``0.2`` versions, and exclude such versions as Another non-standard version specification extension allows one to use a trailing ``+`` to express an upper limit on build versions, e.g. ``VERSION > -"v"0.2-rc1+"`` can be used to mean any version above ``0.2-rc1`` and any of its +v"0.2-rc1+"`` can be used to mean any version above ``0.2-rc1`` and any of its builds: it will return ``false`` for version ``v"0.2-rc1+win64"`` and ``true`` for ``v"0.2-rc2"``. From 358db5d0b48aff2528aa085099660999f87b3864 Mon Sep 17 00:00:00 2001 From: Sacha Verweij Date: Mon, 20 Jun 2016 12:11:22 -0700 Subject: [PATCH 0077/1117] Add a note to CONTRIBUTING.md explaining the required inter-line character alignment of multiline method signatures in doc/stdlib/*.rst. --- CONTRIBUTING.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6789ae822e1b5..e2b891de99b65 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,6 +104,21 @@ If you want to edit an existing docstring signature, you **first** have to chang edit the helpdb.jl or inline method docstrings. The existing signatures in the `doc/stdlib/*.rst` files are pattern matched to base docstrings and the new content overwrites the content in `doc/stdlib/`. The signature definitions **must** be in sync or else the pattern match will fail and documentation will be lost in the result. To add entirely new methods to the `stdlib` documentation, first add the signature in the appropriate `doc/stdlib/*.rst` file before writing the docstring, rebuilding Julia, and re-running `doc/genstdlib.jl`. +Pattern matching requires that multiline method signatures' inter-line character alignment in `doc/stdlib/*.rst` match that in the corresponding docstring. For example, docstring +```julia +""" + foo(bar, baz, + qux, quux) + +Foo `bar`, `baz`, `qux`, and `quux`. +""" +``` +will only match entries in `doc/stdlib/*.rst` of the form +```rst +.. function:: foo(bar, baz, +.............. qux, quux) +``` +where `.` in the preceding snippet's second line is wildcard, and their number preceding indent-` qux, quux)` matches the character count of `.. function:: `. It is encouraged to write all new docstrings in Markdown markup. If you need to write a more complicated docstring that contains cross-references or citations it can be written in a restructured text codeblock. Many of the existing docstrings are currently restructured text codeblocks and these will be transitioned to Markdown over time. RST codeblocks are delineated with the triple-quote (\`\`\`rst \`\`\`) Makdown codeblock syntax. From 632a70ad6c50057710224e603441358b4e3c7ab6 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 Jun 2016 14:07:34 -0400 Subject: [PATCH 0078/1117] Move "Elementary Function" section to improve flow The original location of the "Elementary Function" section broke the flow between "Numerical Conversions" about round and the next section: "Rounding Functions". Additionally moved the "Elementary Function" section to be a subsection of "Numeric Comparisons". --- doc/manual/mathematical-operations.rst | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index 3a2a0050254b4..d6f27627df9bb 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -351,6 +351,22 @@ Control flow ``&&`` followed by ``||`` followed by ``?`` Assignments ``= += -= *= /= //= \= ^= ÷= %= |= &= $= <<= >>= >>>=`` and ``.+= .-= .*= ./= .//= .\= .^= .÷= .%=`` ================= ============================================================================================= +.. _man-elementary-functions: + +Elementary Functions +~~~~~~~~~~~~~~~~~~~~ + +Julia provides a comprehensive collection of mathematical functions and +operators. These mathematical operations are defined over as broad a +class of numerical values as permit sensible definitions, including +integers, floating-point numbers, rationals, and complexes, wherever +such definitions make sense. + +Moreover, these functions (like any Julia function) can be applied +in "vectorized" fashion to arrays and other collections with the +syntax ``f.(A)``, e.g. ``sin.(A)`` will compute the elementwise +sine of each element of an array ``A``. See :ref:`man-dot-vectorizing`:. + .. _man-numerical-conversions: Numerical Conversions @@ -418,22 +434,6 @@ The following examples show the different forms. See :ref:`man-conversion-and-promotion` for how to define your own conversions and promotions. -.. _man-elementary-functions: - -Elementary Functions --------------------- - -Julia provides a comprehensive collection of mathematical functions and -operators. These mathematical operations are defined over as broad a -class of numerical values as permit sensible definitions, including -integers, floating-point numbers, rationals, and complexes, wherever -such definitions make sense. - -Moreover, these functions (like any Julia function) can be applied -in "vectorized" fashion to arrays and other collections with the -syntax ``f.(A)``, e.g. ``sin.(A)`` will compute the elementwise -sine of each element of an array ``A``. See :ref:`man-dot-vectorizing`:. - .. _man-rounding-functions: Rounding functions From bd389d0dc2a0d1997552ab1ec1989dd1c1634d8b Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Wed, 22 Jun 2016 15:45:00 -0400 Subject: [PATCH 0079/1117] fix zero-arg case of ambiguous method error printing --- base/replutil.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/replutil.jl b/base/replutil.jl index 39d6563ccb560..c1ed0f2543953 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -203,9 +203,9 @@ function showerror_ambiguous(io::IO, meth, f, args) p = args.parameters for (i,a) in enumerate(p) print(io, "::", a) - i == length(p) ? print(io, ")") : print(io, ", ") + i < length(p) && print(io, ", ") end - print(io, " is ambiguous. Candidates:") + print(io, ") is ambiguous. Candidates:") for m in meth print(io, "\n ", m) end From 7e1450748b1e3e4691267ba211b906ca15318a55 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 22 Jun 2016 16:13:19 -0400 Subject: [PATCH 0080/1117] fix serializer missing certain typename caches --- src/dump.c | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/dump.c b/src/dump.c index 176697890ae84..fbd6cb2c54f50 100644 --- a/src/dump.c +++ b/src/dump.c @@ -333,6 +333,30 @@ static void jl_serialize_gv_syms(ios_t *s, jl_sym_t *v) if (v->right) jl_serialize_gv_syms(s, v->right); } +static void jl_serialize_gv_svec(ios_t *s, jl_svec_t *svec) +{ + size_t i, l = jl_svec_len(svec); + for (i = 0; i < l; i++) { + jl_value_t *v = jl_svecref(svec, i); + if (v) { + void *bp = ptrhash_get(&backref_table, v); + if (bp == HT_NOTFOUND) { + int32_t gv = jl_get_llvm_gv((jl_value_t*)v); + if (gv != 0) { + jl_serialize_value(s, v); + write_int32(s, gv); + } + } + } + } +} + +static void jl_serialize_gv_tn(ios_t *s, jl_typename_t *tn) +{ + jl_serialize_gv_svec(s, tn->cache); + jl_serialize_gv_svec(s, tn->linearcache); +} + static void jl_serialize_gv_others(ios_t *s) { // ensures all objects referenced in the code have @@ -364,6 +388,15 @@ static void jl_serialize_gv_others(ios_t *s) } } jl_serialize_gv_syms(s, jl_get_root_symbol()); + jl_serialize_gv_tn(s, jl_array_type->name); + jl_serialize_gv_tn(s, jl_ref_type->name); + jl_serialize_gv_tn(s, jl_pointer_type->name); + jl_serialize_gv_tn(s, jl_type_type->name); + jl_serialize_gv_tn(s, jl_simplevector_type->name); + jl_serialize_gv_tn(s, jl_abstractarray_type->name); + jl_serialize_gv_tn(s, jl_densearray_type->name); + jl_serialize_gv_tn(s, jl_tuple_typename); + jl_serialize_gv_tn(s, jl_vararg_type->name); jl_serialize_value(s, NULL); // signal the end of this list } @@ -1267,8 +1300,6 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) // builtin types are not serialized, so their caches aren't // explicitly saved. so we reconstruct the caches of builtin // parametric types here. - // XXX: need to make sure to serialize all leaftypes in this cache - // that were referenced from compiled code jl_array_ptr_1d_push(datatype_list, (jl_value_t*)dt); } } @@ -2013,7 +2044,7 @@ static void jl_restore_system_image_from_stream(ios_t *f) // ensure everything in deser_tag is reassociated with its GlobalValue intptr_t i; - for (i=2; i < 255; i++) { + for (i = 2; i < 255; i++) { jl_deserialize_gv(f, deser_tag[i]); } jl_deserialize_globalvals(f); @@ -2027,7 +2058,7 @@ static void jl_restore_system_image_from_stream(ios_t *f) jl_set_gs_ctr(gs_ctr); // cache builtin parametric types - for(int i=0; i < jl_array_len(datatype_list); i++) { + for (int i = 0; i < jl_array_len(datatype_list); i++) { jl_value_t *v = jl_array_ptr_ref(datatype_list, i); jl_cache_type_((jl_datatype_t*)v); } From 469a23f75a96ad7164fb71532d48b559a01a054d Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Wed, 22 Jun 2016 20:13:36 -0400 Subject: [PATCH 0081/1117] fix sharedarrays for singleton immutables --- base/sharedarray.jl | 4 ++-- test/parallel_exec.jl | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/base/sharedarray.jl b/base/sharedarray.jl index fa9d42412c966..65d95ed7f2e8c 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -89,7 +89,7 @@ function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) end # All good, immediately unlink the segment. - if prod(dims) > 0 + if (prod(dims) > 0) && (sizeof(T) > 0) if onlocalhost rc = shm_unlink(shm_seg_name) else @@ -508,7 +508,7 @@ function shm_mmap_array(T, dims, shm_seg_name, mode) local s = nothing local A = nothing - if prod(dims) == 0 + if (prod(dims) == 0) || (sizeof(T) == 0) return Array{T}(dims) end diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 2e5ba5f35c7df..ecb89640b32ed 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -452,6 +452,13 @@ map!(x->1, d) @test 2.0 == remotecall_fetch(D->D[2], id_other, Base.shmem_fill(2.0, 2; pids=[id_me, id_other])) @test 3.0 == remotecall_fetch(D->D[1], id_other, Base.shmem_fill(3.0, 1; pids=[id_me, id_other])) +# Shared arrays of singleton immutables +@everywhere immutable ShmemFoo end +for T in [Void, ShmemFoo] + s = SharedArray(T, 10) + @test T() === remotecall_fetch(x->x[3], workers()[1], s) +end + # Issue #14664 d = SharedArray(Int,10) @sync @parallel for i=1:10 From f0d48f06d9f1ec2b69c7372237947d1211415175 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 Jun 2016 16:36:37 -0400 Subject: [PATCH 0082/1117] Consistently refer to Julia over julia The manual would sometimes refer to the language as either "Julia" or "julia". This commit changes references to the Julia language as "Julia" and any reference to the command line program as `julia`. --- doc/manual/conversion-and-promotion.rst | 4 ++-- doc/manual/embedding.rst | 8 +++---- doc/manual/faq.rst | 4 ++-- doc/manual/getting-started.rst | 6 ++--- doc/manual/modules.rst | 2 +- doc/manual/packages.rst | 2 +- doc/manual/parallel-computing.rst | 30 ++++++++++++------------- doc/manual/performance-tips.rst | 8 +++---- ui/repl.c | 2 +- 9 files changed, 33 insertions(+), 33 deletions(-) diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 8e3a06c2a2198..1166c92866677 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -153,8 +153,8 @@ to one and zero: The method signatures for conversion methods are often quite a bit more involved than this example, especially for parametric types. The example -above is meant to be pedagogical, and is not the actual julia behaviour. -This is the actual implementation in julia:: +above is meant to be pedagogical, and is not the actual Julia behaviour. +This is the actual implementation in Julia:: convert{T<:Real}(::Type{T}, z::Complex) = (imag(z)==0 ? convert(T,real(z)) : throw(InexactError())) diff --git a/doc/manual/embedding.rst b/doc/manual/embedding.rst index ef1f0768d25b1..dd943f19a2142 100644 --- a/doc/manual/embedding.rst +++ b/doc/manual/embedding.rst @@ -18,15 +18,15 @@ We start with a simple C program that initializes Julia and calls some Julia cod int main(int argc, char *argv[]) { - /* required: setup the julia context */ + /* required: setup the Julia context */ jl_init(NULL); - /* run julia commands */ + /* run Julia commands */ jl_eval_string("print(sqrt(2.0))"); - /* strongly recommended: notify julia that the + /* strongly recommended: notify Julia that the program is about to terminate. this allows - julia time to cleanup pending write requests + Julia time to cleanup pending write requests and run all finalizers */ jl_atexit_hook(0); diff --git a/doc/manual/faq.rst b/doc/manual/faq.rst index 36f1a94c3b8be..4dfcc22e377a2 100644 --- a/doc/manual/faq.rst +++ b/doc/manual/faq.rst @@ -536,13 +536,13 @@ Memory Why does ``x += y`` allocate memory when ``x`` and ``y`` are arrays? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In julia, ``x += y`` gets replaced during parsing by ``x = x + y``. +In Julia, ``x += y`` gets replaced during parsing by ``x = x + y``. For arrays, this has the consequence that, rather than storing the result in the same location in memory as ``x``, it allocates a new array to store the result. While this behavior might surprise some, the choice is deliberate. The -main reason is the presence of ``immutable`` objects within julia, +main reason is the presence of ``immutable`` objects within Julia, which cannot change their value once created. Indeed, a number is an immutable object; the statements ``x = 5; x += 1`` do not modify the meaning of ``5``, they modify the value bound to ``x``. For an diff --git a/doc/manual/getting-started.rst b/doc/manual/getting-started.rst index 4dc95b614aaf0..8d668662cf0db 100644 --- a/doc/manual/getting-started.rst +++ b/doc/manual/getting-started.rst @@ -49,7 +49,7 @@ argument to the julia command:: $ julia script.jl arg1 arg2... -As the example implies, the following command-line arguments to julia +As the example implies, the following command-line arguments to ``julia`` are taken as command-line arguments to the program ``script.jl``, passed in the global constant ``ARGS``. The name of the script itself is passed in as the global ``PROGRAM_FILE``. Note that ``ARGS`` is also set when script @@ -87,7 +87,7 @@ specifies the ip-address and port that other workers should use to connect to this worker. -If you have code that you want executed whenever julia is run, you can +If you have code that you want executed whenever Julia is run, you can put it in ``~/.juliarc.jl``: .. raw:: latex @@ -116,7 +116,7 @@ those available for the ``perl`` and ``ruby`` programs:: -J, --sysimage Start up with the given system image file --precompiled={yes|no} Use precompiled code from system image if available --compilecache={yes|no} Enable/disable incremental precompilation of modules - -H, --home Set location of julia executable + -H, --home Set location of `julia` executable --startup-file={yes|no} Load ~/.juliarc.jl --handle-signals={yes|no} Enable or disable Julia's default signal handlers diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index e8f8ca26462a8..d8129b0860fbc 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -260,7 +260,7 @@ to reduce this time. There are two mechanisms that can achieve this: incremental compile and custom system image. -To create a custom system image that can be used to start julia with the -J option, +To create a custom system image that can be used when starting Julia with the ``-J`` option, recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules. To create an incremental precompiled module file, add diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index 171d8268c7d52..c241aa0e36445 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -466,7 +466,7 @@ are several possible approaches, here is one that is widely used: run the tests: + From Julia, run :func:`Pkg.test("Foo") `: this will run your - tests in a separate (new) julia process. + tests in a separate (new) ``julia`` process. + From Julia, ``include("runtests.jl")`` from the package's ``test/`` folder (it's possible the file has a different name, look for one that runs all the tests): this allows you to run the tests repeatedly in the same session diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 0da062c906453..2acb528b7ee80 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -167,7 +167,7 @@ the following code:: end -Starting julia with ``julia -p 2``, you can use this to verify the following: +Starting Julia with ``julia -p 2``, you can use this to verify the following: - :func:`include("DummyModule.jl") ` loads the file on just a single process (whichever one executes the statement). - ``using DummyModule`` causes the module to be loaded on all processes; however, the module is brought into scope only on the one executing the statement. @@ -202,7 +202,7 @@ The base Julia installation has in-built support for two types of clusters: - A local cluster specified with the ``-p`` option as shown above. - A cluster spanning machines using the ``--machinefile`` option. This uses a passwordless - ``ssh`` login to start julia worker processes (from the same path as the current host) + ``ssh`` login to start Julia worker processes (from the same path as the current host) on the specified machines. Functions :func:`addprocs`, :func:`rmprocs`, :func:`workers`, and others are available as a programmatic means of @@ -729,15 +729,15 @@ handles mapping the shared segment being released sooner. ClusterManagers --------------- -The launching, management and networking of julia processes into a logical +The launching, management and networking of Julia processes into a logical cluster is done via cluster managers. A :obj:`ClusterManager` is responsible for - launching worker processes in a cluster environment - managing events during the lifetime of each worker - optionally, a cluster manager can also provide data transport -A julia cluster has the following characteristics: -- The initial julia process, also called the ``master`` is special and has a julia id of 1. +A Julia cluster has the following characteristics: +- The initial Julia process, also called the ``master`` is special and has a id of 1. - Only the ``master`` process can add or remove worker processes. - All processes can directly communicate with each other. @@ -750,11 +750,11 @@ Connections between workers (using the in-built TCP/IP transport) is established - The cluster manager captures the stdout's of each worker and makes it available to the master process - The master process parses this information and sets up TCP/IP connections to each worker - Every worker is also notified of other workers in the cluster -- Each worker connects to all workers whose julia id is less than its own id +- Each worker connects to all workers whose id is less than its own id - In this way a mesh network is established, wherein every worker is directly connected with every other worker -While the default transport layer uses plain TCP sockets, it is possible for a julia cluster to provide +While the default transport layer uses plain TCP sockets, it is possible for a Julia cluster to provide its own transport. Julia provides two in-built cluster managers: @@ -860,7 +860,7 @@ If ``io`` is not specified, ``host`` and ``port`` are used to connect. For example, a cluster manager may launch a single worker per node, and use that to launch additional workers. ``count`` with an integer value ``n`` will launch a total of ``n`` workers, while a value of ``:auto`` will launch as many workers as cores on that machine. -``exename`` is the name of the julia executable including the full path. +``exename`` is the name of the Julia executable including the full path. ``exeflags`` should be set to the required command line arguments for new workers. ``tunnel``, ``bind_addr``, ``sshflags`` and ``max_parallel`` are used when a ssh tunnel is @@ -884,14 +884,14 @@ Cluster Managers with custom transports --------------------------------------- Replacing the default TCP/IP all-to-all socket connections with a custom transport layer is a little more involved. -Each julia process has as many communication tasks as the workers it is connected to. For example, consider a julia cluster of +Each Julia process has as many communication tasks as the workers it is connected to. For example, consider a Julia cluster of 32 processes in a all-to-all mesh network: -- Each julia process thus has 31 communication tasks +- Each Julia process thus has 31 communication tasks - Each task handles all incoming messages from a single remote worker in a message processing loop - The message processing loop waits on an ``AsyncStream`` object - for example, a TCP socket in the default implementation, reads an entire message, processes it and waits for the next one -- Sending messages to a process is done directly from any julia task - not just communication tasks - again, via the appropriate +- Sending messages to a process is done directly from any Julia task - not just communication tasks - again, via the appropriate ``AsyncStream`` object Replacing the default transport involves the new implementation to setup connections to remote workers, and to provide appropriate @@ -904,17 +904,17 @@ The default implementation (which uses TCP/IP sockets) is implemented as ``conne ``connect`` should return a pair of ``AsyncStream`` objects, one for reading data sent from worker ``pid``, and the other to write data that needs to be sent to worker ``pid``. Custom cluster managers can use an in-memory ``BufferStream`` -as the plumbing to proxy data between the custom, possibly non-AsyncStream transport and julia's in-built parallel infrastructure. +as the plumbing to proxy data between the custom, possibly non-AsyncStream transport and Julia's in-built parallel infrastructure. A ``BufferStream`` is an in-memory ``IOBuffer`` which behaves like an ``AsyncStream``. -Folder ``examples/clustermanager/0mq`` is an example of using ZeroMQ is connect julia workers in a star network with a 0MQ broker in the middle. -Note: The julia processes are still all *logically* connected to each other - any worker can message any other worker directly without any +Folder ``examples/clustermanager/0mq`` is an example of using ZeroMQ is connect Julia workers in a star network with a 0MQ broker in the middle. +Note: The Julia processes are still all *logically* connected to each other - any worker can message any other worker directly without any awareness of 0MQ being used as the transport layer. When using custom transports: -- julia workers must NOT be started with ``--worker``. Starting with ``--worker`` will result in the newly launched +- Julia workers must NOT be started with ``--worker``. Starting with ``--worker`` will result in the newly launched workers defaulting to the TCP/IP socket transport implementation - For every incoming logical connection with a worker, ``Base.process_messages(rd::AsyncStream, wr::AsyncStream)`` must be called. This launches a new task that handles reading and writing of messages from/to the worker represented by the ``AsyncStream`` objects diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index f93a3e4df7227..c44dd59a47a30 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -751,16 +751,16 @@ This might be worthwhile when the following are true: ``Array{Car{:Honda,:Accord},N}``. When the latter holds, a function processing such a homogenous array -can be productively specialized: julia knows the type of each element +can be productively specialized: Julia knows the type of each element in advance (all objects in the container have the same concrete type), -so julia can "look up" the correct method calls when the function is +so Julia can "look up" the correct method calls when the function is being compiled (obviating the need to check at run-time) and thereby emit efficient code for processing the whole list. When these do not hold, then it's likely that you'll get no benefit; worse, the resulting "combinatorial explosion of types" will be counterproductive. If ``items[i+1]`` has a different type than -``item[i]``, julia has to look up the type at run-time, search for the +``item[i]``, Julia has to look up the type at run-time, search for the appropriate method in method tables, decide (via type intersection) which one matches, determine whether it has been JIT-compiled yet (and do so if not), and then make the call. In essence, you're asking the @@ -773,7 +773,7 @@ lookup, and (3) a "switch" statement can be found `on the mailing list `_. Perhaps even worse than the run-time impact is the compile-time -impact: julia will compile specialized functions for each different +impact: Julia will compile specialized functions for each different ``Car{Make, Model}``; if you have hundreds or thousands of such types, then every function that accepts such an object as a parameter (from a custom ``get_year`` function you might write yourself, to the generic diff --git a/ui/repl.c b/ui/repl.c index c7f391a8189b4..a4e726ff464d5 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -60,7 +60,7 @@ static const char opts[] = " -J, --sysimage Start up with the given system image file\n" " --precompiled={yes|no} Use precompiled code from system image if available\n" " --compilecache={yes|no} Enable/disable incremental precompilation of modules\n" - " -H, --home Set location of julia executable\n" + " -H, --home Set location of `julia` executable\n" " --startup-file={yes|no} Load ~/.juliarc.jl\n" " --handle-signals={yes|no} Enable or disable Julia's default signal handlers\n\n" From ebfca4663a0b168ba99f4e86ebeb9f6ac9eef7c2 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 Jun 2016 22:45:04 -0400 Subject: [PATCH 0083/1117] Disable syntax highlighting of `julia` options --- doc/manual/getting-started.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/getting-started.rst b/doc/manual/getting-started.rst index 8d668662cf0db..a8114ffc683a1 100644 --- a/doc/manual/getting-started.rst +++ b/doc/manual/getting-started.rst @@ -107,7 +107,9 @@ put it in ``~/.juliarc.jl``: \end{CJK*} There are various ways to run Julia code and provide options, similar to -those available for the ``perl`` and ``ruby`` programs:: +those available for the ``perl`` and ``ruby`` programs: + +.. code-block:: none julia [switches] -- [programfile] [args...] -v, --version Display version information From 36af20429d540e341cd2c0c585a83c01187a189a Mon Sep 17 00:00:00 2001 From: Fengyang Wang Date: Wed, 22 Jun 2016 23:01:00 -0400 Subject: [PATCH 0084/1117] =?UTF-8?q?Typo:=20second=20=E2=87=92=20seconds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ci skip] --- doc/manual/modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index d8129b0860fbc..aab14db34fb52 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -252,7 +252,7 @@ prevent name conflicts for globals initialized after load time. Module initialization and precompilation ---------------------------------------- -Large modules can take several second to load because executing all of +Large modules can take several seconds to load because executing all of the statements in a module often involves compiling a large amount of code. Julia provides the ability to create precompiled versions of modules to reduce this time. From 0769628f8b7590ed55970effd32fd97685ba289f Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 23 Jun 2016 20:29:33 -0500 Subject: [PATCH 0085/1117] Fix specificity with bound varargs --- src/jltypes.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/typemap.c | 35 ----------------- test/core.jl | 12 ++++++ 3 files changed, 114 insertions(+), 35 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index a02635d78c77b..104a4dba5be16 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2777,6 +2777,21 @@ static int type_eqv_with_ANY(jl_value_t *a, jl_value_t *b) static int jl_type_morespecific_(jl_value_t *a, jl_value_t *b, int invariant); +jl_datatype_t *jl_fix_vararg_bound(jl_datatype_t *tt, int nfix) +{ + assert(jl_is_va_tuple(tt)); + assert(nfix >= 0); + jl_svec_t *tp = tt->parameters; + size_t ntp = jl_svec_len(tp); + jl_value_t *env[2] = {NULL, NULL}; + JL_GC_PUSH2(&env[0], &env[1]); + env[0] = jl_tparam1(jl_tparam(tt, ntp-1)); + env[1] = jl_box_long(nfix); + jl_datatype_t *ret = (jl_datatype_t*)jl_instantiate_type_with((jl_value_t*)tt, env, 2); + JL_GC_POP(); + return ret; +} + static int jl_tuple_morespecific(jl_datatype_t *cdt, jl_datatype_t *pdt, int invariant) { size_t clenr = jl_nparams(cdt); @@ -2999,6 +3014,93 @@ JL_DLLEXPORT int jl_type_morespecific(jl_value_t *a, jl_value_t *b) return jl_type_morespecific_(a, b, 0); } +int jl_args_morespecific_(jl_value_t *a, jl_value_t *b) +{ + int msp = jl_type_morespecific(a,b); + int btv = jl_has_typevars(b); + if (btv) { + if (jl_type_match_morespecific(a,b) == (jl_value_t*)jl_false) { + if (jl_has_typevars(a)) + return 0; + return msp; + } + if (jl_has_typevars(a)) { + type_match_invariance_mask = 0; + //int result = jl_type_match_morespecific(b,a) == (jl_value_t*)jl_false); + // this rule seems to work better: + int result = jl_type_match(b,a) == (jl_value_t*)jl_false; + type_match_invariance_mask = 1; + if (result) + return 1; + } + int nmsp = jl_type_morespecific(b,a); + if (nmsp == msp) + return 0; + } + if (jl_has_typevars((jl_value_t*)a)) { + int nmsp = jl_type_morespecific(b,a); + if (nmsp && msp) + return 1; + if (!btv && jl_types_equal(a,b)) + return 1; + if (jl_type_match_morespecific(b,a) != (jl_value_t*)jl_false) + return 0; + } + return msp; +} + +// Called when a is a bound-vararg and b is not a vararg. Sets the +// vararg length in a to match b, as long as this makes some earlier +// argument more specific. +int jl_args_morespecific_fix1(jl_value_t *a, jl_value_t *b, int swap) +{ + jl_datatype_t *tta = (jl_datatype_t*)a; + jl_datatype_t *ttb = (jl_datatype_t*)b; + size_t n = jl_nparams(tta); + jl_datatype_t *newtta = jl_fix_vararg_bound(tta, jl_nparams(ttb)-n+1); + int changed = 0; + for (size_t i = 0; i < n-1; i++) { + if (jl_tparam(tta, i) != jl_tparam(newtta, i)) { + changed = 1; + break; + } + } + if (changed) { + JL_GC_PUSH1(&newtta); + int ret; + if (swap) + ret = jl_args_morespecific_(b, (jl_value_t*)newtta); + else + ret = jl_args_morespecific_((jl_value_t*)newtta, b); + JL_GC_POP(); + return ret; + } + if (swap) + return jl_args_morespecific_(b, a); + return jl_args_morespecific_(a, b); +} + +JL_DLLEXPORT int jl_args_morespecific(jl_value_t *a, jl_value_t *b) +{ + if (jl_is_tuple_type(a) && jl_is_tuple_type(b)) { + jl_datatype_t *tta = (jl_datatype_t*)a; + jl_datatype_t *ttb = (jl_datatype_t*)b; + size_t alenf, blenf; + jl_vararg_kind_t akind, bkind; + jl_tuple_lenkind_t alenkind, blenkind; + alenf = tuple_vararg_params(tta->parameters, NULL, &akind, &alenkind); + blenf = tuple_vararg_params(ttb->parameters, NULL, &bkind, &blenkind); + // When one is JL_VARARG_BOUND and the other has fixed length, + // allow the argument length to fix the tvar + if (akind == JL_VARARG_BOUND && blenkind == JL_TUPLE_FIXED && blenf >= alenf) { + return jl_args_morespecific_fix1(a, b, 0); + } + if (bkind == JL_VARARG_BOUND && alenkind == JL_TUPLE_FIXED && alenf >= blenf) { + return jl_args_morespecific_fix1(b, a, 1); + } + } + return jl_args_morespecific_(a, b); +} // ---------------------------------------------------------------------------- diff --git a/src/typemap.c b/src/typemap.c index f2b603face54e..922f004f4dcc9 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -1008,41 +1008,6 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par return newrec; } -JL_DLLEXPORT int jl_args_morespecific(jl_value_t *a, jl_value_t *b) -{ - int msp = jl_type_morespecific(a,b); - int btv = jl_has_typevars(b); - if (btv) { - if (jl_type_match_morespecific(a,b) == (jl_value_t*)jl_false) { - if (jl_has_typevars(a)) - return 0; - return msp; - } - if (jl_has_typevars(a)) { - type_match_invariance_mask = 0; - //int result = jl_type_match_morespecific(b,a) == (jl_value_t*)jl_false); - // this rule seems to work better: - int result = jl_type_match(b,a) == (jl_value_t*)jl_false; - type_match_invariance_mask = 1; - if (result) - return 1; - } - int nmsp = jl_type_morespecific(b,a); - if (nmsp == msp) - return 0; - } - if (jl_has_typevars((jl_value_t*)a)) { - int nmsp = jl_type_morespecific(b,a); - if (nmsp && msp) - return 1; - if (!btv && jl_types_equal(a,b)) - return 1; - if (jl_type_match_morespecific(b,a) != (jl_value_t*)jl_false) - return 0; - } - return msp; -} - static int has_unions(jl_tupletype_t *type) { int i; diff --git a/test/core.jl b/test/core.jl index f89d31fa8fa8d..c7b05769be013 100644 --- a/test/core.jl +++ b/test/core.jl @@ -206,6 +206,18 @@ let T = TypeVar(:T, Tuple{Vararg{RangeIndex}}, true) @test args_morespecific(t2, t1) end +let T = TypeVar(:T, Any, true), N = TypeVar(:N, Any, true) + a = Tuple{Array{T,N}, Vararg{Int,N}} + b = Tuple{Array,Int} + @test args_morespecific(a, b) + @test !args_morespecific(b, a) + a = Tuple{Array, Vararg{Int,N}} + @test !args_morespecific(a, b) + @test args_morespecific(b, a) +end + +# with bound varargs + # issue #11840 typealias TT11840{T} Tuple{T,T} f11840(::Type) = "Type" From 27c288d40d5dbd311a964cbbfec1890c3e07b3ad Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 24 Jun 2016 05:35:56 -0500 Subject: [PATCH 0086/1117] PermutedDimsArray: move iperm into the type parameters This leads to a 10% speedup on getindex-heavy benchmarks, and makes row-major sums over row-major matrices indistinguishable (performance-wise) from column-major sums over column-major matrices. --- base/permuteddimsarray.jl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index 887c10898cf27..49060263fc835 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -7,41 +7,41 @@ using Base: IndicesStartAt1, IndicesBehavior, indicesbehavior export permutedims # Some day we will want storage-order-aware iteration, so put perm in the parameters -immutable PermutedDimsArray{T,N,AA<:AbstractArray,perm} <: AbstractArray{T,N} +immutable PermutedDimsArray{T,N,perm,iperm,AA<:AbstractArray} <: AbstractArray{T,N} parent::AA - iperm::NTuple{N,Int} dims::NTuple{N,Int} function PermutedDimsArray(data::AA) # TODO optimize isperm & invperm for low dimensions? - isa(perm, NTuple{N,Int}) || error("perm must be an NTuple{$N,Int}") - (length(perm) == N && isperm(perm)) || throw(ArgumentError(string(perm, " is not a valid permutation of dimensions 1:", N))) - iperm = invperm(perm) - new(data, iperm, genperm(size(data), perm)) + (isa(perm, NTuple{N,Int}) && isa(iperm, NTuple{N,Int})) || error("perm and iperm must both be NTuple{$N,Int}") + isperm(perm) || throw(ArgumentError(string(perm, " is not a valid permutation of dimensions 1:", N))) + all(map(d->iperm[perm[d]]==d, 1:N)) || throw(ArgumentError(string(perm, " and ", iperm, " must be inverses"))) + new(data, genperm(size(data), perm)) end end function PermutedDimsArray{T,N}(data::AbstractArray{T,N}, perm) length(perm) == N || throw(ArgumentError(string(p, " is not a valid permutation of dimensions 1:", N))) - PermutedDimsArray{T,N,typeof(data),(perm...,)}(data) + iperm = invperm(perm) + PermutedDimsArray{T,N,(perm...,),(iperm...,),typeof(data)}(data) end Base.parent(A::PermutedDimsArray) = A.parent Base.size(A::PermutedDimsArray) = A.dims -Base.indices{T,N,AA,perm}(A::PermutedDimsArray{T,N,AA,perm}, d) = indices(parent(A), perm[d]) +Base.indices{T,N,perm}(A::PermutedDimsArray{T,N,perm}, d) = indices(parent(A), perm[d]) -@inline function Base.getindex{T,N}(A::PermutedDimsArray{T,N}, I::Vararg{Int,N}) +@inline function Base.getindex{T,N,perm,iperm}(A::PermutedDimsArray{T,N,perm,iperm}, I::Vararg{Int,N}) @boundscheck checkbounds(A, I...) - @inbounds val = getindex(A.parent, genperm(I, A.iperm)...) + @inbounds val = getindex(A.parent, genperm(I, iperm)...) val end -@inline function Base.setindex!{T,N}(A::PermutedDimsArray{T,N}, val, I::Vararg{Int,N}) +@inline function Base.setindex!{T,N,perm,iperm}(A::PermutedDimsArray{T,N,perm,iperm}, val, I::Vararg{Int,N}) @boundscheck checkbounds(A, I...) - @inbounds setindex!(A.parent, val, genperm(I, A.iperm)...) + @inbounds setindex!(A.parent, val, genperm(I, iperm)...) val end -# Could use ntuple(d->I[perm[d]], Val{N}) once #15276 is solved +# For some reason this is faster than ntuple(d->I[perm[d]], Val{N}) (#15276?) @inline genperm{N}(I::NTuple{N}, perm::Dims{N}) = _genperm((), I, perm...) _genperm(out, I) = out @inline _genperm(out, I, p, perm...) = _genperm((out..., I[p]), I, perm...) @@ -69,7 +69,7 @@ function Base.copy!{T,N}(dest::PermutedDimsArray{T,N}, src::AbstractArray{T,N}) end Base.copy!(dest::PermutedDimsArray, src::AbstractArray) = _copy!(dest, src) -function _copy!{T,N,AA,perm}(P::PermutedDimsArray{T,N,AA,perm}, src) +function _copy!{T,N,perm}(P::PermutedDimsArray{T,N,perm}, src) # If dest/src are "close to dense," then it pays to be cache-friendly. # Determine the first permuted dimension d = 0 # d+1 will hold the first permuted dimension of src From fdef0a922317fd9148a0d9323ccdb28b8458b438 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 24 Jun 2016 06:31:47 -0500 Subject: [PATCH 0087/1117] Specialize isperm and invperm on tuples of lengths 0, 1, and 2 This speeds up construction of PermutedDimsArrays --- base/combinatorics.jl | 9 +++++++++ base/permuteddimsarray.jl | 1 - test/combinatorics.jl | 12 ++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/base/combinatorics.jl b/base/combinatorics.jl index dde0d857f6f38..31407a4939b89 100644 --- a/base/combinatorics.jl +++ b/base/combinatorics.jl @@ -70,6 +70,10 @@ function isperm(A) true end +isperm(p::Tuple{}) = true +isperm(p::Tuple{Int}) = p[1] == 1 +isperm(p::Tuple{Int,Int}) = ((p[1] == 1) & (p[2] == 2)) | ((p[1] == 2) & (p[2] == 1)) + function permute!!{T<:Integer}(a, p::AbstractVector{T}) count = 0 start = 0 @@ -147,6 +151,11 @@ function invperm(a::AbstractVector) end b end + +function invperm(p::Union{Tuple{},Tuple{Int},Tuple{Int,Int}}) + isperm(p) || throw(ArgumentError("argument is not a permutation")) + p # in dimensions 0-2, every permutation is its own inverse +end invperm(a::Tuple) = (invperm([a...])...,) #XXX This function should be moved to Combinatorics.jl but is currently used by Base.DSP. diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index 49060263fc835..c5a461509d494 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -12,7 +12,6 @@ immutable PermutedDimsArray{T,N,perm,iperm,AA<:AbstractArray} <: AbstractArray{T dims::NTuple{N,Int} function PermutedDimsArray(data::AA) - # TODO optimize isperm & invperm for low dimensions? (isa(perm, NTuple{N,Int}) && isa(iperm, NTuple{N,Int})) || error("perm and iperm must both be NTuple{$N,Int}") isperm(perm) || throw(ArgumentError(string(perm, " is not a valid permutation of dimensions 1:", N))) all(map(d->iperm[perm[d]]==d, 1:N)) || throw(ArgumentError(string(perm, " and ", iperm, " must be inverses"))) diff --git a/test/combinatorics.jl b/test/combinatorics.jl index d54903402cf6a..3e4975b6fe0ef 100644 --- a/test/combinatorics.jl +++ b/test/combinatorics.jl @@ -17,6 +17,18 @@ p = shuffle([1:1000;]) @test isperm(p) @test all(invperm(invperm(p)) .== p) +@test isperm(()) == true +@test isperm((1,)) == true +@test isperm((2,)) == false +@test isperm((1,2)) == true +@test isperm((2,1)) == true +@test isperm((2,2)) == false +@test isperm((1,3)) == false +@test invperm(()) == () +@test invperm((1,)) == (1,) +@test invperm((1,2)) == (1,2) +@test invperm((2,1)) == (2,1) +@test_throws ArgumentError invperm((1,3)) push!(p, 1) @test !isperm(p) From d158159505934226ae4a98590fa560aa47460ebb Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 26 May 2016 20:57:23 -0400 Subject: [PATCH 0088/1117] Special case codegen of ccall(:jl_get_ptls_states) --- base/boot.jl | 3 +++ src/ccall.cpp | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/base/boot.jl b/base/boot.jl index 4c93011525708..8056568824fdf 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -225,6 +225,9 @@ immutable String <: AbstractString String(d::Array{UInt8,1}) = new(d) end +# This should always be inlined +getptls() = ccall(:jl_get_ptls_states, Ptr{Void}, ()) + include(fname::String) = ccall(:jl_load_, Any, (Any,), fname) eval(e::ANY) = eval(Main, e) diff --git a/src/ccall.cpp b/src/ccall.cpp index 1103b4e918ad5..95d6fc5c06c62 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1235,6 +1235,17 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) emit_signal_fence(); return ghostValue(jl_void_type); } + if (fptr == (void(*)(void))&jl_get_ptls_states || + ((!f_lib || (intptr_t)f_lib == 2) && f_name && + strcmp(f_name, "jl_get_ptls_states") == 0)) { + assert(lrt == T_pint8); + assert(!isVa); + assert(nargt == 0); + JL_GC_POP(); + return mark_or_box_ccall_result( + builder.CreateBitCast(ctx->ptlsStates, lrt), + retboxed, args[2], rt, static_rt, ctx); + } if (fptr == &jl_sigatomic_begin || ((!f_lib || (intptr_t)f_lib == 2) && f_name && strcmp(f_name, "jl_sigatomic_begin") == 0)) { From acf282fa8ff887ba2c9d08ac4aef7e6c7eff1d89 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Wed, 22 Jun 2016 11:34:33 -0400 Subject: [PATCH 0089/1117] Optimize C finalizer --- base/base.jl | 7 ++++ src/gc.c | 97 ++++++++++++++++++++++++++++++++++------------------ src/gc.h | 12 ++++++- 3 files changed, 81 insertions(+), 35 deletions(-) diff --git a/base/base.jl b/base/base.jl index 6486aaf3acab7..07b8899b7dcaf 100644 --- a/base/base.jl +++ b/base/base.jl @@ -72,6 +72,13 @@ function finalizer(o::ANY, f::ANY) end ccall(:jl_gc_add_finalizer, Void, (Any,Any), o, f) end +function finalizer{T}(o::T, f::Ptr{Void}) + @_inline_meta + if isimmutable(T) + error("objects of type ", T, " cannot be finalized") + end + ccall(:jl_gc_add_ptr_finalizer, Void, (Any, Ptr{Void}), o, f) +end finalize(o::ANY) = ccall(:jl_finalize, Void, (Any,), o) diff --git a/src/gc.c b/src/gc.c index 498f524d0d129..c5d7a230a2e08 100644 --- a/src/gc.c +++ b/src/gc.c @@ -51,6 +51,10 @@ region_t regions[REGION_COUNT]; bigval_t *big_objects_marked = NULL; // finalization +// `finalizer_list` and `finalizer_list_marked` might have tagged pointers. +// If an object pointer has the lowest bit set, the next pointer is an unboxed +// c function pointer. +// `to_finalize` should not have tagged pointers. arraylist_t finalizer_list; arraylist_t finalizer_list_marked; arraylist_t to_finalize; @@ -97,38 +101,38 @@ static void schedule_finalization(void *o, void *f) static void run_finalizer(jl_value_t *o, jl_value_t *ff) { - if (jl_typeis(ff, jl_voidpointer_type)) { - void *p = jl_unbox_voidpointer(ff); - if (p) - ((void (*)(void*))p)(jl_data_ptr(o)); + assert(!jl_typeis(ff, jl_voidpointer_type)); + jl_value_t *args[2] = {ff,o}; + JL_TRY { + jl_apply(args, 2); } - else { - jl_value_t *args[2] = {ff,o}; - JL_TRY { - jl_apply(args, 2); - } - JL_CATCH { - jl_printf(JL_STDERR, "error in running finalizer: "); - jl_static_show(JL_STDERR, jl_exception_in_transit); - jl_printf(JL_STDERR, "\n"); - } + JL_CATCH { + jl_printf(JL_STDERR, "error in running finalizer: "); + jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_printf(JL_STDERR, "\n"); } } static void finalize_object(arraylist_t *list, jl_value_t *o, arraylist_t *copied_list) { - for(int i = 0; i < list->len; i+=2) { - if (o == (jl_value_t*)list->items[i]) { - void *f = list->items[i+1]; + for (int i = 0; i < list->len; i+=2) { + void *v = list->items[i]; + if (o == (jl_value_t*)gc_ptr_clear_tag(v, 1)) { + void *f = list->items[i + 1]; if (i < list->len - 2) { list->items[i] = list->items[list->len-2]; list->items[i+1] = list->items[list->len-1]; i -= 2; } list->len -= 2; - arraylist_push(copied_list, o); - arraylist_push(copied_list, f); + if (gc_ptr_tag(v, 1)) { + ((void (*)(void*))f)(o); + } + else { + arraylist_push(copied_list, o); + arraylist_push(copied_list, f); + } } } } @@ -195,9 +199,10 @@ static void schedule_all_finalizers(arraylist_t *flist) { // Multi-thread version should steal the entire list while holding a lock. for(size_t i=0; i < flist->len; i+=2) { - jl_value_t *f = (jl_value_t*)flist->items[i+1]; - if (f != HT_NOTFOUND && !jl_is_cpointer(f)) { - schedule_finalization(flist->items[i], flist->items[i+1]); + void *v = flist->items[i]; + void *f = flist->items[i + 1]; + if (!gc_ptr_tag(v, 1)) { + schedule_finalization(v, f); } } flist->len = 0; @@ -212,11 +217,29 @@ void jl_gc_run_all_finalizers(void) run_finalizers(); } +static void gc_add_ptr_finalizer(jl_value_t *v, void *f) +{ + arraylist_push(&finalizer_list, (void*)(((uintptr_t)v) | 1)); + arraylist_push(&finalizer_list, f); +} + JL_DLLEXPORT void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) { JL_LOCK_NOGC(&finalizers_lock); - arraylist_push(&finalizer_list, (void*)v); - arraylist_push(&finalizer_list, (void*)f); + if (__unlikely(jl_typeis(f, jl_voidpointer_type))) { + gc_add_ptr_finalizer(v, jl_unbox_voidpointer(f)); + } + else { + arraylist_push(&finalizer_list, v); + arraylist_push(&finalizer_list, f); + } + JL_UNLOCK_NOGC(&finalizers_lock); +} + +JL_DLLEXPORT void jl_gc_add_ptr_finalizer(jl_value_t *v, void *f) +{ + JL_LOCK_NOGC(&finalizers_lock); + gc_add_ptr_finalizer(v, f); JL_UNLOCK_NOGC(&finalizers_lock); } @@ -1238,7 +1261,13 @@ NOINLINE static void gc_mark_task(jl_task_t *ta, int d) void gc_mark_object_list(arraylist_t *list, size_t start) { for (size_t i = start;i < list->len;i++) { - gc_push_root(list->items[i], 0); + void *v = list->items[i]; + if (gc_ptr_tag(v, 1)) { + v = gc_ptr_clear_tag(v, 1); + i++; + assert(i < list->len); + } + gc_push_root(v, 0); } } @@ -1484,13 +1513,15 @@ void pre_mark(void) // this must happen last in the mark phase. static void post_mark(arraylist_t *list) { - for(size_t i=0; i < list->len; i+=2) { - jl_value_t *v = (jl_value_t*)list->items[i]; - jl_value_t *fin = (jl_value_t*)list->items[i+1]; + for (size_t i=0; i < list->len; i+=2) { + void *v0 = list->items[i]; + int is_cptr = gc_ptr_tag(v0, 1); + void *v = gc_ptr_clear_tag(v0, 1); + void *fin = list->items[i+1]; int isfreed = !gc_marked(jl_astaggedvalue(v)->bits.gc); int isold = (list != &finalizer_list_marked && jl_astaggedvalue(v)->bits.gc == GC_OLD_MARKED && - jl_astaggedvalue(fin)->bits.gc == GC_OLD_MARKED); + (is_cptr || jl_astaggedvalue(fin)->bits.gc == GC_OLD_MARKED)); if (isfreed || isold) { // remove from this list if (i < list->len - 2) { @@ -1502,10 +1533,8 @@ static void post_mark(arraylist_t *list) } if (isfreed) { // schedule finalizer or execute right away if it is not julia code - if (jl_typeof(fin) == (jl_value_t*)jl_voidpointer_type) { - void *p = jl_unbox_voidpointer(fin); - if (p) - ((void (*)(void*))p)(jl_data_ptr(v)); + if (is_cptr) { + ((void (*)(void*))fin)(jl_data_ptr(v)); continue; } schedule_finalization(v, fin); @@ -1513,7 +1542,7 @@ static void post_mark(arraylist_t *list) if (isold) { // The caller relies on the new objects to be pushed to the end of // the list!! - arraylist_push(&finalizer_list_marked, v); + arraylist_push(&finalizer_list_marked, v0); arraylist_push(&finalizer_list_marked, fin); } } diff --git a/src/gc.h b/src/gc.h index 4e00639f206de..57d518eb807dd 100644 --- a/src/gc.h +++ b/src/gc.h @@ -219,6 +219,16 @@ STATIC_INLINE int gc_old(int bits) return (bits & GC_OLD) != 0; } +STATIC_INLINE uintptr_t gc_ptr_tag(void *v, uintptr_t mask) +{ + return ((uintptr_t)v) & mask; +} + +STATIC_INLINE void *gc_ptr_clear_tag(void *v, uintptr_t mask) +{ + return (void*)(((uintptr_t)v) & ~mask); +} + NOINLINE uintptr_t gc_get_stack_ptr(void); STATIC_INLINE region_t *find_region(void *ptr, int maybe) @@ -355,7 +365,7 @@ void add_lostval_parent(jl_value_t *parent); } while(0); #define verify_parent(ty, obj, slot, args...) do { \ - if (*(jl_value_t**)(slot) == lostval && \ + if (gc_ptr_clear_tag(*(void**)(slot), 3) == (void*)lostval && \ (jl_value_t*)(obj) != lostval) { \ jl_printf(JL_STDOUT, "Found parent %p %p at %s:%d\n", \ (void*)(ty), (void*)(obj), __FILE__, __LINE__); \ From 3878f81364bb9caee39daa7c7074e6bad1c8a581 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 23 Jun 2016 21:41:15 -0400 Subject: [PATCH 0090/1117] Optimize `jl_add_*_finalizer` fast path to avoid a lock * Make `finalizer_list` thread local * Optimize usage of `arraylist_t` in `gc.c` * Also starts to explicitly pass `ptls` around to reduce TLS access overhead More consistently use `ptls` for the tls pointer of the current thread and `ptls2` for the tls pointer of other threads. --- base/base.jl | 6 +- src/codegen.cpp | 5 +- src/debuginfo.cpp | 22 +-- src/gc-debug.c | 10 +- src/gc.c | 311 +++++++++++++++++++++++++--------------- src/gc.h | 1 - src/jl_uv.c | 9 +- src/jlapi.c | 15 +- src/julia.h | 2 +- src/julia_internal.h | 10 +- src/julia_threads.h | 45 +++--- src/stackwalk.c | 5 +- src/support/arraylist.c | 24 ++-- src/task.c | 19 +-- src/threading.c | 18 +-- 15 files changed, 308 insertions(+), 194 deletions(-) diff --git a/base/base.jl b/base/base.jl index 07b8899b7dcaf..61bd5006e67e3 100644 --- a/base/base.jl +++ b/base/base.jl @@ -70,14 +70,16 @@ function finalizer(o::ANY, f::ANY) if isimmutable(o) error("objects of type ", typeof(o), " cannot be finalized") end - ccall(:jl_gc_add_finalizer, Void, (Any,Any), o, f) + ccall(:jl_gc_add_finalizer_th, Void, (Ptr{Void}, Any, Any), + Core.getptls(), o, f) end function finalizer{T}(o::T, f::Ptr{Void}) @_inline_meta if isimmutable(T) error("objects of type ", T, " cannot be finalized") end - ccall(:jl_gc_add_ptr_finalizer, Void, (Any, Ptr{Void}), o, f) + ccall(:jl_gc_add_ptr_finalizer, Void, (Ptr{Void}, Any, Ptr{Void}), + Core.getptls(), o, f) end finalize(o::ANY) = ccall(:jl_finalize, Void, (Any,), o) diff --git a/src/codegen.cpp b/src/codegen.cpp index e33d370bca767..3b9c0d0ffc854 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1318,6 +1318,7 @@ static uint64_t compute_obj_symsize(const object::ObjectFile *obj, uint64_t offs extern "C" JL_DLLEXPORT const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) { + jl_tls_states_t *ptls = jl_get_ptls_states(); std::string code; llvm::raw_string_ostream stream(code); #ifndef LLVM37 @@ -1362,7 +1363,7 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) return (jl_value_t*)jl_pchar_to_array((char*)fptr, symsize); } - int8_t gc_state = jl_gc_safe_enter(); + int8_t gc_state = jl_gc_safe_enter(ptls); jl_dump_asm_internal(fptr, symsize, slide, #ifndef USE_MCJIT context, @@ -1378,7 +1379,7 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) #ifndef LLVM37 fstream.flush(); #endif - jl_gc_safe_leave(gc_state); + jl_gc_safe_leave(ptls, gc_state); return jl_cstr_to_string(const_cast(stream.str().c_str())); } diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 10184e317c85e..29d21a61f97b3 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -251,9 +251,10 @@ class JuliaJITEventListener: public JITEventListener virtual void NotifyFunctionEmitted(const Function &F, void *Code, size_t Size, const EmittedFunctionDetails &Details) { + jl_tls_states_t *ptls = jl_get_ptls_states(); // This function modify linfo->fptr in GC safe region. // This should be fine since the GC won't scan this field. - int8_t gc_state = jl_gc_safe_enter(); + int8_t gc_state = jl_gc_safe_enter(ptls); uv_rwlock_wrlock(&threadsafe); StringRef sName = F.getName(); StringMap::iterator linfo_it = linfo_in_flight.find(sName); @@ -274,7 +275,7 @@ class JuliaJITEventListener: public JITEventListener const_cast(&F)->deleteBody(); #endif uv_rwlock_wrunlock(&threadsafe); - jl_gc_safe_leave(gc_state); + jl_gc_safe_leave(ptls, gc_state); } std::map& getMap() @@ -309,9 +310,10 @@ class JuliaJITEventListener: public JITEventListener virtual void NotifyObjectEmitted(const ObjectImage &obj) #endif { + jl_tls_states_t *ptls = jl_get_ptls_states(); // This function modify linfo->fptr in GC safe region. // This should be fine since the GC won't scan this field. - int8_t gc_state = jl_gc_safe_enter(); + int8_t gc_state = jl_gc_safe_enter(ptls); uv_rwlock_wrlock(&threadsafe); #ifdef LLVM36 object::section_iterator Section = debugObj.section_begin(); @@ -601,7 +603,7 @@ class JuliaJITEventListener: public JITEventListener #endif #endif uv_rwlock_wrunlock(&threadsafe); - jl_gc_safe_leave(gc_state); + jl_gc_safe_leave(ptls, gc_state); } // must implement if we ever start freeing code @@ -1174,17 +1176,18 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int64_t *se extern "C" JL_DLLEXPORT jl_value_t *jl_get_dobj_data(uint64_t fptr) { + jl_tls_states_t *ptls = jl_get_ptls_states(); // Used by Gallium.jl const object::ObjectFile *object = NULL; DIContext *context; int64_t slide, section_slide; - int8_t gc_state = jl_gc_safe_enter(); + int8_t gc_state = jl_gc_safe_enter(ptls); if (!jl_DI_for_fptr(fptr, NULL, &slide, NULL, &object, NULL)) if (!jl_dylib_DI_for_fptr(fptr, &object, &context, &slide, §ion_slide, false, NULL, NULL, NULL, NULL)) { - jl_gc_safe_leave(gc_state); + jl_gc_safe_leave(ptls, gc_state); return jl_nothing; } - jl_gc_safe_leave(gc_state); + jl_gc_safe_leave(ptls, gc_state); if (object == NULL) return jl_nothing; return (jl_value_t*)jl_ptr_to_array_1d((jl_value_t*)jl_array_uint8_type, @@ -1195,8 +1198,9 @@ JL_DLLEXPORT jl_value_t *jl_get_dobj_data(uint64_t fptr) extern "C" JL_DLLEXPORT uint64_t jl_get_section_start(uint64_t fptr) { + jl_tls_states_t *ptls = jl_get_ptls_states(); // Used by Gallium.jl - int8_t gc_state = jl_gc_safe_enter(); + int8_t gc_state = jl_gc_safe_enter(ptls); std::map &objmap = jl_jit_events->getObjectMap(); std::map::iterator fit = objmap.lower_bound(fptr); @@ -1212,7 +1216,7 @@ JL_DLLEXPORT uint64_t jl_get_section_start(uint64_t fptr) } } uv_rwlock_rdunlock(&threadsafe); - jl_gc_safe_leave(gc_state); + jl_gc_safe_leave(ptls, gc_state); return ret; } diff --git a/src/gc-debug.c b/src/gc-debug.c index e504a2a4ec749..bb7d63dfe54c9 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -172,7 +172,10 @@ static void gc_verify_track(void) clear_mark(GC_CLEAN); pre_mark(); gc_mark_object_list(&to_finalize, 0); - gc_mark_object_list(&finalizer_list, 0); + for (int i = 0;i < jl_n_threads;i++) { + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + gc_mark_object_list(&ptls2->finalizers, 0); + } gc_mark_object_list(&finalizer_list_marked, 0); visit_mark_stack(); if (lostval_parents.len == 0) { @@ -215,7 +218,10 @@ void gc_verify(void) gc_verifying = 1; pre_mark(); gc_mark_object_list(&to_finalize, 0); - gc_mark_object_list(&finalizer_list, 0); + for (int i = 0;i < jl_n_threads;i++) { + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + gc_mark_object_list(&ptls2->finalizers, 0); + } gc_mark_object_list(&finalizer_list_marked, 0); visit_mark_stack(); int clean_len = bits_save[GC_CLEAN].len; diff --git a/src/gc.c b/src/gc.c index c5d7a230a2e08..48ed64850aedb 100644 --- a/src/gc.c +++ b/src/gc.c @@ -6,8 +6,10 @@ extern "C" { #endif -// Protect all access to `finalizer_list`, `finalizer_list_marked` and -// `to_finalize`. +// Protect all access to `finalizer_list_marked` and `to_finalize`. +// For accessing `ptls->finalizers`, the lock is needed if a thread is +// is going to realloc the buffer (of it's own list) or accessing the +// list of another thread static jl_mutex_t finalizers_lock; /** @@ -51,11 +53,10 @@ region_t regions[REGION_COUNT]; bigval_t *big_objects_marked = NULL; // finalization -// `finalizer_list` and `finalizer_list_marked` might have tagged pointers. +// `ptls->finalizers` and `finalizer_list_marked` might have tagged pointers. // If an object pointer has the lowest bit set, the next pointer is an unboxed // c function pointer. // `to_finalize` should not have tagged pointers. -arraylist_t finalizer_list; arraylist_t finalizer_list_marked; arraylist_t to_finalize; @@ -70,14 +71,14 @@ NOINLINE uintptr_t gc_get_stack_ptr(void) static void jl_gc_wait_for_the_world(void) { for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls = jl_all_tls_states[i]; + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; // FIXME: The acquire load pairs with the release stores // in the signal handler of safepoint so we are sure that // all the stores on those threads are visible. However, // we're currently not using atomic stores in mutator threads. // We should either use atomic store release there too or use signals // to flush the memory operations on those threads. - while (!ptls->gc_state || !jl_atomic_load_acquire(&ptls->gc_state)) { + while (!ptls2->gc_state || !jl_atomic_load_acquire(&ptls2->gc_state)) { jl_cpu_pause(); // yield? } } @@ -113,19 +114,21 @@ static void run_finalizer(jl_value_t *o, jl_value_t *ff) } } +// if `need_sync` is true, the `list` is the `finalizers` list of another +// thead and we need additional synchronizations static void finalize_object(arraylist_t *list, jl_value_t *o, - arraylist_t *copied_list) -{ - for (int i = 0; i < list->len; i+=2) { - void *v = list->items[i]; + arraylist_t *copied_list, int need_sync) +{ + // The acquire load makes sure that the first `len` objects are valid + size_t len = need_sync ? jl_atomic_load_acquire(&list->len) : list->len; + size_t oldlen = len; + void **items = list->items; + for (size_t i = 0; i < len; i += 2) { + void *v = items[i]; + int move = 0; if (o == (jl_value_t*)gc_ptr_clear_tag(v, 1)) { - void *f = list->items[i + 1]; - if (i < list->len - 2) { - list->items[i] = list->items[list->len-2]; - list->items[i+1] = list->items[list->len-1]; - i -= 2; - } - list->len -= 2; + void *f = items[i + 1]; + move = 1; if (gc_ptr_tag(v, 1)) { ((void (*)(void*))f)(o); } @@ -134,6 +137,25 @@ static void finalize_object(arraylist_t *list, jl_value_t *o, arraylist_push(copied_list, f); } } + if (move || __unlikely(!v)) { + if (i < len - 2) { + items[i] = items[len - 2]; + items[i + 1] = items[len - 1]; + i -= 2; + } + len -= 2; + } + } + if (oldlen == len) + return; + if (need_sync) { + if (__unlikely(jl_atomic_compare_exchange(&list->len, + oldlen, len) != oldlen)) { + memset(items[len], 0, (oldlen - len) * sizeof(void*)); + } + } + else { + list->len = len; } } @@ -141,9 +163,10 @@ static void finalize_object(arraylist_t *list, jl_value_t *o, // be pointers to `jl_value_t` objects static void jl_gc_push_arraylist(arraylist_t *list) { - list->items[0] = (void*)(((uintptr_t)list->len - 2) << 1); - list->items[1] = jl_pgcstack; - jl_pgcstack = (jl_gcframe_t*)list->items; + void **items = list->items; + items[0] = (void*)(((uintptr_t)list->len - 2) << 1); + items[1] = jl_pgcstack; + jl_pgcstack = (jl_gcframe_t*)items; } // Same assumption as `jl_gc_push_arraylist`. Requires the finalizers lock @@ -197,10 +220,13 @@ JL_DLLEXPORT void jl_gc_enable_finalizers(int on) static void schedule_all_finalizers(arraylist_t *flist) { - // Multi-thread version should steal the entire list while holding a lock. - for(size_t i=0; i < flist->len; i+=2) { - void *v = flist->items[i]; - void *f = flist->items[i + 1]; + void **items = flist->items; + size_t len = flist->len; + for(size_t i = 0; i < len; i+=2) { + void *v = items[i]; + void *f = items[i + 1]; + if (__unlikely(!v)) + continue; if (!gc_ptr_tag(v, 1)) { schedule_finalization(v, f); } @@ -210,41 +236,64 @@ static void schedule_all_finalizers(arraylist_t *flist) void jl_gc_run_all_finalizers(void) { - JL_LOCK_NOGC(&finalizers_lock); - schedule_all_finalizers(&finalizer_list); + for (int i = 0;i < jl_n_threads;i++) { + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + schedule_all_finalizers(&ptls2->finalizers); + } schedule_all_finalizers(&finalizer_list_marked); - JL_UNLOCK_NOGC(&finalizers_lock); run_finalizers(); } -static void gc_add_ptr_finalizer(jl_value_t *v, void *f) +static void gc_add_finalizer_(jl_tls_states_t *ptls, void *v, void *f) +{ + int8_t gc_state = jl_gc_unsafe_enter(ptls); + arraylist_t *a = &ptls->finalizers; + size_t oldlen = jl_atomic_load_acquire(&a->len); + if (__unlikely(oldlen + 2 > a->max)) { + JL_LOCK_NOGC(&finalizers_lock); + arraylist_grow(a, 2); + a->len = oldlen; + JL_UNLOCK_NOGC(&finalizers_lock); + } + void **items = a->items; + items[oldlen] = v; + items[oldlen + 1] = f; + jl_atomic_store_release(&a->len, oldlen + 2); + jl_gc_unsafe_leave(ptls, gc_state); +} + +STATIC_INLINE void gc_add_ptr_finalizer(jl_tls_states_t *ptls, + jl_value_t *v, void *f) { - arraylist_push(&finalizer_list, (void*)(((uintptr_t)v) | 1)); - arraylist_push(&finalizer_list, f); + gc_add_finalizer_(ptls, (void*)(((uintptr_t)v) | 1), f); } -JL_DLLEXPORT void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) +JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_tls_states_t *ptls, + jl_value_t *v, jl_function_t *f) { - JL_LOCK_NOGC(&finalizers_lock); if (__unlikely(jl_typeis(f, jl_voidpointer_type))) { - gc_add_ptr_finalizer(v, jl_unbox_voidpointer(f)); + gc_add_ptr_finalizer(ptls, v, jl_unbox_voidpointer(f)); } else { - arraylist_push(&finalizer_list, v); - arraylist_push(&finalizer_list, f); + gc_add_finalizer_(ptls, v, f); } - JL_UNLOCK_NOGC(&finalizers_lock); } -JL_DLLEXPORT void jl_gc_add_ptr_finalizer(jl_value_t *v, void *f) +JL_DLLEXPORT void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) { - JL_LOCK_NOGC(&finalizers_lock); - gc_add_ptr_finalizer(v, f); - JL_UNLOCK_NOGC(&finalizers_lock); + jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_gc_add_finalizer_th(ptls, v, f); +} + +JL_DLLEXPORT void jl_gc_add_ptr_finalizer(jl_tls_states_t *ptls, + jl_value_t *v, void *f) +{ + gc_add_ptr_finalizer(ptls, v, f); } JL_DLLEXPORT void jl_finalize(jl_value_t *o) { + jl_tls_states_t *ptls = jl_get_ptls_states(); JL_LOCK_NOGC(&finalizers_lock); // Copy the finalizers into a temporary list so that code in the finalizer // won't change the list as we loop through them. @@ -255,8 +304,11 @@ JL_DLLEXPORT void jl_finalize(jl_value_t *o) arraylist_push(&copied_list, NULL); // pgcstack to be filled later // No need to check the to_finalize list since the user is apparently // still holding a reference to the object - finalize_object(&finalizer_list, o, &copied_list); - finalize_object(&finalizer_list_marked, o, &copied_list); + for (int i = 0;i < jl_n_threads;i++) { + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + finalize_object(&ptls2->finalizers, o, &copied_list, ptls != ptls2); + } + finalize_object(&finalizer_list_marked, o, &copied_list, 0); if (copied_list.len > 2) { // This releases the finalizers lock. jl_gc_run_finalizers_in_list(&copied_list); @@ -497,7 +549,8 @@ static inline int maybe_collect(void) jl_gc_collect(0); return 1; } - jl_gc_safepoint(); + jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_gc_safepoint_(ptls); return 0; } @@ -515,11 +568,11 @@ JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) static void sweep_weak_refs(void) { for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls = jl_all_tls_states[i]; + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; size_t n = 0; size_t ndel = 0; - size_t l = ptls->heap.weak_refs.len; - void **lst = ptls->heap.weak_refs.items; + size_t l = ptls2->heap.weak_refs.len; + void **lst = ptls2->heap.weak_refs.items; if (l == 0) continue; while (1) { @@ -539,7 +592,7 @@ static void sweep_weak_refs(void) lst[n] = lst[n + ndel]; lst[n+ndel] = tmp; } - ptls->heap.weak_refs.len -= ndel; + ptls2->heap.weak_refs.len -= ndel; } } @@ -675,9 +728,9 @@ static void sweep_malloced_arrays(void) { gc_time_mallocd_array_start(); for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; - mallocarray_t *ma = ptls->heap.mallocarrays; - mallocarray_t **pma = &ptls->heap.mallocarrays; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + mallocarray_t *ma = ptls2->heap.mallocarrays; + mallocarray_t **pma = &ptls2->heap.mallocarrays; while (ma != NULL) { mallocarray_t *nxt = ma->next; int bits = jl_astaggedvalue(ma->a)->bits.gc; @@ -688,8 +741,8 @@ static void sweep_malloced_arrays(void) *pma = nxt; assert(ma->a->flags.how == 2); jl_gc_free_array(ma->a); - ma->next = ptls->heap.mafreelist; - ptls->heap.mafreelist = ma; + ma->next = ptls2->heap.mafreelist; + ptls2->heap.mafreelist = ma; } gc_time_count_mallocd_array(bits); ma = nxt; @@ -702,8 +755,8 @@ static void sweep_malloced_arrays(void) static inline jl_taggedvalue_t *reset_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_taggedvalue_t *fl) { pg->nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / p->osize; - jl_tls_states_t *ptls = jl_all_tls_states[pg->thread_n]; - pg->pool_n = p - ptls->heap.norm_pools; + jl_tls_states_t *ptls2 = jl_all_tls_states[pg->thread_n]; + pg->pool_n = p - ptls2->heap.norm_pools; memset(pg->ages, 0, GC_PAGE_SZ / 8 / p->osize + 1); jl_taggedvalue_t *beg = (jl_taggedvalue_t*)(pg->data + GC_PAGE_OFFSET); jl_taggedvalue_t *end = (jl_taggedvalue_t*)((char*)beg + (pg->nfree - 1)*p->osize); @@ -732,6 +785,7 @@ static NOINLINE void add_page(jl_gc_pool_t *p) static inline jl_taggedvalue_t *__pool_alloc(jl_gc_pool_t *p, int osize, int end_offset) { + jl_tls_states_t *ptls = jl_get_ptls_states(); #ifdef MEMDEBUG return alloc_big(osize); #endif @@ -743,7 +797,7 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_gc_pool_t *p, int osize, //gc_num.allocd += osize; } else { - jl_gc_safepoint(); + jl_gc_safepoint_(ptls); } gc_num.poolalloc++; // first try to use the freelist @@ -986,8 +1040,8 @@ static void sweep_pool_region(jl_taggedvalue_t ***pfl, int region_i, int sweep_f jl_gc_pagemeta_t *pg = ®ion->meta[pg_i*32 + j]; int p_n = pg->pool_n; int t_n = pg->thread_n; - jl_tls_states_t *ptls = jl_all_tls_states[t_n]; - jl_gc_pool_t *p = &ptls->heap.norm_pools[p_n]; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_n]; + jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; int osize = pg->osize; pfl[t_n * JL_GC_N_POOLS + p_n] = sweep_page(p, pg, pfl[t_n * JL_GC_N_POOLS + p_n], sweep_full, osize); } @@ -1017,9 +1071,9 @@ static void gc_sweep_pool(int sweep_full) // update metadata of pages that were pointed to by freelist or newpages from a pool // i.e. pages being the current allocation target for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; for (int i = 0; i < JL_GC_N_POOLS; i++) { - jl_gc_pool_t *p = &ptls->heap.norm_pools[i]; + jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; jl_taggedvalue_t *last = p->freelist; if (last) { jl_gc_pagemeta_t *pg = page_metadata(last); @@ -1048,9 +1102,9 @@ static void gc_sweep_pool(int sweep_full) // null out terminal pointers of free lists and cache back pg->nfree in the jl_gc_pool_t for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; for (int i = 0; i < JL_GC_N_POOLS; i++) { - jl_gc_pool_t *p = &ptls->heap.norm_pools[i]; + jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; *pfl[t_i * JL_GC_N_POOLS + i] = NULL; if (p->freelist) { p->nfree = page_metadata(p->freelist)->nfree; @@ -1086,12 +1140,12 @@ static void grow_mark_stack(void) static void reset_remset(void) { for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; - arraylist_t *tmp = ptls->heap.remset; - ptls->heap.remset = ptls->heap.last_remset; - ptls->heap.last_remset = tmp; - ptls->heap.remset->len = 0; - ptls->heap.remset_nptr = 0; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + arraylist_t *tmp = ptls2->heap.remset; + ptls2->heap.remset = ptls2->heap.last_remset; + ptls2->heap.last_remset = tmp; + ptls2->heap.remset->len = 0; + ptls2->heap.remset_nptr = 0; } } @@ -1219,25 +1273,25 @@ static void gc_mark_task_stack(jl_task_t *ta, int d) { int stkbuf = (ta->stkbuf != (void*)(intptr_t)-1 && ta->stkbuf != NULL); int16_t tid = ta->tid; - jl_tls_states_t *ptls = jl_all_tls_states[tid]; + jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; if (stkbuf) { #ifdef COPY_STACKS gc_setmark_buf(ta->stkbuf, jl_astaggedvalue(ta)->bits.gc, ta->bufsz); #else // stkbuf isn't owned by julia for the root task - if (ta != ptls->root_task) { + if (ta != ptls2->root_task) { gc_setmark_buf(ta->stkbuf, jl_astaggedvalue(ta)->bits.gc, ta->ssize); } #endif } - if (ta == ptls->current_task) { - gc_mark_stack((jl_value_t*)ta, ptls->pgcstack, 0, d); + if (ta == ptls2->current_task) { + gc_mark_stack((jl_value_t*)ta, ptls2->pgcstack, 0, d); } else if (stkbuf) { intptr_t offset; #ifdef COPY_STACKS - offset = (char *)ta->stkbuf - ((char *)ptls->stackbase - ta->ssize); + offset = (char *)ta->stkbuf - ((char *)ptls2->stackbase - ta->ssize); #else offset = 0; #endif @@ -1260,12 +1314,16 @@ NOINLINE static void gc_mark_task(jl_task_t *ta, int d) void gc_mark_object_list(arraylist_t *list, size_t start) { - for (size_t i = start;i < list->len;i++) { - void *v = list->items[i]; + void **items = list->items; + size_t len = list->len; + for (size_t i = start;i < len;i++) { + void *v = items[i]; + if (__unlikely(!v)) + continue; if (gc_ptr_tag(v, 1)) { v = gc_ptr_clear_tag(v, 1); i++; - assert(i < list->len); + assert(i < len); } gc_push_root(v, 0); } @@ -1474,15 +1532,15 @@ void pre_mark(void) size_t i; for(i=0; i < jl_n_threads; i++) { - jl_tls_states_t *ptls = jl_all_tls_states[i]; + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; // current_module might not have a value when the thread is not // running. - if (ptls->current_module) - gc_push_root(ptls->current_module, 0); - gc_push_root(ptls->current_task, 0); - gc_push_root(ptls->root_task, 0); - gc_push_root(ptls->exception_in_transit, 0); - gc_push_root(ptls->task_arg_in_transit, 0); + if (ptls2->current_module) + gc_push_root(ptls2->current_module, 0); + gc_push_root(ptls2->current_task, 0); + gc_push_root(ptls2->root_task, 0); + gc_push_root(ptls2->exception_in_transit, 0); + gc_push_root(ptls2->task_arg_in_transit, 0); } // invisible builtin values @@ -1513,23 +1571,36 @@ void pre_mark(void) // this must happen last in the mark phase. static void post_mark(arraylist_t *list) { - for (size_t i=0; i < list->len; i+=2) { - void *v0 = list->items[i]; + void **items = list->items; + size_t len = list->len; + for (size_t i=0; i < len; i+=2) { + void *v0 = items[i]; int is_cptr = gc_ptr_tag(v0, 1); void *v = gc_ptr_clear_tag(v0, 1); - void *fin = list->items[i+1]; + if (__unlikely(!v0)) { + // remove from this list + if (i < len - 2) { + items[i] = items[len - 2]; + items[i + 1] = items[len - 1]; + i -= 2; + } + len -= 2; + continue; + } + + void *fin = items[i+1]; int isfreed = !gc_marked(jl_astaggedvalue(v)->bits.gc); int isold = (list != &finalizer_list_marked && jl_astaggedvalue(v)->bits.gc == GC_OLD_MARKED && (is_cptr || jl_astaggedvalue(fin)->bits.gc == GC_OLD_MARKED)); if (isfreed || isold) { // remove from this list - if (i < list->len - 2) { - list->items[i] = list->items[list->len-2]; - list->items[i+1] = list->items[list->len-1]; + if (i < len - 2) { + items[i] = items[len - 2]; + items[i + 1] = items[len - 1]; i -= 2; } - list->len -= 2; + len -= 2; } if (isfreed) { // schedule finalizer or execute right away if it is not julia code @@ -1546,6 +1617,7 @@ static void post_mark(arraylist_t *list) arraylist_push(&finalizer_list_marked, fin); } } + list->len = len; } // collector entry point and control @@ -1564,7 +1636,7 @@ JL_DLLEXPORT int jl_gc_enable(int on) // enable -> disable jl_atomic_fetch_add(&jl_gc_disable_counter, 1); // check if the GC is running and wait for it to finish - jl_gc_safepoint(); + jl_gc_safepoint_(ptls); } return prev; } @@ -1608,39 +1680,39 @@ static void _jl_gc_collect(int full, char *stack_hi) // 1. mark every object in the remset reset_remset(); for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; // avoid counting remembered objects & bindings twice in perm_scanned_bytes - for (int i = 0; i < ptls->heap.last_remset->len; i++) { - jl_value_t *item = (jl_value_t*)ptls->heap.last_remset->items[i]; + for (int i = 0; i < ptls2->heap.last_remset->len; i++) { + jl_value_t *item = (jl_value_t*)ptls2->heap.last_remset->items[i]; objprofile_count(jl_typeof(item), 2, 0); jl_astaggedvalue(item)->bits.gc = GC_OLD_MARKED; } - for (int i = 0; i < ptls->heap.rem_bindings.len; i++) { - void *ptr = ptls->heap.rem_bindings.items[i]; + for (int i = 0; i < ptls2->heap.rem_bindings.len; i++) { + void *ptr = ptls2->heap.rem_bindings.items[i]; jl_astaggedvalue(ptr)->bits.gc = GC_OLD_MARKED; } - for (int i = 0; i < ptls->heap.last_remset->len; i++) { - jl_value_t *item = (jl_value_t*)ptls->heap.last_remset->items[i]; + for (int i = 0; i < ptls2->heap.last_remset->len; i++) { + jl_value_t *item = (jl_value_t*)ptls2->heap.last_remset->items[i]; push_root(item, 0, GC_OLD_MARKED); } } // 2. mark every object in a remembered binding for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; int n_bnd_refyoung = 0; - for (int i = 0; i < ptls->heap.rem_bindings.len; i++) { - jl_binding_t *ptr = (jl_binding_t*)ptls->heap.rem_bindings.items[i]; + for (int i = 0; i < ptls2->heap.rem_bindings.len; i++) { + jl_binding_t *ptr = (jl_binding_t*)ptls2->heap.rem_bindings.items[i]; // A null pointer can happen here when the binding is cleaned up // as an exception is thrown after it was already queued (#10221) if (!ptr->value) continue; if (gc_push_root(ptr->value, 0)) { - ptls->heap.rem_bindings.items[n_bnd_refyoung] = ptr; + ptls2->heap.rem_bindings.items[n_bnd_refyoung] = ptr; n_bnd_refyoung++; } } - ptls->heap.rem_bindings.len = n_bnd_refyoung; + ptls2->heap.rem_bindings.len = n_bnd_refyoung; } // 3. walk roots @@ -1656,12 +1728,18 @@ static void _jl_gc_collect(int full, char *stack_hi) // mark the object moved to the marked list from the // `finalizer_list` by `post_mark` size_t orig_marked_len = finalizer_list_marked.len; - post_mark(&finalizer_list); + for (int i = 0;i < jl_n_threads;i++) { + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + post_mark(&ptls2->finalizers); + } if (prev_sweep_full) { post_mark(&finalizer_list_marked); orig_marked_len = 0; } - gc_mark_object_list(&finalizer_list, 0); + for (int i = 0;i < jl_n_threads;i++) { + jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + gc_mark_object_list(&ptls2->finalizers, 0); + } gc_mark_object_list(&finalizer_list_marked, orig_marked_len); // "Flush" the mark stack before flipping the reset_age bit // so that the objects are not incorrectly resetted. @@ -1734,19 +1812,19 @@ static void _jl_gc_collect(int full, char *stack_hi) // 6. if it is a quick sweep, put back the remembered objects in queued state // so that we don't trigger the barrier again on them. for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; if (!sweep_full) { - for (int i = 0; i < ptls->heap.remset->len; i++) { - jl_astaggedvalue(ptls->heap.remset->items[i])->bits.gc = GC_MARKED; + for (int i = 0; i < ptls2->heap.remset->len; i++) { + jl_astaggedvalue(ptls2->heap.remset->items[i])->bits.gc = GC_MARKED; } - for (int i = 0; i < ptls->heap.rem_bindings.len; i++) { - void *ptr = ptls->heap.rem_bindings.items[i]; + for (int i = 0; i < ptls2->heap.rem_bindings.len; i++) { + void *ptr = ptls2->heap.rem_bindings.items[i]; jl_astaggedvalue(ptr)->bits.gc = GC_MARKED; } } else { - ptls->heap.remset->len = 0; - ptls->heap.rem_bindings.len = 0; + ptls2->heap.remset->len = 0; + ptls2->heap.rem_bindings.len = 0; } } @@ -1777,13 +1855,13 @@ JL_DLLEXPORT void jl_gc_collect(int full) char *stack_hi = (char*)gc_get_stack_ptr(); gc_debug_print(); - int8_t old_state = jl_gc_state(); + int8_t old_state = jl_gc_state(ptls); ptls->gc_state = JL_GC_STATE_WAITING; // `jl_safepoint_start_gc()` makes sure only one thread can // run the GC. if (!jl_safepoint_start_gc()) { // Multithread only. See assertion in `safepoint.c` - jl_gc_state_set(old_state, JL_GC_STATE_WAITING); + jl_gc_state_set(ptls, old_state, JL_GC_STATE_WAITING); return; } // no-op for non-threading @@ -1797,7 +1875,7 @@ JL_DLLEXPORT void jl_gc_collect(int full) // no-op for non-threading jl_safepoint_end_gc(); - jl_gc_state_set(old_state, JL_GC_STATE_WAITING); + jl_gc_state_set(ptls, old_state, JL_GC_STATE_WAITING); // Only disable finalizers on current thread // Doing this on all threads is racy (it's impossible to check @@ -1869,8 +1947,9 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) } // Per-thread initialization (when threading is fully implemented) -void jl_mk_thread_heap(jl_thread_heap_t *heap) +void jl_mk_thread_heap(jl_tls_states_t *ptls) { + jl_thread_heap_t *heap = &ptls->heap; const int *szc = sizeclasses; jl_gc_pool_t *p = heap->norm_pools; for(int i=0; i < JL_GC_N_POOLS; i++) { @@ -1890,6 +1969,7 @@ void jl_mk_thread_heap(jl_thread_heap_t *heap) heap->last_remset = &heap->_remset[1]; arraylist_new(heap->remset, 0); arraylist_new(heap->last_remset, 0); + arraylist_new(&ptls->finalizers, 0); } // System-wide initializations @@ -1898,7 +1978,6 @@ void jl_gc_init(void) jl_gc_init_page(); gc_debug_init(); - arraylist_new(&finalizer_list, 0); arraylist_new(&finalizer_list_marked, 0); arraylist_new(&to_finalize, 0); diff --git a/src/gc.h b/src/gc.h index 57d518eb807dd..73421acd5a69c 100644 --- a/src/gc.h +++ b/src/gc.h @@ -178,7 +178,6 @@ typedef struct { extern jl_gc_num_t gc_num; extern region_t regions[REGION_COUNT]; extern bigval_t *big_objects_marked; -extern arraylist_t finalizer_list; extern arraylist_t finalizer_list_marked; extern arraylist_t to_finalize; extern int64_t lazy_freed_pages; diff --git a/src/jl_uv.c b/src/jl_uv.c index 0e862db4c77cc..e4beeafc1f178 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -136,9 +136,10 @@ JL_DLLEXPORT void *jl_uv_write_handle(uv_write_t *req) { return req->handle; } JL_DLLEXPORT int jl_run_once(uv_loop_t *loop) { + jl_tls_states_t *ptls = jl_get_ptls_states(); if (loop) { loop->stop_flag = 0; - jl_gc_safepoint(); + jl_gc_safepoint_(ptls); return uv_run(loop,UV_RUN_ONCE); } else return 0; @@ -146,18 +147,20 @@ JL_DLLEXPORT int jl_run_once(uv_loop_t *loop) JL_DLLEXPORT void jl_run_event_loop(uv_loop_t *loop) { + jl_tls_states_t *ptls = jl_get_ptls_states(); if (loop) { loop->stop_flag = 0; - jl_gc_safepoint(); + jl_gc_safepoint_(ptls); uv_run(loop,UV_RUN_DEFAULT); } } JL_DLLEXPORT int jl_process_events(uv_loop_t *loop) { + jl_tls_states_t *ptls = jl_get_ptls_states(); if (loop) { loop->stop_flag = 0; - jl_gc_safepoint(); + jl_gc_safepoint_(ptls); return uv_run(loop,UV_RUN_NOWAIT); } else return 0; diff --git a/src/jlapi.c b/src/jlapi.c index a169b8bc302f2..250726dbd70a7 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -326,27 +326,32 @@ JL_DLLEXPORT jl_value_t *(jl_typeof)(jl_value_t *v) JL_DLLEXPORT int8_t (jl_gc_unsafe_enter)(void) { - return jl_gc_unsafe_enter(); + jl_tls_states_t *ptls = jl_get_ptls_states(); + return jl_gc_unsafe_enter(ptls); } JL_DLLEXPORT void (jl_gc_unsafe_leave)(int8_t state) { - jl_gc_unsafe_leave(state); + jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_gc_unsafe_leave(ptls, state); } JL_DLLEXPORT int8_t (jl_gc_safe_enter)(void) { - return jl_gc_safe_enter(); + jl_tls_states_t *ptls = jl_get_ptls_states(); + return jl_gc_safe_enter(ptls); } JL_DLLEXPORT void (jl_gc_safe_leave)(int8_t state) { - jl_gc_safe_leave(state); + jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_gc_safe_leave(ptls, state); } JL_DLLEXPORT void (jl_gc_safepoint)(void) { - jl_gc_safepoint(); + jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_gc_safepoint_(ptls); } JL_DLLEXPORT void (jl_cpu_pause)(void) diff --git a/src/julia.h b/src/julia.h index 4604fbff2c0cc..7feae969021e4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1520,7 +1520,7 @@ STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh) ptls->gc_state = eh->gc_state; ptls->finalizers_inhibited = eh->finalizers_inhibited; if (old_gc_state && !eh->gc_state) { - jl_gc_safepoint(); + jl_gc_safepoint_(ptls); } if (old_defer_signal && !eh->defer_signal) { jl_sigint_safepoint(); diff --git a/src/julia_internal.h b/src/julia_internal.h index ec6ee07329704..b011e4abfd08f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -238,7 +238,7 @@ void jl_init_restored_modules(jl_array_t *init_order); void jl_init_signal_async(void); void jl_init_debuginfo(void); void jl_init_runtime_ccall(void); -void jl_mk_thread_heap(jl_thread_heap_t *heap); +void jl_mk_thread_heap(jl_tls_states_t *ptls); void _julia_init(JL_IMAGE_SEARCH rel); @@ -296,13 +296,13 @@ void jl_wake_libuv(void); jl_get_ptls_states_func jl_get_ptls_states_getter(void); static inline void jl_set_gc_and_wait(void) { + jl_tls_states_t *ptls = jl_get_ptls_states(); // reading own gc state doesn't need atomic ops since no one else // should store to it. - int8_t state = jl_gc_state(); - jl_atomic_store_release(&jl_get_ptls_states()->gc_state, - JL_GC_STATE_WAITING); + int8_t state = jl_gc_state(ptls); + jl_atomic_store_release(&ptls->gc_state, JL_GC_STATE_WAITING); jl_safepoint_wait_gc(); - jl_atomic_store_release(&jl_get_ptls_states()->gc_state, state); + jl_atomic_store_release(&ptls->gc_state, state); } #endif diff --git a/src/julia_threads.h b/src/julia_threads.h index 64c709d1aaada..61af77a7e7cb3 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -109,6 +109,7 @@ typedef struct _jl_tls_states_t { #endif // Counter to disable finalizer **on the current thread** int finalizers_inhibited; + arraylist_t finalizers; } jl_tls_states_t; #ifdef __MIC__ @@ -340,11 +341,11 @@ JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void); // This triggers a SegFault when we are in GC // Assign it to a variable to make sure the compiler emit the load // and to avoid Clang warning for -Wunused-volatile-lvalue -#define jl_gc_safepoint() do { \ - jl_signal_fence(); \ - size_t safepoint_load = *jl_get_ptls_states()->safepoint; \ - jl_signal_fence(); \ - (void)safepoint_load; \ +#define jl_gc_safepoint_(ptls) do { \ + jl_signal_fence(); \ + size_t safepoint_load = *ptls->safepoint; \ + jl_signal_fence(); \ + (void)safepoint_load; \ } while (0) #define jl_sigint_safepoint() do { \ jl_signal_fence(); \ @@ -355,9 +356,11 @@ JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void); #ifndef JULIA_ENABLE_THREADING extern JL_DLLEXPORT jl_tls_states_t jl_tls_states; #define jl_get_ptls_states() (&jl_tls_states) -#define jl_gc_state() ((int8_t)0) -STATIC_INLINE int8_t jl_gc_state_set(int8_t state, int8_t old_state) +#define jl_gc_state(ptls) ((int8_t)0) +STATIC_INLINE int8_t jl_gc_state_set(jl_tls_states_t *ptls, + int8_t state, int8_t old_state) { + (void)ptls; (void)state; return old_state; } @@ -365,25 +368,27 @@ STATIC_INLINE int8_t jl_gc_state_set(int8_t state, int8_t old_state) typedef jl_tls_states_t *(*jl_get_ptls_states_func)(void); JL_DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f); // Make sure jl_gc_state() is always a rvalue -#define jl_gc_state() ((int8_t)(jl_get_ptls_states()->gc_state)) -STATIC_INLINE int8_t jl_gc_state_set(int8_t state, int8_t old_state) +#define jl_gc_state(ptls) ((int8_t)ptls->gc_state) +STATIC_INLINE int8_t jl_gc_state_set(jl_tls_states_t *ptls, + int8_t state, int8_t old_state) { - jl_get_ptls_states()->gc_state = state; + ptls->gc_state = state; // A safe point is required if we transition from GC-safe region to // non GC-safe region. if (old_state && !state) - jl_gc_safepoint(); + jl_gc_safepoint_(ptls); return old_state; } #endif // ifndef JULIA_ENABLE_THREADING -STATIC_INLINE int8_t jl_gc_state_save_and_set(int8_t state) +STATIC_INLINE int8_t jl_gc_state_save_and_set(jl_tls_states_t *ptls, + int8_t state) { - return jl_gc_state_set(state, jl_gc_state()); + return jl_gc_state_set(ptls, state, jl_gc_state(ptls)); } -#define jl_gc_unsafe_enter() jl_gc_state_save_and_set(0) -#define jl_gc_unsafe_leave(state) ((void)jl_gc_state_set((state), 0)) -#define jl_gc_safe_enter() jl_gc_state_save_and_set(JL_GC_STATE_SAFE) -#define jl_gc_safe_leave(state) ((void)jl_gc_state_set((state), JL_GC_STATE_SAFE)) +#define jl_gc_unsafe_enter(ptls) jl_gc_state_save_and_set(ptls, 0) +#define jl_gc_unsafe_leave(ptls, state) ((void)jl_gc_state_set(ptls, (state), 0)) +#define jl_gc_safe_enter(ptls) jl_gc_state_save_and_set(ptls, JL_GC_STATE_SAFE) +#define jl_gc_safe_leave(ptls, state) ((void)jl_gc_state_set(ptls, (state), JL_GC_STATE_SAFE)) JL_DLLEXPORT void (jl_gc_safepoint)(void); #define JL_SIGATOMIC_BEGIN() do { \ @@ -428,8 +433,10 @@ static inline void jl_mutex_wait(jl_mutex_t *lock, int safepoint) lock->count = 1; return; } - if (safepoint) - jl_gc_safepoint(); + if (safepoint) { + jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_gc_safepoint_(ptls); + } jl_cpu_pause(); owner = lock->owner; } diff --git a/src/stackwalk.c b/src/stackwalk.c index 1cea12c1f1c3a..d9025e99903f3 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -350,10 +350,11 @@ size_t rec_backtrace_ctx_dwarf(uintptr_t *data, size_t maxsize, JL_DLLEXPORT jl_value_t *jl_lookup_code_address(void *ip, int skipC) { + jl_tls_states_t *ptls = jl_get_ptls_states(); jl_frame_t *frames = NULL; - int8_t gc_state = jl_gc_safe_enter(); + int8_t gc_state = jl_gc_safe_enter(ptls); int n = jl_getFunctionInfo(&frames, (uintptr_t)ip, skipC, 0); - jl_gc_safe_leave(gc_state); + jl_gc_safe_leave(ptls, gc_state); jl_value_t *rs = (jl_value_t*)jl_alloc_svec(n); JL_GC_PUSH1(&rs); for (int i = 0; i < n; i++) { diff --git a/src/support/arraylist.c b/src/support/arraylist.c index 25e26b481efc7..7b7229a63bd36 100644 --- a/src/support/arraylist.c +++ b/src/support/arraylist.c @@ -22,9 +22,9 @@ arraylist_t *arraylist_new(arraylist_t *a, size_t size) } else { a->items = (void**)LLT_ALLOC(size*sizeof(void*)); + if (a->items == NULL) return NULL; a->max = size; } - if (a->items == NULL) return NULL; return a; } @@ -39,31 +39,35 @@ void arraylist_free(arraylist_t *a) void arraylist_grow(arraylist_t *a, size_t n) { - if (a->len+n > a->max) { + size_t len = a->len; + size_t newlen = len + n; + if (newlen > a->max) { if (a->items == &a->_space[0]) { void **p = (void**)LLT_ALLOC((a->len+n)*sizeof(void*)); if (p == NULL) return; - memcpy(p, a->items, a->len*sizeof(void*)); + memcpy(p, a->items, len * sizeof(void*)); a->items = p; - a->max = a->len+n; + a->max = newlen; } else { - size_t nm = a->max*2; - if (nm == 0) nm = 1; - while (a->len+n > nm) nm*=2; - void **p = (void**)LLT_REALLOC(a->items, nm*sizeof(void*)); + size_t nm = a->max * 2; + if (nm == 0) + nm = 1; + while (newlen > nm) + nm *= 2; + void **p = (void**)LLT_REALLOC(a->items, nm * sizeof(void*)); if (p == NULL) return; a->items = p; a->max = nm; } } - a->len += n; + a->len = newlen; } void arraylist_push(arraylist_t *a, void *elt) { arraylist_grow(a, 1); - a->items[a->len-1] = elt; + a->items[a->len - 1] = elt; } void *arraylist_pop(arraylist_t *a) diff --git a/src/task.c b/src/task.c index c72c13a7f792f..4d8cc8617b87f 100644 --- a/src/task.c +++ b/src/task.c @@ -381,13 +381,13 @@ JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) if (in_pure_callback) jl_error("task switch not allowed from inside staged function"); sig_atomic_t defer_signal = ptls->defer_signal; - int8_t gc_state = jl_gc_unsafe_enter(); + int8_t gc_state = jl_gc_unsafe_enter(ptls); ptls->task_arg_in_transit = arg; ctx_switch(ptls, t, &t->ctx); jl_value_t *val = ptls->task_arg_in_transit; ptls->task_arg_in_transit = jl_nothing; throw_if_exception_set(ptls->current_task); - jl_gc_unsafe_leave(gc_state); + jl_gc_unsafe_leave(ptls, gc_state); sig_atomic_t other_defer_signal = ptls->defer_signal; ptls->defer_signal = defer_signal; if (other_defer_signal && !defer_signal) @@ -495,14 +495,15 @@ static void init_task(jl_task_t *t, char *stack) // yield to exception handler void JL_NORETURN throw_internal(jl_value_t *e) { - jl_get_ptls_states()->io_wait = 0; - if (jl_safe_restore) - jl_longjmp(*jl_safe_restore, 1); - jl_gc_unsafe_enter(); + jl_tls_states_t *ptls = jl_get_ptls_states(); + ptls->io_wait = 0; + if (ptls->safe_restore) + jl_longjmp(*ptls->safe_restore, 1); + jl_gc_unsafe_enter(ptls); assert(e != NULL); - jl_exception_in_transit = e; - if (jl_current_task->eh != NULL) { - jl_longjmp(jl_current_task->eh->eh_ctx, 1); + ptls->exception_in_transit = e; + if (ptls->current_task->eh != NULL) { + jl_longjmp(ptls->current_task->eh->eh_ctx, 1); } else { jl_printf(JL_STDERR, "fatal: error thrown and no exception handler available.\n"); diff --git a/src/threading.c b/src/threading.c index 8acf03b726047..3e9c8a4544db1 100644 --- a/src/threading.c +++ b/src/threading.c @@ -122,7 +122,7 @@ static void ti_initthread(int16_t tid) abort(); } ptls->bt_data = (uintptr_t*)bt_data; - jl_mk_thread_heap(&ptls->heap); + jl_mk_thread_heap(ptls); jl_install_thread_signal_handler(); jl_all_tls_states[tid] = ptls; @@ -178,6 +178,7 @@ static uv_barrier_t thread_init_done; // thread function: used by all except the main thread void ti_threadfun(void *arg) { + jl_tls_states_t *ptls = jl_get_ptls_states(); ti_threadarg_t *ta = (ti_threadarg_t *)arg; ti_threadgroup_t *tg; ti_threadwork_t *work; @@ -200,7 +201,7 @@ void ti_threadfun(void *arg) // critical region. In general, the following part of this function // shouldn't call any managed code without calling `jl_gc_unsafe_enter` // first. - jl_gc_state_set(JL_GC_STATE_SAFE, 0); + jl_gc_state_set(ptls, JL_GC_STATE_SAFE, 0); uv_barrier_wait(&thread_init_done); // initialize this thread in the thread group tg = ta->tg; @@ -232,7 +233,7 @@ void ti_threadfun(void *arg) // the work, and after we have proper GC transition // support in the codegen and runtime we don't need to // enter GC unsafe region when starting the work. - int8_t gc_state = jl_gc_unsafe_enter(); + int8_t gc_state = jl_gc_unsafe_enter(ptls); // This is probably always NULL for now jl_module_t *last_m = jl_current_module; JL_GC_PUSH1(&last_m); @@ -240,7 +241,7 @@ void ti_threadfun(void *arg) ti_run_fun(work->args); jl_current_module = last_m; JL_GC_POP(); - jl_gc_unsafe_leave(gc_state); + jl_gc_unsafe_leave(ptls, gc_state); } } @@ -384,6 +385,7 @@ JL_DLLEXPORT void *jl_threadgroup(void) { return (void *)tgworld; } // and run it in all threads JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) { + jl_tls_states_t *ptls = jl_get_ptls_states(); // GC safe #if PROFILE_JL_THREADING uint64_t tstart = uv_hrtime(); @@ -392,7 +394,7 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) jl_tupletype_t *argtypes = NULL; JL_TYPECHK(jl_threading_run, simplevector, (jl_value_t*)args); - int8_t gc_state = jl_gc_unsafe_enter(); + int8_t gc_state = jl_gc_unsafe_enter(ptls); JL_GC_PUSH1(&argtypes); argtypes = arg_type_tuple(jl_svec_data(args), jl_svec_len(args)); jl_compile_hint(argtypes); @@ -426,10 +428,10 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) user_ns[ti_tid] += (trun - tfork); #endif - jl_gc_state_set(JL_GC_STATE_SAFE, 0); + jl_gc_state_set(ptls, JL_GC_STATE_SAFE, 0); // wait for completion (TODO: nowait?) ti_threadgroup_join(tgworld, ti_tid); - jl_gc_state_set(0, JL_GC_STATE_SAFE); + jl_gc_state_set(ptls, 0, JL_GC_STATE_SAFE); #if PROFILE_JL_THREADING uint64_t tjoin = uv_hrtime(); @@ -437,7 +439,7 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) #endif JL_GC_POP(); - jl_gc_unsafe_leave(gc_state); + jl_gc_unsafe_leave(ptls, gc_state); return tw->ret; } From 302d9c5e1a4bd682757370394dd9c0cd4256cd8a Mon Sep 17 00:00:00 2001 From: Patrick Kofod Mogensen Date: Fri, 24 Jun 2016 16:23:48 +0200 Subject: [PATCH 0091/1117] Make concatenation of vectors of matrices return dense matrix. --- base/array.jl | 2 ++ test/abstractarray.jl | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/base/array.jl b/base/array.jl index 0c568126ed17a..ec23d42237891 100644 --- a/base/array.jl +++ b/base/array.jl @@ -731,6 +731,8 @@ hcat{T}(A::Union{Matrix{T}, Vector{T}}...) = typed_hcat(T, A...) vcat(A::Union{Matrix, Vector}...) = typed_vcat(promote_eltype(A...), A...) vcat{T}(A::Union{Matrix{T}, Vector{T}}...) = typed_vcat(T, A...) +hvcat(rows::Tuple{Vararg{Int}}, xs::Vector...) = typed_hvcat(promote_eltype(xs...), rows, xs...) +hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Vector{T}...) = typed_hvcat(T, rows, xs...) hvcat(rows::Tuple{Vararg{Int}}, xs::Matrix...) = typed_hvcat(promote_eltype(xs...), rows, xs...) hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Matrix{T}...) = typed_hvcat(T, rows, xs...) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 633c59d6e2ac3..877137c8c463a 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -612,3 +612,8 @@ A = TSlowNIndexes(rand(2,2)) @test @inferred(indices(rand(3,2), 1)) == 1:3 @test @inferred(indices(rand(3,2), 2)) == 1:2 @test @inferred(indices(rand(3,2), 3)) == 1:1 + +#17088 +let M = rand(10,10) + @test !issparse([[M] [M];]) +end From 92072ed5c86409b852dacb8a4c6b24b93faccc5c Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Fri, 24 Jun 2016 16:32:45 -0400 Subject: [PATCH 0092/1117] initialize underlying array in sharedarray at construction time --- base/sharedarray.jl | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/base/sharedarray.jl b/base/sharedarray.jl index 65d95ed7f2e8c..9e4b0a98de2d7 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -22,7 +22,9 @@ type SharedArray{T,N} <: DenseArray{T,N} # a subset of workers. loc_subarr_1d::SubArray{T,1,Array{T,1},Tuple{UnitRange{Int}},true} - SharedArray(d,p,r,sn) = new(d,p,r,sn) + function SharedArray(d,p,r,sn,s) + new(d,p,r,sn,s,0,view(Array{T}(ntuple(d->0,N)), 1:0)) + end end (::Type{SharedArray{T}}){T,N}(d::NTuple{N,Int}; kwargs...) = @@ -58,7 +60,7 @@ function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) pids, onlocalhost = shared_pids(pids) local shm_seg_name = "" - local s + local s = Array{T}(ntuple(d->0,N)) local S = nothing local shmmem_create_pid try @@ -97,8 +99,8 @@ function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) end systemerror("Error unlinking shmem segment " * shm_seg_name, rc != 0) end - S = SharedArray{T,N}(dims, pids, refs, shm_seg_name) - initialize_shared_array(S, s, onlocalhost, init, pids) + S = SharedArray{T,N}(dims, pids, refs, shm_seg_name, s) + initialize_shared_array(S, onlocalhost, init, pids) shm_seg_name = "" finally @@ -167,7 +169,7 @@ function SharedArray{T,N}(filename::AbstractString, ::Type{T}, dims::NTuple{N,In func_mmap = mode -> open(filename, mode) do io Mmap.mmap(io, Array{T,N}, dims, offset; shared=true) end - local s + s = Array{T}(ntuple(d->0,N)) if onlocalhost s = func_mmap(mode) refs[1] = remotecall(pids[1]) do @@ -191,17 +193,14 @@ function SharedArray{T,N}(filename::AbstractString, ::Type{T}, dims::NTuple{N,In wait(ref) end - S = SharedArray{T,N}(dims, pids, refs, filename) - initialize_shared_array(S, s, onlocalhost, init, pids) + S = SharedArray{T,N}(dims, pids, refs, filename, s) + initialize_shared_array(S, onlocalhost, init, pids) S end -function initialize_shared_array(S, s, onlocalhost, init, pids) +function initialize_shared_array(S, onlocalhost, init, pids) if onlocalhost init_loc_flds(S) - # In the event that myid() is not part of pids, s will not be set - # in the init function above, hence setting it here if available. - S.s = s else S.pidx = 0 end @@ -248,9 +247,8 @@ function reshape{T,N}(a::SharedArray{T}, dims::NTuple{N,Int}) end end - A = SharedArray{T,N}(dims, a.pids, refs, a.segname) + A = SharedArray{T,N}(dims, a.pids, refs, a.segname, reshape(a.s, dims)) init_loc_flds(A) - (a.pidx == 0) && isdefined(a, :s) && (A.s = reshape(a.s, dims)) A end From fa2ac9c35899517eca49e7a24152c509c269e472 Mon Sep 17 00:00:00 2001 From: Patrick Kofod Mogensen Date: Sat, 25 Jun 2016 11:52:02 +0200 Subject: [PATCH 0093/1117] Further tests. --- test/abstractarray.jl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 877137c8c463a..438e7b70e0274 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -614,6 +614,18 @@ A = TSlowNIndexes(rand(2,2)) @test @inferred(indices(rand(3,2), 3)) == 1:1 #17088 -let M = rand(10,10) - @test !issparse([[M] [M];]) +let + n = 10 + M = rand(n, n) + # vector of vectors + v = [[M]; [M]] # using vcat + @test size(v) == (2,) + @test !issparse(v) + # matrix of vectors + m1 = [[M] [M]] # using hcat + m2 = [[M] [M];] # using hvcat + @test m1 == m2 + @test size(m1) == (1,2) + @test !issparse(m1) + @test !issparse(m2) end From 7239c95ccf80e971d616cfc6a932c82cd57c1801 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 25 Jun 2016 06:19:29 -0500 Subject: [PATCH 0094/1117] Make the SharedArray constructor inferrable. Fixes #17101 --- base/sharedarray.jl | 5 ++--- test/parallel_exec.jl | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/base/sharedarray.jl b/base/sharedarray.jl index 9e4b0a98de2d7..dca0bf88c0e0e 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -52,8 +52,7 @@ computation with the master process acting as a driver. If an `init` function of the type `initfn(S::SharedArray)` is specified, it is called on all the participating workers. """ -function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) - N = length(dims) +function SharedArray{T,N}(::Type{T}, dims::Dims{N}; init=false, pids=Int[]) isbits(T) || throw(ArgumentError("type of SharedArray elements must be bits types, got $(T)")) @@ -61,7 +60,7 @@ function SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) local shm_seg_name = "" local s = Array{T}(ntuple(d->0,N)) - local S = nothing + local S local shmmem_create_pid try # On OSX, the shm_seg_name length must be <= 31 characters (including the terminating NULL character) diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index ecb89640b32ed..4ec27ae662cd8 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -294,7 +294,7 @@ copy!(s, sdata(d)) a = rand(dims) @test sdata(a) == a -d = SharedArray(Int, dims; init = D->fill!(D.loc_subarr_1d, myid())) +d = SharedArray(Int, dims, init = D->fill!(D.loc_subarr_1d, myid())) for p in procs(d) idxes_in_p = remotecall_fetch(p, d) do D parentindexes(D.loc_subarr_1d)[1] @@ -305,7 +305,7 @@ for p in procs(d) @test d[idxl] == p end -d = SharedArray(Float64, (2,3)) +d = @inferred(SharedArray(Float64, (2,3))) @test isa(d[:,2], Vector{Float64}) ### SharedArrays from a file @@ -316,7 +316,7 @@ write(fn, 1:30) sz = (6,5) Atrue = reshape(1:30, sz) -S = SharedArray(fn, Int, sz) +S = @inferred(SharedArray(fn, Int, sz)) @test S == Atrue @test length(procs(S)) > 1 @sync begin @@ -370,16 +370,16 @@ rm(fn); rm(fn2); rm(fn3) ### Utility functions # construct PR #13514 -S = SharedArray{Int}((1,2,3)) +S = @inferred(SharedArray{Int}((1,2,3))) @test size(S) == (1,2,3) @test typeof(S) <: SharedArray{Int} -S = SharedArray{Int}(2) +S = @inferred(SharedArray{Int}(2)) @test size(S) == (2,) @test typeof(S) <: SharedArray{Int} -S = SharedArray{Int}(1,2) +S = @inferred(SharedArray{Int}(1,2)) @test size(S) == (1,2) @test typeof(S) <: SharedArray{Int} -S = SharedArray{Int}(1,2,3) +S = @inferred(SharedArray{Int}(1,2,3)) @test size(S) == (1,2,3) @test typeof(S) <: SharedArray{Int} @@ -430,8 +430,8 @@ d[2:4] = 7 d[5,1:2:4,8] = 19 AA = rand(4,2) -A = convert(SharedArray, AA) -B = convert(SharedArray, AA') +A = @inferred(convert(SharedArray, AA)) +B = @inferred(convert(SharedArray, AA')) @test B*A == ctranspose(AA)*AA d=SharedArray(Int64, (10,10); init = D->fill!(D.loc_subarr_1d, myid()), pids=[id_me, id_other]) @@ -455,7 +455,7 @@ map!(x->1, d) # Shared arrays of singleton immutables @everywhere immutable ShmemFoo end for T in [Void, ShmemFoo] - s = SharedArray(T, 10) + s = @inferred(SharedArray(T, 10)) @test T() === remotecall_fetch(x->x[3], workers()[1], s) end From f4fc1a2f5b8149a3e0dbd4293abe2c5f1e855750 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 24 Jun 2016 08:02:39 -0400 Subject: [PATCH 0095/1117] Optimize allocation in generated code. * Passing TLS pointer explicitly to the GC * Inline pool offset and metadata for all pool sizes instead of only few special cases. --- src/ccall.cpp | 2 +- src/cgutils.cpp | 44 +++++++++++++---------------- src/codegen.cpp | 58 ++++++++++++++------------------------ src/gc-debug.c | 18 ++++++------ src/gc.c | 66 +++++++++++++++++++++++++++++++++----------- src/init.c | 12 ++++---- src/intrinsics.cpp | 6 ++-- src/julia_internal.h | 5 ++++ src/signals-mach.c | 58 +++++++++++++++++++------------------- src/signals-unix.c | 20 +++++++------- src/signals-win.c | 4 +-- 11 files changed, 154 insertions(+), 139 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 95d6fc5c06c62..d63fdec235c8e 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -854,7 +854,7 @@ static jl_cgval_t mark_or_box_ccall_result(Value *result, bool isboxed, jl_value int nb = sizeof(void*); // TODO: can this be tighter than tbaa_value? return mark_julia_type( - init_bits_value(emit_allocobj(nb), runtime_bt, result, tbaa_value), + init_bits_value(emit_allocobj(ctx, nb), runtime_bt, result, tbaa_value), true, (jl_value_t*)jl_pointer_type, ctx); } return mark_julia_type(result, isboxed, rt, ctx); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 600d40ce3b4ab..221cb0ce3d11f 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1345,7 +1345,7 @@ static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, size_ // --- boxing --- -static Value *emit_allocobj(size_t static_size); +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size); static void init_tag(Value *v, Value *jt) { tbaa_decorate(tbaa_tag, builder.CreateStore(jt, emit_typeptr_addr(v))); @@ -1532,7 +1532,7 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) return literal_pointer_val(jb->instance); } else { - box = init_bits_cgval(emit_allocobj(jl_datatype_size(jt)), vinfo, jb->mutabl ? tbaa_mutab : tbaa_immut, ctx); + box = init_bits_cgval(emit_allocobj(ctx, jl_datatype_size(jt)), vinfo, jb->mutabl ? tbaa_mutab : tbaa_immut, ctx); } if (gcrooted) { @@ -1566,29 +1566,23 @@ static void emit_cpointercheck(const jl_cgval_t &x, const std::string &msg, jl_c } // allocation for known size object -static Value *emit_allocobj(size_t static_size) +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size) { - if (static_size == sizeof(void*)*1) - return builder.CreateCall(prepare_call(jlalloc1w_func) -#ifdef LLVM37 - , {} -#endif - ); - else if (static_size == sizeof(void*)*2) - return builder.CreateCall(prepare_call(jlalloc2w_func) -#ifdef LLVM37 - , {} -#endif - ); - else if (static_size == sizeof(void*)*3) - return builder.CreateCall(prepare_call(jlalloc3w_func) -#ifdef LLVM37 - , {} -#endif - ); - else - return builder.CreateCall(prepare_call(jlallocobj_func), - ConstantInt::get(T_size, static_size)); + int osize; + int end_offset; + int offset = jl_gc_classify_pools(static_size, &osize, &end_offset); + Value *ptls_ptr = builder.CreateBitCast(ctx->ptlsStates, T_pint8); + if (offset < 0) { + Value *args[] = {ptls_ptr, + ConstantInt::get(T_size, static_size + sizeof(void*))}; + return builder.CreateCall(prepare_call(jlalloc_big_func), + ArrayRef(args, 2)); + } + Value *pool_ptr = builder.CreateConstGEP1_32(ptls_ptr, offset); + Value *args[] = {ptls_ptr, pool_ptr, ConstantInt::get(T_int32, osize), + ConstantInt::get(T_int32, end_offset)}; + return builder.CreateCall(prepare_call(jlalloc_pool_func), + ArrayRef(args, 4)); } // if ptr is NULL this emits a write barrier _back_ @@ -1739,7 +1733,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg f1 = boxed(fval_info, ctx); j++; } - Value *strct = emit_allocobj(sty->size); + Value *strct = emit_allocobj(ctx, sty->size); jl_cgval_t strctinfo = mark_julia_type(strct, true, ty, ctx); tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)ty), emit_typeptr_addr(strct))); diff --git a/src/codegen.cpp b/src/codegen.cpp index 3b9c0d0ffc854..f6ad5461e14c9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -378,11 +378,8 @@ static Function *jlgenericfunction_func; static Function *jlenter_func; static Function *jlleave_func; static Function *jlegal_func; -static Function *jlallocobj_func; -static Function *jlalloc1w_func; -static Function *jlalloc2w_func; -static Function *jlalloc3w_func; -static Function *jl_alloc_svec_func; +static Function *jlalloc_pool_func; +static Function *jlalloc_big_func; static Function *jlsubtype_func; static Function *setjmp_func; static Function *memcmp_func; @@ -3659,7 +3656,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t (void)julia_type_to_llvm(jargty, &isboxed); if (isboxed) { // passed an unboxed T, but want something boxed - Value *mem = emit_allocobj(jl_datatype_size(jargty)); + Value *mem = emit_allocobj(&ctx, jl_datatype_size(jargty)); tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)jargty), emit_typeptr_addr(mem))); tbaa_decorate(jl_is_mutable(jargty) ? tbaa_mutab : tbaa_immut, @@ -5465,40 +5462,25 @@ static void init_julia_llvm_env(Module *m) "jl_subtype", m); add_named_global(jlsubtype_func, &jl_subtype); - std::vector aoargs(0); - aoargs.push_back(T_size); - jlallocobj_func = - Function::Create(FunctionType::get(T_pjlvalue, aoargs, false), + std::vector alloc_pool_args(0); + alloc_pool_args.push_back(T_pint8); + alloc_pool_args.push_back(T_pint8); + alloc_pool_args.push_back(T_int32); + alloc_pool_args.push_back(T_int32); + jlalloc_pool_func = + Function::Create(FunctionType::get(T_pjlvalue, alloc_pool_args, false), Function::ExternalLinkage, - "jl_gc_allocobj", m); - add_named_global(jlallocobj_func, &jl_gc_allocobj); - - std::vector empty_args(0); - jlalloc1w_func = - Function::Create(FunctionType::get(T_pjlvalue, empty_args, false), - Function::ExternalLinkage, - "jl_gc_alloc_1w", m); - add_named_global(jlalloc1w_func, &jl_gc_alloc_1w); - - jlalloc2w_func = - Function::Create(FunctionType::get(T_pjlvalue, empty_args, false), - Function::ExternalLinkage, - "jl_gc_alloc_2w", m); - add_named_global(jlalloc2w_func, &jl_gc_alloc_2w); - - jlalloc3w_func = - Function::Create(FunctionType::get(T_pjlvalue, empty_args, false), - Function::ExternalLinkage, - "jl_gc_alloc_3w", m); - add_named_global(jlalloc3w_func, &jl_gc_alloc_3w); - - std::vector atargs(0); - atargs.push_back(T_size); - jl_alloc_svec_func = - Function::Create(FunctionType::get(T_pjlvalue, atargs, false), + "jl_gc_pool_alloc", m); + add_named_global(jlalloc_pool_func, &jl_gc_pool_alloc); + + std::vector alloc_big_args(0); + alloc_big_args.push_back(T_pint8); + alloc_big_args.push_back(T_size); + jlalloc_big_func = + Function::Create(FunctionType::get(T_pjlvalue, alloc_big_args, false), Function::ExternalLinkage, - "jl_alloc_svec", m); - add_named_global(jl_alloc_svec_func, &jl_alloc_svec); + "jl_gc_big_alloc", m); + add_named_global(jlalloc_big_func, &jl_gc_big_alloc); std::vector dlsym_args(0); dlsym_args.push_back(T_pint8); diff --git a/src/gc-debug.c b/src/gc-debug.c index bb7d63dfe54c9..91670730e66b4 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -139,8 +139,8 @@ static void clear_mark(int bits) for (int j = 0; j < 32; j++) { if ((line >> j) & 1) { jl_gc_pagemeta_t *pg = page_metadata(region->pages[pg_i*32 + j].data + GC_PAGE_OFFSET); - jl_tls_states_t *ptls = jl_all_tls_states[pg->thread_n]; - jl_gc_pool_t *pool = &ptls->heap.norm_pools[pg->pool_n]; + jl_tls_states_t *ptls2 = jl_all_tls_states[pg->thread_n]; + jl_gc_pool_t *pool = &ptls2->heap.norm_pools[pg->pool_n]; pv = (jl_taggedvalue_t*)(pg->data + GC_PAGE_OFFSET); char *lim = (char*)pv + GC_PAGE_SZ - GC_PAGE_OFFSET - pool->osize; while ((char*)pv <= lim) { @@ -660,9 +660,9 @@ void gc_time_mark_pause(int64_t t0, int64_t scanned_bytes, int64_t last_remset_len = 0; int64_t remset_nptr = 0; for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; - last_remset_len += ptls->heap.last_remset->len; - remset_nptr = ptls->heap.remset_nptr; + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + last_remset_len += ptls2->heap.last_remset->len; + remset_nptr = ptls2->heap.remset_nptr; } jl_printf(JL_STDOUT, "GC mark pause %.2f ms | " "scanned %" PRId64 " kB = %" PRId64 " + %" PRId64 " | " @@ -779,14 +779,14 @@ void gc_stats_all_pool(void) size_t nb=0, w, tw=0, no=0,tp=0, nold=0,noldbytes=0, np, nol; for (int i = 0; i < JL_GC_N_POOLS; i++) { for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls = jl_all_tls_states[t_i]; - size_t b = pool_stats(&ptls->heap.norm_pools[i], &w, &np, &nol); + jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + size_t b = pool_stats(&ptls2->heap.norm_pools[i], &w, &np, &nol); nb += b; - no += (b / ptls->heap.norm_pools[i].osize); + no += (b / ptls2->heap.norm_pools[i].osize); tw += w; tp += np; nold += nol; - noldbytes += nol * ptls->heap.norm_pools[i].osize; + noldbytes += nol * ptls2->heap.norm_pools[i].osize; } } jl_printf(JL_STDOUT, diff --git a/src/gc.c b/src/gc.c index 48ed64850aedb..5775435a254b2 100644 --- a/src/gc.c +++ b/src/gc.c @@ -782,10 +782,10 @@ static NOINLINE void add_page(jl_gc_pool_t *p) p->newpages = fl; } -static inline jl_taggedvalue_t *__pool_alloc(jl_gc_pool_t *p, int osize, +static inline jl_taggedvalue_t *__pool_alloc(jl_tls_states_t *ptls, + jl_gc_pool_t *p, int osize, int end_offset) { - jl_tls_states_t *ptls = jl_get_ptls_states(); #ifdef MEMDEBUG return alloc_big(osize); #endif @@ -844,14 +844,29 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_gc_pool_t *p, int osize, // use this variant when osize is statically known // and is definitely in sizeclasses // GC_POOL_END_OFS uses an integer division -static inline jl_taggedvalue_t *_pool_alloc(jl_gc_pool_t *p, int osize) +static inline jl_taggedvalue_t *_pool_alloc(jl_tls_states_t *ptls, + jl_gc_pool_t *p, int osize) +{ + return __pool_alloc(ptls, p, osize, GC_POOL_END_OFS(osize)); +} + +static inline jl_taggedvalue_t *pool_alloc(jl_tls_states_t *ptls, + jl_gc_pool_t *p) +{ + return __pool_alloc(ptls, p, p->osize, p->end_offset); +} + +// Size includes the tag!! +JL_DLLEXPORT void *jl_gc_pool_alloc(jl_tls_states_t *ptls, jl_gc_pool_t *p, + int osize, int end_offset) { - return __pool_alloc(p, osize, GC_POOL_END_OFS(osize)); + return jl_valueof(__pool_alloc(ptls, p, osize, end_offset)); } -static inline jl_taggedvalue_t *pool_alloc(jl_gc_pool_t *p) +// Size includes the tag!! +JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_tls_states_t *ptls, size_t allocsz) { - return __pool_alloc(p, p->osize, p->end_offset); + return jl_valueof(alloc_big(allocsz)); } // pools are 16376 bytes large (GC_POOL_SZ - GC_PAGE_OFFSET) @@ -885,7 +900,6 @@ static const int sizeclasses[JL_GC_N_POOLS] = { // 64, 32, 160, 64, 16, 64, 112, 128, bytes lost }; - static inline int szclass(size_t sz) { #ifdef _P64 @@ -907,6 +921,17 @@ static inline int szclass(size_t sz) return 16 - 16376 / 1 / LLT_ALIGN(sz, 16 * 1) + 32 + N; } +int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset) +{ + if (sz > GC_MAX_SZCLASS) + return -1; + size_t allocsz = sz + sizeof(jl_taggedvalue_t); + int klass = szclass(allocsz); + *osize = sizeclasses[klass]; + *end_offset = GC_POOL_END_OFS(*osize); + return (int)(intptr_t)(&((jl_tls_states_t*)0)->heap.norm_pools[klass]); +} + // sweep phase int64_t lazy_freed_pages = 0; @@ -1892,6 +1917,7 @@ JL_DLLEXPORT void jl_gc_collect(int full) void *allocb(size_t sz) { + jl_tls_states_t *ptls = jl_get_ptls_states(); jl_taggedvalue_t *b = NULL; size_t allocsz = sz + sizeof(jl_taggedvalue_t); if (allocsz < sz) // overflow in adding offs, size was "negative" @@ -1901,7 +1927,7 @@ void *allocb(size_t sz) b->header = jl_buff_tag; } else { - b = pool_alloc(&jl_thread_heap.norm_pools[szclass(allocsz)]); + b = pool_alloc(ptls, &ptls->heap.norm_pools[szclass(allocsz)]); b->header = jl_buff_tag; } return jl_valueof(b); @@ -1909,40 +1935,48 @@ void *allocb(size_t sz) JL_DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) { + jl_tls_states_t *ptls = jl_get_ptls_states(); size_t allocsz = sz + sizeof(jl_taggedvalue_t); if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); if (allocsz <= GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { - return jl_valueof(pool_alloc(&jl_thread_heap.norm_pools[szclass(allocsz)])); + return jl_valueof(pool_alloc(ptls, &ptls->heap.norm_pools[szclass(allocsz)])); } return jl_valueof(alloc_big(allocsz)); } JL_DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) { + jl_tls_states_t *ptls = jl_get_ptls_states(); const int sz = sizeof(jl_taggedvalue_t); - void *tag = _pool_alloc(&jl_thread_heap.norm_pools[szclass(sz)], sz); + void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); return jl_valueof(tag); } JL_DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) { - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(&jl_thread_heap.norm_pools[szclass(sz)], sz); + jl_tls_states_t *ptls = jl_get_ptls_states(); + const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*), + JL_SMALL_BYTE_ALIGNMENT); + void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); return jl_valueof(tag); } JL_DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) { - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 2, JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(&jl_thread_heap.norm_pools[szclass(sz)], sz); + jl_tls_states_t *ptls = jl_get_ptls_states(); + const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 2, + JL_SMALL_BYTE_ALIGNMENT); + void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); return jl_valueof(tag); } JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) { - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 3, JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(&jl_thread_heap.norm_pools[szclass(sz)], sz); + jl_tls_states_t *ptls = jl_get_ptls_states(); + const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 3, + JL_SMALL_BYTE_ALIGNMENT); + void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); return jl_valueof(tag); } diff --git a/src/init.c b/src/init.c index 9db6f36df98bc..06df5fb4112e1 100644 --- a/src/init.c +++ b/src/init.c @@ -798,12 +798,12 @@ void jl_get_builtin_hooks(void) { int t; for (t = 0; t < jl_n_threads; t++) { - jl_tls_states_t *ptls = jl_all_tls_states[t]; - ptls->root_task->tls = jl_nothing; - ptls->root_task->consumers = jl_nothing; - ptls->root_task->donenotify = jl_nothing; - ptls->root_task->exception = jl_nothing; - ptls->root_task->result = jl_nothing; + jl_tls_states_t *ptls2 = jl_all_tls_states[t]; + ptls2->root_task->tls = jl_nothing; + ptls2->root_task->consumers = jl_nothing; + ptls2->root_task->donenotify = jl_nothing; + ptls2->root_task->exception = jl_nothing; + ptls2->root_task->result = jl_nothing; } jl_char_type = (jl_datatype_t*)core("Char"); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index a16a774c2373f..a7c9af2a7b84b 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -509,7 +509,7 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx return mark_julia_type(vx, false, bt, ctx); else return mark_julia_type( - init_bits_value(emit_allocobj(nb), boxed(bt_value, ctx), vx, tbaa_immut), + init_bits_value(emit_allocobj(ctx, nb), boxed(bt_value, ctx), vx, tbaa_immut), true, bt, ctx); } @@ -550,7 +550,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c Value *runtime_bt = boxed(bt_value, ctx); // XXX: emit type validity check on runtime_bt (bitstype of size nb) - Value *newobj = emit_allocobj(nb); + Value *newobj = emit_allocobj(ctx, nb); tbaa_decorate(tbaa_tag, builder.CreateStore(runtime_bt, emit_typeptr_addr(newobj))); if (!v.ispointer()) { tbaa_decorate(tbaa_value, builder.CreateAlignedStore(emit_unbox(llvmt, v, v.typ), builder.CreatePointerCast(newobj, llvmt->getPointerTo()), alignment)); @@ -774,7 +774,7 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct } assert(jl_is_datatype(ety)); uint64_t size = jl_datatype_size(ety); - Value *strct = emit_allocobj(size); + Value *strct = emit_allocobj(ctx, size); tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)ety), emit_typeptr_addr(strct))); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, diff --git a/src/julia_internal.h b/src/julia_internal.h index b011e4abfd08f..2edfe782d1ad4 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -43,6 +43,11 @@ extern unsigned sig_stack_size; JL_DLLEXPORT extern int jl_lineno; JL_DLLEXPORT extern const char *jl_filename; +JL_DLLEXPORT void *jl_gc_pool_alloc(jl_tls_states_t *ptls, jl_gc_pool_t *p, + int osize, int end_offset); +JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_tls_states_t *ptls, size_t allocsz); +int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset); + STATIC_INLINE jl_value_t *newobj(jl_value_t *type, size_t nfields) { jl_value_t *jv = NULL; diff --git a/src/signals-mach.c b/src/signals-mach.c index e8291fbd54c4c..1f5a9950f4357 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -27,16 +27,16 @@ void jl_mach_gc_end(void) uintptr_t item = (uintptr_t)suspended_threads.items[i]; int16_t tid = (int16_t)item; int8_t gc_state = (int8_t)(item >> 8); - jl_tls_states_t *ptls = jl_all_tls_states[tid]; - jl_atomic_store_release(&ptls->gc_state, gc_state); - thread_resume(pthread_mach_thread_np(ptls->system_id)); + jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_atomic_store_release(&ptls2->gc_state, gc_state); + thread_resume(pthread_mach_thread_np(ptls2->system_id)); } suspended_threads.len = 0; } // Suspend the thread and return `1` if the GC is running. // Otherwise return `0` -static int jl_mach_gc_wait(jl_tls_states_t *ptls, +static int jl_mach_gc_wait(jl_tls_states_t *ptls2, mach_port_t thread, int16_t tid) { jl_mutex_lock_nogc(&safepoint_lock); @@ -47,8 +47,8 @@ static int jl_mach_gc_wait(jl_tls_states_t *ptls, return 0; } // Otherwise, set the gc state of the thread, suspend and record it - int8_t gc_state = ptls->gc_state; - jl_atomic_store_release(&ptls->gc_state, JL_GC_STATE_WAITING); + int8_t gc_state = ptls2->gc_state; + jl_atomic_store_release(&ptls2->gc_state, JL_GC_STATE_WAITING); uintptr_t item = tid | (((uintptr_t)gc_state) << 16); arraylist_push(&suspended_threads, (void*)item); thread_suspend(thread); @@ -126,13 +126,13 @@ void jl_throw_in_thread(int tid, mach_port_t thread, jl_value_t *exception) x86_thread_state64_t state; kern_return_t ret = thread_get_state(thread, x86_THREAD_STATE64, (thread_state_t)&state, &count); HANDLE_MACH_ERROR("thread_get_state", ret); - jl_tls_states_t *ptls = jl_all_tls_states[tid]; + jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; - ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, - (bt_context_t*)&state); - ptls->exception_in_transit = exception; + ptls2->bt_size = rec_backtrace_ctx(ptls2->bt_data, JL_MAX_BT_SIZE, + (bt_context_t*)&state); + ptls2->exception_in_transit = exception; - uint64_t rsp = (uint64_t)ptls->signal_stack + sig_stack_size; + uint64_t rsp = (uint64_t)ptls2->signal_stack + sig_stack_size; rsp &= -16; // ensure 16-byte alignment // push (null) $RIP onto the stack @@ -166,15 +166,15 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, #endif int16_t tid; #ifdef JULIA_ENABLE_THREADING - jl_tls_states_t *ptls = NULL; + jl_tls_states_t *ptls2 = NULL; for (tid = 0;tid < jl_n_threads;tid++) { - jl_tls_states_t *_ptls = jl_all_tls_states[tid]; - if (pthread_mach_thread_np(_ptls->system_id) == thread) { - ptls = _ptls; + jl_tls_states_t *_ptls2 = jl_all_tls_states[tid]; + if (pthread_mach_thread_np(_ptls2->system_id) == thread) { + ptls2 = _ptls2; break; } } - if (!ptls) { + if (!ptls2) { // We don't know about this thread, let the kernel try another handler // instead. This shouldn't actually happen since we only register the // handler for the threads we know about. @@ -182,7 +182,7 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, return KERN_INVALID_ARGUMENT; } #else - jl_tls_states_t *ptls = &jl_tls_states; + jl_tls_states_t *ptls2 = &jl_tls_states; tid = 0; #endif kern_return_t ret = thread_get_state(thread, x86_EXCEPTION_STATE64, (thread_state_t)&exc_state, &exc_count); @@ -190,12 +190,12 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, uint64_t fault_addr = exc_state.__faultvaddr; if (jl_addr_is_safepoint(fault_addr)) { #ifdef JULIA_ENABLE_THREADING - if (jl_mach_gc_wait(ptls, thread, tid)) + if (jl_mach_gc_wait(ptls2, thread, tid)) return KERN_SUCCESS; - if (ptls->tid != 0) + if (ptls2->tid != 0) return KERN_SUCCESS; #endif - if (ptls->defer_signal) { + if (ptls2->defer_signal) { jl_safepoint_defer_sigint(); } else if (jl_safepoint_consume_sigint()) { @@ -210,7 +210,7 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, if (msync((void*)(fault_addr & ~(jl_page_size - 1)), 1, MS_ASYNC) == 0) { // check if this was a valid address #endif jl_value_t *excpt; - if (is_addr_on_stack(ptls, (void*)fault_addr)) { + if (is_addr_on_stack(ptls2, (void*)fault_addr)) { excpt = jl_stackovf_exception; } #ifdef SEGV_EXCEPTION @@ -232,7 +232,7 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, kern_return_t ret = thread_get_state(thread, x86_THREAD_STATE64, (thread_state_t)&state, &count); HANDLE_MACH_ERROR("thread_get_state", ret); jl_critical_error(SIGSEGV, (unw_context_t*)&state, - ptls->bt_data, &ptls->bt_size); + ptls2->bt_data, &ptls2->bt_size); return KERN_INVALID_ARGUMENT; } } @@ -247,8 +247,8 @@ static void attach_exception_port(thread_port_t thread) static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) { - jl_tls_states_t *ptls = jl_all_tls_states[tid]; - mach_port_t tid_port = pthread_mach_thread_np(ptls->system_id); + jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + mach_port_t tid_port = pthread_mach_thread_np(ptls2->system_id); kern_return_t ret = thread_suspend(tid_port); HANDLE_MACH_ERROR("thread_suspend", ret); @@ -267,8 +267,8 @@ static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) static void jl_thread_resume(int tid, int sig) { - jl_tls_states_t *ptls = jl_all_tls_states[tid]; - mach_port_t thread = pthread_mach_thread_np(ptls->system_id); + jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + mach_port_t thread = pthread_mach_thread_np(ptls2->system_id); kern_return_t ret = thread_resume(thread); HANDLE_MACH_ERROR("thread_resume", ret); } @@ -277,8 +277,8 @@ static void jl_thread_resume(int tid, int sig) // or if SIGINT happens too often. static void jl_try_deliver_sigint(void) { - jl_tls_states_t *ptls = jl_all_tls_states[0]; - mach_port_t thread = pthread_mach_thread_np(ptls->system_id); + jl_tls_states_t *ptls2 = jl_all_tls_states[0]; + mach_port_t thread = pthread_mach_thread_np(ptls2->system_id); kern_return_t ret = thread_suspend(thread); HANDLE_MACH_ERROR("thread_suspend", ret); @@ -289,7 +289,7 @@ static void jl_try_deliver_sigint(void) jl_safepoint_enable_sigint(); int force = jl_check_force_sigint(); - if (force || (!ptls->defer_signal && ptls->io_wait)) { + if (force || (!ptls2->defer_signal && ptls2->io_wait)) { jl_safepoint_consume_sigint(); if (force) jl_safe_printf("WARNING: Force throwing a SIGINT\n"); diff --git a/src/signals-unix.c b/src/signals-unix.c index d084d1e22a668..fd66d6334f985 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -164,22 +164,22 @@ static pthread_cond_t signal_caught_cond; static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) { pthread_mutex_lock(&in_signal_lock); - jl_tls_states_t *ptls = jl_all_tls_states[tid]; - jl_atomic_store_release(&ptls->signal_request, 1); - pthread_kill(ptls->system_id, SIGUSR2); + jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_atomic_store_release(&ptls2->signal_request, 1); + pthread_kill(ptls2->system_id, SIGUSR2); pthread_cond_wait(&signal_caught_cond, &in_signal_lock); // wait for thread to acknowledge - assert(jl_atomic_load_acquire(&ptls->signal_request) == 0); + assert(jl_atomic_load_acquire(&ptls2->signal_request) == 0); *ctx = signal_context; } static void jl_thread_resume(int tid, int sig) { (void)sig; - jl_tls_states_t *ptls = jl_all_tls_states[tid]; - jl_atomic_store_release(&ptls->signal_request, 1); + jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_atomic_store_release(&ptls2->signal_request, 1); pthread_cond_broadcast(&exit_signal_cond); pthread_cond_wait(&signal_caught_cond, &in_signal_lock); // wait for thread to acknowledge - assert(jl_atomic_load_acquire(&ptls->signal_request) == 0); + assert(jl_atomic_load_acquire(&ptls2->signal_request) == 0); pthread_mutex_unlock(&in_signal_lock); } @@ -187,12 +187,12 @@ static void jl_thread_resume(int tid, int sig) // or if SIGINT happens too often. static void jl_try_deliver_sigint(void) { - jl_tls_states_t *ptls = jl_all_tls_states[0]; + jl_tls_states_t *ptls2 = jl_all_tls_states[0]; jl_safepoint_enable_sigint(); jl_wake_libuv(); - jl_atomic_store_release(&ptls->signal_request, 2); + jl_atomic_store_release(&ptls2->signal_request, 2); // This also makes sure `sleep` is aborted. - pthread_kill(ptls->system_id, SIGUSR2); + pthread_kill(ptls2->system_id, SIGUSR2); } // request: diff --git a/src/signals-win.c b/src/signals-win.c index 9e4a966f4da55..4950e0c6d016b 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -126,7 +126,7 @@ HANDLE hMainThread = INVALID_HANDLE_VALUE; // Try to throw the exception in the master thread. static void jl_try_deliver_sigint(void) { - jl_tls_states_t *ptls = jl_all_tls_states[0]; + jl_tls_states_t *ptls2 = jl_all_tls_states[0]; jl_safepoint_enable_sigint(); jl_wake_libuv(); if ((DWORD)-1 == SuspendThread(hMainThread)) { @@ -135,7 +135,7 @@ static void jl_try_deliver_sigint(void) return; } int force = jl_check_force_sigint(); - if (force || (!ptls->defer_signal && ptls->io_wait)) { + if (force || (!ptls2->defer_signal && ptls2->io_wait)) { jl_safepoint_consume_sigint(); if (force) jl_safe_printf("WARNING: Force throwing a SIGINT\n"); From 657509b8a9fbbe9fdadf5003945f2b4b933279ae Mon Sep 17 00:00:00 2001 From: ranjanan Date: Sat, 25 Jun 2016 16:32:06 -0400 Subject: [PATCH 0096/1117] Allow removing header --- base/show.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/show.jl b/base/show.jl index 1f1f9fb0c0f4c..1a7c481d7651b 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1536,7 +1536,7 @@ end show(io::IO, X::AbstractArray) = showarray(io, X) -function showarray(io::IO, X::AbstractArray) +function showarray(io::IO, X::AbstractArray; header = true) repr = !get(io, :multiline, false) if repr && ndims(X) == 1 return show_vector(io, X, "[", "]") @@ -1549,9 +1549,9 @@ function showarray(io::IO, X::AbstractArray) # override usual show method for Vector{Method}: don't abbreviate long lists io = IOContext(io, :limit => false) end - !repr && print(io, summary(X)) + (!repr && header) && print(io, summary(X)) if !isempty(X) - !repr && println(io, ":") + (!repr && header) && println(io, ":") if ndims(X) == 0 if isassigned(X) return show(io, X[]) From edfd3d0a00400d97aec7448e2c0e20fead487cc9 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Mon, 18 Jan 2016 21:44:06 -0500 Subject: [PATCH 0097/1117] Fix type instability in generic matmul Also make sure that intermediate values are promoted to the output type --- base/linalg/matmul.jl | 69 +++++++++++++++---------------------------- test/linalg/matmul.jl | 12 ++++++++ 2 files changed, 35 insertions(+), 46 deletions(-) diff --git a/base/linalg/matmul.jl b/base/linalg/matmul.jl index 4cc46ac75ffd8..0f998330395b6 100644 --- a/base/linalg/matmul.jl +++ b/base/linalg/matmul.jl @@ -457,6 +457,9 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV if size(C,1) != mA || size(C,2) != nB throw(DimensionMismatch("result C has dimensions $(size(C)), needs ($mA,$nB)")) end + if isempty(A) || isempty(B) + return fill!(C, zero(R)) + end tile_size = 0 if isbits(R) && isbits(T) && isbits(S) && (tA == 'N' || tB != 'N') @@ -468,7 +471,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV Atile = unsafe_wrap(Array, convert(Ptr{T}, pointer(Abuf)), sz) Btile = unsafe_wrap(Array, convert(Ptr{S}, pointer(Bbuf)), sz) - z = zero(R) + z1 = zero(A[1, 1]*B[1, 1] + A[1, 1]*B[1, 1]) + z = convert(promote_type(typeof(z1), R), z1) if mA < tile_size && nA < tile_size && nB < tile_size Base.copy_transpose!(Atile, 1:nA, 1:mA, tA, A, 1:mA, 1:nA) @@ -520,11 +524,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV if tA == 'N' if tB == 'N' for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[i, 1]*B[1, j] + A[i, 1]*B[1, j]) - end + z2 = zero(A[i, 1]*B[1, j] + A[i, 1]*B[1, j]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[i, k]*B[k, j] end @@ -532,11 +533,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV end elseif tB == 'T' for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[i, 1]*B[j, 1] + A[i, 1]*B[j, 1]) - end + z2 = zero(A[i, 1]*B[j, 1] + A[i, 1]*B[j, 1]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[i, k]*B[j, k].' end @@ -544,11 +542,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV end else for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[i, 1]*B[j, 1] + A[i, 1]*B[j, 1]) - end + z2 = zero(A[i, 1]*B[j, 1] + A[i, 1]*B[j, 1]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[i, k]*B[j, k]' end @@ -558,11 +553,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV elseif tA == 'T' if tB == 'N' for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[1, i]*B[1, j] + A[1, i]*B[1, j]) - end + z2 = zero(A[1, i]*B[1, j] + A[1, i]*B[1, j]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[k, i].'B[k, j] end @@ -570,11 +562,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV end elseif tB == 'T' for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[1, i]*B[j, 1] + A[1, i]*B[j, 1]) - end + z2 = zero(A[1, i]*B[j, 1] + A[1, i]*B[j, 1]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[k, i].'B[j, k].' end @@ -582,11 +571,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV end else for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[1, i]*B[j, 1] + A[1, i]*B[j, 1]) - end + z2 = zero(A[1, i]*B[j, 1] + A[1, i]*B[j, 1]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[k, i].'B[j, k]' end @@ -596,11 +582,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV else if tB == 'N' for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[1, i]*B[1, j] + A[1, i]*B[1, j]) - end + z2 = zero(A[1, i]*B[1, j] + A[1, i]*B[1, j]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[k, i]'B[k, j] end @@ -608,11 +591,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV end elseif tB == 'T' for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[1, i]*B[j, 1] + A[1, i]*B[j, 1]) - end + z2 = zero(A[1, i]*B[j, 1] + A[1, i]*B[j, 1]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[k, i]'B[j, k].' end @@ -620,11 +600,8 @@ function _generic_matmatmul!{T,S,R}(C::AbstractVecOrMat{R}, tA, tB, A::AbstractV end else for i = 1:mA, j = 1:nB - if isempty(A) || isempty(B) - Ctmp = zero(R) - else - Ctmp = zero(A[1, i]*B[j, 1] + A[1, i]*B[j, 1]) - end + z2 = zero(A[1, i]*B[j, 1] + A[1, i]*B[j, 1]) + Ctmp = convert(promote_type(R, typeof(z2)), z2) for k = 1:nA Ctmp += A[k, i]'B[j, k]' end diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 53d1368506e95..3daa60e3746e6 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -388,3 +388,15 @@ let @test_throws DimensionMismatch A_mul_B!(full43, full43, tri44) end end + +# Ensure that matrix multiplication with a narrower output type than input type does not +# produce allocation in the inner loop (#14722), by ensuring allocation does not change +# with the size of the input. +C1 = Array(Float32, 5, 5) +A1 = rand(Float64, 5, 5) +B1 = rand(Float64, 5, 5) +C2 = Array(Float32, 6, 6) +A2 = rand(Float64, 6, 6) +B2 = rand(Float64, 6, 6) +A_mul_B!(C1, A1, B1) +@test @allocated(A_mul_B!(C1, A1, B1)) == @allocated(A_mul_B!(C2, A2, B2)) From 3677950797967bfaaab7832f1a4955c72bafbbf7 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 Jun 2016 23:01:35 -0400 Subject: [PATCH 0098/1117] Made splice example into a doctest --- doc/manual/functions.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 786f9f12d1c8e..2b29becbadc8a 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -389,9 +389,11 @@ be a tuple: (1,2,(3,4)) Also, the function that arguments are spliced into need not be a varargs -function (although it often is):: +function (although it often is): - baz(a,b) = a + b +.. doctest:: + + julia> baz(a,b) = a + b; julia> args = [1,2] 2-element Array{Int64,1}: @@ -408,7 +410,10 @@ function (although it often is):: 3 julia> baz(args...) - no method baz(Int64,Int64,Int64) + ERROR: MethodError: no method matching baz(::Int64, ::Int64, ::Int64) + Closest candidates are: + baz(::Any, ::Any) + ... As you can see, if the wrong number of elements are in the spliced container, then the function call will fail, just as it would if too From 8ce52c8f7819cbe78c168817dc7bb0239adb3045 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 22 Jun 2016 23:49:45 -0400 Subject: [PATCH 0099/1117] Removed trailing colon in reference Also prepended "see" to the references and fixed a triple space after a period. --- doc/manual/functions.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 2b29becbadc8a..7bd58491e71e3 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -622,8 +622,8 @@ convenient for data processing, but in other languages vectorization is also often required for performance: if loops are slow, the "vectorized" version of a function can call fast library code written in a low-level language. In Julia, vectorized functions are *not* required for performance, and indeed it is often -beneficial to write your own loops (:ref:`man-performance-tips`:), but they can -still be convenient. Therefore, *any* Julia function ``f`` can be applied +beneficial to write your own loops (see :ref:`man-performance-tips`), but they +can still be convenient. Therefore, *any* Julia function ``f`` can be applied elementwise to any array (or other collection) with the syntax ``f.(A)``. Of course, you can omit the dot if you write a specialized "vector" method @@ -634,7 +634,7 @@ which functions you want to vectorize. More generally, ``f.(args...)`` is actually equivalent to ``broadcast(f, args...)``, which allows you to operate on multiple arrays (even of different shapes), or a mix of arrays and scalars -(:ref:`man-broadcasting`:). For example, if you have ``f(x,y) = 3x + 4y``, +(see :ref:`man-broadcasting`). For example, if you have ``f(x,y) = 3x + 4y``, then ``f.(pi,A)`` will return a new array consisting of ``f(pi,a)`` for each ``a`` in ``A``, and ``f.(vector1,vector2)`` will return a new vector consisting of ``f(vector1[i],vector2[i])`` for each index ``i`` From 623acaca8da3a2190038cf1c941a5557f117395c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 23 Jun 2016 07:27:04 -0400 Subject: [PATCH 0100/1117] Remove unnecessary punctuation --- doc/manual/control-flow.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 25486060dbbb7..98ff5413e13dd 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -769,7 +769,7 @@ Warnings and informational messages Julia also provides other functions that write messages to the standard error I/O, but do not throw any :exc:`Exception`\ s and hence do not interrupt -execution.: +execution: .. doctest:: From 2648ba26d8ccb9d6be1237ad2d401c6fa8dd20bc Mon Sep 17 00:00:00 2001 From: ranjanan Date: Sat, 25 Jun 2016 20:05:13 -0400 Subject: [PATCH 0101/1117] Add test for showarray without header --- test/show.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/show.jl b/test/show.jl index 2c0ae4d479531..0d0dfc61eaf25 100644 --- a/test/show.jl +++ b/test/show.jl @@ -497,3 +497,11 @@ let io = IOBuffer() showcompact(IOContext(IOContext(io, :compact=>true), :multiline=>true), x) @test takebuf_string(io) == "[1,2]" end + +# PR 17117 +# test show array +let s = IOBuffer(Array{UInt8}(0), true, true) + io = IOContext(s, multiline=true) + Base.showarray(io, [1,2,3], header = false) + @test String(resize!(s.data, s.size)) == " 1\n 2\n 3" +end From 3cb48560c3b50b3da292a72797dd7c454ea045d7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 13 Apr 2016 04:29:09 -0400 Subject: [PATCH 0102/1117] patch llvm win64 dwarf4 relocations afaict, this is how other platforms treat this flag --- deps/llvm.mk | 2 + deps/patches/llvm-win64-reloc-dwarf.patch | 163 ++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 deps/patches/llvm-win64-reloc-dwarf.patch diff --git a/deps/llvm.mk b/deps/llvm.mk index 8b835beec197b..e11e2d4ad0a28 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -430,6 +430,7 @@ $(eval $(call LLVM_PATCH,llvm-3.8.0_bindir)) $(eval $(call LLVM_PATCH,llvm-D14260)) $(eval $(call LLVM_PATCH,llvm-nodllalias)) $(eval $(call LLVM_PATCH,llvm-D21271-instcombine-tbaa-3.7)) +$(eval $(call LLVM_PATCH,llvm-win64-reloc-dwarf)) else ifeq ($(LLVM_VER),3.8.0) $(eval $(call LLVM_PATCH,llvm-3.7.1_3)) $(eval $(call LLVM_PATCH,llvm-D14260)) @@ -447,6 +448,7 @@ $(eval $(call LLVM_PATCH,llvm-PR26180)) $(eval $(call LLVM_PATCH,llvm-PR27046)) $(eval $(call LLVM_PATCH,llvm-3.8.0_ppc64_SUBFC8)) $(eval $(call LLVM_PATCH,llvm-D21271-instcombine-tbaa-3.8)) +$(eval $(call LLVM_PATCH,llvm-win64-reloc-dwarf)) endif # LLVM_VER ifeq ($(LLVM_VER),3.7.1) diff --git a/deps/patches/llvm-win64-reloc-dwarf.patch b/deps/patches/llvm-win64-reloc-dwarf.patch new file mode 100644 index 0000000000000..45bd2604e8488 --- /dev/null +++ b/deps/patches/llvm-win64-reloc-dwarf.patch @@ -0,0 +1,163 @@ +diff -rpu llvm-3.8.0/include/llvm/MC/MCStreamer.h llvm-3.8.0-reloc/include/llvm/MC/MCStreamer.h +--- llvm-3.8.0/include/llvm/MC/MCStreamer.h 2016-01-12 08:38:15.000000000 -0500 ++++ llvm-3.8.0-reloc/include/llvm/MC/MCStreamer.h 2016-05-23 18:54:01.830143100 -0400 +@@ -456,7 +456,7 @@ public: + /// \brief Emits a COFF section relative relocation. + /// + /// \param Symbol - Symbol the section relative relocation should point to. +- virtual void EmitCOFFSecRel32(MCSymbol const *Symbol); ++ virtual void EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset); + + /// \brief Emit an ELF .size directive. + /// +diff -rpu llvm-3.8.0/include/llvm/MC/MCWinCOFFStreamer.h llvm-3.8.0-reloc/include/llvm/MC/MCWinCOFFStreamer.h +--- llvm-3.8.0/include/llvm/MC/MCWinCOFFStreamer.h 2015-11-17 05:00:43.000000000 -0500 ++++ llvm-3.8.0-reloc/include/llvm/MC/MCWinCOFFStreamer.h 2016-05-23 18:54:01.845722400 -0400 +@@ -52,7 +52,7 @@ public: + void EndCOFFSymbolDef() override; + void EmitCOFFSafeSEH(MCSymbol const *Symbol) override; + void EmitCOFFSectionIndex(MCSymbol const *Symbol) override; +- void EmitCOFFSecRel32(MCSymbol const *Symbol) override; ++ void EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) override; + void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size, + unsigned ByteAlignment) override; + void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size, +diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinter.cpp llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +--- llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinter.cpp 2016-01-12 20:18:13.000000000 -0500 ++++ llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/AsmPrinter.cpp 2016-05-23 18:54:01.845722400 -0400 +@@ -1729,7 +1729,7 @@ void AsmPrinter::EmitLabelPlusOffset(con + unsigned Size, + bool IsSectionRelative) const { + if (MAI->needsDwarfSectionOffsetDirective() && IsSectionRelative) { +- OutStreamer->EmitCOFFSecRel32(Label); ++ OutStreamer->EmitCOFFSecRel32(Label, Offset); + return; + } + +diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp +--- llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp 2015-11-17 19:34:10.000000000 -0500 ++++ llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp 2016-05-23 20:17:42.105695800 -0400 +@@ -150,7 +150,7 @@ void AsmPrinter::emitDwarfSymbolReferenc + if (!ForceOffset) { + // On COFF targets, we have to emit the special .secrel32 directive. + if (MAI->needsDwarfSectionOffsetDirective()) { +- OutStreamer->EmitCOFFSecRel32(Label); ++ OutStreamer->EmitCOFFSecRel32(Label, /*offset*/0); + return; + } + +diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/DIE.cpp llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/DIE.cpp +--- llvm-3.8.0/lib/CodeGen/AsmPrinter/DIE.cpp 2016-01-07 09:28:20.000000000 -0500 ++++ llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/DIE.cpp 2016-05-23 19:08:35.434799900 -0400 +@@ -487,7 +487,7 @@ void DIEEntry::EmitValue(const AsmPrinte + Addr += CU->getDebugInfoOffset(); + if (AP->MAI->doesDwarfUseRelocationsAcrossSections()) + AP->EmitLabelPlusOffset(CU->getSectionSym(), Addr, +- DIEEntry::getRefAddrSize(AP)); ++ DIEEntry::getRefAddrSize(AP), true); + else + AP->OutStreamer->EmitIntValue(Addr, DIEEntry::getRefAddrSize(AP)); + } else +diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp +--- llvm-3.8.0/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp 2016-01-12 20:05:23.000000000 -0500 ++++ llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp 2016-05-23 21:19:31.454460900 -0400 +@@ -231,7 +231,7 @@ void WinCodeViewLineTables::emitDebugInf + // code is located and what's its size: + EmitLabelDiff(*Asm->OutStreamer, Fn, FI.End); + Asm->OutStreamer->EmitFill(12, 0); +- Asm->OutStreamer->EmitCOFFSecRel32(Fn); ++ Asm->OutStreamer->EmitCOFFSecRel32(Fn, /*offset*/0); + Asm->OutStreamer->EmitCOFFSectionIndex(Fn); + Asm->EmitInt8(0); + // Emit the function display name as a null-terminated string. +@@ -272,7 +272,7 @@ void WinCodeViewLineTables::emitDebugInf + Asm->OutStreamer->EmitLabel(LineTableBegin); + + // Identify the function this subsection is for. +- Asm->OutStreamer->EmitCOFFSecRel32(Fn); ++ Asm->OutStreamer->EmitCOFFSecRel32(Fn, /*offset*/0); + Asm->OutStreamer->EmitCOFFSectionIndex(Fn); + // Insert flags after a 16-bit section index. + Asm->EmitInt16(COFF::DEBUG_LINE_TABLES_HAVE_COLUMN_RECORDS); +diff -rpu llvm-3.8.0/lib/MC/MCAsmStreamer.cpp llvm-3.8.0-reloc/lib/MC/MCAsmStreamer.cpp +--- llvm-3.8.0/lib/MC/MCAsmStreamer.cpp 2015-11-12 08:33:00.000000000 -0500 ++++ llvm-3.8.0-reloc/lib/MC/MCAsmStreamer.cpp 2016-05-23 18:54:01.859727900 -0400 +@@ -143,7 +143,7 @@ public: + void EndCOFFSymbolDef() override; + void EmitCOFFSafeSEH(MCSymbol const *Symbol) override; + void EmitCOFFSectionIndex(MCSymbol const *Symbol) override; +- void EmitCOFFSecRel32(MCSymbol const *Symbol) override; ++ void EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) override; + void emitELFSize(MCSymbolELF *Symbol, const MCExpr *Value) override; + void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size, + unsigned ByteAlignment) override; +@@ -525,7 +525,7 @@ void MCAsmStreamer::EmitCOFFSectionIndex + EmitEOL(); + } + +-void MCAsmStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol) { ++void MCAsmStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) { + OS << "\t.secrel32\t"; + Symbol->print(OS, MAI); + EmitEOL(); +diff -rpu llvm-3.8.0/lib/MC/MCParser/COFFAsmParser.cpp llvm-3.8.0-reloc/lib/MC/MCParser/COFFAsmParser.cpp +--- llvm-3.8.0/lib/MC/MCParser/COFFAsmParser.cpp 2015-11-18 01:02:15.000000000 -0500 ++++ llvm-3.8.0-reloc/lib/MC/MCParser/COFFAsmParser.cpp 2016-05-23 18:54:15.937859600 -0400 +@@ -450,7 +450,7 @@ bool COFFAsmParser::ParseDirectiveSecRel + MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); + + Lex(); +- getStreamer().EmitCOFFSecRel32(Symbol); ++ getStreamer().EmitCOFFSecRel32(Symbol, /*Offset=*/0); + return false; + } + +diff -rpu llvm-3.8.0/lib/MC/MCStreamer.cpp llvm-3.8.0-reloc/lib/MC/MCStreamer.cpp +--- llvm-3.8.0/lib/MC/MCStreamer.cpp 2015-11-04 18:59:18.000000000 -0500 ++++ llvm-3.8.0-reloc/lib/MC/MCStreamer.cpp 2016-05-23 18:54:15.953281800 -0400 +@@ -119,7 +119,7 @@ void MCStreamer::EmitSymbolValue(const M + if (!IsSectionRelative) + EmitValueImpl(MCSymbolRefExpr::create(Sym, getContext()), Size); + else +- EmitCOFFSecRel32(Sym); ++ EmitCOFFSecRel32(Sym, /*Offset=*/0); + } + + void MCStreamer::EmitGPRel64Value(const MCExpr *Value) { +@@ -570,7 +570,7 @@ void MCStreamer::EmitCOFFSafeSEH(MCSymbo + void MCStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) { + } + +-void MCStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol) { ++void MCStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) { + } + + /// EmitRawText - If this file is backed by an assembly streamer, this dumps +diff -rpu llvm-3.8.0/lib/MC/WinCOFFStreamer.cpp llvm-3.8.0-reloc/lib/MC/WinCOFFStreamer.cpp +--- llvm-3.8.0/lib/MC/WinCOFFStreamer.cpp 2015-11-17 05:00:43.000000000 -0500 ++++ llvm-3.8.0-reloc/lib/MC/WinCOFFStreamer.cpp 2016-06-25 22:00:26.530421900 -0400 +@@ -199,14 +199,21 @@ void MCWinCOFFStreamer::EmitCOFFSectionI + DF->getContents().resize(DF->getContents().size() + 2, 0); + } + +-void MCWinCOFFStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol) { ++void MCWinCOFFStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) { + MCDataFragment *DF = getOrCreateDataFragment(); +- const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(Symbol, getContext()); +- MCFixup Fixup = MCFixup::create(DF->getContents().size(), SRE, FK_SecRel_4); ++ // Create Symbol A for the relocation relative reference. ++ const MCExpr *MCE = MCSymbolRefExpr::create(Symbol, getContext()); ++ // Add the constant offset, if given ++ if (Offset) ++ MCE = MCBinaryExpr::createAdd(MCE, MCConstantExpr::create(Offset, getContext()), getContext()); ++ // Build the secrel32 relocation. ++ MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_SecRel_4); ++ // Record the relocation. + DF->getFixups().push_back(Fixup); ++ // Emit 4 bytes (zeros) to the object file. + DF->getContents().resize(DF->getContents().size() + 4, 0); + } + + void MCWinCOFFStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size, + unsigned ByteAlignment) { + assert((!Symbol->isInSection() || From 23288f2c5b5a5cc6222d8bc54110f5b9d130745a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 26 Jun 2016 00:18:33 +0200 Subject: [PATCH 0103/1117] specialize on dict key --- base/dict.jl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 0d5dbe321b642..7c473f211185f 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -696,12 +696,15 @@ function _setindex!(h::Dict, v, key, index) end function setindex!{K,V}(h::Dict{K,V}, v0, key0) - key = convert(K,key0) - if !isequal(key,key0) + key = convert(K, key0) + if !isequal(key, key0) throw(ArgumentError("$key0 is not a valid key for type $K")) end - v = convert(V, v0) + setindex!(h, v0, key) +end +function setindex!{K,V}(h::Dict{K,V}, v0, key::K) + v = convert(V, v0) index = ht_keyindex2(h, key) if index > 0 @@ -719,12 +722,15 @@ function get!{K,V}(h::Dict{K,V}, key0, default) if !isequal(key,key0) throw(ArgumentError("$key0 is not a valid key for type $K")) end + get!(h, key, default) +end +function get!{K, V}(h::Dict{K,V}, key::K, default) index = ht_keyindex2(h, key) index > 0 && return h.vals[index] - v = convert(V, default) + v = convert(V, default) _setindex!(h, v, key, -index) return v end @@ -734,7 +740,10 @@ function get!{K,V}(default::Callable, h::Dict{K,V}, key0) if !isequal(key,key0) throw(ArgumentError("$key0 is not a valid key for type $K")) end + get!(default, h, key) +end +function get!{K,V}(default::Callable, h::Dict{K,V}, key::K) index = ht_keyindex2(h, key) index > 0 && return h.vals[index] From cd2f3cc6d467dc78fee8d4d626027ccd8479cd91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Mon, 30 May 2016 21:59:24 +0200 Subject: [PATCH 0104/1117] Fix so method errors caused by keywords is displayed as the original method in show_method_candidates with a text stating the not matching keyword. Fix #15639 --- base/replutil.jl | 54 +++++++++++++++++++++++++++++++++++++++++------- test/replutil.jl | 54 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 93 insertions(+), 15 deletions(-) diff --git a/base/replutil.jl b/base/replutil.jl index c1ed0f2543953..90260b515363c 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -123,6 +123,16 @@ function showerror(io::IO, ex::MethodError) ft = typeof(f) name = ft.name.mt.name f_is_function = false + kwargs = Any[] + if startswith(string(ft.name), "#kw#") + f = ex.args[2] + ft = typeof(f) + name = ft.name.mt.name + arg_types_param = arg_types_param[3:end] + temp = ex.args[1] + kwargs = Any[(temp[i*2-1], temp[i*2]) for i in 1:(length(temp) ÷ 2)] + ex = MethodError(f, ex.args[3:end]) + end if f == Base.convert && length(arg_types_param) == 2 && !is_arg_types f_is_function = true # See #13033 @@ -150,6 +160,14 @@ function showerror(io::IO, ex::MethodError) print(io, "::$typ") i == length(arg_types_param) || print(io, ", ") end + if !isempty(kwargs) + print(io, "; ") + for (i, (k, v)) in enumerate(kwargs) + print(io, k, "=") + show(IOContext(io, :limit=>true), v) + i == length(kwargs) || print(io, ", ") + end + end print(io, ")") end if ft <: AbstractArray @@ -189,7 +207,7 @@ function showerror(io::IO, ex::MethodError) "\nsince type constructors fall back to convert methods.") end try - show_method_candidates(io, ex) + show_method_candidates(io, ex, kwargs) catch warn(io, "Error showing method candidates, aborted") end @@ -222,7 +240,7 @@ function showerror_nostdio(err, msg::AbstractString) ccall(:jl_printf, Cint, (Ptr{Void},Cstring), stderr_stream, "\n") end -function show_method_candidates(io::IO, ex::MethodError) +function show_method_candidates(io::IO, ex::MethodError, kwargs::Vector=Any[]) is_arg_types = isa(ex.args, DataType) arg_types = is_arg_types ? ex.args : typesof(ex.args...) arg_types_param = Any[arg_types.parameters...] @@ -233,7 +251,7 @@ function show_method_candidates(io::IO, ex::MethodError) else f = ex.f end - + ft = typeof(f) lines = [] # These functions are special cased to only show if first argument is matched. special = f in [convert, getindex, setindex!] @@ -262,7 +280,6 @@ function show_method_candidates(io::IO, ex::MethodError) use_constructor_syntax = isa(func, Type) print(buf, use_constructor_syntax ? func : typeof(func).name.mt.name) end - right_matches = 0 tv = method.tvars if !isa(tv,SimpleVector) tv = Any[tv] @@ -318,17 +335,19 @@ function show_method_candidates(io::IO, ex::MethodError) end end - if right_matches > 0 + if right_matches > 0 || length(ex.args) < 2 if length(t_i) < length(sig) # If the methods args is longer than input then the method # arguments is printed as not a match - for sigtype in sig[length(t_i)+1:end] + for (k, sigtype) in enumerate(sig[length(t_i)+1:end]) if Base.isvarargtype(sigtype) sigstr = string(sigtype.parameters[1], "...") else sigstr = string(sigtype) end - print(buf, ", ") + if !((min(length(t_i), length(sig)) == 0) && k==1) + print(buf, ", ") + end if Base.have_color Base.with_output_color(:red, buf) do buf print(buf, "::$sigstr") @@ -338,7 +357,28 @@ function show_method_candidates(io::IO, ex::MethodError) end end end + kwords = Symbol[] + if isdefined(ft.name.mt, :kwsorter) + kwsorter_t = typeof(ft.name.mt.kwsorter) + kwords = kwarg_decl(method.sig, kwsorter_t) + length(kwords) > 0 && print(buf, "; ", join(kwords, ", ")) + end print(buf, ")") + if !isempty(kwargs) + unexpected = Symbol[] + if isempty(kwords) || !(any(endswith(string(kword), "...") for kword in kwords)) + for (k, v) in kwargs + if !(k in kwords) + push!(unexpected, k) + end + end + end + if !isempty(unexpected) + Base.with_output_color(:red, buf) do buf + print(buf, " got an unsupported keyword argument \"", join(unexpected, "\", \""), "\"") + end + end + end push!(lines, (buf, right_matches)) end end diff --git a/test/replutil.jl b/test/replutil.jl index 40d69ef76f15f..a8e0d2acead96 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -52,16 +52,19 @@ no_color = no_color = "\nClosest candidates are:\n method_c3(::Float64, !Matche test_have_color(buf, color, no_color) # Test for the method error in issue #8651 -Base.show_method_candidates(buf, MethodError(readline,("",))) -test_have_color(buf, "\e[0m\nClosest candidates are:\n readline(::AbstractString)\e[0m", "\nClosest candidates are:\n readline(::AbstractString)") +method_c4() = true +method_c4(x::AbstractString) = false +Base.show_method_candidates(buf, MethodError(method_c4,("",))) +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c4(::AbstractString)\n method_c4()\e[0m", "\nClosest candidates are:\n method_c4(::AbstractString)\n method_c4()") -method_c4(::Type{Float64}) = true -Base.show_method_candidates(buf, MethodError(method_c4,(Float64,))) -test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c4(::Type{Float64})\e[0m", - "\nClosest candidates are:\n method_c4(::Type{Float64})") +method_c5(::Type{Float64}) = true +Base.show_method_candidates(buf, MethodError(method_c5,(Float64,))) +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c5(::Type{Float64})\e[0m", + "\nClosest candidates are:\n method_c5(::Type{Float64})") -Base.show_method_candidates(buf, MethodError(method_c4,(Int32,))) -test_have_color(buf, "", "") +Base.show_method_candidates(buf, MethodError(method_c5,(Int32,))) +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c5(\e[1m\e[31m::Type{Float64}\e[0m)\e[0m", + "\nClosest candidates are:\n method_c5(!Matched::Type{Float64})") type Test_type end test_type = Test_type() @@ -83,6 +86,41 @@ Base.show_method_candidates(buf, MethodError(PR16155,(Int64(3), 2.0, Int64(3)))) test_have_color(buf, "\e[0m\nClosest candidates are:\n PR16155(::Int64, ::Any)\n PR16155(::Any, ::Any)\n PR16155{T}(::Any)\e[0m", "\nClosest candidates are:\n PR16155(::Int64, ::Any)\n PR16155(::Any, ::Any)\n PR16155{T}(::Any)") +method_c6(; x=1) = x +method_c6(a; y=1) = y +m_error = try method_c6(y=1) catch e; e; end +showerror(buf, m_error) +error_out = takebuf_string(buf) +m_error = try method_c6(1, x=1) catch e; e; end +showerror(buf, m_error) +error_out1 = takebuf_string(buf) + +if Base.have_color + @test contains(error_out, "method_c6(; x)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") + @test contains(error_out, "method_c6(\e[1m\e[31m::Any\e[0m; y)") + @test contains(error_out1, "method_c6(::Any; y)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") +else + @test contains(error_out, "method_c6(; x) got an unsupported keyword argument \"y\"") + @test contains(error_out, "method_c6(!Matched::Any; y)") + @test contains(error_out1, "method_c6(::Any; y) got an unsupported keyword argument \"x\"") +end + +method_c7(a, b; kargs...) = a +Base.show_method_candidates(buf, MethodError(method_c7, (1, 1)), [(:x, 1), (:y, 2)]) +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c7(::Any, ::Any; kargs...)\e[0m", + "\nClosest candidates are:\n method_c7(::Any, ::Any; kargs...)") +method_c8(a, b; y=1, w=1) = a +Base.show_method_candidates(buf, MethodError(method_c8, (1, 1)), [(:x, 1), (:y, 2), (:z, 1), (:w, 1)]) +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)\e[1m\e[31m got an unsupported keyword argument \"x\", \"z\"\e[0m\e[0m", + "\nClosest candidates are:\n method_c8(::Any, ::Any; y, w) got an unsupported keyword argument \"x\", \"z\"") + +addConstraint_15639(c::Int32) = c +addConstraint_15639(c::Int64; uncset=nothing) = addConstraint_15639(Int32(c), uncset=uncset) + +Base.show_method_candidates(buf, MethodError(addConstraint_15639, (Int32(1),)), [(:uncset, nothing)]) +test_have_color(buf, "\e[0m\nClosest candidates are:\n addConstraint_15639(::Int32)\e[1m\e[31m got an unsupported keyword argument \"uncset\"\e[0m\n addConstraint_15639(\e[1m\e[31m::Int64\e[0m; uncset)\e[0m", + "\nClosest candidates are:\n addConstraint_15639(::Int32) got an unsupported keyword argument \"uncset\"\n addConstraint_15639(!Matched::Int64; uncset)") + macro except_str(expr, err_type) return quote let err = nothing From 97eb3adc412f46a2591545f30f7b39d1a419073c Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sun, 26 Jun 2016 07:17:39 -0400 Subject: [PATCH 0105/1117] fix makefile bug in LLVM_PATCH from #16823 was resulting in the last patch file not getting applied, my bad - needed doubled `$` so the variable expansion happens after the LLVM_PATCH macro gets expanded, not at its definition (makefiles should not be written by humans) --- deps/llvm.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index e11e2d4ad0a28..bf33bc4ca354b 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -406,11 +406,11 @@ endif # LLVM_VER LLVM_PATCH_PREV:= LLVM_PATCH_LIST:= define LLVM_PATCH -$$(LLVM_SRC_DIR)/$1.patch-applied: $(LLVM_SRC_DIR)/configure | $$(SRCDIR)/patches/$1.patch $(LLVM_PATCH_PREV) +$$(LLVM_SRC_DIR)/$1.patch-applied: $$(LLVM_SRC_DIR)/configure | $$(SRCDIR)/patches/$1.patch $$(LLVM_PATCH_PREV) cd $$(LLVM_SRC_DIR) && patch -p1 < $$(SRCDIR)/patches/$1.patch echo 1 > $$@ LLVM_PATCH_PREV := $$(LLVM_SRC_DIR)/$1.patch-applied -LLVM_PATCH_LIST += $(LLVM_PATCH_PREV) +LLVM_PATCH_LIST += $$(LLVM_PATCH_PREV) endef ifeq ($(LLVM_VER),3.3) $(eval $(call LLVM_PATCH,llvm-3.3)) From dd327f9d9f5358bd5bfbf9d45016106e9e98fd86 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 26 Jun 2016 09:31:31 -0400 Subject: [PATCH 0106/1117] Add last remaining bessel tests Missed a few DomainErrors and Float32 fallbacks. --- test/math.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/math.jl b/test/math.jl index 51bcd915edddb..b273af6debba3 100644 --- a/test/math.jl +++ b/test/math.jl @@ -375,6 +375,7 @@ true_i33 = 0.95975362949600785698 @test_approx_eq besseli(-3,3) true_i33 @test_approx_eq besseli(3,-3) -true_i33 @test_approx_eq besseli(-3,-3) -true_i33 +@test_approx_eq besseli(Float32(-3),Complex64(-3,0)) -true_i33 @test_throws Base.Math.AmosException besseli(1,1000) @test_throws DomainError besseli(0.4,-1.0) @@ -454,6 +455,8 @@ y33 = bessely(3,3.) @test_throws DomainError bessely(0.4,-1.0) @test_throws DomainError bessely(0.4,Float32(-1.0)) @test_throws DomainError bessely(1,Float32(-1.0)) +@test_throws DomainError bessely(Cint(3),Float32(-3.)) +@test_throws DomainError bessely(Cint(3),Float64(-3.)) @test_throws MethodError bessely(1.2,big(1.0)) @test_throws MethodError bessely(1,complex(big(1.0))) @@ -465,6 +468,7 @@ y33 = bessely(3,3.) for elty in [Complex64,Complex128] z = convert(elty, 1.0 + 1.9im) @test_approx_eq besselhx(1.0, 1, z) convert(elty,-0.5949634147786144 - 0.18451272807835967im) + @test_approx_eq besselhx(Float32(1.0), 1, z) convert(elty,-0.5949634147786144 - 0.18451272807835967im) end @test_throws MethodError besselh(1,1,big(1.0)) From 3410bb68b2a0ca5f198306dc01e68977c2cec133 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 26 Jun 2016 12:13:36 -0400 Subject: [PATCH 0107/1117] Few missing tests for Numbers --- test/numbers.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/numbers.jl b/test/numbers.jl index 217faa977d90a..f8377601f7ad2 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2361,6 +2361,7 @@ end @test widen(Int32(42)) === Int64(42) @test widen(Int8) === Int32 @test widen(Float32) === Float64 +@test widen(Float16) === Float32 ## Note: this should change to e.g. Float128 at some point @test widen(Float64) === BigFloat @test widen(BigInt) === BigInt @@ -2742,3 +2743,10 @@ testmi(typemin(Int)+1:typemin(Int)+1000, -100:100) @test_throws ArgumentError Base.multiplicativeinverse(0) testmi(map(UInt32, 0:1000), map(UInt32, 1:100)) testmi(typemax(UInt32)-UInt32(1000):typemax(UInt32), map(UInt32, 1:100)) + +@test ndims(1) == 0 +@test size(1,1) == 1 +@test_throws BoundsError size(1,-1) +@test indices(1) == () +@test indices(1,1) == 1:1 +@test_throws BoundsError indices(1,-1) From 7947d1ca72167024082d20bf015f7be752db2778 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Sun, 26 Jun 2016 13:29:37 -0400 Subject: [PATCH 0108/1117] Introduce AbstractWorkerPool, CachingPool (#16808) * Introduce CachingPool * doc fixes --- base/exports.jl | 2 + base/multi.jl | 2 +- base/pmap.jl | 8 ++- base/workerpool.jl | 150 +++++++++++++++++++++++++++++++++------- doc/stdlib/parallel.rst | 90 +++++++++++++++++------- doc/stdlib/strings.rst | 1 + doc/stdlib/test.rst | 2 +- test/parallel_exec.jl | 7 ++ 8 files changed, 208 insertions(+), 54 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 838410226e02c..000d2b4a96d2f 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1185,6 +1185,8 @@ export # multiprocessing addprocs, asyncmap, + CachingPool, + clear!, ClusterManager, default_worker_pool, fetch, diff --git a/base/multi.jl b/base/multi.jl index e7ae912babed2..38aab509dc45f 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -1157,7 +1157,7 @@ function handle_msg(msg::JoinCompleteMsg, r_stream, w_stream, version) ntfy_channel = lookup_ref(msg.notify_oid) put!(ntfy_channel, w.id) - put!(default_worker_pool(), w) + push!(default_worker_pool(), w) end function disable_threaded_libs() diff --git a/base/pmap.jl b/base/pmap.jl index 01f5811902d03..a8ba5605e54af 100644 --- a/base/pmap.jl +++ b/base/pmap.jl @@ -31,7 +31,7 @@ pgenerate(f, c) = pgenerate(default_worker_pool(), f, c) pgenerate(f, c1, c...) = pgenerate(a->f(a...), zip(c1, c...)) """ - pmap([::WorkerPool], f, c...; distributed=true, batch_size=1, on_error=nothing, retry_n=0, retry_max_delay=DEFAULT_RETRY_MAX_DELAY, retry_on=DEFAULT_RETRY_ON) -> collection + pmap([::AbstractWorkerPool], f, c...; distributed=true, batch_size=1, on_error=nothing, retry_n=0, retry_max_delay=DEFAULT_RETRY_MAX_DELAY, retry_on=DEFAULT_RETRY_ON) -> collection Transform collection `c` by applying `f` to each element using available workers and tasks. @@ -70,7 +70,7 @@ The following are equivalent: * `pmap(f, c; retry_n=1)` and `asyncmap(retry(remote(f)),c)` * `pmap(f, c; retry_n=1, on_error=e->e)` and `asyncmap(x->try retry(remote(f))(x) catch e; e end, c)` """ -function pmap(p::WorkerPool, f, c; distributed=true, batch_size=1, on_error=nothing, +function pmap(p::AbstractWorkerPool, f, c; distributed=true, batch_size=1, on_error=nothing, retry_n=0, retry_max_delay=DEFAULT_RETRY_MAX_DELAY, retry_on=DEFAULT_RETRY_ON) @@ -97,6 +97,7 @@ function pmap(p::WorkerPool, f, c; distributed=true, batch_size=1, on_error=not if on_error !== nothing f = wrap_on_error(f, on_error) end + return collect(AsyncGenerator(f, c)) else batches = batchsplit(c, min_batch_count = length(p) * 3, @@ -116,11 +117,12 @@ function pmap(p::WorkerPool, f, c; distributed=true, batch_size=1, on_error=not if (on_error !== nothing) || (retry_n > 0) process_batch_errors!(p, f_orig, results, on_error, retry_on, retry_n, retry_max_delay) end + return results end end -pmap(p::WorkerPool, f, c1, c...; kwargs...) = pmap(p, a->f(a...), zip(c1, c...); kwargs...) +pmap(p::AbstractWorkerPool, f, c1, c...; kwargs...) = pmap(p, a->f(a...), zip(c1, c...); kwargs...) pmap(f, c; kwargs...) = pmap(default_worker_pool(), f, c; kwargs...) pmap(f, c1, c...; kwargs...) = pmap(a->f(a...), zip(c1, c...); kwargs...) diff --git a/base/workerpool.jl b/base/workerpool.jl index d0d697d9d9a39..f82eac5fbdeea 100644 --- a/base/workerpool.jl +++ b/base/workerpool.jl @@ -1,11 +1,26 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -type WorkerPool +abstract AbstractWorkerPool + +# An AbstractWorkerPool should implement +# +# `push!` - add a new worker to the overall pool (available + busy) +# `put!` - put back a worker to the available pool +# `take!` - take a worker from the available pool (to be used for remote function execution) +# `length` - number of workers available in the overall pool +# `isready` - return false if a `take!` on the pool would block, else true +# +# The default implementations of the above (on a AbstractWorkerPool) require fields +# channel::RemoteChannel{Channel{Int}} +# workers::Set{Int} +# + +type WorkerPool <: AbstractWorkerPool channel::RemoteChannel{Channel{Int}} - count::Int + workers::Set{Int} # Create a shared queue of available workers - WorkerPool() = new(RemoteChannel(()->Channel{Int}(typemax(Int))), 0) + WorkerPool() = new(RemoteChannel(()->Channel{Int}(typemax(Int))), Set{Int}()) end @@ -19,25 +34,26 @@ function WorkerPool(workers::Vector{Int}) # Add workers to the pool for w in workers - put!(pool, w) + push!(pool, w) end return pool end +push!(pool::AbstractWorkerPool, w::Int) = (push!(pool.workers, w); put!(pool.channel, w); pool) +push!(pool::AbstractWorkerPool, w::Worker) = push!(pool, w.id) +length(pool::AbstractWorkerPool) = length(pool.workers) +isready(pool::AbstractWorkerPool) = isready(pool.channel) -put!(pool::WorkerPool, w::Int) = (pool.count += 1; put!(pool.channel, w)) -put!(pool::WorkerPool, w::Worker) = put!(pool, w.id) +put!(pool::AbstractWorkerPool, w::Int) = (put!(pool.channel, w); pool) -length(pool::WorkerPool) = pool.count +workers(pool::AbstractWorkerPool) = collect(pool.workers) -isready(pool::WorkerPool) = isready(pool.channel) - -function remotecall_pool(rc_f, f, pool::WorkerPool, args...; kwargs...) +function take!(pool::AbstractWorkerPool) # Find an active worker worker = 0 while true - if pool.count == 0 + if length(pool) == 0 if pool === default_worker_pool() # No workers, the master process is used as a worker worker = 1 @@ -51,42 +67,47 @@ function remotecall_pool(rc_f, f, pool::WorkerPool, args...; kwargs...) if worker in procs() break else - pool.count = pool.count - 1 + delete!(pool.workers, worker) # Remove invalid worker from pool end end + return worker +end +function remotecall_pool(rc_f, f, pool::AbstractWorkerPool, args...; kwargs...) + worker = take!(pool) try rc_f(f, worker, args...; kwargs...) finally - if worker != 1 - put!(pool.channel, worker) - end + # In case of default_worker_pool, the master is implictly considered a worker + # till the time new workers are added, and it is not added back to the available pool. + # However, it is perfectly valid for other pools to `push!` any worker (including 1) + # to the pool. Confirm the same before making a worker available. + worker in pool.workers && put!(pool, worker) end end - """ - remotecall(f, pool::WorkerPool, args...; kwargs...) + remotecall(f, pool::AbstractWorkerPool, args...; kwargs...) Call `f(args...; kwargs...)` on one of the workers in `pool`. Returns a `Future`. """ -remotecall(f, pool::WorkerPool, args...; kwargs...) = remotecall_pool(remotecall, f, pool, args...; kwargs...) +remotecall(f, pool::AbstractWorkerPool, args...; kwargs...) = remotecall_pool(remotecall, f, pool, args...; kwargs...) """ - remotecall_wait(f, pool::WorkerPool, args...; kwargs...) + remotecall_wait(f, pool::AbstractWorkerPool, args...; kwargs...) Call `f(args...; kwargs...)` on one of the workers in `pool`. Waits for completion, returns a `Future`. """ -remotecall_wait(f, pool::WorkerPool, args...; kwargs...) = remotecall_pool(remotecall_wait, f, pool, args...; kwargs...) +remotecall_wait(f, pool::AbstractWorkerPool, args...; kwargs...) = remotecall_pool(remotecall_wait, f, pool, args...; kwargs...) """ - remotecall_fetch(f, pool::WorkerPool, args...; kwargs...) + remotecall_fetch(f, pool::AbstractWorkerPool, args...; kwargs...) Call `f(args...; kwargs...)` on one of the workers in `pool`. Waits for completion and returns the result. """ -remotecall_fetch(f, pool::WorkerPool, args...; kwargs...) = remotecall_pool(remotecall_fetch, f, pool, args...; kwargs...) +remotecall_fetch(f, pool::AbstractWorkerPool, args...; kwargs...) = remotecall_pool(remotecall_fetch, f, pool, args...; kwargs...) """ default_worker_pool() @@ -107,10 +128,91 @@ end """ - remote([::WorkerPool], f) -> Function + remote([::AbstractWorkerPool], f) -> Function Returns a lambda that executes function `f` on an available worker using `remotecall_fetch`. """ remote(f) = (args...; kwargs...)->remotecall_fetch(f, default_worker_pool(), args...; kwargs...) -remote(p::WorkerPool, f) = (args...; kwargs...)->remotecall_fetch(f, p, args...; kwargs...) +remote(p::AbstractWorkerPool, f) = (args...; kwargs...)->remotecall_fetch(f, p, args...; kwargs...) + +type CachingPool <: AbstractWorkerPool + channel::RemoteChannel{Channel{Int}} + workers::Set{Int} + + # Mapping between a tuple (worker_id, f) and a remote_ref + map_obj2ref::Dict{Tuple{Int, Function}, RemoteChannel} + + function CachingPool() + wp = new(RemoteChannel(()->Channel{Int}(typemax(Int))), Set{Int}(), Dict{Int, Function}()) + finalizer(wp, clear!) + wp + end +end + +""" + CachingPool(workers::Vector{Int}) + +An implementation of an `AbstractWorkerPool`. `remote`, `remotecall_fetch`, `pmap` and other +remote calls which execute functions remotely, benefit from caching the serialized/deserialized +functions on the worker nodes, especially for closures which capture large amounts of data. + +The remote cache is maintained for the lifetime of the returned `CachingPool` object. To clear the +cache earlier, use `clear!(pool)`. + +For global variables, only the bindings are captured in a closure, not the data. +`let` blocks can be used to capture global data. + +For example: +``` +const foo=rand(10^8); +wp=CachingPool(workers()) +let foo=foo + pmap(wp, i->sum(foo)+i, 1:100); +end +``` + +The above would transfer `foo` only once to each worker. + +""" +function CachingPool(workers::Vector{Int}) + pool = CachingPool() + for w in workers + push!(pool, w) + end + return pool +end + +CachingPool(wp::WorkerPool) = CachingPool(workers(wp)) + +""" + clear!(pool::CachingPool) -> pool + +Removes all cached functions from all participating workers. +""" +function clear!(pool::CachingPool) + for (_,rr) in pool.map_obj2ref + finalize(rr) + end + empty!(pool.map_obj2ref) + pool +end + +exec_from_cache(rr::RemoteChannel, args...; kwargs...) = fetch(rr)(args...; kwargs...) +function exec_from_cache(f_ref::Tuple{Function, RemoteChannel}, args...; kwargs...) + put!(f_ref[2], f_ref[1]) # Cache locally + f_ref[1](args...; kwargs...) +end + +function remotecall_pool(rc_f, f, pool::CachingPool, args...; kwargs...) + worker = take!(pool) + f_ref = get(pool.map_obj2ref, (worker, f), (f, RemoteChannel(worker))) + isa(f_ref, Tuple) && (pool.map_obj2ref[(worker, f)] = f_ref[2]) # Add to tracker + + try + rc_f(exec_from_cache, worker, f_ref, args...; kwargs...) + finally + # ensure that we do not add pid 1 back if it is not registered. + worker in pool.workers && put!(pool, worker) + end +end diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index cc7c53235181a..bba11c5ce0102 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -235,18 +235,6 @@ General Parallel Computing Support Returns a list of all worker process identifiers. -.. function:: default_worker_pool() - - .. Docstring generated from Julia source - - WorkerPool containing idle ``workers()`` (used by ``remote(f)``\ ). - -.. function:: WorkerPool(workers) - - .. Docstring generated from Julia source - - Create a WorkerPool from a vector of worker ids. - .. function:: rmprocs(pids...) .. Docstring generated from Julia source @@ -273,7 +261,7 @@ General Parallel Computing Support For multiple collection arguments, apply f elementwise. -.. function:: pmap([::WorkerPool], f, c...; distributed=true, batch_size=1, on_error=nothing, retry_n=0, retry_max_delay=DEFAULT_RETRY_MAX_DELAY, retry_on=DEFAULT_RETRY_ON) -> collection +.. function:: pmap([::AbstractWorkerPool], f, c...; distributed=true, batch_size=1, on_error=nothing, retry_n=0, retry_max_delay=DEFAULT_RETRY_MAX_DELAY, retry_on=DEFAULT_RETRY_ON) -> collection .. Docstring generated from Julia source @@ -377,18 +365,6 @@ General Parallel Computing Support Perform ``fetch(remotecall(...))`` in one message. Keyword arguments, if any, are passed through to ``func``\ . Any remote exceptions are captured in a ``RemoteException`` and thrown. -.. function:: remotecall_fetch(f, pool::WorkerPool, args...; kwargs...) - - .. Docstring generated from Julia source - - Call ``f(args...; kwargs...)`` on one of the workers in ``pool``\ . Waits for completion and returns the result. - -.. function:: remote([::WorkerPool], f) -> Function - - .. Docstring generated from Julia source - - Returns a lambda that executes function ``f`` on an available worker using ``remotecall_fetch``\ . - .. function:: put!(RemoteChannel, value) .. Docstring generated from Julia source @@ -448,6 +424,64 @@ General Parallel Computing Support * ``put!`` on a closed channel. * ``take!`` and ``fetch`` on an empty, closed channel. +.. function:: WorkerPool(workers) + + .. Docstring generated from Julia source + + Create a WorkerPool from a vector of worker ids. + +.. function:: CachingPool(workers::Vector{Int}) + + .. Docstring generated from Julia source + + An implementation of an ``AbstractWorkerPool``\ . ``remote``\ , ``remotecall_fetch``\ , ``pmap`` and other remote calls which execute functions remotely, benefit from caching the serialized/deserialized functions on the worker nodes, especially for closures which capture large amounts of data. + + The remote cache is maintained for the lifetime of the returned ``CachingPool`` object. To clear the cache earlier, use ``clear!(pool)``\ . + + For global variables, only the bindings are captured in a closure, not the data. ``let`` blocks can be used to capture global data. + + For example: + + .. code-block:: julia + + const foo=rand(10^8); + wp=CachingPool(workers()) + let foo=foo + pmap(wp, i->sum(foo)+i, 1:100); + end + + The above would transfer ``foo`` only once to each worker. + +.. function:: default_worker_pool() + + .. Docstring generated from Julia source + + WorkerPool containing idle ``workers()`` (used by ``remote(f)``\ ). + +.. function:: remote([::AbstractWorkerPool], f) -> Function + + .. Docstring generated from Julia source + + Returns a lambda that executes function ``f`` on an available worker using ``remotecall_fetch``\ . + +.. function:: remotecall(f, pool::AbstractWorkerPool, args...; kwargs...) + + .. Docstring generated from Julia source + + Call ``f(args...; kwargs...)`` on one of the workers in ``pool``\ . Returns a ``Future``\ . + +.. function:: remotecall_wait(f, pool::AbstractWorkerPool, args...; kwargs...) + + .. Docstring generated from Julia source + + Call ``f(args...; kwargs...)`` on one of the workers in ``pool``\ . Waits for completion, returns a ``Future``\ . + +.. function:: remotecall_fetch(f, pool::AbstractWorkerPool, args...; kwargs...) + + .. Docstring generated from Julia source + + Call ``f(args...; kwargs...)`` on one of the workers in ``pool``\ . Waits for completion and returns the result. + .. function:: timedwait(testcb::Function, secs::Float64; pollint::Float64=0.1) .. Docstring generated from Julia source @@ -531,6 +565,12 @@ General Parallel Computing Support foo = 1 @eval @everywhere bar=$foo +.. function:: clear!(pool::CachingPool) -> pool + + .. Docstring generated from Julia source + + Removes all cached functions from all participating workers. + .. function:: Base.remoteref_id(r::AbstractRemoteRef) -> (whence, id) .. Docstring generated from Julia source diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index b92a95c3d3040..e5bb975d8745a 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -500,3 +500,4 @@ .. Docstring generated from Julia source Create a string from the address of a NUL-terminated UTF-32 string. A copy is made; the pointer can be safely freed. If ``length`` is specified, the string does not have to be NUL-terminated. + diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index 0006659bd7dd4..ef5de78ce02f9 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -306,7 +306,7 @@ gives a `Broken` `Result`. .. Docstring generated from Julia source - For use to indicate a test that should pass but currently intermittently fails. Does not evaluate the expression. + For use to indicate a test that should pass but currently intermittently fails. Does not evaluate the expression, which makes it useful for tests of not-yet-implemented functionality. Creating Custom ``AbstractTestSet`` Types ----------------------------------------- diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 4ec27ae662cd8..978204b210628 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -803,6 +803,13 @@ end # Test asyncmap @test allunique(asyncmap(x->object_id(current_task()), 1:100)) +# CachingPool tests +wp = CachingPool(workers()) +@test [1:100...] == pmap(wp, x->x, 1:100) + +clear!(wp) +@test length(wp.map_obj2ref) == 0 + # The below block of tests are usually run only on local development systems, since: # - tests which print errors From 358b2d744a8fd40fc0e62486742047adf0520dab Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 26 Jun 2016 14:24:17 -0400 Subject: [PATCH 0109/1117] Tests for contains --- test/reduce.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/reduce.jl b/test/reduce.jl index 8db11f0a4014a..81be7086fa23d 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -287,3 +287,10 @@ A = reshape(map(UInt8, 1:100), (10,10)) @test sum([-0.0]) === -0.0 @test sum([-0.0, -0.0]) === -0.0 @test prod([-0.0, -0.0]) === 0.0 + +#contains +let A = collect(1:10) + @test A ∋ 5 + @test A ∌ 11 + @test contains(==,A,6) +end From 4ff1a06394e886cc948718600254eff1a49381ca Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 26 Jun 2016 14:24:24 -0400 Subject: [PATCH 0110/1117] Few missing BigInt function tests --- test/bigint.jl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/bigint.jl b/test/bigint.jl index 4e4554e736788..53cb5a9ccc7e2 100644 --- a/test/bigint.jl +++ b/test/bigint.jl @@ -300,3 +300,14 @@ ndigits(rand(big(-999:999)), big(2)^rand(2:999)) @test_throws InexactError round(BigInt,Inf) @test_throws InexactError floor(BigInt,Inf) @test_throws InexactError ceil(BigInt,Inf) + +@test bin(big(3)) == "11" +@test oct(big(9)) == "11" +@test oct(-big(9)) == "-11" +@test hex(big(12)) == "c" + +@test isqrt(big(4)) == 2 +@test isqrt(big(5)) == 2 + +@test big(5)^true == big(5) +@test big(5)^false == one(BigInt) From a84d66be83c4992dcc512a809c7970171ae8d22f Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 26 Jun 2016 14:24:34 -0400 Subject: [PATCH 0111/1117] Improved error message and tests for tuple --- base/tuple.jl | 2 +- test/tuple.jl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/base/tuple.jl b/base/tuple.jl index 2e69bf217bff2..08c245f837977 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -49,7 +49,7 @@ function front(t::Tuple) @_inline_meta _front((), t...) end -front(::Tuple{}) = error("Cannot call front on an empty tuple") +front(::Tuple{}) = throw(ArgumentError("Cannot call front on an empty tuple")) _front(out, v) = out function _front(out, v, t...) @_inline_meta diff --git a/test/tuple.jl b/test/tuple.jl index 252c5e0dc2202..5d7240f15bbed 100644 --- a/test/tuple.jl +++ b/test/tuple.jl @@ -6,6 +6,9 @@ @test length((1,)) === 1 @test length((1,2)) === 2 +@test_throws ArgumentError Base.front(()) +@test_throws ArgumentError first(()) + @test endof(()) === 0 @test endof((1,)) === 1 @test endof((1,2)) === 2 From c51c6b8da468b0979543980b15e3c616833ad0e4 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Mon, 27 Jun 2016 00:17:19 -0400 Subject: [PATCH 0112/1117] Fix compilation on LLVM svn `GlobalValue::setUnnamedAddr` Ref https://github.com/llvm-mirror/llvm/commit/63b34cdf342b8265d98357008765b2563dd04e6c --- src/cgutils.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 221cb0ce3d11f..981fe3edd482b 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -59,7 +59,11 @@ static Value *stringConstPtr(const std::string &txt) (const unsigned char*)pooledtxt.data(), pooledtxt.size())), ssno.str()); +#ifdef LLVM39 + gv->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); +#else gv->setUnnamedAddr(true); +#endif pooledval->second = gv; jl_ExecutionEngine->addGlobalMapping(gv, (void*)(uintptr_t)pooledtxt.data()); } From afd7be432b1514a609e6f8f40c18601db9476669 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Tue, 14 Jun 2016 21:40:32 -0400 Subject: [PATCH 0113/1117] Array micro optimizations * Make sure that newly allocated arrays are always young * Micro optimize `sizehint!` * Implement `copy(::Array)` in C to avoid calling `memcpy` that bypasses the write barrier. --- base/array.jl | 8 +++---- src/array.c | 66 ++++++++++++++++++++++++++++++--------------------- src/gc.c | 2 ++ src/jltypes.c | 2 +- src/julia.h | 8 +++++++ 5 files changed, 53 insertions(+), 33 deletions(-) diff --git a/base/array.jl b/base/array.jl index ec23d42237891..f85b1b6a8fb49 100644 --- a/base/array.jl +++ b/base/array.jl @@ -36,6 +36,8 @@ end ## copy ## function unsafe_copy!{T}(dest::Ptr{T}, src::Ptr{T}, n) + # Do not use this to copy data between pointer arrays. + # It can't be made safe no matter how carefully you checked. ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt), dest, src, n*sizeof(T)) return dest @@ -63,11 +65,7 @@ end copy!{T}(dest::Array{T}, src::Array{T}) = copy!(dest, 1, src, 1, length(src)) -function copy(a::Array) - b = similar(a) - ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt), b, a, sizeof(a)) - return b -end +copy{T<:Array}(a::T) = ccall(:jl_array_copy, Ref{T}, (Any,), a) function reinterpret{T,S}(::Type{T}, a::Array{S,1}) nel = Int(div(length(a)*sizeof(S),sizeof(T))) diff --git a/src/array.c b/src/array.c index ea3727cfec258..9656498997a29 100644 --- a/src/array.c +++ b/src/array.c @@ -22,7 +22,7 @@ extern "C" { static inline int store_unboxed(jl_value_t *el_type) { - return jl_is_datatype(el_type) && jl_is_leaf_type(el_type) && jl_is_immutable(el_type) && jl_is_pointerfree((jl_datatype_t*)el_type); + return jl_is_leaf_type(el_type) && jl_is_immutable(el_type) && jl_is_pointerfree((jl_datatype_t*)el_type); } int jl_array_store_unboxed(jl_value_t *el_type) @@ -80,6 +80,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, tsz += tot; tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align whole object a = (jl_array_t*)jl_gc_allocobj(tsz); + // No allocation or safepoint allowed after this jl_set_typeof(a, atype); a->flags.how = 0; data = (char*)a + doffs; @@ -89,21 +90,16 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, } else { tsz = JL_ARRAY_ALIGN(tsz, JL_CACHE_BYTE_ALIGNMENT); // align whole object + data = jl_gc_managed_malloc(tot); + // Allocate the Array **after** allocating the data + // to make sure the array is still young a = (jl_array_t*)jl_gc_allocobj(tsz); - JL_GC_PUSH1(&a); + // No allocation or safepoint allowed after this jl_set_typeof(a, atype); - // temporarily initialize to make gc-safe - a->data = NULL; a->flags.how = 2; - // Make sure the GC can correctly identify if this is pool allocated - // and mark the page accordingly - a->flags.pooled = tsz <= GC_MAX_SZCLASS; - - data = jl_gc_managed_malloc(tot); jl_gc_track_malloced_array(a); if (!isunboxed) memset(data, 0, tot); - JL_GC_POP(); } a->flags.pooled = tsz <= GC_MAX_SZCLASS; @@ -173,6 +169,7 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_allocobj(tsz); + // No allocation or safepoint allowed after this jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->flags.ndims = ndims; @@ -180,7 +177,8 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, a->data = NULL; a->flags.isaligned = data->flags.isaligned; jl_value_t *el_type = jl_tparam0(atype); - if (store_unboxed(el_type)) { + assert(store_unboxed(el_type) == !data->flags.ptrarray); + if (!data->flags.ptrarray) { a->elsize = jl_datatype_size(el_type); a->flags.ptrarray = 0; } @@ -247,6 +245,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_allocobj(tsz); + // No allocation or safepoint allowed after this jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; @@ -301,6 +300,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); a = (jl_array_t*)jl_gc_allocobj(tsz); + // No allocation or safepoint allowed after this jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; @@ -607,7 +607,7 @@ static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, siz a->maxsize = newlen; } -static void array_try_unshare(jl_array_t *a) +static void NOINLINE array_try_unshare(jl_array_t *a) { if (a->flags.isshared) { if (a->flags.how != 3) @@ -668,18 +668,6 @@ JL_DLLEXPORT void jl_array_del_end(jl_array_t *a, size_t dec) a->nrows -= dec; } -JL_DLLEXPORT void jl_array_sizehint(jl_array_t *a, size_t sz) -{ - if (sz <= jl_array_len(a)) - return; - size_t inc = sz - jl_array_len(a); - jl_array_grow_end(a, inc); -#ifdef STORE_ARRAY_LEN - a->length -= inc; -#endif - a->nrows -= inc; -} - JL_DLLEXPORT void jl_array_grow_beg(jl_array_t *a, size_t inc) { // For pointer array the memory grown should be zero'd @@ -759,19 +747,43 @@ JL_DLLEXPORT void jl_array_del_beg(jl_array_t *a, size_t dec) a->offset = newoffs; } +JL_DLLEXPORT void jl_array_sizehint(jl_array_t *a, size_t sz) +{ + size_t n = jl_array_nrows(a); + if (sz <= n) + return; + size_t inc = sz - n; + jl_array_grow_end(a, inc); + a->nrows = n; +#ifdef STORE_ARRAY_LEN + a->length = n; +#endif +} + +JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary) +{ + size_t elsz = ary->elsize; + jl_array_t *new_ary = _new_array_(jl_typeof(ary), jl_array_ndims(ary), + &ary->nrows, !ary->flags.ptrarray, elsz); + memcpy(new_ary->data, ary->data, jl_array_len(ary) * elsz); + return new_ary; +} + JL_DLLEXPORT void jl_array_ptr_1d_push(jl_array_t *a, jl_value_t *item) { assert(jl_typeis(a, jl_array_any_type)); jl_array_grow_end(a, 1); - jl_array_ptr_set(a, jl_array_dim(a,0)-1, item); + size_t n = jl_array_nrows(a); + jl_array_ptr_set(a, n - 1, item); } JL_DLLEXPORT void jl_array_ptr_1d_push2(jl_array_t *a, jl_value_t *b, jl_value_t *c) { assert(jl_typeis(a, jl_array_any_type)); jl_array_grow_end(a, 2); - jl_array_ptr_set(a, jl_array_dim(a,0)-2, b); - jl_array_ptr_set(a, jl_array_dim(a,0)-1, c); + size_t n = jl_array_nrows(a); + jl_array_ptr_set(a, n - 2, b); + jl_array_ptr_set(a, n - 1, c); } JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a) diff --git a/src/gc.c b/src/gc.c index 5775435a254b2..8ac80fea4a8f9 100644 --- a/src/gc.c +++ b/src/gc.c @@ -684,6 +684,7 @@ static void sweep_big(int sweep_full) void jl_gc_track_malloced_array(jl_array_t *a) { + // This is **NOT** a GC safe point. mallocarray_t *ma; if (jl_thread_heap.mafreelist == NULL) { ma = (mallocarray_t*)malloc(sizeof(mallocarray_t)); @@ -699,6 +700,7 @@ void jl_gc_track_malloced_array(jl_array_t *a) void jl_gc_count_allocd(size_t sz) { + // This is **NOT** a GC safe point. gc_num.allocd += sz; } diff --git a/src/jltypes.c b/src/jltypes.c index 104a4dba5be16..887ad84ede3ac 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -159,7 +159,7 @@ JL_DLLEXPORT int jl_has_typevars(jl_value_t *v) return jl_has_typevars__(v, 0, NULL, 0); } -JL_DLLEXPORT int jl_is_leaf_type(jl_value_t *v) +JL_DLLEXPORT int (jl_is_leaf_type)(jl_value_t *v) { if (jl_is_datatype(v)) { int isleaf = ((jl_datatype_t*)v)->isleaftype; diff --git a/src/julia.h b/src/julia.h index 7feae969021e4..c80cb6ebe62ad 100644 --- a/src/julia.h +++ b/src/julia.h @@ -985,6 +985,14 @@ JL_DLLEXPORT const char *jl_typename_str(jl_value_t *v); JL_DLLEXPORT const char *jl_typeof_str(jl_value_t *v); JL_DLLEXPORT int jl_type_morespecific(jl_value_t *a, jl_value_t *b); +#ifdef NDEBUG +STATIC_INLINE int jl_is_leaf_type_(jl_value_t *v) +{ + return jl_is_datatype(v) && ((jl_datatype_t*)v)->isleaftype; +} +#define jl_is_leaf_type(v) jl_is_leaf_type_(v) +#endif + // type constructors JL_DLLEXPORT jl_typename_t *jl_new_typename(jl_sym_t *name); JL_DLLEXPORT jl_tvar_t *jl_new_typevar(jl_sym_t *name,jl_value_t *lb,jl_value_t *ub); From 43f8144ed15f4fe4458d17c5b9578f2b4c89fa0f Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Tue, 14 Jun 2016 23:31:13 -0400 Subject: [PATCH 0114/1117] A few Cstring related fix * Always call `cconvert` before calling `unsafe_convert`. * Optimize parse functions for non-NUL terminated input. * Use `Cstring` for `ccall`s that's expecting a NUL terminated string --- base/c.jl | 14 +++----------- base/libgit2/libgit2.jl | 2 +- base/libgit2/oid.jl | 4 ++-- base/libgit2/strarray.jl | 6 +++++- base/loading.jl | 2 +- base/process.jl | 4 ++-- base/refpointer.jl | 17 ++++++++++++++--- src/builtins.c | 26 ++++++++++++++++++++------ test/file.jl | 4 ++-- test/strings/basic.jl | 4 ++-- 10 files changed, 52 insertions(+), 31 deletions(-) diff --git a/base/c.jl b/base/c.jl index 53ed7e4f03f01..878bffff8b186 100644 --- a/base/c.jl +++ b/base/c.jl @@ -46,17 +46,6 @@ if !is_windows() end end -# C NUL-terminated string pointers; these can be used in ccall -# instead of Ptr{Cchar} and Ptr{Cwchar_t}, respectively, to enforce -# a check for embedded NUL chars in the string (to avoid silent truncation). -if Int === Int64 - bitstype 64 Cstring - bitstype 64 Cwstring -else - bitstype 32 Cstring - bitstype 32 Cwstring -end - # construction from typed pointers convert{T<:Union{Int8,UInt8}}(::Type{Cstring}, p::Ptr{T}) = box(Cstring, p) convert(::Type{Cwstring}, p::Ptr{Cwchar_t}) = box(Cwstring, p) @@ -88,6 +77,9 @@ function cconvert(::Type{Cwstring}, s::AbstractString) return v end +eltype(::Type{Cstring}) = UInt8 +eltype(::Type{Cwstring}) = Cwchar_t + containsnul(p::Ptr, len) = C_NULL != ccall(:memchr, Ptr{Cchar}, (Ptr{Cchar}, Cint, Csize_t), p, 0, len) containsnul(s::String) = containsnul(unsafe_convert(Ptr{Cchar}, s), sizeof(s)) diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index 5b579511a76bb..89f05da2d84c6 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -297,7 +297,7 @@ function clone{P<:AbstractPayload}(repo_url::AbstractString, repo_path::Abstract remote_cb::Ptr{Void} = C_NULL, payload::Nullable{P}=Nullable{AbstractPayload}()) # setup clone options - lbranch = Base.cconvert(Cstring, String(branch)) + lbranch = Base.cconvert(Cstring, branch) fetch_opts=FetchOptions(callbacks = RemoteCallbacks(credentials_cb(), payload)) clone_opts = CloneOptions( bare = Cint(isbare), diff --git a/base/libgit2/oid.jl b/base/libgit2/oid.jl index 816be5f800ef1..313f9487fb3dc 100644 --- a/base/libgit2/oid.jl +++ b/base/libgit2/oid.jl @@ -28,7 +28,7 @@ function Oid(id::AbstractString) (Ptr{Oid}, Ptr{UInt8}, Csize_t), oid_ptr, bstr, len) else ccall((:git_oid_fromstrp, :libgit2), Cint, - (Ptr{Oid}, Ptr{Cchar}), oid_ptr, bstr) + (Ptr{Oid}, Cstring), oid_ptr, bstr) end err != 0 && return Oid() return oid_ptr[] @@ -46,7 +46,7 @@ function Oid(repo::GitRepo, ref_name::AbstractString) isempty(repo) && return Oid() oid_ptr = Ref(Oid()) @check ccall((:git_reference_name_to_id, :libgit2), Cint, - (Ptr{Oid}, Ptr{Void}, Ptr{UInt8}), + (Ptr{Oid}, Ptr{Void}, Cstring), oid_ptr, repo.ptr, ref_name) return oid_ptr[] end diff --git a/base/libgit2/strarray.jl b/base/libgit2/strarray.jl index daaf300e42018..ae34ec1da7327 100644 --- a/base/libgit2/strarray.jl +++ b/base/libgit2/strarray.jl @@ -2,7 +2,11 @@ function StrArrayStruct{T<:AbstractString}(strs::T...) strcount = length(strs) - map(s->Base.unsafe_convert(Cstring, String(s)), strs) # check for null-strings + for s in strs + if Base.containsnul(s) + throw("embedded NULs are not allowed in C strings: $(repr(s))") + end + end sa_strings = convert(Ptr{Cstring}, Libc.malloc(sizeof(Cstring) * strcount)) for i=1:strcount len = length(strs[i]) diff --git a/base/loading.jl b/base/loading.jl index 99c049faf5593..528a33ecb4944 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -171,7 +171,7 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t end elseif node == myid() myid() == 1 && recompile_stale(mod, path_to_try) - restored = ccall(:jl_restore_incremental, Any, (Ptr{UInt8},), path_to_try) + restored = ccall(:jl_restore_incremental, Any, (Cstring,), path_to_try) else content = remotecall_fetch(open, node, read, path_to_try) restored = _include_from_serialized(content) diff --git a/base/process.jl b/base/process.jl index e0b1a11e6c23a..55495765febe5 100644 --- a/base/process.jl +++ b/base/process.jl @@ -310,8 +310,8 @@ function _jl_spawn(cmd, argv, loop::Ptr{Void}, pp::Process, proc = Libc.malloc(_sizeof_uv_process) disassociate_julia_struct(proc) error = ccall(:jl_spawn, Int32, - (Ptr{UInt8}, Ptr{Ptr{UInt8}}, Ptr{Void}, Ptr{Void}, Any, Int32, - Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Ptr{UInt8}}, Ptr{UInt8}, Ptr{Void}), + (Cstring, Ptr{Cstring}, Ptr{Void}, Ptr{Void}, Any, Int32, + Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Cstring}, Cstring, Ptr{Void}), cmd, argv, loop, proc, pp, uvtype(in), uvhandle(in), uvtype(out), uvhandle(out), uvtype(err), uvhandle(err), pp.cmd.flags, pp.cmd.env === nothing ? C_NULL : pp.cmd.env, isempty(pp.cmd.dir) ? C_NULL : pp.cmd.dir, diff --git a/base/refpointer.jl b/base/refpointer.jl index 4a65c67ddcf83..a074f7f60f4bc 100644 --- a/base/refpointer.jl +++ b/base/refpointer.jl @@ -1,5 +1,16 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +# C NUL-terminated string pointers; these can be used in ccall +# instead of Ptr{Cchar} and Ptr{Cwchar_t}, respectively, to enforce +# a check for embedded NUL chars in the string (to avoid silent truncation). +if Int === Int64 + bitstype 64 Cstring + bitstype 64 Cwstring +else + bitstype 32 Cstring + bitstype 32 Cwstring +end + ### General Methods for Ref{T} type eltype{T}(x::Type{Ref{T}}) = T @@ -62,10 +73,10 @@ end unsafe_convert{T}(::Type{Ptr{Void}}, b::RefArray{T}) = convert(Ptr{Void}, unsafe_convert(Ptr{T}, b)) # convert Arrays to pointer arrays for ccall -function (::Type{Ref{P}}){P<:Ptr,T<:Ptr}(a::Array{T}) # Ref{P<:Ptr}(a::Array{T<:Ptr}) +function (::Type{Ref{P}}){P<:Union{Ptr,Cwstring,Cstring},T<:Union{Ptr,Cwstring,Cstring}}(a::Array{T}) # Ref{P<:Ptr}(a::Array{T<:Ptr}) return RefArray(a) # effectively a no-op end -function (::Type{Ref{P}}){P<:Ptr,T}(a::Array{T}) # Ref{P<:Ptr}(a::Array) +function (::Type{Ref{P}}){P<:Union{Ptr,Cwstring,Cstring},T}(a::Array{T}) # Ref{P<:Ptr}(a::Array) if (!isbits(T) && T <: eltype(P)) # this Array already has the right memory layout for the requested Ref return RefArray(a,1,false) # root something, so that this function is type-stable @@ -82,7 +93,7 @@ function (::Type{Ref{P}}){P<:Ptr,T}(a::Array{T}) # Ref{P<:Ptr}(a::Array) end end cconvert{P<:Ptr,T<:Ptr}(::Union{Type{Ptr{P}},Type{Ref{P}}}, a::Array{T}) = a -cconvert{P<:Ptr}(::Union{Type{Ptr{P}},Type{Ref{P}}}, a::Array) = Ref{P}(a) +cconvert{P<:Union{Ptr,Cwstring,Cstring}}(::Union{Type{Ptr{P}},Type{Ref{P}}}, a::Array) = Ref{P}(a) ### diff --git a/src/builtins.c b/src/builtins.c index 4c25cd8562ed6..cd91a4fe58382 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -801,12 +801,19 @@ JL_DLLEXPORT jl_nullable_float64_t jl_try_substrtod(char *str, size_t offset, si char *p; char *bstr = str+offset; char *pend = bstr+len; + char *tofree = NULL; int err = 0; errno = 0; if (!(*pend == '\0' || isspace((unsigned char)*pend) || *pend == ',')) { // confusing data outside substring. must copy. - char *newstr = (char*)malloc(len+1); + char *newstr; + if (len + 1 < jl_page_size) { + newstr = (char*)alloca(len + 1); + } + else { + newstr = tofree = (char*)malloc(len + 1); + } memcpy(newstr, bstr, len); newstr[len] = 0; bstr = newstr; @@ -826,8 +833,8 @@ JL_DLLEXPORT jl_nullable_float64_t jl_try_substrtod(char *str, size_t offset, si err = substr_isspace(p, pend) ? 0 : 1; } - if (bstr != str+offset) - free(bstr); + if (__unlikely(tofree)) + free(tofree); jl_nullable_float64_t ret = {(uint8_t)err, out}; return ret; @@ -853,12 +860,19 @@ JL_DLLEXPORT jl_nullable_float32_t jl_try_substrtof(char *str, size_t offset, si char *p; char *bstr = str+offset; char *pend = bstr+len; + char *tofree = NULL; int err = 0; errno = 0; if (!(*pend == '\0' || isspace((unsigned char)*pend) || *pend == ',')) { // confusing data outside substring. must copy. - char *newstr = (char*)malloc(len+1); + char *newstr; + if (len + 1 < jl_page_size) { + newstr = (char*)alloca(len + 1); + } + else { + newstr = tofree = (char*)malloc(len + 1); + } memcpy(newstr, bstr, len); newstr[len] = 0; bstr = newstr; @@ -882,8 +896,8 @@ JL_DLLEXPORT jl_nullable_float32_t jl_try_substrtof(char *str, size_t offset, si err = substr_isspace(p, pend) ? 0 : 1; } - if (bstr != str+offset) - free(bstr); + if (__unlikely(tofree)) + free(tofree); jl_nullable_float32_t ret = {(uint8_t)err, out}; return ret; diff --git a/test/file.jl b/test/file.jl index 3a70e221cf780..3c1e4381473e5 100644 --- a/test/file.jl +++ b/test/file.jl @@ -897,9 +897,9 @@ let f = open(file, "w") test_LibcFILE(convert(Libc.FILE, f)) close(f) if is_windows() - f = RawFD(ccall(:_open, Cint, (Ptr{UInt8}, Cint), file, Base.Filesystem.JL_O_RDONLY)) + f = RawFD(ccall(:_open, Cint, (Cstring, Cint), file, Base.Filesystem.JL_O_RDONLY)) else - f = RawFD(ccall(:open, Cint, (Ptr{UInt8}, Cint), file, Base.Filesystem.JL_O_RDONLY)) + f = RawFD(ccall(:open, Cint, (Cstring, Cint), file, Base.Filesystem.JL_O_RDONLY)) end test_LibcFILE(Libc.FILE(f, Libc.modestr(true, false))) end diff --git a/test/strings/basic.jl b/test/strings/basic.jl index de45bf8405740..940eae91afc7a 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -235,10 +235,10 @@ for T in [BigInt, Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int1 @test isnull(tryparse(T, "1\0")) end let s = normalize_string("tést",:NFKC) - @test unsafe_string(Base.unsafe_convert(Cstring, s)) == s + @test unsafe_string(Base.unsafe_convert(Cstring, Base.cconvert(Cstring, s))) == s @test unsafe_string(convert(Cstring, Symbol(s))) == s end -@test_throws ArgumentError Base.unsafe_convert(Cstring, "ba\0d") +@test_throws ArgumentError Base.unsafe_convert(Cstring, Base.cconvert(Cstring, "ba\0d")) cstrdup(s) = @static is_windows() ? ccall(:_strdup, Cstring, (Cstring,), s) : ccall(:strdup, Cstring, (Cstring,), s) let p = cstrdup("hello") From 7892de0ca71b22c9cf4d10f38a352fe4bb8f7de2 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Wed, 15 Jun 2016 22:59:44 -0400 Subject: [PATCH 0115/1117] Implement implicit byte checker in C. * Use it in `jl_load_` * Remove the length parameter from `jl_load`, `jl_load_file_string` and `jl_parse_eval_all` to NOT pretend they support non-NUL-terminated strings. --- base/loading.jl | 4 ++-- src/array.c | 40 ++++++++++++++++++++++++++++++++++++++++ src/ast.c | 7 ++++--- src/init.c | 2 +- src/julia.h | 4 ++-- src/julia_internal.h | 4 +++- src/toplevel.c | 18 ++++++++++-------- test/loading.jl | 4 +--- ui/repl.c | 2 +- 9 files changed, 64 insertions(+), 21 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 528a33ecb4944..2803460cadb4d 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -378,8 +378,8 @@ end # remote/parallel load include_string(txt::String, fname::String) = - ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Ptr{UInt8},Csize_t), - txt, sizeof(txt), fname, sizeof(fname)) + ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Cstring), + txt, sizeof(txt), fname) include_string(txt::AbstractString, fname::AbstractString="string") = include_string(String(txt), String(fname)) diff --git a/src/array.c b/src/array.c index 9656498997a29..bc014ff6c5dd8 100644 --- a/src/array.c +++ b/src/array.c @@ -791,6 +791,46 @@ JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a) return jl_array_data_owner(a); } +STATIC_INLINE int jl_has_implicit_byte_owned(jl_array_t *a) +{ + assert(a->flags.how != 3); + if (!a->flags.isshared) + return 1; + return a->flags.how == 1; +} + +STATIC_INLINE int jl_has_implicit_byte(jl_array_t *a) +{ + // * unshared: + // * how: 0-2 + // We own and allocated the data. + // It should have the extra byte. + // * shared: + // * how: 0, 2 + // The data might come from external source without implicit NUL byte. + // There could be an entra byte for a `reinterpreted` array + // but that should be unlikely for strings. + // * how: 1 + // We allocated the data with the extra byte. + // * how: 3 + // We should check the owner. + if (a->flags.how == 3) { + a = (jl_array_t*)jl_array_data_owner(a); + return a->elsize == 1 && jl_has_implicit_byte_owned(a); + } + return jl_has_implicit_byte_owned(a); +} + +// Create an array with the same content +JL_DLLEXPORT jl_array_t *jl_array_cconvert_cstring(jl_array_t *a) +{ + assert(jl_typeof(a) == jl_array_uint8_type); + if (!jl_has_implicit_byte(a)) + a = jl_array_copy(a); + ((char*)a->data)[a->nrows] = 0; + return a; +} + #ifdef __cplusplus } #endif diff --git a/src/ast.c b/src/ast.c index b01980ba100aa..e620d1a94eb4b 100644 --- a/src/ast.c +++ b/src/ast.c @@ -641,7 +641,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len, } // parse and eval a whole file, possibly reading from a string (`content`) -jl_value_t *jl_parse_eval_all(const char *fname, size_t len, +jl_value_t *jl_parse_eval_all(const char *fname, const char *content, size_t contentlen) { if (in_pure_callback) @@ -649,6 +649,7 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len, jl_ast_context_t *ctx = jl_ast_ctx_enter(); fl_context_t *fl_ctx = &ctx->fl; value_t f, ast; + size_t len = strlen(fname); f = cvalue_static_cstrn(fl_ctx, fname, len); fl_gc_handle(fl_ctx, &f); if (content != NULL) { @@ -723,9 +724,9 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len, } JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len, - char *filename, size_t namelen) + char *filename) { - return jl_parse_eval_all(filename, namelen, text, len); + return jl_parse_eval_all(filename, text, len); } JL_DLLEXPORT int jl_parse_depwarn(int warn) diff --git a/src/init.c b/src/init.c index 06df5fb4112e1..6b794c68cf2bf 100644 --- a/src/init.c +++ b/src/init.c @@ -653,7 +653,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_all_tls_states[t]->root_task->current_module = jl_current_module; } - jl_load("boot.jl", sizeof("boot.jl")-1); + jl_load("boot.jl"); jl_get_builtin_hooks(); jl_boot_file_loaded = 1; jl_init_box_caches(); diff --git a/src/julia.h b/src/julia.h index c80cb6ebe62ad..ad021d135319c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1315,7 +1315,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len, int pos0, int greedy); JL_DLLEXPORT int jl_parse_depwarn(int warn); JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len, - char *filename, size_t namelen); + char *filename); JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr); JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str); @@ -1350,7 +1350,7 @@ JL_DLLEXPORT const char *jl_lookup_soname(const char *pfx, size_t n); // compiler JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_value_t *v); JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex); -JL_DLLEXPORT jl_value_t *jl_load(const char *fname, size_t len); +JL_DLLEXPORT jl_value_t *jl_load(const char *fname); JL_DLLEXPORT jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e, jl_lambda_info_t *lam); JL_DLLEXPORT jl_module_t *jl_base_relative_to(jl_module_t *m); diff --git a/src/julia_internal.h b/src/julia_internal.h index 2edfe782d1ad4..40c184c35bba4 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -196,7 +196,7 @@ jl_value_t *jl_toplevel_eval_in_warn(jl_module_t *m, jl_value_t *ex, jl_lambda_info_t *jl_wrap_expr(jl_value_t *expr); jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e); -jl_value_t *jl_parse_eval_all(const char *fname, size_t len, +jl_value_t *jl_parse_eval_all(const char *fname, const char *content, size_t contentlen); jl_value_t *jl_interpret_toplevel_thunk(jl_lambda_info_t *lam); jl_value_t *jl_interpret_toplevel_expr(jl_value_t *e); @@ -662,6 +662,8 @@ STATIC_INLINE void *jl_get_frame_addr(void) #endif } +JL_DLLEXPORT jl_array_t *jl_array_cconvert_cstring(jl_array_t *a); + #ifdef __cplusplus } #endif diff --git a/src/toplevel.c b/src/toplevel.c index 69a0f55f41d3e..ea048c1c8eff5 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -551,28 +551,30 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_value_t *v) return jl_toplevel_eval_flex(v, 1, 0); } -JL_DLLEXPORT jl_value_t *jl_load(const char *fname, size_t len) +JL_DLLEXPORT jl_value_t *jl_load(const char *fname) { - if (NULL != memchr(fname, 0, len)) - jl_exceptionf(jl_argumenterror_type, "file name may not contain \\0"); if (jl_current_module->istopmod) { jl_printf(JL_STDOUT, "%s\r\n", fname); #ifdef _OS_WINDOWS_ uv_run(uv_default_loop(), (uv_run_mode)1); #endif } - char *fpath = (char*)fname; uv_stat_t stbuf; - if (jl_stat(fpath, (char*)&stbuf) != 0 || (stbuf.st_mode & S_IFMT) != S_IFREG) { - jl_errorf("could not open file %s", fpath); + if (jl_stat(fname, (char*)&stbuf) != 0 || (stbuf.st_mode & S_IFMT) != S_IFREG) { + jl_errorf("could not open file %s", fname); } - return jl_parse_eval_all(fpath, len, NULL, 0); + return jl_parse_eval_all(fname, NULL, 0); } // load from filename given as a String object JL_DLLEXPORT jl_value_t *jl_load_(jl_value_t *str) { - return jl_load(jl_string_data(str), jl_string_len(str)); + jl_array_t *ary = + jl_array_cconvert_cstring((jl_array_t*)(jl_data_ptr(str)[0])); + JL_GC_PUSH1(&ary); + jl_value_t *res = jl_load((const char*)ary->data); + JL_GC_POP(); + return res; } // method definition ---------------------------------------------------------- diff --git a/test/loading.jl b/test/loading.jl index b7687d65d71e8..8d40b2c16981c 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -4,10 +4,8 @@ using Base.Test @test @__LINE__ == 5 -@test_throws ArgumentError Core.include("test_sourcepath.jl\0") - include("test_sourcepath.jl") -thefname = "the fname!//\\&\0\1*" +thefname = "the fname!//\\&\1*" @test include_string("include_string_test() = @__FILE__", thefname)() == Base.source_path() @test include_string("Base.source_path()", thefname) == Base.source_path() @test basename(@__FILE__) == "loading.jl" diff --git a/ui/repl.c b/ui/repl.c index a4e726ff464d5..1539eac058005 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -487,7 +487,7 @@ static int exec_program(char *program) JL_EH_POP(); return 1; } - jl_load(program, strlen(program)); + jl_load(program); } JL_CATCH { err = 1; From c93f86499217eb23e09e661a13854c9c478e9133 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 16 Jun 2016 08:53:36 -0400 Subject: [PATCH 0116/1117] Implement `cconvert(Cstring)` in C Add tests for implicit extra byte check. --- base/c.jl | 8 ++++++-- test/core.jl | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/base/c.jl b/base/c.jl index 878bffff8b186..81aecc8c2c64b 100644 --- a/base/c.jl +++ b/base/c.jl @@ -69,7 +69,11 @@ unsafe_wrap(::Type{String}, p::Cstring, len::Integer, own::Bool=false) = unsafe_string(s::Cstring) = unsafe_string(convert(Ptr{UInt8}, s)) # convert strings to String etc. to pass as pointers -cconvert(::Type{Cstring}, s::AbstractString) = String(s) +cconvert(::Type{Cstring}, s::String) = + ccall(:jl_array_cconvert_cstring, Ref{Vector{UInt8}}, + (Vector{UInt8},), s.data) +cconvert(::Type{Cstring}, s::AbstractString) = + cconvert(Cstring, String(s)::String) function cconvert(::Type{Cwstring}, s::AbstractString) v = transcode(Cwchar_t, String(s).data) @@ -85,7 +89,7 @@ containsnul(p::Ptr, len) = containsnul(s::String) = containsnul(unsafe_convert(Ptr{Cchar}, s), sizeof(s)) containsnul(s::AbstractString) = '\0' in s -function unsafe_convert(::Type{Cstring}, s::String) +function unsafe_convert(::Type{Cstring}, s::Vector{UInt8}) p = unsafe_convert(Ptr{Cchar}, s) containsnul(p, sizeof(s)) && throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))")) diff --git a/test/core.jl b/test/core.jl index c7b05769be013..8ec8bb2371445 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3890,6 +3890,51 @@ let arrayset_unknown_dim(Int, 3) end +module TestArrayNUL +using Base.Test +function check_nul(a::Vector{UInt8}) + b = ccall(:jl_array_cconvert_cstring, + Ref{Vector{UInt8}}, (Vector{UInt8},), a) + @test unsafe_load(pointer(b), length(b) + 1) == 0x0 + return b === a +end + +a = UInt8[] +b = "aaa" +c = [0x2, 0x1, 0x3] + +@test check_nul(a) +@test check_nul(b.data) +@test check_nul(c) +d = [0x2, 0x1, 0x3] +@test check_nul(d) +push!(d, 0x3) +@test check_nul(d) +push!(d, 0x3) +@test check_nul(d) +ccall(:jl_array_del_end, Void, (Any, UInt), d, 2) +@test check_nul(d) +ccall(:jl_array_grow_end, Void, (Any, UInt), d, 1) +@test check_nul(d) +ccall(:jl_array_grow_end, Void, (Any, UInt), d, 1) +@test check_nul(d) +ccall(:jl_array_grow_end, Void, (Any, UInt), d, 10) +@test check_nul(d) +ccall(:jl_array_del_beg, Void, (Any, UInt), d, 8) +@test check_nul(d) +ccall(:jl_array_grow_beg, Void, (Any, UInt), d, 8) +@test check_nul(d) +ccall(:jl_array_grow_beg, Void, (Any, UInt), d, 8) +@test check_nul(d) +f = unsafe_wrap(Array, pointer(d), length(d)) +@test !check_nul(f) +f = unsafe_wrap(Array, ccall(:malloc, Ptr{UInt8}, (Csize_t,), 10), 10, true) +@test !check_nul(f) +g = reinterpret(UInt8, UInt16[0x1, 0x2]) +@test !check_nul(g) +@test check_nul(copy(g)) +end + # issue #15370 @test isdefined(Core, :Box) @test !isdefined(Base, :Box) From 81f20795550a05bc5926f937b8f84ab730654bda Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 16 Jun 2016 09:13:35 -0400 Subject: [PATCH 0117/1117] Array resizing fixes and clean up * Move some `memcpy` and `memmove` from julia to C This is dangerous especially for `ptrarray` since it bypasses the write barrier. * Make sure resizing a ptrarray always have the new memory cleared * Add more test for shared array resizing --- base/array.jl | 102 ++---------- base/precompile.jl | 5 +- src/array.c | 390 +++++++++++++++++++++++++++++++-------------- test/core.jl | 61 +++++++ 4 files changed, 346 insertions(+), 212 deletions(-) diff --git a/base/array.jl b/base/array.jl index f85b1b6a8fb49..7a76f11c69e9f 100644 --- a/base/array.jl +++ b/base/array.jl @@ -387,66 +387,13 @@ setindex!{T, N}(A::Array{T, N}, x::Number, ::Vararg{Colon, N}) = fill!(A, x) # efficiently grow an array -function _growat!(a::Vector, i::Integer, delta::Integer) - n = length(a) - if i < div(n,2) - _growat_beg!(a, i, delta) - else - _growat_end!(a, i, delta) - end - return a -end - -function _growat_beg!(a::Vector, i::Integer, delta::Integer) - ccall(:jl_array_grow_beg, Void, (Any, UInt), a, delta) - if i > 1 - ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t), - pointer(a, 1), pointer(a, 1+delta), (i-1)*elsize(a)) - end - return a -end - -function _growat_end!(a::Vector, i::Integer, delta::Integer) - ccall(:jl_array_grow_end, Void, (Any, UInt), a, delta) - n = length(a) - if n >= i+delta - ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t), - pointer(a, i+delta), pointer(a, i), (n-i-delta+1)*elsize(a)) - end - return a -end +_growat!(a::Vector, i::Integer, delta::Integer) = + ccall(:jl_array_grow_at, Void, (Any, Int, UInt), a, i - 1, delta) # efficiently delete part of an array -function _deleteat!(a::Vector, i::Integer, delta::Integer) - n = length(a) - last = i+delta-1 - if i-1 < n-last - _deleteat_beg!(a, i, delta) - else - _deleteat_end!(a, i, delta) - end - return a -end - -function _deleteat_beg!(a::Vector, i::Integer, delta::Integer) - if i > 1 - ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t), - pointer(a, 1+delta), pointer(a, 1), (i-1)*elsize(a)) - end - ccall(:jl_array_del_beg, Void, (Any, UInt), a, delta) - return a -end - -function _deleteat_end!(a::Vector, i::Integer, delta::Integer) - n = length(a) - if n >= i+delta - ccall(:memmove, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t), - pointer(a, i), pointer(a, i+delta), (n-i-delta+1)*elsize(a)) - end - ccall(:jl_array_del_end, Void, (Any, UInt), a, delta) - return a -end +_deleteat!(a::Vector, i::Integer, delta::Integer) = + ccall(:jl_array_del_at, Void, (Any, Int, UInt), a, i - 1, delta) ## Dequeue functionality ## @@ -526,34 +473,20 @@ function shift!(a::Vector) end function insert!{T}(a::Array{T,1}, i::Integer, item) - if !(1 <= i <= length(a)+1) - throw(BoundsError()) - end - if i == length(a)+1 - return push!(a, item) - end - item = convert(T, item) + # Throw convert error before changing the shape of the array + _item = convert(T, item) _growat!(a, i, 1) - a[i] = item + # _growat! already did bound check + @inbounds a[i] = _item return a end -function deleteat!(a::Vector, i::Integer) - if !(1 <= i <= length(a)) - throw(BoundsError()) - end - return _deleteat!(a, i, 1) -end +deleteat!(a::Vector, i::Integer) = (_deleteat!(a, i, 1); a) function deleteat!{T<:Integer}(a::Vector, r::UnitRange{T}) n = length(a) - isempty(r) && return a - f = first(r) - l = last(r) - if !(1 <= f && l <= n) - throw(BoundsError()) - end - return _deleteat!(a, f, length(r)) + isempty(r) || _deleteat!(a, first(r), length(r)) + return a end function deleteat!(a::Vector, inds) @@ -620,18 +553,9 @@ function splice!{T<:Integer}(a::Vector, r::UnitRange{T}, ins=_default_splice) if m < d delta = d - m - if f-1 < n-l - _deleteat_beg!(a, f, delta) - else - _deleteat_end!(a, l-delta+1, delta) - end + _deleteat!(a, (f - 1 < n - l) ? f : (l - delta + 1), delta) elseif m > d - delta = m - d - if f-1 < n-l - _growat_beg!(a, f, delta) - else - _growat_end!(a, l+1, delta) - end + _growat!(a, (f - 1 < n - l) ? f : (l + 1), m - d) end k = 1 diff --git a/base/precompile.jl b/base/precompile.jl index d0f5da4d166b2..54d5300fa770a 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -154,10 +154,7 @@ precompile(Libc.TmStruct, (Float64,)) precompile(Base.VersionNumber, (Int, Int, Int, Tuple{}, Tuple{String})) precompile(Base._atexit, ()) precompile(Base._deleteat!, (Array{UInt8, 1}, Int, Int)) -precompile(Base._deleteat_beg!, (Array{UInt8, 1}, Int, Int)) -precompile(Base._deleteat_end!, (Array{UInt8, 1}, Int, Int)) -precompile(Base._growat_beg!, (Array{UInt8, 1}, Int, Int)) -precompile(Base._growat_end!, (Array{UInt8, 1}, Int, Int)) +precompile(Base._growat!, (Array{UInt8, 1}, Int, Int)) precompile(Base._setindex!, (Base.Dict{Symbol, Any}, Base.LineEdit.Prompt, Symbol, Int)) precompile(Base._setindex!, (Dict{Any, Any}, Base.LineEdit.PromptState, Base.LineEdit.Prompt, Int)) precompile(Base._setindex!, (Dict{Any, Any}, Bool, WeakRef, Int)) diff --git a/src/array.c b/src/array.c index bc014ff6c5dd8..8fcca4066626f 100644 --- a/src/array.c +++ b/src/array.c @@ -16,6 +16,8 @@ extern "C" { #endif +#define JL_ARRAY_IMPL_NUL 1 + #define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes) // array constructors --------------------------------------------------------- @@ -60,7 +62,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, jl_error("invalid Array size"); tot = prod; if (elsz == 1) { - // hidden 0 terminator for all byte arrays + // extra byte for all julia allocated byte arrays tot++; } } @@ -104,7 +106,8 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; - if (elsz == 1) ((char*)data)[tot-1] = '\0'; + if (JL_ARRAY_IMPL_NUL && elsz == 1) + ((char*)data)[tot - 1] = '\0'; #ifdef STORE_ARRAY_LEN a->length = nel; #endif @@ -553,58 +556,61 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i) // at this size and bigger, allocate resized array data with malloc #define MALLOC_THRESH 1048576 -// allocate buffer of newlen elements, placing old data at given offset (in #elts) -// newlen: new length (#elts), including offset -// oldlen: old length (#elts), excluding offset -// offs: new offset -static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, size_t offs) -{ - size_t es = a->elsize; - size_t nbytes = newlen * es; - size_t offsnb = offs * es; - size_t oldnbytes = oldlen * es; - size_t oldoffsnb = a->offset * es; - if (es == 1) +// Resize the buffer to a max size of `newlen` +// The buffer can either be newly allocated or realloc'd, the return +// value is 1 if a new buffer is allocated and 0 if it is realloc'd. +// the caller needs to take care of moving the data from the old buffer +// to the new one if necessary. +// When this function returns, the `->data` pointer always points to +// the **beginning** of the new buffer. +static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen) +{ + assert(!a->flags.isshared || a->flags.how == 3); + size_t elsz = a->elsize; + size_t nbytes = newlen * elsz; + size_t oldnbytes = a->maxsize * elsz; + size_t oldoffsnb = a->offset * elsz; + size_t oldlen = a->nrows; + assert(nbytes >= oldnbytes); + if (elsz == 1) { nbytes++; - assert(!a->flags.isshared || a->flags.how==3); - char *newdata; + oldnbytes++; + } + int newbuf = 0; if (a->flags.how == 2) { // already malloc'd - use realloc - newdata = (char*)jl_gc_managed_realloc((char*)a->data - oldoffsnb, nbytes, - oldnbytes+oldoffsnb, a->flags.isaligned, (jl_value_t*)a); - if (offs != a->offset) { - memmove(&newdata[offsnb], &newdata[oldoffsnb], oldnbytes); - } + char *olddata = (char*)a->data - oldoffsnb; + a->data = jl_gc_managed_realloc(olddata, nbytes, oldnbytes, + a->flags.isaligned, (jl_value_t*)a); } else { + newbuf = 1; if ( #ifdef _P64 nbytes >= MALLOC_THRESH #else - es > 4 + elsz > 4 #endif ) { - newdata = (char*)jl_gc_managed_malloc(nbytes); + a->data = jl_gc_managed_malloc(nbytes); jl_gc_track_malloced_array(a); a->flags.how = 2; a->flags.isaligned = 1; } else { - newdata = (char*)allocb(nbytes); + a->data = allocb(nbytes); a->flags.how = 1; + jl_gc_wb_buf(a, a->data, nbytes); } - memcpy(newdata + offsnb, (char*)a->data, oldnbytes); } + if (JL_ARRAY_IMPL_NUL && elsz == 1) + memset((char*)a->data + oldnbytes - 1, 0, nbytes - oldnbytes + 1); + (void)oldlen; assert(oldlen == a->nrows && "Race condition detected: recursive resizing on the same array."); - - a->data = newdata + offsnb; a->flags.isshared = 0; - if (a->flags.ptrarray || es==1) - memset(newdata+offsnb+oldnbytes, 0, nbytes-oldnbytes-offsnb); - if (a->flags.how == 1) - jl_gc_wb_buf(a, newdata, newlen * es); a->maxsize = newlen; + return newbuf; } static void NOINLINE array_try_unshare(jl_array_t *a) @@ -612,8 +618,15 @@ static void NOINLINE array_try_unshare(jl_array_t *a) if (a->flags.isshared) { if (a->flags.how != 3) jl_error("cannot resize array with shared data"); + assert(a->offset == 0); size_t len = jl_array_nrows(a); - array_resize_buffer(a, len, len, a->offset); + size_t es = a->elsize; + size_t nbytes = len * es; + char *olddata = (char*)a->data; + int newbuf = array_resize_buffer(a, len); + assert(newbuf); + (void)newbuf; + memcpy(a->data, olddata, nbytes); } } @@ -629,122 +642,261 @@ static size_t limit_overallocation(jl_array_t *a, size_t alen, size_t newlen, si return newlen; } -JL_DLLEXPORT void jl_array_grow_end(jl_array_t *a, size_t inc) +STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc, + size_t n) { - if (a->flags.isshared && a->flags.how!=3) jl_error("cannot resize array with shared data"); - // optimized for the case of only growing and shrinking at the end - size_t alen = jl_array_nrows(a); - if ((alen + inc) > a->maxsize - a->offset) { - size_t newlen = a->maxsize==0 ? (inc<4?4:inc) : a->maxsize*2; - while ((alen + inc) > newlen - a->offset) - newlen *= 2; - - newlen = limit_overallocation(a, alen, newlen, inc); - array_resize_buffer(a, newlen, alen, a->offset); + // designed to handle the case of growing and shrinking at both ends + if (__unlikely(a->flags.isshared)) { + if (a->flags.how != 3) + jl_error("cannot resize array with shared data"); + if (inc == 0) { + // If inc > 0, it will always trigger the slow path and unshare the + // buffer + array_try_unshare(a); + return; + } + } + size_t newnrows = n + inc; + size_t elsz = a->elsize; + size_t nbinc = inc * elsz; + char *data = (char*)a->data; + char *newdata; + if (a->offset >= inc) { + assert(!a->flags.isshared); + newdata = data - nbinc; + a->offset -= inc; + if (idx > 0) { + memmove(newdata, data, idx * elsz); + } + } + else { + size_t oldoffsnb = a->offset * elsz; + size_t nb1 = idx * elsz; + if (inc > (a->maxsize - n) / 2 - (a->maxsize - n) / 20) { + size_t newlen = a->maxsize == 0 ? inc * 2 : a->maxsize * 2; + while (n + 2 * inc > newlen - a->offset) + newlen *= 2; + newlen = limit_overallocation(a, n, newlen, 2 * inc); + size_t newoffset = (newlen - newnrows) / 2; + if (!array_resize_buffer(a, newlen)) + data = (char*)a->data + oldoffsnb; + newdata = (char*)a->data + newoffset * elsz; + // We could use memcpy if resizing allocates a new buffer, + // hopefully it's not a particularly important optimization. + if (idx > 0 && newdata < data) + memmove(newdata, data, nb1); + memmove(newdata + nbinc + nb1, data + nb1, n * elsz - nb1); + if (idx > 0 && newdata > data) + memmove(newdata, data, nb1); + a->offset = newoffset; + } + else { + assert(!a->flags.isshared); + a->offset = (a->maxsize - newnrows) / 2; + newdata = data - oldoffsnb + a->offset * elsz; + // We could use memcpy if resizing allocates a new buffer, + // hopefully it's not a particularly important optimization. + if (idx > 0 && newdata < data) + memmove(newdata, data, nb1); + memmove(newdata + nbinc + nb1, data + nb1, n * elsz - nb1); + if (idx > 0 && newdata > data) + memmove(newdata, data, nb1); + } } #ifdef STORE_ARRAY_LEN - a->length += inc; + a->length = newnrows; #endif - a->nrows += inc; + a->nrows = newnrows; + a->data = newdata; + if (a->flags.ptrarray) { + memset(newdata + idx * elsz, 0, nbinc); + } } -JL_DLLEXPORT void jl_array_del_end(jl_array_t *a, size_t dec) +STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx, + size_t inc, size_t n) { - if (dec == 0) return; - if (dec > a->nrows) - jl_bounds_error_int((jl_value_t*)a, a->nrows - dec); - if (a->flags.isshared) array_try_unshare(a); - if (a->elsize > 0) { - char *ptail = (char*)a->data + (a->nrows-dec)*a->elsize; - assert(ptail < (char*)a->data + (a->length*a->elsize)); - if (a->flags.ptrarray) - memset(ptail, 0, dec*a->elsize); - else - ptail[0] = 0; + // optimized for the case of only growing and shrinking at the end + if (__unlikely(a->flags.isshared)) { + if (a->flags.how != 3) + jl_error("cannot resize array with shared data"); + if (inc == 0) { + // If inc > 0, it will always trigger the slow path and unshare the + // buffer + array_try_unshare(a); + return; + } + } + size_t elsz = a->elsize; + char *data = (char*)a->data; + int has_gap = n > idx; + if (__unlikely((n + inc) > a->maxsize - a->offset)) { + size_t nb1 = idx * elsz; + size_t nbinc = inc * elsz; + size_t newlen = a->maxsize == 0 ? (inc < 4 ? 4 : inc) : a->maxsize * 2; + while ((n + inc) > newlen - a->offset) + newlen *= 2; + newlen = limit_overallocation(a, n, newlen, inc); + int newbuf = array_resize_buffer(a, newlen); + char *newdata = (char*)a->data + a->offset * elsz; + if (newbuf) { + memcpy(newdata, data, nb1); + if (has_gap) { + memcpy(newdata + nb1 + nbinc, data + nb1, n * elsz - nb1); + } + } + else if (has_gap) { + memmove(newdata + nb1 + nbinc, newdata + nb1, n * elsz - nb1); + } + a->data = data = newdata; + } + else if (has_gap) { + size_t nb1 = idx * elsz; + memmove(data + nb1 + inc * elsz, data + nb1, n * elsz - nb1); } + size_t newnrows = n + inc; #ifdef STORE_ARRAY_LEN - a->length -= dec; + a->length = newnrows; #endif - a->nrows -= dec; + a->nrows = newnrows; + if (a->flags.ptrarray) { + memset(data + idx * elsz, 0, inc * elsz); + } } -JL_DLLEXPORT void jl_array_grow_beg(jl_array_t *a, size_t inc) +JL_DLLEXPORT void jl_array_grow_at(jl_array_t *a, ssize_t idx, size_t inc) { - // For pointer array the memory grown should be zero'd - if (inc == 0) return; - // designed to handle the case of growing and shrinking at both ends - if (a->flags.isshared) array_try_unshare(a); - size_t es = a->elsize; - size_t incnb = inc*es; - if (a->offset >= inc) { - a->data = (char*)a->data - incnb; - a->offset -= inc; + // No need to explicitly unshare. + // Shared arrays are guaranteed to trigger the slow path for growing. + size_t n = jl_array_nrows(a); + if (idx < 0 || idx > n) + jl_bounds_error_int((jl_value_t*)a, idx + 1); + if (idx + 1 < n / 2) { + jl_array_grow_at_beg(a, idx, inc, n); } else { - size_t alen = a->nrows; - size_t anb = alen*es; - if (inc > (a->maxsize-alen)/2 - (a->maxsize-alen)/20) { - size_t newlen = a->maxsize==0 ? inc*2 : a->maxsize*2; - while (alen+2*inc > newlen-a->offset) - newlen *= 2; + jl_array_grow_at_end(a, idx, inc, n); + } +} - newlen = limit_overallocation(a, alen, newlen, 2*inc); - size_t center = (newlen - (alen + inc))/2; - array_resize_buffer(a, newlen, alen, center+inc); - a->offset = center; - a->data = (char*)a->data - incnb; - } - else { - size_t center = (a->maxsize - (alen + inc))/2; - char *newdata = (char*)a->data - es*a->offset + es*center; - memmove(&newdata[incnb], a->data, anb); - a->data = newdata; - a->offset = center; - } +JL_DLLEXPORT void jl_array_grow_end(jl_array_t *a, size_t inc) +{ + size_t n = jl_array_nrows(a); + jl_array_grow_at_end(a, n, inc, n); +} + +JL_DLLEXPORT void jl_array_grow_beg(jl_array_t *a, size_t inc) +{ + size_t n = jl_array_nrows(a); + jl_array_grow_at_beg(a, 0, inc, n); +} + +static size_t jl_array_limit_offset(jl_array_t *a, size_t offset) +{ + // make sure offset doesn't grow forever due to deleting at beginning + // and growing at end + if (offset >= 13 * a->maxsize / 20) + offset = 17 * (a->maxsize - a->nrows) / 100; +#ifdef _P64 + while (offset > (size_t)UINT32_MAX) { + offset /= 2; } - if (a->flags.ptrarray) - memset((char*)a->data, 0, incnb); -#ifdef STORE_ARRAY_LEN - a->length += inc; #endif - a->nrows += inc; + return offset; } -JL_DLLEXPORT void jl_array_del_beg(jl_array_t *a, size_t dec) +STATIC_INLINE void jl_array_del_at_beg(jl_array_t *a, size_t idx, size_t dec, + size_t n) { - if (dec == 0) return; - if (dec > a->nrows) - jl_bounds_error_int((jl_value_t*)a, dec); - if (a->flags.isshared) array_try_unshare(a); - size_t es = a->elsize; - size_t nb = dec*es; - memset(a->data, 0, nb); + // no error checking + // assume inbounds, assume unshared + assert(!a->flags.isshared); + size_t elsz = a->elsize; size_t offset = a->offset; offset += dec; - a->data = (char*)a->data + nb; #ifdef STORE_ARRAY_LEN - a->length -= dec; + a->length = n - dec; #endif - a->nrows -= dec; - - // make sure offset doesn't grow forever due to deleting at beginning - // and growing at end - size_t newoffs = offset; - if (offset >= 13*a->maxsize/20) { - newoffs = 17*(a->maxsize - a->nrows)/100; + a->nrows = n - dec; + size_t newoffs = jl_array_limit_offset(a, offset); + assert(newoffs <= offset); + size_t nbdec = dec * elsz; + if (__unlikely(newoffs != offset) || idx > 0) { + char *olddata = (char*)a->data; + char *newdata = olddata - (a->offset - newoffs) * elsz; + size_t nb1 = idx * elsz; // size in bytes of the first block + size_t nbtotal = a->nrows * elsz; // size in bytes of the new array + // Implicit '\0' for byte arrays + if (elsz == 1) + nbtotal++; + if (idx > 0) + memmove(newdata, olddata, nb1); + memmove(newdata + nb1, olddata + nb1 + nbdec, nbtotal - nb1); + a->data = newdata; } -#ifdef _P64 - while (newoffs > (size_t)((uint32_t)-1)) { - newoffs = newoffs/2; + else { + a->data = (char*)a->data + nbdec; } + a->offset = newoffs; +} + +STATIC_INLINE void jl_array_del_at_end(jl_array_t *a, size_t idx, size_t dec, + size_t n) +{ + // no error checking + // assume inbounds, assume unshared + assert(!a->flags.isshared); + char *data = (char*)a->data; + size_t elsz = a->elsize; + size_t last = idx + dec; + if (n > last) + memmove(data + idx * elsz, data + last * elsz, (n - last) * elsz); + n -= dec; + if (elsz == 1) + data[n] = 0; + a->nrows = n; +#ifdef STORE_ARRAY_LEN + a->length = n; #endif - if (newoffs != offset) { - size_t anb = a->nrows*es; - size_t delta = (offset - newoffs)*es; - a->data = (char*)a->data - delta; - memmove(a->data, (char*)a->data + delta, anb); +} + +JL_DLLEXPORT void jl_array_del_at(jl_array_t *a, ssize_t idx, size_t dec) +{ + size_t n = jl_array_nrows(a); + size_t last = idx + dec; + if (__unlikely(idx < 0)) + jl_bounds_error_int((jl_value_t*)a, idx + 1); + if (__unlikely(last > n)) + jl_bounds_error_int((jl_value_t*)a, last); + // The unsharing needs to happen before we modify the buffer + if (__unlikely(a->flags.isshared)) + array_try_unshare(a); + if (idx < n - last) { + jl_array_del_at_beg(a, idx, dec, n); } - a->offset = newoffs; + else { + jl_array_del_at_end(a, idx, dec, n); + } +} + +JL_DLLEXPORT void jl_array_del_beg(jl_array_t *a, size_t dec) +{ + size_t n = jl_array_nrows(a); + if (__unlikely(dec > n)) + jl_bounds_error_int((jl_value_t*)a, dec); + if (__unlikely(a->flags.isshared)) + array_try_unshare(a); + jl_array_del_at_beg(a, 0, dec, n); +} + +JL_DLLEXPORT void jl_array_del_end(jl_array_t *a, size_t dec) +{ + size_t n = jl_array_nrows(a); + if (__unlikely(n < dec)) + jl_bounds_error_int((jl_value_t*)a, 0); + if (__unlikely(a->flags.isshared)) + array_try_unshare(a); + jl_array_del_at_end(a, n - dec, dec, n); } JL_DLLEXPORT void jl_array_sizehint(jl_array_t *a, size_t sz) diff --git a/test/core.jl b/test/core.jl index 8ec8bb2371445..030e181ffe1f1 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3823,6 +3823,27 @@ let ary = Vector{Any}(10) ccall(:jl_array_grow_beg, Void, (Any, Csize_t), ary, 4) check_undef_and_fill(ary, 1:(2n + 4)) end + + ary = Vector{Any}(100) + ccall(:jl_array_grow_end, Void, (Any, Csize_t), ary, 10000) + ary[:] = 1:length(ary) + ccall(:jl_array_del_beg, Void, (Any, Csize_t), ary, 10000) + # grow on the back until a buffer reallocation happens + cur_ptr = pointer(ary) + while cur_ptr == pointer(ary) + len = length(ary) + ccall(:jl_array_grow_end, Void, (Any, Csize_t), ary, 10) + for i in (len + 1):(len + 10) + @test !isdefined(ary, i) + end + end + + ary = Vector{Any}(100) + ary[:] = 1:length(ary) + ccall(:jl_array_grow_at, Void, (Any, Csize_t, Csize_t), ary, 50, 10) + for i in 51:60 + @test !isdefined(ary, i) + end end # check if we can run multiple finalizers at the same time @@ -3890,6 +3911,46 @@ let arrayset_unknown_dim(Int, 3) end +module TestSharedArrayResize +using Base.Test +# Attempting to change the shape of a shared array should unshare it and +# not modify the original data +function test_shared_array_resize{T}(::Type{T}) + len = 100 + a = Vector{T}(len) + function test_unshare(f) + a′ = reshape(reshape(a, (len ÷ 2, 2)), len) + a[:] = 1:length(a) + # The operation should fail on the owner shared array + # and has no side effect. + @test_throws ErrorException f(a) + @test a == [1:len;] + @test a′ == [1:len;] + @test pointer(a) == pointer(a′) + # The operation should pass on the non-owner shared array + # and should unshare the arrays with no effect on the original one. + f(a′) + @test a == [1:len;] + @test pointer(a) != pointer(a′) + end + + test_unshare(a->ccall(:jl_array_del_end, Void, (Any, Csize_t), a, 0)) + test_unshare(a->ccall(:jl_array_del_end, Void, (Any, Csize_t), a, 1)) + test_unshare(a->ccall(:jl_array_del_beg, Void, (Any, Csize_t), a, 0)) + test_unshare(a->ccall(:jl_array_del_beg, Void, (Any, Csize_t), a, 1)) + test_unshare(a->deleteat!(a, 10)) + test_unshare(a->deleteat!(a, 90)) + test_unshare(a->ccall(:jl_array_grow_end, Void, (Any, Csize_t), a, 0)) + test_unshare(a->ccall(:jl_array_grow_end, Void, (Any, Csize_t), a, 1)) + test_unshare(a->ccall(:jl_array_grow_beg, Void, (Any, Csize_t), a, 0)) + test_unshare(a->ccall(:jl_array_grow_beg, Void, (Any, Csize_t), a, 1)) + test_unshare(a->insert!(a, 10, 10)) + test_unshare(a->insert!(a, 90, 90)) +end +test_shared_array_resize(Int) +test_shared_array_resize(Any) +end + module TestArrayNUL using Base.Test function check_nul(a::Vector{UInt8}) From 8eb03e6c417f8e10389f700f05c64eaee5ae832f Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 18 Jun 2016 09:13:21 -0400 Subject: [PATCH 0118/1117] Implement faster and safer `unsafe_copy!` in C * Use `unsafe_copy!` instead of `memcpy` in `vcat` to avoid bypassing the write barrier. * Add test for `copy!` on `#undef` and `unsafe_copy!` with memory alias. --- base/array.jl | 22 ++++++++------ src/array.c | 83 ++++++++++++++++++++++++++++++++++++++++++++------- test/core.jl | 13 ++++++++ 3 files changed, 98 insertions(+), 20 deletions(-) diff --git a/base/array.jl b/base/array.jl index 7a76f11c69e9f..270ddff15924a 100644 --- a/base/array.jl +++ b/base/array.jl @@ -47,9 +47,8 @@ function unsafe_copy!{T}(dest::Array{T}, doffs, src::Array{T}, soffs, n) if isbits(T) unsafe_copy!(pointer(dest, doffs), pointer(src, soffs), n) else - for i=0:n-1 - @inbounds arrayset(dest, src[i+soffs], i+doffs) - end + ccall(:jl_array_ptr_copy, Void, (Any, Ptr{Void}, Any, Ptr{Void}, Int), + dest, pointer(dest, doffs), src, pointer(src, soffs), n) end return dest end @@ -615,17 +614,22 @@ function vcat{T}(arrays::Vector{T}...) end arr = Array{T}(n) ptr = pointer(arr) - offset = 0 if isbits(T) - elsz = sizeof(T) + elsz = Core.sizeof(T) else elsz = Core.sizeof(Ptr{Void}) end for a in arrays - nba = length(a)*elsz - ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt), - ptr+offset, a, nba) - offset += nba + na = length(a) + nba = na * elsz + if isbits(T) + ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt), + ptr, a, nba) + else + ccall(:jl_array_ptr_copy, Void, (Any, Ptr{Void}, Any, Ptr{Void}, Int), + arr, ptr, a, pointer(a), na) + end + ptr += nba end return arr end diff --git a/src/array.c b/src/array.c index 8fcca4066626f..9580a3eb74d55 100644 --- a/src/array.c +++ b/src/array.c @@ -32,6 +32,15 @@ int jl_array_store_unboxed(jl_value_t *el_type) return store_unboxed(el_type); } +STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a) +{ + if (a->flags.how == 3) { + a = (jl_array_t*)jl_array_data_owner(a); + assert(a->flags.how != 3); + } + return (jl_value_t*)a; +} + #if defined(_P64) && defined(UINT128MAX) typedef __uint128_t wideint_t; #else @@ -190,14 +199,9 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, a->flags.ptrarray = 1; } - jl_array_t *owner = data; // if data is itself a shared wrapper, // owner should point back to the original array - if (owner->flags.how == 3) { - owner = (jl_array_t*)jl_array_data_owner(owner); - } - assert(owner->flags.how != 3); - jl_array_data_owner(a) = (jl_value_t*)owner; + jl_array_data_owner(a) = jl_array_owner(data); a->flags.how = 3; a->data = data->data; @@ -526,11 +530,7 @@ JL_DLLEXPORT void jl_arrayset(jl_array_t *a, jl_value_t *rhs, size_t i) } else { ((jl_value_t**)a->data)[i] = rhs; - jl_value_t *owner = (jl_value_t*)a; - if (a->flags.how == 3) { - owner = jl_array_data_owner(a); - } - jl_gc_wb(owner, rhs); + jl_gc_wb(jl_array_owner(a), rhs); } } @@ -921,6 +921,67 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary) return new_ary; } +// Copy element by element until we hit a young object, at which point +// we can continue using `memmove`. +static NOINLINE ssize_t jl_array_ptr_copy_forward(jl_value_t *owner, + void **src_p, void **dest_p, + ssize_t n) +{ + for (ssize_t i = 0; i < n; i++) { + void *val = src_p[i]; + dest_p[i] = val; + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { + jl_gc_queue_root(owner); + return i; + } + } + return n; +} + +static NOINLINE ssize_t jl_array_ptr_copy_backward(jl_value_t *owner, + void **src_p, void **dest_p, + ssize_t n) +{ + for (ssize_t i = 0; i < n; i++) { + void *val = src_p[n - i - 1]; + dest_p[n - i - 1] = val; + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { + jl_gc_queue_root(owner); + return i; + } + } + return n; +} + +// Unsafe, assume inbounds and that dest and src have the same eltype +// `doffs` and `soffs` are zero based. +JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p, + jl_array_t *src, void **src_p, ssize_t n) +{ + assert(dest->flags.ptrarray && src->flags.ptrarray); + jl_value_t *owner = jl_array_owner(dest); + // Destination is old and doesn't refer any young object + if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { + jl_value_t *src_owner = jl_array_owner(src); + // Source is young or might refer young objects + if (!(jl_astaggedvalue(src_owner)->bits.gc & GC_OLD)) { + ssize_t done; + if (dest_p < src_p || dest_p > src_p + n) { + done = jl_array_ptr_copy_forward(owner, src_p, dest_p, n); + dest_p += done; + src_p += done; + } + else { + done = jl_array_ptr_copy_backward(owner, src_p, dest_p, n); + } + n -= done; + } + } + memmove(dest_p, src_p, n * sizeof(void*)); +} + JL_DLLEXPORT void jl_array_ptr_1d_push(jl_array_t *a, jl_value_t *item) { assert(jl_typeis(a, jl_array_any_type)); diff --git a/test/core.jl b/test/core.jl index 030e181ffe1f1..b270850d2716f 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3996,6 +3996,19 @@ g = reinterpret(UInt8, UInt16[0x1, 0x2]) @test check_nul(copy(g)) end +# Copy of `#undef` +copy!(Vector{Any}(10), Vector{Any}(10)) +function test_copy_alias{T}(::Type{T}) + ary = T[1:100;] + unsafe_copy!(ary, 1, ary, 11, 90) + @test ary == [11:100; 91:100] + ary = T[1:100;] + unsafe_copy!(ary, 11, ary, 1, 90) + @test ary == [1:10; 1:90] +end +test_copy_alias(Int) +test_copy_alias(Any) + # issue #15370 @test isdefined(Core, :Box) @test !isdefined(Base, :Box) From 325ec0ec02e71ebc23c4e6ad8d08b26139da579e Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Fri, 17 Jun 2016 21:04:27 +0200 Subject: [PATCH 0119/1117] Fix bitshift operators on Bool Previously triggered a stack overflow due to recursive definition. Multiple definitions are needed to fix ambiguities and inference issues with the generci integer method. --- base/bool.jl | 12 ++++++++++++ test/int.jl | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/base/bool.jl b/base/bool.jl index cef413455db46..753d838abeae9 100644 --- a/base/bool.jl +++ b/base/bool.jl @@ -20,6 +20,18 @@ typemax(::Type{Bool}) = true (|)(x::Bool, y::Bool) = box(Bool,or_int(unbox(Bool,x),unbox(Bool,y))) ($)(x::Bool, y::Bool) = (x!=y) +>>(x::Bool, c::Unsigned) = Int(x) >> c +<<(x::Bool, c::Unsigned) = Int(x) << c +>>>(x::Bool, c::Unsigned) = Int(x) >>> c + +>>(x::Bool, c::Int) = Int(x) >> c +<<(x::Bool, c::Int) = Int(x) << c +>>>(x::Bool, c::Int) = Int(x) >>> c + +>>(x::Bool, c::Integer) = Int(x) >> c +<<(x::Bool, c::Integer) = Int(x) << c +>>>(x::Bool, c::Integer) = Int(x) >>> c + signbit(x::Bool) = false sign(x::Bool) = x abs(x::Bool) = x diff --git a/test/int.jl b/test/int.jl index 40b79c726d2ac..cc724bac92173 100644 --- a/test/int.jl +++ b/test/int.jl @@ -190,3 +190,8 @@ end # issue #16700 @test_throws MethodError 1.0 >> 8 + +# PR #16988 +@test true << 2 === 1 << 2 +@test true >> 2 === 1 >> 2 +@test true >>> 2 === 1 >>> 2 From 02a39a1e4604664c6425f213d93ab6e3c79633d0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 27 Jun 2016 10:53:14 -0400 Subject: [PATCH 0120/1117] win: add support for stackwalk from a null starting ip ref #16137 --- src/stackwalk.c | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/src/stackwalk.c b/src/stackwalk.c index d9025e99903f3..fcac8a1d9294b 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -227,41 +227,60 @@ static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) #endif } +static int readable_pointer(LPCVOID pointer) { + // Check whether the pointer is valid and executable before dereferencing + // to avoid segfault while recording. See #10638. + MEMORY_BASIC_INFORMATION mInfo; + if (VirtualQuery(pointer, &mInfo, sizeof(MEMORY_BASIC_INFORMATION)) == 0) + return 0; + DWORD X = mInfo.AllocationProtect; + if (!((X&PAGE_READONLY) || (X&PAGE_READWRITE) || (X&PAGE_WRITECOPY) || (X&PAGE_EXECUTE_READ)) || + (X&PAGE_GUARD) || (X&PAGE_NOACCESS)) + return 0; + return 1; +} + static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp) { // Might be called from unmanaged thread. #ifndef _CPU_X86_64_ *ip = (uintptr_t)cursor->stackframe.AddrPC.Offset; *sp = (uintptr_t)cursor->stackframe.AddrStack.Offset; + if (*ip == 0 || *ip == ((uintptr_t)0)-1) { + if (!readable_pointer((LPCVOID)*sp)) + return 0; + cursor->stackframe.AddrPC.Offset = *(DWORD32*)*sp; // POP EIP (aka RET) + cursor->stackframe.AddrStack.Offset += sizeof(void*); + return cursor->stackframe.AddrPC.Offset != 0; + } + BOOL result = StackWalk64(IMAGE_FILE_MACHINE_I386, GetCurrentProcess(), hMainThread, &cursor->stackframe, &cursor->context, NULL, JuliaFunctionTableAccess64, JuliaGetModuleBase64, NULL); return result; #else *ip = (uintptr_t)cursor->Rip; *sp = (uintptr_t)cursor->Rsp; + if (*ip == 0 || *ip == ((uintptr_t)0)-1) { + if (!readable_pointer((LPCVOID)*sp)) + return 0; + cursor->Rip = *(DWORD64*)*sp; // POP RIP (aka RET) + cursor->Rsp += sizeof(void*); + return cursor->Rip != 0; + } + DWORD64 ImageBase = JuliaGetModuleBase64(GetCurrentProcess(), cursor->Rip); if (!ImageBase) return 0; PRUNTIME_FUNCTION FunctionEntry = (PRUNTIME_FUNCTION)JuliaFunctionTableAccess64(GetCurrentProcess(), cursor->Rip); if (!FunctionEntry) { // assume this is a NO_FPO RBP-based function - MEMORY_BASIC_INFORMATION mInfo; - cursor->Rsp = cursor->Rbp; // MOV RSP, RBP - - // Check whether the pointer is valid and executable before dereferencing - // to avoid segfault while recording. See #10638. - if (VirtualQuery((LPCVOID)cursor->Rsp, &mInfo, sizeof(MEMORY_BASIC_INFORMATION)) == 0) - return 0; - DWORD X = mInfo.AllocationProtect; - if (!((X&PAGE_READONLY) || (X&PAGE_READWRITE) || (X&PAGE_WRITECOPY) || (X&PAGE_EXECUTE_READ)) || - (X&PAGE_GUARD) || (X&PAGE_NOACCESS)) + if (!readable_pointer((LPCVOID)cursor->Rsp)) return 0; - cursor->Rbp = *(DWORD64*)cursor->Rsp; // POP RBP - cursor->Rsp = cursor->Rsp + sizeof(void*); + cursor->Rsp += sizeof(void*); cursor->Rip = *(DWORD64*)cursor->Rsp; // POP RIP (aka RET) - cursor->Rsp = cursor->Rsp + sizeof(void*); + cursor->Rsp += sizeof(void*); } else { PVOID HandlerData; From b4f295d0d3b2944a3d45fd8205242ef61ea9589b Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 19 Jun 2016 11:50:35 -0400 Subject: [PATCH 0121/1117] Added more docs for factorize, cleaned up LDLt and cholfact Added a table to the `factorize` docs, explaining what types a user can expect the function to output depending on what they put in. Also removed a very long and (IMO) unclear type signature for sparse `ldltfact` and `cholfact`. --- base/docs/helpdb/Base.jl | 9 ------- base/linalg/dense.jl | 34 ++++++++++++++++++++++++++ base/sparse/cholmod.jl | 26 ++++++++++++++++---- doc/stdlib/linalg.rst | 53 +++++++++++++++++++++++++++++++++------- 4 files changed, 99 insertions(+), 23 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 41be192c95955..ff4e5c0290f13 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -7949,15 +7949,6 @@ Return the index of the first element of `A` for which `predicate` returns `true """ findfirst -""" - factorize(A) - -Compute a convenient factorization (including LU, Cholesky, Bunch-Kaufman, LowerTriangular, -UpperTriangular) of `A`, based upon the type of the input matrix. The return value can then -be reused for efficient solving of multiple systems. For example: `A=factorize(A); x=A\\b; y=A\\C`. -""" -factorize - """ promote_rule(type1, type2) diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index ffae38ec2d57c..2a68cc7473c6d 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -398,6 +398,40 @@ function inv{T}(A::StridedMatrix{T}) return convert(typeof(parent(Ai)), Ai) end +""" + factorize(A) + +Compute a convenient factorization of `A`, based upon the type of the input matrix. +`factorize` checks `A` to see if it is symmetric/triangular/etc. if `A` is passed +as a generic matrix. `factorize` checks every element of `A` to verify/rule out +each property. It will short-circuit as soon as it can rule out symmetry/triangular +structure. The return value can be reused for efficient solving of multiple +systems. For example: `A=factorize(A); x=A\\b; y=A\\C`. + +| Properties of `A` | type of factorization | +|:---------------------------|:-----------------------------------------------| +| Positive-definite | Cholesky (see [`cholfact`](:func:`cholfact`)) | +| Dense Symmetric/Hermitian | Bunch-Kaufman (see [`bkfact`](:func:`bkfact`)) | +| Sparse Symmetric/Hermitian | LDLt (see [`ldltfact`](:func:`ldltfact`)) | +| Triangular | Triangular | +| Diagonal | Diagonal | +| Bidiagonal | Bidiagonal | +| Tridiagonal | LU (see [`lufact`](:func:`lufact`)) | +| Symmetric real tridiagonal | LDLt (see [`ldltfact`](:func:`ldltfact`)) | +| General square | LU (see [`lufact`](:func:`lufact`)) | +| General non-square | QR (see [`qrfact`](:func:`qrfact`)) | + +If `factorize` is called on a Hermitian positive-definite matrix, for instance, then `factorize` +will return a Cholesky factorization. + +Example: +```julia +A = diagm(rand(5)) + diagm(rand(4),1); #A is really bidiagonal +factorize(A) #factorize will check to see that A is already factorized +``` +This returns a `5×5 Bidiagonal{Float64}`, which can now be passed to other linear algebra functions +(e.g. eigensolvers) which will use specialized methods for `Bidiagonal` types. +""" function factorize{T}(A::StridedMatrix{T}) m, n = size(A) if m == n diff --git a/base/sparse/cholmod.jl b/base/sparse/cholmod.jl index f596b26d40ae0..87c0d30c59141 100644 --- a/base/sparse/cholmod.jl +++ b/base/sparse/cholmod.jl @@ -1252,9 +1252,13 @@ function cholfact!{Tv}(F::Factor{Tv}, A::Sparse{Tv}; shift::Real=0.0) end """ - cholfact!(F::Factor, A::Union{SparseMatrixCSC{<:Real},SparseMatrixCSC{Complex{<:Real}},Symmetric{<:Real,SparseMatrixCSC{<:Real,SuiteSparse_long}},Hermitian{Complex{<:Real},SparseMatrixCSC{Complex{<:Real},SuiteSparse_long}}}; shift = 0.0) -> CHOLMOD.Factor + cholfact!(F::Factor, A; shift = 0.0) -> CHOLMOD.Factor Compute the Cholesky (``LL'``) factorization of `A`, reusing the symbolic factorization `F`. +`A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, or +`Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't +have the type tag, its structure and values must still be +symmetric/Hermitian. ** Note ** @@ -1289,9 +1293,13 @@ function cholfact(A::Sparse; shift::Real=0.0, end """ - cholfact(::Union{SparseMatrixCSC{<:Real},SparseMatrixCSC{Complex{<:Real}},Symmetric{<:Real,SparseMatrixCSC{<:Real,SuiteSparse_long}},Hermitian{Complex{<:Real},SparseMatrixCSC{Complex{<:Real},SuiteSparse_long}}}; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor + cholfact(A; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor Compute the Cholesky factorization of a sparse positive definite matrix `A`. +`A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, or +`Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't +have the type tag, its structure and values must still be +symmetric/Hermitian. A fill-reducing permutation is used. `F = cholfact(A)` is most frequently used to solve systems of equations with `F\\b`, but also the methods `diag`, `det`, `logdet` are defined for `F`. @@ -1342,9 +1350,13 @@ function ldltfact!{Tv}(F::Factor{Tv}, A::Sparse{Tv}; shift::Real=0.0) end """ - ldltfact!(F::Factor, A::Union{SparseMatrixCSC{<:Real},SparseMatrixCSC{Complex{<:Real}},Symmetric{<:Real,SparseMatrixCSC{<:Real,SuiteSparse_long}},Hermitian{Complex{<:Real},SparseMatrixCSC{Complex{<:Real},SuiteSparse_long}}}; shift = 0.0) -> CHOLMOD.Factor + ldltfact!(F::Factor, A; shift = 0.0) -> CHOLMOD.Factor Compute the ``LDL'`` factorization of `A`, reusing the symbolic factorization `F`. +`A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, or +`Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't +have the type tag, its structure and values must still be +symmetric/Hermitian. ** Note ** @@ -1379,9 +1391,13 @@ function ldltfact(A::Sparse; shift::Real=0.0, end """ - ldltfact(::Union{SparseMatrixCSC{<:Real},SparseMatrixCSC{Complex{<:Real}},Symmetric{<:Real,SparseMatrixCSC{<:Real,SuiteSparse_long}},Hermitian{Complex{<:Real},SparseMatrixCSC{Complex{<:Real},SuiteSparse_long}}}; shift = 0.0, perm=Int[]) -> CHOLMOD.Factor + ldltfact(A; shift = 0.0, perm=Int[]) -> CHOLMOD.Factor -Compute the ``LDL'`` factorization of a sparse symmetric or Hermitian matrix. +Compute the ``LDL'`` factorization of a sparse matrix `A`. +`A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, or +`Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't +have the type tag, its structure and values must still be +symmetric/Hermitian. A fill-reducing permutation is used. `F = ldltfact(A)` is most frequently used to solve systems of equations `A*x = b` with `F\\b`. The returned factorization object `F` also supports the methods `diag`, diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 63ec01e493d7d..f2be13a8b3930 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -51,7 +51,42 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute a convenient factorization (including LU, Cholesky, Bunch-Kaufman, LowerTriangular, UpperTriangular) of ``A``\ , based upon the type of the input matrix. The return value can then be reused for efficient solving of multiple systems. For example: ``A=factorize(A); x=A\b; y=A\C``\ . + Compute a convenient factorization of ``A``\ , based upon the type of the input matrix. ``factorize`` checks ``A`` to see if it is symmetric/triangular/etc. if ``A`` is passed as a generic matrix. ``factorize`` checks every element of ``A`` to verify/rule out each property. It will short-circuit as soon as it can rule out symmetry/triangular structure. The return value can be reused for efficient solving of multiple systems. For example: ``A=factorize(A); x=A\b; y=A\C``\ . + + +----------------------------+--------------------------------------+ + | Properties of ``A`` | type of factorization | + +============================+======================================+ + | Positive-definite | Cholesky (see :func:`cholfact`\ ) | + +----------------------------+--------------------------------------+ + | Dense Symmetric/Hermitian | Bunch-Kaufman (see :func:`bkfact`\ ) | + +----------------------------+--------------------------------------+ + | Sparse Symmetric/Hermitian | LDLt (see :func:`ldltfact`\ ) | + +----------------------------+--------------------------------------+ + | Triangular | Triangular | + +----------------------------+--------------------------------------+ + | Diagonal | Diagonal | + +----------------------------+--------------------------------------+ + | Bidiagonal | Bidiagonal | + +----------------------------+--------------------------------------+ + | Tridiagonal | LU (see :func:`lufact`\ ) | + +----------------------------+--------------------------------------+ + | Symmetric real tridiagonal | LDLt (see :func:`ldltfact`\ ) | + +----------------------------+--------------------------------------+ + | General square | LU (see :func:`lufact`\ ) | + +----------------------------+--------------------------------------+ + | General non-square | QR (see :func:`qrfact`\ ) | + +----------------------------+--------------------------------------+ + + If ``factorize`` is called on a Hermitian positive-definite matrix, for instance, then ``factorize`` will return a Cholesky factorization. + + Example: + + .. code-block:: julia + + A = diagm(rand(5)) + diagm(rand(4),1); #A is really bidiagonal + factorize(A) #factorize will check to see that A is already factorized + + This returns a ``5×5 Bidiagonal{Float64}``\ , which can now be passed to other linear algebra functions (e.g. eigensolvers) which will use specialized methods for ``Bidiagonal`` types. .. function:: full(F) @@ -283,11 +318,11 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. -.. function:: cholfact(::Union{SparseMatrixCSC{<:Real},SparseMatrixCSC{Complex{<:Real}},Symmetric{<:Real,SparseMatrixCSC{<:Real,SuiteSparse_long}},Hermitian{Complex{<:Real},SparseMatrixCSC{Complex{<:Real},SuiteSparse_long}}}; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor +.. function:: cholfact(A; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor .. Docstring generated from Julia source - Compute the Cholesky factorization of a sparse positive definite matrix ``A``\ . A fill-reducing permutation is used. ``F = cholfact(A)`` is most frequently used to solve systems of equations with ``F\b``\ , but also the methods ``diag``\ , ``det``\ , ``logdet`` are defined for ``F``\ . You can also extract individual factors from ``F``\ , using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). + Compute the Cholesky factorization of a sparse positive definite matrix ``A``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . A fill-reducing permutation is used. ``F = cholfact(A)`` is most frequently used to solve systems of equations with ``F\b``\ , but also the methods ``diag``\ , ``det``\ , ``logdet`` are defined for ``F``\ . You can also extract individual factors from ``F``\ , using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). Setting optional ``shift`` keyword argument computes the factorization of ``A+shift*I`` instead of ``A``\ . If the ``perm`` argument is nonempty, it should be a permutation of ``1:size(A,1)`` giving the ordering to use (instead of CHOLMOD's default AMD ordering). @@ -297,11 +332,11 @@ Linear algebra functions in Julia are largely implemented by calling functions f Many other functions from CHOLMOD are wrapped but not exported from the ``Base.SparseArrays.CHOLMOD`` module. -.. function:: cholfact!(F::Factor, A::Union{SparseMatrixCSC{<:Real},SparseMatrixCSC{Complex{<:Real}},Symmetric{<:Real,SparseMatrixCSC{<:Real,SuiteSparse_long}},Hermitian{Complex{<:Real},SparseMatrixCSC{Complex{<:Real},SuiteSparse_long}}}; shift = 0.0) -> CHOLMOD.Factor +.. function:: cholfact!(F::Factor, A; shift = 0.0) -> CHOLMOD.Factor .. Docstring generated from Julia source - Compute the Cholesky (:math:`LL'`\ ) factorization of ``A``\ , reusing the symbolic factorization ``F``\ . + Compute the Cholesky (:math:`LL'`\ ) factorization of ``A``\ , reusing the symbolic factorization ``F``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . ** Note ** @@ -353,11 +388,11 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute an ``LDLt`` factorization of a real symmetric tridiagonal matrix such that ``A = L*Diagonal(d)*L'`` where ``L`` is a unit lower triangular matrix and ``d`` is a vector. The main use of an ``LDLt`` factorization ``F = ldltfact(A)`` is to solve the linear system of equations ``Ax = b`` with ``F\b``\ . -.. function:: ldltfact(::Union{SparseMatrixCSC{<:Real},SparseMatrixCSC{Complex{<:Real}},Symmetric{<:Real,SparseMatrixCSC{<:Real,SuiteSparse_long}},Hermitian{Complex{<:Real},SparseMatrixCSC{Complex{<:Real},SuiteSparse_long}}}; shift = 0.0, perm=Int[]) -> CHOLMOD.Factor +.. function:: ldltfact(A; shift = 0.0, perm=Int[]) -> CHOLMOD.Factor .. Docstring generated from Julia source - Compute the :math:`LDL'` factorization of a sparse symmetric or Hermitian matrix. A fill-reducing permutation is used. ``F = ldltfact(A)`` is most frequently used to solve systems of equations ``A*x = b`` with ``F\b``\ . The returned factorization object ``F`` also supports the methods ``diag``\ , ``det``\ , and ``logdet``\ . You can extract individual factors from ``F`` using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*D*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). The complete list of supported factors is ``:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP``\ . + Compute the :math:`LDL'` factorization of a sparse matrix ``A``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . Note that even if ``A`` doesn't have the type tag, its structure must still be symmetric/Hermitian. A fill-reducing permutation is used. ``F = ldltfact(A)`` is most frequently used to solve systems of equations ``A*x = b`` with ``F\b``\ . The returned factorization object ``F`` also supports the methods ``diag``\ , ``det``\ , and ``logdet``\ . You can extract individual factors from ``F`` using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*D*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). The complete list of supported factors is ``:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP``\ . Setting optional ``shift`` keyword argument computes the factorization of ``A+shift*I`` instead of ``A``\ . If the ``perm`` argument is nonempty, it should be a permutation of ``1:size(A,1)`` giving the ordering to use (instead of CHOLMOD's default AMD ordering). @@ -367,11 +402,11 @@ Linear algebra functions in Julia are largely implemented by calling functions f Many other functions from CHOLMOD are wrapped but not exported from the ``Base.SparseArrays.CHOLMOD`` module. -.. function:: ldltfact!(F::Factor, A::Union{SparseMatrixCSC{<:Real},SparseMatrixCSC{Complex{<:Real}},Symmetric{<:Real,SparseMatrixCSC{<:Real,SuiteSparse_long}},Hermitian{Complex{<:Real},SparseMatrixCSC{Complex{<:Real},SuiteSparse_long}}}; shift = 0.0) -> CHOLMOD.Factor +.. function:: ldltfact!(F::Factor, A; shift = 0.0) -> CHOLMOD.Factor .. Docstring generated from Julia source - Compute the :math:`LDL'` factorization of ``A``\ , reusing the symbolic factorization ``F``\ . + Compute the :math:`LDL'` factorization of ``A``\ , reusing the symbolic factorization ``F``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . ** Note ** From e053b2fc93bff02f03225e4e127f4244115d89d5 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Mon, 27 Jun 2016 11:27:36 -0400 Subject: [PATCH 0122/1117] Fixed LQ docstring too --- base/linalg/lq.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/linalg/lq.jl b/base/linalg/lq.jl index 476b2c195ed29..b49ece2abe93a 100644 --- a/base/linalg/lq.jl +++ b/base/linalg/lq.jl @@ -21,13 +21,13 @@ LQPackedQ{T}(factors::AbstractMatrix{T}, τ::Vector{T}) = LQPackedQ{T,typeof(fac lqfact!(A) -> LQ Compute the LQ factorization of `A`, using the input -matrix as a workspace. See also [`lq`](:func:`lq). +matrix as a workspace. See also [`lq`](:func:`lq`). """ lqfact!{T<:BlasFloat}(A::StridedMatrix{T}) = LQ(LAPACK.gelqf!(A)...) """ lqfact(A) -> LQ -Compute the LQ factorization of `A`. See also [`lq`](:func:`lq). +Compute the LQ factorization of `A`. See also [`lq`](:func:`lq`). """ lqfact{T<:BlasFloat}(A::StridedMatrix{T}) = lqfact!(copy(A)) lqfact(x::Number) = lqfact(fill(x,1,1)) From c37906007f6d184dcbe943534fdc98c562ce9bb8 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Mon, 27 Jun 2016 12:47:15 -0400 Subject: [PATCH 0123/1117] Fix conversion Irrational => Rational (#16527) Fixes #16513. Changes generated macro to pure macro, so as to avoid breaking in future compiler changes. --- base/irrationals.jl | 24 ++++++++++++++++++++---- test/numbers.jl | 6 ++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/base/irrationals.jl b/base/irrationals.jl index 794d0ec91a748..e4347f6b8dedd 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -13,11 +13,27 @@ promote_rule{s,T<:Number}(::Type{Irrational{s}}, ::Type{T}) = promote_type(Float convert(::Type{AbstractFloat}, x::Irrational) = Float64(x) convert(::Type{Float16}, x::Irrational) = Float16(Float32(x)) convert{T<:Real}(::Type{Complex{T}}, x::Irrational) = convert(Complex{T}, convert(T,x)) -convert{T<:Integer}(::Type{Rational{T}}, x::Irrational) = convert(Rational{T}, Float64(x)) -@generated function (t::Type{T}){T<:Union{Float32,Float64},s}(c::Irrational{s},r::RoundingMode) - f = T(big(c()),r()) - :($f) +@pure function convert{T<:Integer}(::Type{Rational{T}}, x::Irrational) + o = precision(BigFloat) + p = 256 + while true + setprecision(BigFloat, p) + bx = BigFloat(x) + r = rationalize(T, bx, tol=0) + if abs(BigFloat(r) - bx) > eps(bx) + setprecision(BigFloat, o) + return r + end + p += 32 + end +end +convert(::Type{Rational{BigInt}}, x::Irrational) = throw(ArgumentError("Cannot convert an Irrational to a Rational{BigInt}: use rationalize(Rational{BigInt}, x) instead")) + +@pure function (t::Type{T}){T<:Union{Float32,Float64}}(x::Irrational, r::RoundingMode) + setprecision(BigFloat, 256) do + T(BigFloat(x), r) + end end =={s}(::Irrational{s}, ::Irrational{s}) = true diff --git a/test/numbers.jl b/test/numbers.jl index f8377601f7ad2..550cc6981639a 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2082,6 +2082,12 @@ for f in (trunc, round, floor, ceil) @test isa(convert(Float64, big(1)//2), Float64) +# issue 16513 +@test convert(Rational{Int32}, pi) == 1068966896 // 340262731 +@test convert(Rational{Int64}, pi) == 2646693125139304345 // 842468587426513207 +@test convert(Rational{Int128}, pi) == 60728338969805745700507212595448411044 // 19330430665609526556707216376512714945 +@test_throws ArgumentError convert(Rational{BigInt}, pi) + # issue 5935 @test rationalize(Int8, nextfloat(0.1)) == 1//10 @test rationalize(Int64, nextfloat(0.1)) == 300239975158034//3002399751580339 From 4da6c4f70bf3ed25133ee9ca888ae5274a73e288 Mon Sep 17 00:00:00 2001 From: Patrick Kofod Mogensen Date: Mon, 27 Jun 2016 20:45:52 +0200 Subject: [PATCH 0124/1117] Fix bug in several Float16 methods where the second positional argument wasn't used. Add test based on original issue #17148. --- base/float16.jl | 2 +- test/float16.jl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/base/float16.jl b/base/float16.jl index a58a36a4c7a85..46dcb8bfaa0d7 100644 --- a/base/float16.jl +++ b/base/float16.jl @@ -154,7 +154,7 @@ end for func in (:div,:fld,:cld,:rem,:mod,:atan2,:hypot) @eval begin - $func(a::Float16,b::Float16) = Float16($func(Float32(a),Float32(a))) + $func(a::Float16,b::Float16) = Float16($func(Float32(a),Float32(b))) end end diff --git a/test/float16.jl b/test/float16.jl index 22705ae9131a5..2031c056040b7 100644 --- a/test/float16.jl +++ b/test/float16.jl @@ -143,3 +143,6 @@ end # #9939 (and #9897) @test rationalize(Float16(0.1)) == 1//10 + +# issue #17148 +@test rem(Float16(1.2), Float16(one(1.2))) == 0.20019531f0 From bc733d8143f3a7eddc60e134c1690fa53618b136 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Mon, 27 Jun 2016 18:12:36 -0400 Subject: [PATCH 0125/1117] Make `median` non-mutating on arrays. Due to a change in the behaviour of `mapslices` (#16260), `median(X,k)` would mutate the underlying array. Fixes #17153. --- NEWS.md | 2 ++ base/statistics.jl | 5 +++-- test/statistics.jl | 7 +++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index d6a32e4271dd1..12db8fff3237e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -84,6 +84,8 @@ Breaking changes If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of the desired shape ([#4211]). + * `mapslices` will always pass a view, so passing mutating functions will mutate the underlying array ([#16260]) + * Local variables and arguments are represented in lowered code as numbered `Slot` objects instead of as symbols ([#15609]). diff --git a/base/statistics.jl b/base/statistics.jl index b474af31fed46..9fff861eda492 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -490,9 +490,10 @@ function median!{T}(v::AbstractVector{T}) end end median!{T}(v::AbstractArray{T}) = median!(vec(v)) - median{T}(v::AbstractArray{T}) = median!(copy!(Array(T, length(v)), v)) -median{T}(v::AbstractArray{T}, region) = mapslices(median!, v, region) + +median!{T}(v::AbstractArray{T}, region) = mapslices(median!, v, region) +median{T}(v::AbstractArray{T}, region) = median!(copy(v), region) # for now, use the R/S definition of quantile; may want variants later # see ?quantile in R -- this is type 7 diff --git a/test/statistics.jl b/test/statistics.jl index d2429cdf1d0ec..0ea874926257f 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -32,8 +32,10 @@ end @test median([1.,-1.,Inf,-Inf]) == 0.0 @test isnan(median([-Inf,Inf])) -@test all(median([2 3 1 -1; 7 4 5 -4], 2) .== [1.5, 4.5]) -@test all(median([2 3 1 -1; 7 4 5 -4], 1) .== [4.5 3.5 3.0 -2.5]) +X = [2 3 1 -1; 7 4 5 -4] +@test all(median(X, 2) .== [1.5, 4.5]) +@test all(median(X, 1) .== [4.5 3.5 3.0 -2.5]) +@test X == [2 3 1 -1; 7 4 5 -4] # issue #17153 @test_throws ArgumentError median([]) @test isnan(median([NaN])) @@ -44,6 +46,7 @@ end @test median!([1 2 3 4]) == 2.5 @test median!([1 2; 3 4]) == 2.5 + @test invoke(median, (AbstractVector,), 1:10) == median(1:10) == 5.5 # mean From d09cd9bce6e9d0c0b6b170a45c9d865768d0a145 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Mon, 27 Jun 2016 18:36:19 -0400 Subject: [PATCH 0126/1117] Remove VC90 from julia-manifest.xml --- contrib/windows/julia-manifest.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/contrib/windows/julia-manifest.xml b/contrib/windows/julia-manifest.xml index ab9b812592bd4..5e76b7b8e76f8 100644 --- a/contrib/windows/julia-manifest.xml +++ b/contrib/windows/julia-manifest.xml @@ -7,11 +7,6 @@ - - - - - From ef97de2f06323b1ec5a881bab4a53d13e84dea59 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Mon, 27 Jun 2016 22:53:13 -0400 Subject: [PATCH 0127/1117] =?UTF-8?q?replace=20`@test=5Fapprox=5Feq`=20wit?= =?UTF-8?q?h=20=E2=89=88=20(#17151)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also fix bug in hex2num, exposed due to ≈ not being broken like `@test_approx_eq`. --- base/floatfuncs.jl | 3 + test/arrayops.jl | 2 +- test/blas.jl | 52 ++-- test/broadcast.jl | 4 +- test/ccall.jl | 32 +-- test/complex.jl | 152 ++++++------ test/dsp.jl | 70 +++--- test/fft.jl | 110 ++++----- test/float16.jl | 8 +- test/floatfuncs.jl | 10 +- test/linalg/arnoldi.jl | 52 ++-- test/linalg/bidiag.jl | 20 +- test/linalg/bunchkaufman.jl | 8 +- test/linalg/cholesky.jl | 22 +- test/linalg/dense.jl | 156 ++++++------ test/linalg/eigen.jl | 18 +- test/linalg/generic.jl | 14 +- test/linalg/givens.jl | 12 +- test/linalg/hessenberg.jl | 8 +- test/linalg/lapack.jl | 78 +++--- test/linalg/lu.jl | 18 +- test/linalg/matmul.jl | 4 +- test/linalg/pinv.jl | 30 +-- test/linalg/qr.jl | 68 +++--- test/linalg/schur.jl | 22 +- test/linalg/special.jl | 12 +- test/linalg/triangular.jl | 102 ++++---- test/linalg/tridiag.jl | 64 ++--- test/llvmcall.jl | 10 +- test/math.jl | 433 +++++++++++++++++---------------- test/mod2pi.jl | 14 +- test/mpfr.jl | 34 +-- test/numbers.jl | 14 +- test/ranges.jl | 6 +- test/reduce.jl | 16 +- test/reducedim.jl | 70 +++--- test/sparsedir/cholmod.jl | 208 ++++++++-------- test/sparsedir/sparse.jl | 48 ++-- test/sparsedir/sparsevector.jl | 58 ++--- test/sparsedir/spqr.jl | 8 +- test/statistics.jl | 94 +++---- 41 files changed, 1084 insertions(+), 1080 deletions(-) diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index d81c1b5503c9b..3630501c43863 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -31,6 +31,9 @@ num2hex(x::Float32) = hex(box(UInt32,unbox(Float32,x)),8) num2hex(x::Float64) = hex(box(UInt64,unbox(Float64,x)),16) function hex2num(s::AbstractString) + if length(s) <= 4 + return box(Float16,unbox(UInt16,parse(UInt16,s,16))) + end if length(s) <= 8 return box(Float32,unbox(UInt32,parse(UInt32,s,16))) end diff --git a/test/arrayops.jl b/test/arrayops.jl index 74c0f8ae9f8a4..171cbda41e1bf 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1346,7 +1346,7 @@ end # PR #11080 let x = fill(0.9, 1000) - @test_approx_eq prod(x) cumprod(x)[end] + @test prod(x) ≈ cumprod(x)[end] end #binary ops on bool arrays diff --git a/test/blas.jl b/test/blas.jl index f44a5ae900b51..f7a4684c40abc 100644 --- a/test/blas.jl +++ b/test/blas.jl @@ -13,10 +13,10 @@ for elty in (Float32, Float64, Complex64, Complex128) end U = convert(Array{elty, 2}, U) V = convert(Array{elty, 2}, V) - @test_approx_eq tril(LinAlg.BLAS.syr2k('L','N',U,V)) tril(U*V.' + V*U.') - @test_approx_eq triu(LinAlg.BLAS.syr2k('U','N',U,V)) triu(U*V.' + V*U.') - @test_approx_eq tril(LinAlg.BLAS.syr2k('L','T',U,V)) tril(U.'*V + V.'*U) - @test_approx_eq triu(LinAlg.BLAS.syr2k('U','T',U,V)) triu(U.'*V + V.'*U) + @test tril(LinAlg.BLAS.syr2k('L','N',U,V)) ≈ tril(U*V.' + V*U.') + @test triu(LinAlg.BLAS.syr2k('U','N',U,V)) ≈ triu(U*V.' + V*U.') + @test tril(LinAlg.BLAS.syr2k('L','T',U,V)) ≈ tril(U.'*V + V.'*U) + @test triu(LinAlg.BLAS.syr2k('U','T',U,V)) ≈ triu(U.'*V + V.'*U) end for elty in (Complex64, Complex128) @@ -28,10 +28,10 @@ for elty in (Complex64, Complex128) end U = convert(Array{elty, 2}, U) V = convert(Array{elty, 2}, V) - @test_approx_eq tril(LinAlg.BLAS.her2k('L','N',U,V)) tril(U*V' + V*U') - @test_approx_eq triu(LinAlg.BLAS.her2k('U','N',U,V)) triu(U*V' + V*U') - @test_approx_eq tril(LinAlg.BLAS.her2k('L','C',U,V)) tril(U'*V + V'*U) - @test_approx_eq triu(LinAlg.BLAS.her2k('U','C',U,V)) triu(U'*V + V'*U) + @test tril(LinAlg.BLAS.her2k('L','N',U,V)) ≈ tril(U*V' + V*U') + @test triu(LinAlg.BLAS.her2k('U','N',U,V)) ≈ triu(U*V' + V*U') + @test tril(LinAlg.BLAS.her2k('L','C',U,V)) ≈ tril(U'*V + V'*U) + @test triu(LinAlg.BLAS.her2k('U','C',U,V)) ≈ triu(U'*V + V'*U) end ## BLAS tests - testing the interface code to BLAS routines @@ -55,13 +55,13 @@ for elty in [Float32, Float64, Complex64, Complex128] if elty <: Real x1 = convert(Vector{elty}, randn(n)) x2 = convert(Vector{elty}, randn(n)) - @test_approx_eq BLAS.dot(x1,x2) sum(x1.*x2) + @test BLAS.dot(x1,x2) ≈ sum(x1.*x2) @test_throws DimensionMismatch BLAS.dot(x1,rand(elty, n + 1)) else z1 = convert(Vector{elty}, complex(randn(n),randn(n))) z2 = convert(Vector{elty}, complex(randn(n),randn(n))) - @test_approx_eq BLAS.dotc(z1,z2) sum(conj(z1).*z2) - @test_approx_eq BLAS.dotu(z1,z2) sum(z1.*z2) + @test BLAS.dotc(z1,z2) ≈ sum(conj(z1).*z2) + @test BLAS.dotu(z1,z2) ≈ sum(z1.*z2) @test_throws DimensionMismatch BLAS.dotc(z1,rand(elty, n + 1)) @test_throws DimensionMismatch BLAS.dotu(z1,rand(elty, n + 1)) end @@ -80,22 +80,22 @@ for elty in [Float32, Float64, Complex64, Complex128] x1 = convert(Vector{elty}, randn(n)) x2 = convert(Vector{elty}, randn(n)) α = rand(elty) - @test_approx_eq BLAS.axpy!(α,copy(x1),copy(x2)) x2 + α*x1 + @test BLAS.axpy!(α,copy(x1),copy(x2)) ≈ x2 + α*x1 @test_throws DimensionMismatch BLAS.axpy!(α, copy(x1), rand(elty, n + 1)) @test_throws DimensionMismatch BLAS.axpy!(α, copy(x1), 1:div(n,2), copy(x2), 1:n) @test_throws ArgumentError BLAS.axpy!(α, copy(x1), 0:div(n,2), copy(x2), 1:(div(n, 2) + 1)) @test_throws ArgumentError BLAS.axpy!(α, copy(x1), 1:div(n,2), copy(x2), 0:(div(n, 2) - 1)) - @test_approx_eq BLAS.axpy!(α,copy(x1),1:n,copy(x2),1:n) x2 + α*x1 + @test BLAS.axpy!(α,copy(x1),1:n,copy(x2),1:n) ≈ x2 + α*x1 else z1 = convert(Vector{elty}, complex(randn(n), randn(n))) z2 = convert(Vector{elty}, complex(randn(n), randn(n))) α = rand(elty) - @test_approx_eq BLAS.axpy!(α, copy(z1), copy(z2)) z2 + α * z1 + @test BLAS.axpy!(α, copy(z1), copy(z2)) ≈ z2 + α * z1 @test_throws DimensionMismatch BLAS.axpy!(α, copy(z1), rand(elty, n + 1)) @test_throws DimensionMismatch BLAS.axpy!(α, copy(z1), 1:div(n, 2), copy(z2), 1:(div(n, 2) + 1)) @test_throws ArgumentError BLAS.axpy!(α, copy(z1), 0:div(n,2), copy(z2), 1:(div(n, 2) + 1)) @test_throws ArgumentError BLAS.axpy!(α, copy(z1), 1:div(n,2), copy(z2), 0:(div(n, 2) - 1)) - @test_approx_eq BLAS.axpy!(α,copy(z1),1:n,copy(z2),1:n) z2 + α*z1 + @test BLAS.axpy!(α,copy(z1),1:n,copy(z2),1:n) ≈ z2 + α*z1 end # nrm2, iamax, and asum for StridedVectors @@ -117,7 +117,7 @@ for elty in [Float32, Float64, Complex64, Complex128] # trsv A = triu(rand(elty,n,n)) x = rand(elty,n) - @test_approx_eq A\x BLAS.trsv('U','N','N',A,x) + @test A\x ≈ BLAS.trsv('U','N','N',A,x) @test_throws DimensionMismatch BLAS.trsv('U','N','N',A,ones(elty,n+1)) # ger, her, syr @@ -126,20 +126,20 @@ for elty in [Float32, Float64, Complex64, Complex128] x = rand(elty,n) y = rand(elty,n) - @test_approx_eq BLAS.ger!(α,x,y,copy(A)) A + α*x*y' + @test BLAS.ger!(α,x,y,copy(A)) ≈ A + α*x*y' @test_throws DimensionMismatch BLAS.ger!(α,ones(elty,n+1),y,copy(A)) A = rand(elty,n,n) A = A + A.' @test issymmetric(A) - @test_approx_eq triu(BLAS.syr!('U',α,x,copy(A))) triu(A + α*x*x.') + @test triu(BLAS.syr!('U',α,x,copy(A))) ≈ triu(A + α*x*x.') @test_throws DimensionMismatch BLAS.syr!('U',α,ones(elty,n+1),copy(A)) if elty <: Complex A = rand(elty,n,n) A = A + A' α = real(α) - @test_approx_eq triu(BLAS.her!('U',α,x,copy(A))) triu(A + α*x*x') + @test triu(BLAS.her!('U',α,x,copy(A))) ≈ triu(A + α*x*x') @test_throws DimensionMismatch BLAS.her!('U',α,ones(elty,n+1),copy(A)) end @@ -159,11 +159,11 @@ for elty in [Float32, Float64, Complex64, Complex128] Aherm = A + A' Asymm = A + A.' - @test_approx_eq BLAS.symv('U',Asymm,x) Asymm*x + @test BLAS.symv('U',Asymm,x) ≈ Asymm*x @test_throws DimensionMismatch BLAS.symv!('U',one(elty),Asymm,x,one(elty),ones(elty,n+1)) @test_throws DimensionMismatch BLAS.symv('U',ones(elty,n,n+1),x) if elty <: BlasComplex - @test_approx_eq BLAS.hemv('U',Aherm,x) Aherm*x + @test BLAS.hemv('U',Aherm,x) ≈ Aherm*x @test_throws DimensionMismatch BLAS.hemv('U',ones(elty,n,n+1),x) @test_throws DimensionMismatch BLAS.hemv!('U',one(elty),Aherm,x,one(elty),ones(elty,n+1)) end @@ -171,7 +171,7 @@ for elty in [Float32, Float64, Complex64, Complex128] # trmv A = triu(rand(elty,n,n)) x = rand(elty,n) - @test_approx_eq BLAS.trmv('U','N','N',A,x) A*x + @test BLAS.trmv('U','N','N',A,x) ≈ A*x # symm error throwing @test_throws DimensionMismatch BLAS.symm('L','U',ones(elty,n,n-1),rand(elty,n,n)) @@ -192,7 +192,7 @@ for elty in [Float32, Float64, Complex64, Complex128] #trsm A = triu(rand(elty,n,n)) B = rand(elty,(n,n)) - @test_approx_eq BLAS.trsm('L','U','N','N',one(elty),A,B) A\B + @test BLAS.trsm('L','U','N','N',one(elty),A,B) ≈ A\B #gbmv - will work for SymTridiagonal,Tridiagonal,Bidiagonal! TD = Tridiagonal(rand(elty,n-1),rand(elty,n),rand(elty,n-1)) @@ -202,7 +202,7 @@ for elty in [Float32, Float64, Complex64, Complex128] fTD[1,2:n] = TD.du fTD[2,:] = TD.d fTD[3,1:n-1] = TD.dl - @test_approx_eq BLAS.gbmv('N',n,1,1,fTD,x) TD*x + @test BLAS.gbmv('N',n,1,1,fTD,x) ≈ TD*x #sbmv - will work for SymTridiagonal only! if elty <: BlasReal @@ -212,7 +212,7 @@ for elty in [Float32, Float64, Complex64, Complex128] fST = zeros(elty,2,n) fST[1,2:n] = ST.ev fST[2,:] = ST.dv - @test_approx_eq BLAS.sbmv('U',1,fST,x) ST*x + @test BLAS.sbmv('U',1,fST,x) ≈ ST*x else dv = real(rand(elty,n)) ev = rand(elty,n-1) @@ -220,7 +220,7 @@ for elty in [Float32, Float64, Complex64, Complex128] bH[1,2:n] = ev bH[2,:] = dv fullH = diagm(dv) + diagm(conj(ev),-1) + diagm(ev,1) - @test_approx_eq BLAS.hbmv('U',1,bH,x) fullH*x + @test BLAS.hbmv('U',1,bH,x) ≈ fullH*x end end diff --git a/test/broadcast.jl b/test/broadcast.jl index 2484a43d29701..05a8e42e8d60b 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -159,8 +159,8 @@ ratio = [1,1/2,1/3,1/4,1/5] @test r1./r2 == ratio m = [1:2;]' @test m.*r2 == [1:5 2:2:10] -@test_approx_eq m./r2 [ratio 2ratio] -@test_approx_eq m./[r2;] [ratio 2ratio] +@test m./r2 ≈ [ratio 2ratio] +@test m./[r2;] ≈ [ratio 2ratio] @test @inferred([0,1.2].+reshape([0,-2],1,1,2)) == reshape([0 -2; 1.2 -0.8],2,1,2) rt = Base.return_types(.+, Tuple{Array{Float64, 3}, Array{Int, 1}}) diff --git a/test/ccall.jl b/test/ccall.jl index f37c541c0a4f5..b4fd41f3e3c5a 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -123,8 +123,8 @@ let @test a2.x == a.x && a2.y == a.y @test !(a2 === x) - @test_approx_eq x.x a.x + 1*b - @test_approx_eq x.y a.y - 2*b + @test x.x ≈ a.x + 1*b + @test x.y ≈ a.y - 2*b end let @@ -283,7 +283,7 @@ let x = ccall((:test_11, libccalltest), Struct11, (Struct11,Float32), a, b) - @test_approx_eq x.x a.x + b*1 - b*2im + @test x.x ≈ a.x + b*1 - b*2im end type Struct12 @@ -297,8 +297,8 @@ let x = ccall((:test_12, libccalltest), Struct12, (Struct12,Float32), a, b) - @test_approx_eq x.x a.x + b*1 - b*2im - @test_approx_eq x.y a.y + b*3 - b*4im + @test x.x ≈ a.x + b*1 - b*2im + @test x.y ≈ a.y + b*3 - b*4im end type Struct13 @@ -311,7 +311,7 @@ let x = ccall((:test_13, libccalltest), Struct13, (Struct13,Float64), a, b) - @test_approx_eq x.x a.x + b*1 - b*2im + @test x.x ≈ a.x + b*1 - b*2im end type Struct14 @@ -325,8 +325,8 @@ let x = ccall((:test_14, libccalltest), Struct14, (Struct14,Float32), a, b) - @test_approx_eq x.x a.x + b*1 - @test_approx_eq x.y a.y - b*2 + @test x.x ≈ a.x + b*1 + @test x.y ≈ a.y - b*2 end type Struct15 @@ -340,8 +340,8 @@ let x = ccall((:test_15, libccalltest), Struct15, (Struct15,Float64), a, b) - @test_approx_eq x.x a.x + b*1 - @test_approx_eq x.y a.y - b*2 + @test x.x ≈ a.x + b*1 + @test x.y ≈ a.y - b*2 end type Struct16 @@ -360,12 +360,12 @@ let x = ccall((:test_16, libccalltest), Struct16, (Struct16,Float32), a, b) - @test_approx_eq x.x a.x + b*1 - @test_approx_eq x.y a.y - b*2 - @test_approx_eq x.z a.z + b*3 - @test_approx_eq x.a a.a - b*4 - @test_approx_eq x.b a.b + b*5 - @test_approx_eq x.c a.c - b*6 + @test x.x ≈ a.x + b*1 + @test x.y ≈ a.y - b*2 + @test x.z ≈ a.z + b*3 + @test x.a ≈ a.a - b*4 + @test x.b ≈ a.b + b*5 + @test x.c ≈ a.c - b*6 end let diff --git a/test/complex.jl b/test/complex.jl index 84a6988445648..89c3c15862e6f 100644 --- a/test/complex.jl +++ b/test/complex.jl @@ -69,67 +69,67 @@ for T in (Float32, Float64) y = Complex{T}(1//2 + 1//5*im) yi = 4 # Test random values - @test_approx_eq x^y big(x)^big(y) - @test_approx_eq x^yi big(x)^yi - @test_approx_eq x^true big(x)^true - @test_approx_eq x^false big(x)^false - @test_approx_eq x^1 big(x)^1 - @test_approx_eq abs(x) abs(big(x)) - @test_approx_eq abs2(x) abs2(big(x)) - @test_approx_eq acos(x) acos(big(x)) - @test_approx_eq acosh(1+x) acosh(1+big(x)) - @test_approx_eq angle(x) angle(big(x)) - @test_approx_eq asin(x) asin(big(x)) - @test_approx_eq asinh(x) asinh(big(x)) - @test_approx_eq atan(x) atan(big(x)) - @test_approx_eq atanh(x) atanh(big(x)) - @test_approx_eq cis(real(x)) cis(real(big(x))) - @test_approx_eq cis(x) cis(big(x)) - @test_approx_eq cos(x) cos(big(x)) - @test_approx_eq cosh(x) cosh(big(x)) - @test_approx_eq exp(x) exp(big(x)) - @test_approx_eq exp10(x) exp10(big(x)) - @test_approx_eq exp2(x) exp2(big(x)) + @test x^y ≈ big(x)^big(y) + @test x^yi ≈ big(x)^yi + @test x^true ≈ big(x)^true + @test x^false ≈ big(x)^false + @test x^1 ≈ big(x)^1 + @test abs(x) ≈ abs(big(x)) + @test abs2(x) ≈ abs2(big(x)) + @test acos(x) ≈ acos(big(x)) + @test acosh(1+x) ≈ acosh(1+big(x)) + @test angle(x) ≈ angle(big(x)) + @test asin(x) ≈ asin(big(x)) + @test asinh(x) ≈ asinh(big(x)) + @test atan(x) ≈ atan(big(x)) + @test atanh(x) ≈ atanh(big(x)) + @test cis(real(x)) ≈ cis(real(big(x))) + @test cis(x) ≈ cis(big(x)) + @test cos(x) ≈ cos(big(x)) + @test cosh(x) ≈ cosh(big(x)) + @test exp(x) ≈ exp(big(x)) + @test exp10(x) ≈ exp10(big(x)) + @test exp2(x) ≈ exp2(big(x)) @test_approx_eq_eps expm1(x) expm1(big(x)) eps(T) - @test_approx_eq log(x) log(big(x)) - @test_approx_eq log10(x) log10(big(x)) - @test_approx_eq log1p(x) log1p(big(x)) - @test_approx_eq log2(x) log2(big(x)) - @test_approx_eq sin(x) sin(big(x)) - @test_approx_eq sinh(x) sinh(big(x)) - @test_approx_eq sqrt(x) sqrt(big(x)) - @test_approx_eq tan(x) tan(big(x)) - @test_approx_eq tanh(x) tanh(big(x)) + @test log(x) ≈ log(big(x)) + @test log10(x) ≈ log10(big(x)) + @test log1p(x) ≈ log1p(big(x)) + @test log2(x) ≈ log2(big(x)) + @test sin(x) ≈ sin(big(x)) + @test sinh(x) ≈ sinh(big(x)) + @test sqrt(x) ≈ sqrt(big(x)) + @test tan(x) ≈ tan(big(x)) + @test tanh(x) ≈ tanh(big(x)) # Test inverses - @test_approx_eq acos(cos(x)) x - @test_approx_eq acosh(cosh(x)) x - @test_approx_eq asin(sin(x)) x - @test_approx_eq asinh(sinh(x)) x - @test_approx_eq atan(tan(x)) x - @test_approx_eq atanh(tanh(x)) x - @test_approx_eq cos(acos(x)) x - @test_approx_eq cosh(acosh(1+x)) 1+x - @test_approx_eq exp(log(x)) x - @test_approx_eq exp10(log10(x)) x - @test_approx_eq exp2(log2(x)) x - @test_approx_eq expm1(log1p(x)) x - @test_approx_eq log(exp(x)) x - @test_approx_eq log10(exp10(x)) x - @test_approx_eq log1p(expm1(x)) x - @test_approx_eq log2(exp2(x)) x - @test_approx_eq sin(asin(x)) x - @test_approx_eq sinh(asinh(x)) x - @test_approx_eq sqrt(x)^2 x - @test_approx_eq sqrt(x^2) x - @test_approx_eq tan(atan(x)) x - @test_approx_eq tanh(atanh(x)) x + @test acos(cos(x)) ≈ x + @test acosh(cosh(x)) ≈ x + @test asin(sin(x)) ≈ x + @test asinh(sinh(x)) ≈ x + @test atan(tan(x)) ≈ x + @test atanh(tanh(x)) ≈ x + @test cos(acos(x)) ≈ x + @test cosh(acosh(1+x)) ≈ 1+x + @test exp(log(x)) ≈ x + @test exp10(log10(x)) ≈ x + @test exp2(log2(x)) ≈ x + @test expm1(log1p(x)) ≈ x + @test log(exp(x)) ≈ x + @test log10(exp10(x)) ≈ x + @test log1p(expm1(x)) ≈ x + @test log2(exp2(x)) ≈ x + @test sin(asin(x)) ≈ x + @test sinh(asinh(x)) ≈ x + @test sqrt(x)^2 ≈ x + @test sqrt(x^2) ≈ x + @test tan(atan(x)) ≈ x + @test tanh(atanh(x)) ≈ x # Test some properties - @test_approx_eq cosh(x) (exp(x)+exp(-x))/2 - @test_approx_eq cosh(x)^2-sinh(x)^2 1 - @test_approx_eq sin(x)^2+cos(x)^2 1 - @test_approx_eq sinh(x) (exp(x)-exp(-x))/2 - @test_approx_eq tan(x) sin(x)/cos(x) - @test_approx_eq tanh(x) sinh(x)/cosh(x) + @test cosh(x) ≈ (exp(x)+exp(-x))/2 + @test cosh(x)^2-sinh(x)^2 ≈ 1 + @test sin(x)^2+cos(x)^2 ≈ 1 + @test sinh(x) ≈ (exp(x)-exp(-x))/2 + @test tan(x) ≈ sin(x)/cos(x) + @test tanh(x) ≈ sinh(x)/cosh(x) end #isimag and isinf @@ -295,15 +295,15 @@ end @test isequal(expm1(complex( 1e-20, 0.0)), complex(expm1( 1e-20), 0.0)) @test isequal(expm1(complex(-1e-20, 0.0)), complex(expm1(-1e-20), 0.0)) -@test_approx_eq expm1(complex( 1e-20, 1e-10)) complex( 5e-21, 1e-10) -@test_approx_eq expm1(complex( 1e-20,-1e-10)) complex( 5e-21,-1e-10) -@test_approx_eq expm1(complex(-1e-20, 1e-10)) complex(-1.5e-20, 1e-10) -@test_approx_eq expm1(complex(-1e-20,-1e-10)) complex(-1.5e-20,-1e-10) +@test expm1(complex( 1e-20, 1e-10)) ≈ complex( 5e-21, 1e-10) +@test expm1(complex( 1e-20,-1e-10)) ≈ complex( 5e-21,-1e-10) +@test expm1(complex(-1e-20, 1e-10)) ≈ complex(-1.5e-20, 1e-10) +@test expm1(complex(-1e-20,-1e-10)) ≈ complex(-1.5e-20,-1e-10) -@test_approx_eq expm1(complex( 10.0, 10.0)) exp(complex( 10.0, 10.0))-1 -@test_approx_eq expm1(complex( 10.0,-10.0)) exp(complex( 10.0,-10.0))-1 -@test_approx_eq expm1(complex(-10.0, 10.0)) exp(complex(-10.0, 10.0))-1 -@test_approx_eq expm1(complex(-10.0,-10.0)) exp(complex(-10.0,-10.0))-1 +@test expm1(complex( 10.0, 10.0)) ≈ exp(complex( 10.0, 10.0))-1 +@test expm1(complex( 10.0,-10.0)) ≈ exp(complex( 10.0,-10.0))-1 +@test expm1(complex(-10.0, 10.0)) ≈ exp(complex(-10.0, 10.0))-1 +@test expm1(complex(-10.0,-10.0)) ≈ exp(complex(-10.0,-10.0))-1 # log1p: @test isequal(log1p(complex(Inf, 3)), complex(Inf, +0.)) @@ -320,10 +320,10 @@ end @test isequal(log1p(complex(1, -Inf)), complex(Inf, -pi/2)) import Base.Math.@horner for z in (1e-10+1e-9im, 1e-10-1e-9im, -1e-10+1e-9im, -1e-10-1e-9im) - @test_approx_eq log1p(z) @horner(z, 0, 1, -0.5, 1/3, -0.25, 0.2) + @test log1p(z) ≈ @horner(z, 0, 1, -0.5, 1/3, -0.25, 0.2) end for z in (15+4im, 0.2+3im, 0.08-0.9im) - @test_approx_eq log1p(z) log(1+z) + @test log1p(z) ≈ log(1+z) end @@ -351,8 +351,8 @@ end @test isequal(complex(-0.0,-0.0)^complex(-0.0, 0.0), complex(1.0, 0.0)) @test isequal(complex(-0.0,-0.0)^complex(-0.0,-0.0), complex(1.0, 0.0)) -@test_approx_eq complex(0.0,1.0)^complex(2.0,0) complex(-1.0, 0.0) -@test_approx_eq complex(1.0,2.0)^complex(3.0,0) complex(-11.0, -2.0) +@test complex(0.0,1.0)^complex(2.0,0) ≈ complex(-1.0, 0.0) +@test complex(1.0,2.0)^complex(3.0,0) ≈ complex(-11.0, -2.0) @test isequal(complex(0.0,0.0)^false, complex(1.0,0.0)) @test isequal(complex(0.0,0.0)^0, complex(1.0,0.0)) @@ -864,22 +864,22 @@ for T in (Float16, Float32, Float64) end # cis -@test_approx_eq cis(0.0+1.0im) 0.367879441171442321595523770161460867445811131031767834507836+0.0im -@test_approx_eq cis(1.0+0.0im) 0.54030230586813971740093660744297660373231042061+0.84147098480789650665250232163029899962256306079im -@test_approx_eq cis(pi) -1.0+0.0im -@test_approx_eq cis(pi/2) 0.0+1.0im +@test cis(0.0+1.0im) ≈ 0.367879441171442321595523770161460867445811131031767834507836+0.0im +@test cis(1.0+0.0im) ≈ 0.54030230586813971740093660744297660373231042061+0.84147098480789650665250232163029899962256306079im +@test cis(pi) ≈ -1.0+0.0im +@test cis(pi/2) ≈ 0.0+1.0im # exp2 @test exp2(0.0+0.0im) == 1.0+0.0im @test exp2(1.0+0.0im) == 2.0+0.0im #wolframalpha -@test_approx_eq exp2(1.0+3.0im) -0.9739888359315627962096198412+1.74681016354974281701922im +@test exp2(1.0+3.0im) ≈ -0.9739888359315627962096198412+1.74681016354974281701922im # exp10 @test exp10(0.0+0.0im) == 1.0+0.0im @test exp10(1.0+0.0im) == 10.0+0.0im #wolframalpha -@test_approx_eq exp10(1.0+2.0im) -1.0701348355877020772086517528518239460495529361-9.9425756941378968736161937190915602112878340717im +@test exp10(1.0+2.0im) ≈ -1.0701348355877020772086517528518239460495529361-9.9425756941378968736161937190915602112878340717im # round #8291 @test round(Complex(1.125, 0.875), 2) == Complex(1.12, 0.88) diff --git a/test/dsp.jl b/test/dsp.jl index a55ad5faa2a1c..b9c62581a2aff 100644 --- a/test/dsp.jl +++ b/test/dsp.jl @@ -21,7 +21,7 @@ si = [zeros(3) ones(3)] b = [0.003279216306360201,0.016396081531801006,0.03279216306360201,0.03279216306360201,0.016396081531801006,0.003279216306360201] a = [1.0,-2.4744161749781606,2.8110063119115782,-1.703772240915465,0.5444326948885326,-0.07231566910295834] si = [0.9967207836936347,-1.4940914728163142,1.2841226760316475,-0.4524417279474106,0.07559488540931815] -@test_approx_eq filt(b, a, ones(10), si) ones(10) # Shouldn't affect DC offset +@test filt(b, a, ones(10), si) ≈ ones(10) # Shouldn't affect DC offset @test_throws ArgumentError filt!([1, 2], [1], [1], [1]) @test xcorr([1, 2], [3, 4]) == [4, 11, 6] @@ -36,8 +36,8 @@ si = [0.9967207836936347,-1.4940914728163142,1.2841226760316475,-0.4524417279474 # Convolution a = [1., 2., 1., 2.] b = [1., 2., 3.] -@test_approx_eq conv(a, b) [1., 4., 8., 10., 7., 6.] -@test_approx_eq conv(complex(a, ones(4)), complex(b)) complex([1., 4., 8., 10., 7., 6.], [1., 3., 6., 6., 5., 3.]) +@test conv(a, b) ≈ [1., 4., 8., 10., 7., 6.] +@test conv(complex(a, ones(4)), complex(b)) ≈ complex([1., 4., 8., 10., 7., 6.], [1., 3., 6., 6., 5., 3.]) # Discrete cosine transform (DCT) tests @@ -91,37 +91,37 @@ if Base.fftw_vendor() != :mkl psYdct! = copy(Y); psXdct! = view(psYdct!,3:5,9:12); plan_dct!(psXdct!)*(psXdct!) for i = 1:length(X) - @test_approx_eq Xdct[i] true_Xdct[i] - @test_approx_eq Xdct![i] true_Xdct[i] - @test_approx_eq Xdct_1[i] true_Xdct_1[i] - @test_approx_eq Xdct!_1[i] true_Xdct_1[i] - @test_approx_eq Xdct_2[i] true_Xdct_2[i] - @test_approx_eq Xdct!_2[i] true_Xdct_2[i] - - @test_approx_eq pXdct[i] true_Xdct[i] - @test_approx_eq pXdct![i] true_Xdct[i] - @test_approx_eq pXdct_1[i] true_Xdct_1[i] - @test_approx_eq pXdct!_1[i] true_Xdct_1[i] - @test_approx_eq pXdct_2[i] true_Xdct_2[i] - @test_approx_eq pXdct!_2[i] true_Xdct_2[i] - - @test_approx_eq Xidct[i] X[i] - @test_approx_eq Xidct![i] X[i] - @test_approx_eq Xidct_1[i] X[i] - @test_approx_eq Xidct!_1[i] X[i] - @test_approx_eq Xidct_2[i] X[i] - @test_approx_eq Xidct!_2[i] X[i] - - @test_approx_eq pXidct[i] X[i] - @test_approx_eq pXidct![i] X[i] - @test_approx_eq pXidct_1[i] X[i] - @test_approx_eq pXidct!_1[i] X[i] - @test_approx_eq pXidct_2[i] X[i] - @test_approx_eq pXidct!_2[i] X[i] - - @test_approx_eq sXdct[i] true_Xdct[i] - @test_approx_eq psXdct[i] true_Xdct[i] - @test_approx_eq sXdct![i] true_Xdct[i] - @test_approx_eq psXdct![i] true_Xdct[i] + @test Xdct[i] ≈ true_Xdct[i] + @test Xdct![i] ≈ true_Xdct[i] + @test Xdct_1[i] ≈ true_Xdct_1[i] + @test Xdct!_1[i] ≈ true_Xdct_1[i] + @test Xdct_2[i] ≈ true_Xdct_2[i] + @test Xdct!_2[i] ≈ true_Xdct_2[i] + + @test pXdct[i] ≈ true_Xdct[i] + @test pXdct![i] ≈ true_Xdct[i] + @test pXdct_1[i] ≈ true_Xdct_1[i] + @test pXdct!_1[i] ≈ true_Xdct_1[i] + @test pXdct_2[i] ≈ true_Xdct_2[i] + @test pXdct!_2[i] ≈ true_Xdct_2[i] + + @test Xidct[i] ≈ X[i] + @test Xidct![i] ≈ X[i] + @test Xidct_1[i] ≈ X[i] + @test Xidct!_1[i] ≈ X[i] + @test Xidct_2[i] ≈ X[i] + @test Xidct!_2[i] ≈ X[i] + + @test pXidct[i] ≈ X[i] + @test pXidct![i] ≈ X[i] + @test pXidct_1[i] ≈ X[i] + @test pXidct!_1[i] ≈ X[i] + @test pXidct_2[i] ≈ X[i] + @test pXidct!_2[i] ≈ X[i] + + @test sXdct[i] ≈ true_Xdct[i] + @test psXdct[i] ≈ true_Xdct[i] + @test sXdct![i] ≈ true_Xdct[i] + @test psXdct![i] ≈ true_Xdct[i] end end # fftw_vendor() != :mkl diff --git a/test/fft.jl b/test/fft.jl index 9ebc93c353dfc..b851def9aff9f 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -83,34 +83,34 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), psfft!n_m4 = view(psfft!n_b,3:6,9:12); plan_fft!(psfft!n_m4)*psfft!n_m4 for i = 1:length(m4) - @test_approx_eq fft_m4[i] true_fft_m4[i] - @test_approx_eq fftd2_m4[i] true_fftd2_m4[i] - @test_approx_eq ifft_fft_m4[i] m4[i] - @test_approx_eq fftn_m4[i] true_fftn_m4[i] - @test_approx_eq ifftn_fftn_m4[i] m4[i] - - @test_approx_eq fft!_m4[i] true_fft_m4[i] - @test_approx_eq fft!d2_m4[i] true_fftd2_m4[i] - @test_approx_eq ifft!_fft_m4[i] m4[i] - @test_approx_eq fft!n_m4[i] true_fftn_m4[i] - @test_approx_eq ifft!n_fftn_m4[i] m4[i] - - @test_approx_eq pfft_m4[i] true_fft_m4[i] - @test_approx_eq pfftd2_m4[i] true_fftd2_m4[i] - @test_approx_eq pifft_fft_m4[i] m4[i] - @test_approx_eq pfftn_m4[i] true_fftn_m4[i] - @test_approx_eq pifftn_fftn_m4[i] m4[i] - - @test_approx_eq pfft!_m4[i] true_fft_m4[i] - @test_approx_eq pfft!d2_m4[i] true_fftd2_m4[i] - @test_approx_eq pifft!_fft_m4[i] m4[i] - @test_approx_eq pfft!n_m4[i] true_fftn_m4[i] - @test_approx_eq pifft!n_fftn_m4[i] m4[i] - - @test_approx_eq sfftn_m4[i] true_fftn_m4[i] - @test_approx_eq sfft!n_m4[i] true_fftn_m4[i] - @test_approx_eq psfftn_m4[i] true_fftn_m4[i] - @test_approx_eq psfft!n_m4[i] true_fftn_m4[i] + @test fft_m4[i] ≈ true_fft_m4[i] + @test fftd2_m4[i] ≈ true_fftd2_m4[i] + @test ifft_fft_m4[i] ≈ m4[i] + @test fftn_m4[i] ≈ true_fftn_m4[i] + @test ifftn_fftn_m4[i] ≈ m4[i] + + @test fft!_m4[i] ≈ true_fft_m4[i] + @test fft!d2_m4[i] ≈ true_fftd2_m4[i] + @test ifft!_fft_m4[i] ≈ m4[i] + @test fft!n_m4[i] ≈ true_fftn_m4[i] + @test ifft!n_fftn_m4[i] ≈ m4[i] + + @test pfft_m4[i] ≈ true_fft_m4[i] + @test pfftd2_m4[i] ≈ true_fftd2_m4[i] + @test pifft_fft_m4[i] ≈ m4[i] + @test pfftn_m4[i] ≈ true_fftn_m4[i] + @test pifftn_fftn_m4[i] ≈ m4[i] + + @test pfft!_m4[i] ≈ true_fft_m4[i] + @test pfft!d2_m4[i] ≈ true_fftd2_m4[i] + @test pifft!_fft_m4[i] ≈ m4[i] + @test pfft!n_m4[i] ≈ true_fftn_m4[i] + @test pifft!n_fftn_m4[i] ≈ m4[i] + + @test sfftn_m4[i] ≈ true_fftn_m4[i] + @test sfft!n_m4[i] ≈ true_fftn_m4[i] + @test psfftn_m4[i] ≈ true_fftn_m4[i] + @test psfft!n_m4[i] ≈ true_fftn_m4[i] end ifft!(sfft!n_m4) @@ -146,17 +146,17 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), @test isa(pifft!d3_fftd3_m3d, Array{Complex64,3}) for i = 1:length(m3d) - @test_approx_eq fftd3_m3d[i] true_fftd3_m3d[i] - @test_approx_eq ifftd3_fftd3_m3d[i] m3d[i] - @test_approx_eq ifft3_fft3_m3d[i] m3d[i] + @test fftd3_m3d[i] ≈ true_fftd3_m3d[i] + @test ifftd3_fftd3_m3d[i] ≈ m3d[i] + @test ifft3_fft3_m3d[i] ≈ m3d[i] - @test_approx_eq fft!d3_m3d[i] true_fftd3_m3d[i] - @test_approx_eq ifft!d3_fftd3_m3d[i] m3d[i] + @test fft!d3_m3d[i] ≈ true_fftd3_m3d[i] + @test ifft!d3_fftd3_m3d[i] ≈ m3d[i] - @test_approx_eq pfftd3_m3d[i] true_fftd3_m3d[i] - @test_approx_eq pifftd3_fftd3_m3d[i] m3d[i] - @test_approx_eq pfft!d3_m3d[i] true_fftd3_m3d[i] - @test_approx_eq pifft!d3_fftd3_m3d[i] m3d[i] + @test pfftd3_m3d[i] ≈ true_fftd3_m3d[i] + @test pifftd3_fftd3_m3d[i] ≈ m3d[i] + @test pfft!d3_m3d[i] ≈ true_fftd3_m3d[i] + @test pifft!d3_fftd3_m3d[i] ≈ m3d[i] end end # if fftw_vendor() != :mkl @@ -175,16 +175,16 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), psrfftn_m4 = plan_rfft(sm4)*sm4 for i = 1:3, j = 1:4 - @test_approx_eq rfft_m4[i,j] true_fft_m4[i,j] - @test_approx_eq rfftd2_m4[j,i] true_fftd2_m4[j,i] - @test_approx_eq rfftn_m4[i,j] true_fftn_m4[i,j] + @test rfft_m4[i,j] ≈ true_fft_m4[i,j] + @test rfftd2_m4[j,i] ≈ true_fftd2_m4[j,i] + @test rfftn_m4[i,j] ≈ true_fftn_m4[i,j] - @test_approx_eq prfft_m4[i,j] true_fft_m4[i,j] - @test_approx_eq prfftd2_m4[j,i] true_fftd2_m4[j,i] - @test_approx_eq prfftn_m4[i,j] true_fftn_m4[i,j] + @test prfft_m4[i,j] ≈ true_fft_m4[i,j] + @test prfftd2_m4[j,i] ≈ true_fftd2_m4[j,i] + @test prfftn_m4[i,j] ≈ true_fftn_m4[i,j] - @test_approx_eq srfftn_m4[i,j] true_fftn_m4[i,j] - @test_approx_eq psrfftn_m4[i,j] true_fftn_m4[i,j] + @test srfftn_m4[i,j] ≈ true_fftn_m4[i,j] + @test psrfftn_m4[i,j] ≈ true_fftn_m4[i,j] end irfft_rfft_m4 = irfft(rfft_m4,size(m4,1),1) @@ -196,13 +196,13 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), pirfftn_rfftn_m4 = plan_irfft(rfftn_m4,size(m4,1))*rfftn_m4 for i = 1:length(m4) - @test_approx_eq irfft_rfft_m4[i] m4[i] - @test_approx_eq irfft_rfftd2_m4[i] m4[i] - @test_approx_eq irfftn_rfftn_m4[i] m4[i] + @test irfft_rfft_m4[i] ≈ m4[i] + @test irfft_rfftd2_m4[i] ≈ m4[i] + @test irfftn_rfftn_m4[i] ≈ m4[i] - @test_approx_eq pirfft_rfft_m4[i] m4[i] - @test_approx_eq pirfft_rfftd2_m4[i] m4[i] - @test_approx_eq pirfftn_rfftn_m4[i] m4[i] + @test pirfft_rfft_m4[i] ≈ m4[i] + @test pirfft_rfftd2_m4[i] ≈ m4[i] + @test pirfftn_rfftn_m4[i] ≈ m4[i] end if Base.fftw_vendor() != :mkl @@ -213,9 +213,9 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), irfft_rfftd3_m3d = irfft(rfftd3_m3d,size(m3d,3),3) irfftn_rfftn_m3d = irfft(rfftn_m3d,size(m3d,1)) for i = 1:length(m3d) - @test_approx_eq rfftd3_m3d[i] true_fftd3_m3d[i] - @test_approx_eq irfft_rfftd3_m3d[i] m3d[i] - @test_approx_eq irfftn_rfftn_m3d[i] m3d[i] + @test rfftd3_m3d[i] ≈ true_fftd3_m3d[i] + @test irfft_rfftd3_m3d[i] ≈ m3d[i] + @test irfftn_rfftn_m3d[i] ≈ m3d[i] end fftn_m3d = fft(m3d) @@ -223,7 +223,7 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), rfftn_m3d = rfft(m3d) @test size(rfftn_m3d) == (3,3,2) for i = 1:3, j = 1:3, k = 1:2 - @test_approx_eq rfftn_m3d[i,j,k] fftn_m3d[i,j,k] + @test rfftn_m3d[i,j,k] ≈ fftn_m3d[i,j,k] end end # !mkl diff --git a/test/float16.jl b/test/float16.jl index 2031c056040b7..8ada5d87f1780 100644 --- a/test/float16.jl +++ b/test/float16.jl @@ -33,7 +33,7 @@ g = Float16(1.) @test_throws InexactError convert(UInt128, Float16(3.5)) @test_throws InexactError convert(UInt128, Float16(-1)) -@test_approx_eq Float16(0.5f0)^2 Float16(0.5f0^2) +@test Float16(0.5f0)^2 ≈ Float16(0.5f0^2) @test round(Int,Float16(0.5f0)) == round(Int,0.5f0) @test trunc(Int,Float16(0.9f0)) === trunc(Int,0.9f0) === 0 @test floor(Int,Float16(0.9f0)) === floor(Int,0.9f0) === 0 @@ -49,8 +49,8 @@ g = Float16(1.) @test floor(Float16(1)) === Float16(1) @test ceil(Float16(0.1)) == ceil(0.1) @test ceil(Float16(0.9)) == ceil(0.9) -@test_approx_eq fma(Float16(0.1),Float16(0.9),Float16(0.5)) fma(0.1,0.9,0.5) -@test_approx_eq muladd(Float16(0.1),Float16(0.9),Float16(0.5)) muladd(0.1,0.9,0.5) +@test fma(Float16(0.1),Float16(0.9),Float16(0.5)) ≈ fma(0.1,0.9,0.5) +@test muladd(Float16(0.1),Float16(0.9),Float16(0.5)) ≈ muladd(0.1,0.9,0.5) @test convert(Int128,Float16(-1.0)) == Int128(-1) @test convert(UInt128,Float16(5.0)) == UInt128(5) @@ -72,7 +72,7 @@ g = Float16(1.) @test f*2. === 4. @test f/2. === 1. -@test_approx_eq sin(f) sin(2f0) +@test sin(f) ≈ sin(2f0) @test isnan(NaN16) @test isnan(-NaN16) diff --git a/test/floatfuncs.jl b/test/floatfuncs.jl index fc4a27f204fee..78546eae1b3ff 100644 --- a/test/floatfuncs.jl +++ b/test/floatfuncs.jl @@ -12,7 +12,7 @@ for elty in (Float32,Float64) @test flipsign(x,big(-1.0)) == convert(elty,-2.0) end -#maxintfloat +# maxintfloat @test maxintfloat(Float16) == Float16(2048f0) for elty in (Float16,Float32,Float64) @@ -35,13 +35,13 @@ for elty in (Float16,Float32,Float64) @test !isinteger(elty(NaN)) end -#num2hex -for elty in (Float16,Float32,Float64) +# num2hex, hex2num +for elty in (Float16,Float32,Float64), _ = 1:10 x = rand(elty) - @test_approx_eq hex2num(num2hex(x)) x + @test hex2num(num2hex(x)) ≈ x end -#round +# round for elty in (Float32,Float64) x = rand(elty) A = fill(x,(10,10)) diff --git a/test/linalg/arnoldi.jl b/test/linalg/arnoldi.jl index d9bc48fc00106..81dc9217972ef 100644 --- a/test/linalg/arnoldi.jl +++ b/test/linalg/arnoldi.jl @@ -33,20 +33,20 @@ let bpd = b'*b (d,v) = eigs(a, nev=3) - @test_approx_eq a*v[:,2] d[2]*v[:,2] + @test a*v[:,2] ≈ d[2]*v[:,2] @test norm(v) > testtol # eigenvectors cannot be null vectors # (d,v) = eigs(a, b, nev=3, tol=1e-8) # not handled yet # @test_approx_eq_eps a*v[:,2] d[2]*b*v[:,2] testtol # @test norm(v) > testtol # eigenvectors cannot be null vectors (d,v) = eigs(asym, nev=3) - @test_approx_eq asym*v[:,1] d[1]*v[:,1] - @test_approx_eq eigs(asym; nev=1, sigma=d[3])[1][1] d[3] + @test asym*v[:,1] ≈ d[1]*v[:,1] + @test eigs(asym; nev=1, sigma=d[3])[1][1] ≈ d[3] @test norm(v) > testtol # eigenvectors cannot be null vectors (d,v) = eigs(apd, nev=3) - @test_approx_eq apd*v[:,3] d[3]*v[:,3] - @test_approx_eq eigs(apd; nev=1, sigma=d[3])[1][1] d[3] + @test apd*v[:,3] ≈ d[3]*v[:,3] + @test eigs(apd; nev=1, sigma=d[3])[1][1] ≈ d[3] (d,v) = eigs(apd, bpd, nev=3, tol=1e-8) @test_approx_eq_eps apd*v[:,2] d[2]*bpd*v[:,2] testtol @@ -54,7 +54,7 @@ let # test (shift-and-)invert mode (d,v) = eigs(apd, nev=3, sigma=0) - @test_approx_eq apd*v[:,3] d[3]*v[:,3] + @test apd*v[:,3] ≈ d[3]*v[:,3] @test norm(v) > testtol # eigenvectors cannot be null vectors (d,v) = eigs(apd, bpd, nev=3, sigma=0, tol=1e-8) @@ -89,9 +89,9 @@ let A6965 = [ ] d, = eigs(A6965,which=:SM,nev=2,ncv=4,tol=eps()) - @test_approx_eq d[1] 2.5346936860350002 - @test_approx_eq real(d[2]) 2.6159972444834976 - @test_approx_eq abs(imag(d[2])) 1.2917858749046127 + @test d[1] ≈ 2.5346936860350002 + @test real(d[2]) ≈ 2.6159972444834976 + @test abs(imag(d[2])) ≈ 1.2917858749046127 # Requires ARPACK 3.2 or a patched 3.1.5 #T6965 = [ 0.9 0.05 0.05 @@ -131,7 +131,7 @@ let # Properties: largest eigenvalue should be 1, largest eigenvector, when reshaped as matrix # should be a Hermitian positive definite matrix (up to an arbitrary phase) - @test_approx_eq d[1] 1. # largest eigenvalue should be 1. + @test d[1] ≈ 1. # largest eigenvalue should be 1. v=reshape(v,(50,50)) # reshape to matrix v/=trace(v) # factor out arbitrary phase @test isapprox(vecnorm(imag(v)),0.) # it should be real @@ -146,9 +146,9 @@ let v2=reshape(v2,(50,50)) v2/=trace(v2) @test numiter20 - @test_approx_eq cholfact(apos).factors √apos + @test all(x -> x ≈ √apos, cholfact(apos).factors) @test_throws ArgumentError chol(-one(eltya)) if eltya <: Real capds = cholfact(apds) - @test_approx_eq inv(capds)*apds eye(n) + @test inv(capds)*apds ≈ eye(n) @test abs((det(capds) - det(apd))/det(capds)) <= ε*κ*n end # test chol of 2x2 Strang matrix S = convert(AbstractMatrix{eltya},full(SymTridiagonal([2,2],[-1]))) U = Bidiagonal([2,sqrt(eltya(3))],[-1],true) / sqrt(eltya(2)) - @test_approx_eq full(chol(S)) full(U) + @test full(chol(S)) ≈ full(U) #lower Cholesky factor lapd = cholfact(apd, :L) - @test_approx_eq full(lapd) apd + @test full(lapd) ≈ apd l = lapd[:L] - @test_approx_eq l*l' apd + @test l*l' ≈ apd @test triu(capd.factors) ≈ lapd[:U] @test tril(lapd.factors) ≈ capd[:L] if eltya <: Real capds = cholfact(apds) lapds = cholfact(apds, :L) ls = lapds[:L] - @test_approx_eq ls*ls' apd + @test ls*ls' ≈ apd @test triu(capds.factors) ≈ lapds[:U] @test tril(lapds.factors) ≈ capds[:L] end @@ -93,7 +93,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test rank(cpapd) == n @test all(diff(diag(real(cpapd.factors))).<=0.) # diagonal should be non-increasing if isreal(apd) - @test_approx_eq apd * inv(cpapd) eye(n) + @test apd*inv(cpapd) ≈ eye(n) end @test full(cpapd) ≈ apd #getindex @@ -167,8 +167,8 @@ for elty in (Float32, Float64, Complex{Float32}, Complex{Float64}) A = randn(5,5) end A = convert(Matrix{elty}, A'A) - @test_approx_eq full(cholfact(A)[:L]) full(invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{LowerTriangular}}, copy(A), LowerTriangular)) - @test_approx_eq full(cholfact(A)[:U]) full(invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{UpperTriangular}}, copy(A), UpperTriangular)) + @test full(cholfact(A)[:L]) ≈ full(invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{LowerTriangular}}, copy(A), LowerTriangular)) + @test full(cholfact(A)[:U]) ≈ full(invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{UpperTriangular}}, copy(A), UpperTriangular)) end # Test up- and downdates diff --git a/test/linalg/dense.jl b/test/linalg/dense.jl index a3d8f6f1cfca6..e3593fb5b49cc 100644 --- a/test/linalg/dense.jl +++ b/test/linalg/dense.jl @@ -4,7 +4,7 @@ debug = false using Base.Test # Check that non-floats are correctly promoted -@test_approx_eq [1 0 0; 0 1 0]\[1,1] [1;1;0] +@test [1 0 0; 0 1 0]\[1,1] ≈ [1;1;0] using Base.LinAlg: BlasComplex, BlasFloat, BlasReal @@ -69,24 +69,24 @@ debug && println("Test nullspace") debug && println("Test pinv") pinva15 = pinv(a[:,1:n1]) - @test_approx_eq a[:,1:n1]*pinva15*a[:,1:n1] a[:,1:n1] - @test_approx_eq pinva15*a[:,1:n1]*pinva15 pinva15 + @test a[:,1:n1]*pinva15*a[:,1:n1] ≈ a[:,1:n1] + @test pinva15*a[:,1:n1]*pinva15 ≈ pinva15 @test size(pinv(ones(eltya,0,0))) == (0,0) debug && println("Lyapunov/Sylvester") let x = lyap(a, a2) - @test_approx_eq -a2 a*x + x*a' + @test -a2 ≈ a*x + x*a' x2 = sylvester(a[1:3, 1:3], a[4:n, 4:n], a2[1:3,4:n]) - @test_approx_eq -a2[1:3, 4:n] a[1:3, 1:3]*x2 + x2*a[4:n, 4:n] + @test -a2[1:3, 4:n] ≈ a[1:3, 1:3]*x2 + x2*a[4:n, 4:n] end debug && println("Matrix square root") asq = sqrtm(a) - @test_approx_eq asq*asq a + @test asq*asq ≈ a asymsq = sqrtm(asym) - @test_approx_eq asymsq*asymsq asym + @test asymsq*asymsq ≈ asym debug && println("Numbers") α = rand(eltya) @@ -146,7 +146,7 @@ for elty in (Int32, Int64, Float32, Float64, Complex64, Complex128) x = convert(Vector{elty}, complex([1:3;], [1:3;])) g = convert(Vector{elty}, complex(ones(3), ones(3))) end - @test_approx_eq gradient(x) g + @test gradient(x) ≈ g @test gradient(ones(elty,1)) == zeros(elty,1) end @@ -160,25 +160,25 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com ## Vector x = ones(elty,10) xs = view(x,1:2:10) - @test_approx_eq norm(x, -Inf) 1 - @test_approx_eq norm(x, -1) 1/10 - @test_approx_eq norm(x, 0) 10 - @test_approx_eq norm(x, 1) 10 - @test_approx_eq norm(x, 2) sqrt(10) - @test_approx_eq norm(x, 3) cbrt(10) - @test_approx_eq norm(x, Inf) 1 + @test norm(x, -Inf) ≈ 1 + @test norm(x, -1) ≈ 1/10 + @test norm(x, 0) ≈ 10 + @test norm(x, 1) ≈ 10 + @test norm(x, 2) ≈ sqrt(10) + @test norm(x, 3) ≈ cbrt(10) + @test norm(x, Inf) ≈ 1 if elty <: Base.LinAlg.BlasFloat - @test_approx_eq norm(x, 1:4) 2 + @test norm(x, 1:4) ≈ 2 @test_throws BoundsError norm(x,-1:4) @test_throws BoundsError norm(x,1:11) end - @test_approx_eq norm(xs, -Inf) 1 - @test_approx_eq norm(xs, -1) 1/5 - @test_approx_eq norm(xs, 0) 5 - @test_approx_eq norm(xs, 1) 5 - @test_approx_eq norm(xs, 2) sqrt(5) - @test_approx_eq norm(xs, 3) cbrt(5) - @test_approx_eq norm(xs, Inf) 1 + @test norm(xs, -Inf) ≈ 1 + @test norm(xs, -1) ≈ 1/5 + @test norm(xs, 0) ≈ 5 + @test norm(xs, 1) ≈ 5 + @test norm(xs, 2) ≈ sqrt(5) + @test norm(xs, 3) ≈ cbrt(5) + @test norm(xs, Inf) ≈ 1 # Issue #12552: if real(elty) <: AbstractFloat @@ -208,19 +208,19 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com elty <: Complex ? convert(elty, complex(randn(),randn())) : convert(elty, randn()) # Absolute homogeneity - @test_approx_eq norm(α*x,-Inf) abs(α)*norm(x,-Inf) - @test_approx_eq norm(α*x,-1) abs(α)*norm(x,-1) - @test_approx_eq norm(α*x,1) abs(α)*norm(x,1) - @test_approx_eq norm(α*x) abs(α)*norm(x) # two is default - @test_approx_eq norm(α*x,3) abs(α)*norm(x,3) - @test_approx_eq norm(α*x,Inf) abs(α)*norm(x,Inf) - - @test_approx_eq norm(α*xs,-Inf) abs(α)*norm(xs,-Inf) - @test_approx_eq norm(α*xs,-1) abs(α)*norm(xs,-1) - @test_approx_eq norm(α*xs,1) abs(α)*norm(xs,1) - @test_approx_eq norm(α*xs) abs(α)*norm(xs) # two is default - @test_approx_eq norm(α*xs,3) abs(α)*norm(xs,3) - @test_approx_eq norm(α*xs,Inf) abs(α)*norm(xs,Inf) + @test norm(α*x,-Inf) ≈ abs(α)*norm(x,-Inf) + @test norm(α*x,-1) ≈ abs(α)*norm(x,-1) + @test norm(α*x,1) ≈ abs(α)*norm(x,1) + @test norm(α*x) ≈ abs(α)*norm(x) # two is default + @test norm(α*x,3) ≈ abs(α)*norm(x,3) + @test norm(α*x,Inf) ≈ abs(α)*norm(x,Inf) + + @test norm(α*xs,-Inf) ≈ abs(α)*norm(xs,-Inf) + @test norm(α*xs,-1) ≈ abs(α)*norm(xs,-1) + @test norm(α*xs,1) ≈ abs(α)*norm(xs,1) + @test norm(α*xs) ≈ abs(α)*norm(xs) # two is default + @test norm(α*xs,3) ≈ abs(α)*norm(xs,3) + @test norm(α*xs,Inf) ≈ abs(α)*norm(xs,Inf) # Triangle inequality @test norm(x + y,1) <= norm(x,1) + norm(y,1) @@ -234,23 +234,23 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com @test norm(xs + ys,Inf) <= norm(xs,Inf) + norm(ys,Inf) # Against vectorized versions - @test_approx_eq norm(x,-Inf) minimum(abs(x)) - @test_approx_eq norm(x,-1) inv(sum(1./abs(x))) - @test_approx_eq norm(x,0) sum(x .!= 0) - @test_approx_eq norm(x,1) sum(abs(x)) - @test_approx_eq norm(x) sqrt(sum(abs2(x))) - @test_approx_eq norm(x,3) cbrt(sum(abs(x).^3.)) - @test_approx_eq norm(x,Inf) maximum(abs(x)) + @test norm(x,-Inf) ≈ minimum(abs(x)) + @test norm(x,-1) ≈ inv(sum(1./abs(x))) + @test norm(x,0) ≈ sum(x .!= 0) + @test norm(x,1) ≈ sum(abs(x)) + @test norm(x) ≈ sqrt(sum(abs2(x))) + @test norm(x,3) ≈ cbrt(sum(abs(x).^3.)) + @test norm(x,Inf) ≈ maximum(abs(x)) end ## Matrix (Operator) A = ones(elty,10,10) As = view(A,1:5,1:5) - @test_approx_eq norm(A, 1) 10 - elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test_approx_eq norm(A, 2) 10 - @test_approx_eq norm(A, Inf) 10 - @test_approx_eq norm(As, 1) 5 - elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test_approx_eq norm(As, 2) 5 - @test_approx_eq norm(As, Inf) 5 + @test norm(A, 1) ≈ 10 + elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test norm(A, 2) ≈ 10 + @test norm(A, Inf) ≈ 10 + @test norm(As, 1) ≈ 5 + elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test norm(As, 2) ≈ 5 + @test norm(As, Inf) ≈ 5 for i = 1:10 A = elty <: Integer ? convert(Matrix{elty}, rand(1:10, mmat, nmat)) : @@ -266,13 +266,13 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com convert(elty, randn()) # Absolute homogeneity - @test_approx_eq norm(α*A,1) abs(α)*norm(A,1) - elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test_approx_eq norm(α*A) abs(α)*norm(A) # two is default - @test_approx_eq norm(α*A,Inf) abs(α)*norm(A,Inf) + @test norm(α*A,1) ≈ abs(α)*norm(A,1) + elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test norm(α*A) ≈ abs(α)*norm(A) # two is default + @test norm(α*A,Inf) ≈ abs(α)*norm(A,Inf) - @test_approx_eq norm(α*As,1) abs(α)*norm(As,1) - elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test_approx_eq norm(α*As) abs(α)*norm(As) # two is default - @test_approx_eq norm(α*As,Inf) abs(α)*norm(As,Inf) + @test norm(α*As,1) ≈ abs(α)*norm(As,1) + elty <: Union{BigFloat,Complex{BigFloat},BigInt} || @test norm(α*As) ≈ abs(α)*norm(As) # two is default + @test norm(α*As,Inf) ≈ abs(α)*norm(As,Inf) # Triangle inequality @test norm(A + B,1) <= norm(A,1) + norm(B,1) @@ -305,8 +305,8 @@ end @test norm(Any[Inf],-2) == norm(Any[Inf],-1) == norm(Any[Inf],1) == norm(Any[Inf],1.5) == norm(Any[Inf],2) == norm(Any[Inf],Inf) == Inf # overflow/underflow in norms: -@test_approx_eq norm(Float64[1e-300, 1], -3)*1e300 1 -@test_approx_eq norm(Float64[1e300, 1], 3)*1e-300 1 +@test norm(Float64[1e-300, 1], -3)*1e300 ≈ 1 +@test norm(Float64[1e300, 1], 3)*1e-300 ≈ 1 ## Issue related tests # issue #1447 @@ -314,7 +314,7 @@ let A = [1.+0.0im 0; 0 1] B = pinv(A) for i = 1:4 - @test_approx_eq A[i] B[i] + @test A[i] ≈ B[i] end end @@ -322,15 +322,15 @@ end let A = [1 2 0 0; 0 1 0 0; 0 0 0 0; 0 0 0 0] Asq = sqrtm(A) - @test_approx_eq Asq*Asq A + @test Asq*Asq ≈ A A2 = view(A, 1:2, 1:2) A2sq = sqrtm(A2) - @test_approx_eq A2sq*A2sq A2 + @test A2sq*A2sq ≈ A2 end let N = 3 - @test_approx_eq log(det(eye(N))) logdet(eye(N)) + @test log(det(eye(N))) ≈ logdet(eye(N)) end # issue #2637 @@ -367,7 +367,7 @@ for elty in (Float32, Float64, Complex64, Complex128) eA1 = convert(Matrix{elty}, [147.866622446369 127.781085523181 127.781085523182; 183.765138646367 183.765138646366 163.679601723179; 71.797032399996 91.8825693231832 111.968106246371]') - @test_approx_eq expm(A1) eA1 + @test expm(A1) ≈ eA1 A2 = convert(Matrix{elty}, [29.87942128909879 0.7815750847907159 -2.289519314033932; @@ -377,13 +377,13 @@ for elty in (Float32, Float64, Complex64, Complex128) [ 5496313853692458.0 -18231880972009236.0 -30475770808580460.0; -18231880972009252.0 60605228702221920.0 101291842930249760.0; -30475770808580480.0 101291842930249728.0 169294411240851968.0]) - @test_approx_eq expm(A2) eA2 + @test expm(A2) ≈ eA2 A3 = convert(Matrix{elty}, [-131 19 18;-390 56 54;-387 57 52]) eA3 = convert(Matrix{elty}, [-1.50964415879218 -5.6325707998812 -4.934938326092; 0.367879439109187 1.47151775849686 1.10363831732856; 0.135335281175235 0.406005843524598 0.541341126763207]') - @test_approx_eq expm(A3) eA3 + @test expm(A3) ≈ eA3 A4 = convert(Matrix{elty}, [0.25 0.25; 0 0]) eA4 = convert(Matrix{elty}, [1.2840254166877416 0.2840254166877415; 0 1]) @@ -394,7 +394,7 @@ for elty in (Float32, Float64, Complex64, Complex128) @test expm(A5) ≈ eA5 # Hessenberg - @test_approx_eq hessfact(A1)[:H] convert(Matrix{elty}, + @test hessfact(A1)[:H] ≈ convert(Matrix{elty}, [4.000000000000000 -1.414213562373094 -1.414213562373095 -1.414213562373095 4.999999999999996 -0.000000000000000 0 -0.000000000000002 3.000000000000000]) @@ -406,21 +406,21 @@ for elty in (Float64, Complex{Float64}) 1/3 1/4 1/5 1/6; 1/4 1/5 1/6 1/7; 1/5 1/6 1/7 1/8]) - @test_approx_eq expm(logm(A4)) A4 + @test expm(logm(A4)) ≈ A4 A5 = convert(Matrix{elty}, [1 1 0 1; 0 1 1 0; 0 0 1 1; 1 0 0 1]) - @test_approx_eq expm(logm(A5)) A5 + @test expm(logm(A5)) ≈ A5 A6 = convert(Matrix{elty}, [-5 2 0 0 ; 1/2 -7 3 0; 0 1/3 -9 4; 0 0 1/4 -11]) - @test_approx_eq expm(logm(A6)) A6 + @test expm(logm(A6)) ≈ A6 A7 = convert(Matrix{elty}, [1 0 0 1e-8; 0 1 0 0; 0 0 1 0; 0 0 0 1]) - @test_approx_eq expm(logm(A7)) A7 + @test expm(logm(A7)) ≈ A7 end A8 = 100 * [-1+1im 0 0 1e-8; 0 1 0 0; 0 0 1 0; 0 0 0 1]; -@test_approx_eq expm(logm(A8)) A8 +@test expm(logm(A8)) ≈ A8 # issue 5116 A9 = [0 10 0 0; -1 0 0 0; 0 0 0 0; -2 0 0 0] @@ -428,7 +428,7 @@ eA9 = [-0.999786072879326 -0.065407069689389 0.0 0.0 0.006540706968939 -0.999786072879326 0.0 0.0 0.0 0.0 1.0 0.0 0.013081413937878 -3.999572145758650 0.0 1.0] -@test_approx_eq expm(A9) eA9 +@test expm(A9) ≈ eA9 # issue 5116 A10 = [ 0. 0. 0. 0. ; 0. 0. -im 0.; 0. im 0. 0.; 0. 0. 0. 0.] @@ -436,7 +436,7 @@ eA10 = [ 1.0+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0i 0.0+0.0im 1.543080634815244+0.0im 0.0-1.175201193643801im 0.0+0.0im 0.0+0.0im 0.0+1.175201193643801im 1.543080634815243+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im 1.0+0.0im] -@test_approx_eq expm(A10) eA10 +@test expm(A10) ≈ eA10 for elty in (Float64, Complex{Float64}) A11 = convert(Matrix{elty}, [1 2 3; 4 7 1; 2 1 4]) @@ -511,25 +511,25 @@ for elty in (Float32, Float64, Complex64, Complex128) # Vector rhs x = a[:,1:2]\b[:,1] - @test_approx_eq ((a[:,1:2]*x-b[:,1])'*(a[:,1:2]*x-b[:,1]))[1] convert(elty, 2.546616541353384) + @test ((a[:,1:2]*x-b[:,1])'*(a[:,1:2]*x-b[:,1]))[1] ≈ convert(elty, 2.546616541353384) # Matrix rhs x = a[:,1:2]\b - @test_approx_eq det((a[:,1:2]*x-b)'*(a[:,1:2]*x-b)) convert(elty, 4.437969924812031) + @test det((a[:,1:2]*x-b)'*(a[:,1:2]*x-b)) ≈ convert(elty, 4.437969924812031) # Rank deficient x = a\b - @test_approx_eq det((a*x-b)'*(a*x-b)) convert(elty, 4.437969924812031) + @test det((a*x-b)'*(a*x-b)) ≈ convert(elty, 4.437969924812031) # Underdetermined minimum norm x = convert(Matrix{elty}, [1 0 0; 0 1 -1]) \ convert(Vector{elty}, [1,1]) - @test_approx_eq x convert(Vector{elty}, [1, 0.5, -0.5]) + @test x ≈ convert(Vector{elty}, [1, 0.5, -0.5]) # symmetric, positive definite - @test_approx_eq inv(convert(Matrix{elty}, [6. 2; 2 1])) convert(Matrix{elty}, [0.5 -1; -1 3]) + @test inv(convert(Matrix{elty}, [6. 2; 2 1])) ≈ convert(Matrix{elty}, [0.5 -1; -1 3]) # symmetric, indefinite - @test_approx_eq inv(convert(Matrix{elty}, [1. 2; 2 1])) convert(Matrix{elty}, [-1. 2; 2 -1]/3) + @test inv(convert(Matrix{elty}, [1. 2; 2 1])) ≈ convert(Matrix{elty}, [-1. 2; 2 -1]/3) end # test ops on Numbers diff --git a/test/linalg/eigen.jl b/test/linalg/eigen.jl index 89089f0f422fc..dffc849b90a34 100644 --- a/test/linalg/eigen.jl +++ b/test/linalg/eigen.jl @@ -39,18 +39,18 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) debug && println("\ntype of a: ", eltya,"\n") debug && println("non-symmetric eigen decomposition") d,v = eig(a) - for i in 1:size(a,2) @test_approx_eq a*v[:,i] d[i]*v[:,i] end + for i in 1:size(a,2) @test a*v[:,i] ≈ d[i]*v[:,i] end f = eigfact(a) - @test_approx_eq det(a) det(f) - @test_approx_eq inv(a) inv(f) + @test det(a) ≈ det(f) + @test inv(a) ≈ inv(f) @test eigvals(f) === f[:values] @test eigvecs(f) === f[:vectors] num_fact = eigfact(one(eltya)) @test num_fact.values[1] == one(eltya) h = asym - @test_approx_eq minimum(eigvals(h)) eigmin(h) - @test_approx_eq maximum(eigvals(h)) eigmax(h) + @test minimum(eigvals(h)) ≈ eigmin(h) + @test maximum(eigvals(h)) ≈ eigmax(h) @test_throws DomainError eigmin(a - a') @test_throws DomainError eigmax(a - a') @@ -63,8 +63,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) a_sg = view(a, 1:n, n1+1:n2) end f = eigfact(asym_sg, a_sg'a_sg) - @test_approx_eq asym_sg*f[:vectors] (a_sg'a_sg*f[:vectors]) * Diagonal(f[:values]) - @test_approx_eq f[:values] eigvals(asym_sg, a_sg'a_sg) + @test asym_sg*f[:vectors] ≈ (a_sg'a_sg*f[:vectors]) * Diagonal(f[:values]) + @test f[:values] ≈ eigvals(asym_sg, a_sg'a_sg) @test_approx_eq_eps prod(f[:values]) prod(eigvals(asym_sg/(a_sg'a_sg))) 200ε @test eigvecs(asym_sg, a_sg'a_sg) == f[:vectors] @test eigvals(f) === f[:values] @@ -84,8 +84,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) a2_nsg = view(a, n1+1:n2, n1+1:n2) end f = eigfact(a1_nsg, a2_nsg) - @test_approx_eq a1_nsg*f[:vectors] (a2_nsg*f[:vectors]) * Diagonal(f[:values]) - @test_approx_eq f[:values] eigvals(a1_nsg, a2_nsg) + @test a1_nsg*f[:vectors] ≈ (a2_nsg*f[:vectors]) * Diagonal(f[:values]) + @test f[:values] ≈ eigvals(a1_nsg, a2_nsg) @test_approx_eq_eps prod(f[:values]) prod(eigvals(a1_nsg/a2_nsg)) 50000ε @test eigvecs(a1_nsg, a2_nsg) == f[:vectors] @test_throws KeyError f[:Z] diff --git a/test/linalg/generic.jl b/test/linalg/generic.jl index 1ce2337b0d043..af2e73e525fcf 100644 --- a/test/linalg/generic.jl +++ b/test/linalg/generic.jl @@ -40,8 +40,8 @@ for elty in (Int, Rational{BigInt}, Float32, Float64, BigFloat, Complex{Float32} debug && println("element type: $elty") - @test_approx_eq logdet(A) log(det(A)) - @test_approx_eq logabsdet(A)[1] log(abs(det(A))) + @test logdet(A) ≈ log(det(A)) + @test logabsdet(A)[1] ≈ log(abs(det(A))) @test logabsdet(convert(Matrix{elty}, -eye(n)))[2] == -1 if elty <: Real @test logabsdet(A)[2] == sign(det(A)) @@ -77,23 +77,23 @@ y = [3; 7; 10] # check (UnitRange, Array) x = 1:12 y = [5.5; 6.3; 7.6; 8.8; 10.9; 11.79; 13.48; 15.02; 17.77; 20.81; 22.0; 22.99] -@test_approx_eq [linreg(x,y)...] [2.5559090909090867, 1.6960139860139862] -@test_approx_eq [linreg(view(x,1:6),view(y,1:6))...] [3.8366666666666642,1.3271428571428574] +@test [linreg(x,y)...] ≈ [2.5559090909090867, 1.6960139860139862] +@test [linreg(view(x,1:6),view(y,1:6))...] ≈ [3.8366666666666642,1.3271428571428574] # check (LinSpace, UnitRange) x = linspace(1.0, 12.0, 100) y = -100:-1 -@test_approx_eq [linreg(x, y)...] [-109.0, 9.0] +@test [linreg(x, y)...] ≈ [-109.0, 9.0] # check (UnitRange, UnitRange) x = 1:12 y = 12:-1:1 -@test_approx_eq [linreg(x, y)...] [13.0, -1.0] +@test [linreg(x, y)...] ≈ [13.0, -1.0] # check (LinSpace, LinSpace) x = linspace(-5, 10, 100) y = linspace(50, 200, 100) -@test_approx_eq [linreg(x, y)...] [100.0, 10.0] +@test [linreg(x, y)...] ≈ [100.0, 10.0] # check (Array, Array) # Anscombe's quartet (https://en.wikipedia.org/wiki/Anscombe%27s_quartet) diff --git a/test/linalg/givens.jl b/test/linalg/givens.jl index 1078de8b839de..97bb27246bc4d 100644 --- a/test/linalg/givens.jl +++ b/test/linalg/givens.jl @@ -33,8 +33,8 @@ for elty in (Float32, Float64, Complex64, Complex128) @test A_mul_B!(G,eye(elty,10,10)) == [G[i,j] for i=1:10,j=1:10] # test transposes - @test_approx_eq ctranspose(G)*G*eye(10) eye(elty, 10) - @test_approx_eq ctranspose(R)*(R*eye(10)) eye(elty, 10) + @test ctranspose(G)*G*eye(10) ≈ eye(elty, 10) + @test ctranspose(R)*(R*eye(10)) ≈ eye(elty, 10) @test_throws ErrorException transpose(G) @test_throws ErrorException transpose(R) end @@ -44,13 +44,13 @@ for elty in (Float32, Float64, Complex64, Complex128) G, _ = givens(one(elty),zero(elty),11,12) @test_throws DimensionMismatch A_mul_B!(G, A) @test_throws DimensionMismatch A_mul_Bc!(A,G) - @test_approx_eq abs(A) abs(hessfact(Ac)[:H]) - @test_approx_eq norm(R*eye(elty, 10)) one(elty) + @test abs(A) ≈ abs(hessfact(Ac)[:H]) + @test norm(R*eye(elty, 10)) ≈ one(elty) G, _ = givens(one(elty),zero(elty),9,10) - @test_approx_eq ctranspose(G*eye(elty,10))*(G*eye(elty,10)) eye(elty, 10) + @test ctranspose(G*eye(elty,10))*(G*eye(elty,10)) ≈ eye(elty, 10) K, _ = givens(zero(elty),one(elty),9,10) - @test_approx_eq ctranspose(K*eye(elty,10))*(K*eye(elty,10)) eye(elty, 10) + @test ctranspose(K*eye(elty,10))*(K*eye(elty,10)) ≈ eye(elty, 10) # test that Givens * work for vectors if Atype == "Array" diff --git a/test/linalg/hessenberg.jl b/test/linalg/hessenberg.jl index aabbe4f42c1dd..80d7406f2a801 100644 --- a/test/linalg/hessenberg.jl +++ b/test/linalg/hessenberg.jl @@ -27,11 +27,11 @@ let n = 10 @test size(H[:Q], 2) == size(A, 2) @test size(H[:Q]) == size(A) @test_throws KeyError H[:Z] - @test_approx_eq full(H) A - @test_approx_eq (H[:Q] * H[:H]) * H[:Q]' A - @test_approx_eq (H[:Q]' * A) * H[:Q] H[:H] + @test full(H) ≈ A + @test (H[:Q] * H[:H]) * H[:Q]' ≈ A + @test (H[:Q]' *A) * H[:Q] ≈ H[:H] #getindex for HessenbergQ - @test_approx_eq H[:Q][1,1] full(H[:Q])[1,1] + @test H[:Q][1,1] ≈ full(H[:Q])[1,1] end end end diff --git a/test/linalg/lapack.jl b/test/linalg/lapack.jl index b54d8ab8a8463..51a47eace3a4a 100644 --- a/test/linalg/lapack.jl +++ b/test/linalg/lapack.jl @@ -20,11 +20,11 @@ let # syevr A = convert(Array{elty, 2}, A) Asym = A'A vals, Z = LAPACK.syevr!('V', copy(Asym)) - @test_approx_eq Z * (Diagonal(vals) * Z') Asym + @test Z*(Diagonal(vals)*Z') ≈ Asym @test all(vals .> 0.0) - @test_approx_eq LAPACK.syevr!('N','V','U',copy(Asym),0.0,1.0,4,5,-1.0)[1] vals[vals .< 1.0] - @test_approx_eq LAPACK.syevr!('N','I','U',copy(Asym),0.0,1.0,4,5,-1.0)[1] vals[4:5] - @test_approx_eq vals LAPACK.syev!('N','U',copy(Asym)) + @test LAPACK.syevr!('N','V','U',copy(Asym),0.0,1.0,4,5,-1.0)[1] ≈ vals[vals .< 1.0] + @test LAPACK.syevr!('N','I','U',copy(Asym),0.0,1.0,4,5,-1.0)[1] ≈ vals[4:5] + @test vals ≈ LAPACK.syev!('N','U',copy(Asym)) @test_throws DimensionMismatch LAPACK.sygvd!(1,'V','U',copy(Asym),ones(elty,6,6)) end @@ -36,7 +36,7 @@ let # Test gglse c = convert(Array{elty, 1}, [2, 1, 6, 3, 1]) B = convert(Array{elty, 2}, [1 1 1 -1; 1 -1 1 1; 1 1 -1 1]) d = convert(Array{elty, 1}, [1, 3, -1]) - @test_approx_eq LAPACK.gglse!(A, c, B, d)[1] convert(Array{elty}, [0.5, -0.5, 1.5, 0.5]) + @test LAPACK.gglse!(A, c, B, d)[1] ≈ convert(Array{elty}, [0.5, -0.5, 1.5, 0.5]) end end @@ -46,7 +46,7 @@ let # gebrd, bdsqr & throw for bdsdc d, e = convert(Vector{elty}, randn(n)), convert(Vector{elty}, randn(n - 1)) U, Vt, C = eye(elty, n), eye(elty, n), eye(elty, n) s, _ = LAPACK.bdsqr!('U', copy(d), copy(e), Vt, U, C) - @test_approx_eq full(Bidiagonal(d, e, true)) U*Diagonal(s)*Vt + @test full(Bidiagonal(d, e, true)) ≈ U*Diagonal(s)*Vt @test_throws ArgumentError LAPACK.bdsqr!('A', d, e, Vt, U, C) @test_throws DimensionMismatch LAPACK.bdsqr!('U', d, [e; 1], Vt, U, C) @@ -67,7 +67,7 @@ end let # Issue #7886 x, r = LAPACK.gelsy!([0 1; 0 2; 0 3.], [2, 4, 6.]) - @test_approx_eq x [0,2] + @test x ≈ [0,2] @test r == 1 end @@ -77,7 +77,7 @@ for elty in (Float32, Float64, Complex64, Complex128) B = copy(A) C,T = LAPACK.geqrt!(A,zeros(elty,10,10)) D,S = LAPACK.geqrt3!(A,zeros(elty,10,10)) - @test_approx_eq C D + @test C ≈ D end #gbtrf and gbtrs @@ -133,7 +133,7 @@ for elty in (Float32, Float64, Complex64, Complex128) B = rand(elty,10,10) C, j = LAPACK.gelsd!(copy(A),copy(B)) D, k = LAPACK.gelsy!(copy(A),copy(B)) - @test_approx_eq C D + @test C ≈ D @test_throws DimensionMismatch LAPACK.gelsd!(A,rand(elty,12,10)) @test_throws DimensionMismatch LAPACK.gelsy!(A,rand(elty,12,10)) end @@ -151,9 +151,9 @@ for elty in (Float32, Float64, Complex64, Complex128) A = rand(elty,10,5) U,S,V = svd(A) lU,lS,lVt = LAPACK.gesvd!('S','S',A) - @test_approx_eq U lU - @test_approx_eq S lS - @test_approx_eq V' lVt + @test U ≈ lU + @test S ≈ lS + @test V' ≈ lVt B = rand(elty,10,10) # xggsvd3 replaced xggsvd in LAPACK 3.6.0 if LAPACK.laver() < (3, 6, 0) @@ -192,7 +192,7 @@ for elty in (Float32, Float64, Complex64, Complex128) A = rand(elty,10,10) X = rand(elty,10) B,Y,z = LAPACK.gels!('N',copy(A),copy(X)) - @test_approx_eq A\X Y + @test A\X ≈ Y end #getrf/getri @@ -201,7 +201,7 @@ for elty in (Float32, Float64, Complex64, Complex128) iA = inv(A) A, ipiv = LAPACK.getrf!(A) A = LAPACK.getri!(A, ipiv) - @test_approx_eq A iA + @test A ≈ iA end #geev - complex is easier for now @@ -209,8 +209,8 @@ for elty in (Complex64, Complex128) A = rand(elty,10,10) Aw, Avl, Avr = LAPACK.geev!('N','V',copy(A)) fA = eigfact(A) - @test_approx_eq fA[:values] Aw - @test_approx_eq fA[:vectors] Avr + @test fA[:values] ≈ Aw + @test fA[:vectors] ≈ Avr end #gtsv @@ -221,7 +221,7 @@ for elty in (Float32, Float64, Complex64, Complex128) b = rand(elty,10) c = Tridiagonal(dl,d,du) \ b b = LAPACK.gtsv!(dl,d,du,b) - @test_approx_eq b c + @test b ≈ c @test_throws DimensionMismatch LAPACK.gtsv!(zeros(elty,11),d,du,b) @test_throws DimensionMismatch LAPACK.gtsv!(dl,d,zeros(elty,11),b) @test_throws DimensionMismatch LAPACK.gtsv!(dl,d,du,zeros(elty,11)) @@ -244,7 +244,7 @@ for elty in (Float32, Float64, Complex64, Complex128) c = copy(b) dl,d,du,du2,ipiv = LAPACK.gttrf!(dl,d,du) c = LAPACK.gttrs!('N',dl,d,du,du2,ipiv,c) - @test_approx_eq A\b c + @test A\b ≈ c end #orglq and friends errors @@ -321,7 +321,7 @@ for elty in (Float32, Float64, Complex64, Complex128) A = A + A.' #symmetric! B = copy(A) B,ipiv = LAPACK.sytrf!('U',B) - @test_approx_eq triu(inv(A)) triu(LAPACK.sytri!('U',B,ipiv)) + @test triu(inv(A)) ≈ triu(LAPACK.sytri!('U',B,ipiv)) @test_throws DimensionMismatch LAPACK.sytrs!('U',B,ipiv,rand(elty,11,5)) @test LAPACK.sytrf!('U',zeros(elty,0,0)) == (zeros(elty,0,0),zeros(BlasInt,0)) end @@ -331,7 +331,7 @@ for elty in (Float32, Float64, Complex64, Complex128) A = A + A.' #symmetric! B = copy(A) B,ipiv = LAPACK.sytrf_rook!('U',B) - @test_approx_eq triu(inv(A)) triu(LAPACK.sytri_rook!('U',B,ipiv)) + @test triu(inv(A)) ≈ triu(LAPACK.sytri_rook!('U',B,ipiv)) @test_throws DimensionMismatch LAPACK.sytrs_rook!('U',B,ipiv,rand(elty,11,5)) @test LAPACK.sytrf_rook!('U',zeros(elty,0,0)) == (zeros(elty,0,0),zeros(BlasInt,0)) A = rand(elty,10,10) @@ -339,7 +339,7 @@ for elty in (Float32, Float64, Complex64, Complex128) b = rand(elty,10) c = A \ b b,A = LAPACK.sysv_rook!('U',A,b) - @test_approx_eq b c + @test b ≈ c @test_throws DimensionMismatch LAPACK.sysv_rook!('U',A,rand(elty,11)) end @@ -369,7 +369,7 @@ for elty in (Float32, Float64, Complex64, Complex128) A = rand(elty,10,10) A = triu(A) B = copy(A) - @test_approx_eq inv(A) LAPACK.trtri!('U','N',B) + @test inv(A) ≈ LAPACK.trtri!('U','N',B) @test_throws DimensionMismatch LAPACK.trtrs!('U','N','N',B,zeros(elty,11,10)) end @@ -390,7 +390,7 @@ for elty in (Float32, Float64, Complex64, Complex128) b = rand(elty,10) c = A \ b b,A = LAPACK.sysv!('U',A,b) - @test_approx_eq b c + @test b ≈ c @test_throws DimensionMismatch LAPACK.sysv!('U',A,rand(elty,11)) end @@ -401,14 +401,14 @@ for elty in (Complex64, Complex128) b = rand(elty,10) c = A \ b b,A = LAPACK.hesv!('U',A,b) - @test_approx_eq b c + @test b ≈ c @test_throws DimensionMismatch LAPACK.hesv!('U',A,rand(elty,11)) A = rand(elty,10,10) A = A + A' #hermitian! b = rand(elty,10) c = A \ b b,A = LAPACK.hesv_rook!('U',A,b) - @test_approx_eq b c + @test b ≈ c @test_throws DimensionMismatch LAPACK.hesv_rook!('U',A,rand(elty,11)) end @@ -422,7 +422,7 @@ for elty in (Float32, Float64, Complex64, Complex128) end B = rand(elty,10,10) C = copy(B) - @test_approx_eq A\B LAPACK.ptsv!(dv,ev,C) + @test A\B ≈ LAPACK.ptsv!(dv,ev,C) @test_throws DimensionMismatch LAPACK.ptsv!(dv,ones(elty,10),C) @test_throws DimensionMismatch LAPACK.ptsv!(dv,ev,ones(elty,11,11)) end @@ -440,11 +440,11 @@ for elty in (Float32, Float64, Complex64, Complex128) B = rand(elty,10,10) C = copy(B) if elty <: Complex - @test_approx_eq A\B LAPACK.pttrs!('U',dv,ev,C) + @test A\B ≈ LAPACK.pttrs!('U',dv,ev,C) @test_throws DimensionMismatch LAPACK.pttrs!('U',dv,ones(elty,10),C) @test_throws DimensionMismatch LAPACK.pttrs!('U',dv,ev,rand(elty,11,11)) else - @test_approx_eq A\B LAPACK.pttrs!(dv,ev,C) + @test A\B ≈ LAPACK.pttrs!(dv,ev,C) @test_throws DimensionMismatch LAPACK.pttrs!(dv,ones(elty,10),C) @test_throws DimensionMismatch LAPACK.pttrs!(dv,ev,rand(elty,11,11)) end @@ -463,7 +463,7 @@ for elty in (Float32, Float64, Complex64, Complex128) D = copy(A) C = copy(B) D,C = LAPACK.posv!('U',D,C) - @test_approx_eq A\B C + @test A\B ≈ C @test_throws DimensionMismatch LAPACK.posv!('U',D,ones(elty,12,12)) @test_throws DimensionMismatch LAPACK.potrs!('U',D,ones(elty,12,12)) @@ -477,7 +477,7 @@ for elty in (Float32, Float64, Complex64, Complex128) C = copy(A) D = copy(B) X, rcond, f, b, r = LAPACK.gesvx!(C,D) - @test_approx_eq X A\B + @test X ≈ A\B end #gees, gges error throwing @@ -520,8 +520,8 @@ for elty in (Float32, Float64, Complex64, Complex128) Base.LinAlg.LAPACK.trsen!('N',c,Array{LinAlg.BlasInt}([0,1,0,0]),T,Q) @test d[1] ≈ T[2,2] @test d[2] ≈ T[1,1] - if (c == 'V') - @test Q * T * Q' ≈ A + if c == 'V' + @test Q*T*Q' ≈ A end end end @@ -534,8 +534,8 @@ for elty in (Float32, Float64, Complex64, Complex128) Base.LinAlg.LAPACK.trexc!(c,LinAlg.BlasInt(1),LinAlg.BlasInt(2),T,Q) @test d[1] ≈ T[2,2] @test d[2] ≈ T[1,1] - if (c == 'V') - @test Q * T * Q' ≈ A + if c == 'V' + @test Q*T*Q' ≈ A end end end @@ -551,16 +551,16 @@ for elty in (Float32, Float64, Complex{Float32}, Complex{Float64}) if elty <: Real FJulia = Base.LinAlg.generic_lufact!(copy(A)) FLAPACK = Base.LinAlg.LAPACK.getrf!(copy(A)) - @test_approx_eq FJulia.factors FLAPACK[1] - @test_approx_eq FJulia.ipiv FLAPACK[2] - @test_approx_eq FJulia.info FLAPACK[3] + @test FJulia.factors ≈ FLAPACK[1] + @test FJulia.ipiv ≈ FLAPACK[2] + @test FJulia.info ≈ FLAPACK[3] end ## QR FJulia = LinAlg.qrfactUnblocked!(copy(A)) FLAPACK = Base.LinAlg.LAPACK.geqrf!(copy(A)) - @test_approx_eq FJulia.factors FLAPACK[1] - @test_approx_eq FJulia.τ FLAPACK[2] + @test FJulia.factors ≈ FLAPACK[1] + @test FJulia.τ ≈ FLAPACK[2] end end diff --git a/test/linalg/lu.jl b/test/linalg/lu.jl index 5891fe63a7ea2..ca5831029d204 100644 --- a/test/linalg/lu.jl +++ b/test/linalg/lu.jl @@ -36,7 +36,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) if eltya <: BlasFloat num = rand(eltya) @test lu(num) == (one(eltya),num,1) - @test_approx_eq full(lufact(num)) eltya[num] + @test full(lufact(num)) ≈ eltya[num] end for eltyb in (Float32, Float64, Complex64, Complex128, Int) b = eltyb == Int ? rand(1:5, n, 2) : convert(Matrix{eltyb}, eltyb <: Complex ? complex(breal, bimg) : breal) @@ -68,7 +68,7 @@ debug && println("(Automatic) Square LU decomposition") @test norm(a'*(lua'\a') - a', 1) < ε*κ*n^2 @test norm(a*(lua\c) - c, 1) < ε*κ*n # c is a vector @test norm(a'*(lua'\c) - c, 1) < ε*κ*n # c is a vector - @test_approx_eq full(lua) a + @test full(lua) ≈ a if eltya <: Real && eltyb <: Real @test norm(a.'*(lua.'\b) - b,1) < ε*κ*n*2 # Two because the right hand side has two columns @test norm(a.'*(lua.'\c) - c,1) < ε*κ*n @@ -126,11 +126,11 @@ debug && println("Tridiagonal LU") debug && println("Thin LU") lua = @inferred lufact(a[:,1:n1]) - @test_approx_eq lua[:L]*lua[:U] lua[:P]*a[:,1:n1] + @test lua[:L]*lua[:U] ≈ lua[:P]*a[:,1:n1] debug && println("Fat LU") lua = lufact(a[1:n1,:]) - @test_approx_eq lua[:L]*lua[:U] lua[:P]*a[1:n1,:] + @test lua[:L]*lua[:U] ≈ lua[:P]*a[1:n1,:] end end @@ -149,9 +149,9 @@ b = rand(1:10,n,2) @inferred lufact(a) lua = factorize(a) l,u,p = lua[:L], lua[:U], lua[:p] -@test_approx_eq l*u a[p,:] -@test_approx_eq l[invperm(p),:]*u a -@test_approx_eq a * inv(lua) eye(n) +@test l*u ≈ a[p,:] +@test l[invperm(p),:]*u ≈ a +@test a*inv(lua) ≈ eye(n) let Bs = b for atype in ("Array", "SubArray") if atype == "Array" @@ -162,7 +162,7 @@ let Bs = b @test a*(lua\b) ≈ b end end -@test_approx_eq @inferred(det(a)) det(Array{Float64}(a)) +@test @inferred(det(a)) ≈ det(Array{Float64}(a)) ## Hilbert Matrix (very ill conditioned) ## Testing Rational{BigInt} and BigFloat version nHilbert = 50 @@ -181,7 +181,7 @@ for elty in (Float32, Float64, Complex64, Complex128) -0.5 -0.5 0.1 1.0]) F = eigfact(A, permute=false, scale=false) eig(A, permute=false, scale=false) - @test_approx_eq F[:vectors]*Diagonal(F[:values])/F[:vectors] A + @test F[:vectors]*Diagonal(F[:values])/F[:vectors] ≈ A F = eigfact(A) # @test norm(F[:vectors]*Diagonal(F[:values])/F[:vectors] - A) > 0.01 end diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 3daa60e3746e6..0f44b8cf8479c 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -76,7 +76,7 @@ module MyArray15367 A = MyArray(rand(4,5)) b = rand(5) - @test_approx_eq A*b A.data*b + @test A*b ≈ A.data*b end let @@ -268,7 +268,7 @@ let AA = [1+2im 3+4im; 5+6im 7+8im], BB = [2+7im 4+1im; 3+8im 6+5im] if n1 != n2 @test_throws DimensionMismatch d(1:n1, 1:n2) else - @test_approx_eq d(1:n1, 1:n2) vecnorm(1:n1)^2 + @test d(1:n1, 1:n2) ≈ vecnorm(1:n1)^2 end end end diff --git a/test/linalg/pinv.jl b/test/linalg/pinv.jl index 77d9d29749036..221a17339e0f2 100644 --- a/test/linalg/pinv.jl +++ b/test/linalg/pinv.jl @@ -269,23 +269,23 @@ for eltya in (Float32, Float64, Complex64, Complex128) debug && println("\n\n<<<<<", eltya, ">>>>>") debug && println("\n--- zero constant ---") a = pinv(zero(eltya)) - @test_approx_eq a 0.0 + @test a ≈ 0.0 end for eltya in (Float32, Float64, Complex64, Complex128) debug && println("\n\n<<<<<", eltya, ">>>>>") debug && println("\n--- zero vector ---") a = pinv([zero(eltya); zero(eltya)]) - @test_approx_eq a[1] 0.0 - @test_approx_eq a[2] 0.0 + @test a[1] ≈ 0.0 + @test a[2] ≈ 0.0 end for eltya in (Float32, Float64, Complex64, Complex128) debug && println("\n\n<<<<<", eltya, ">>>>>") debug && println("\n--- zero Diagonal matrix ---") a = pinv(Diagonal([zero(eltya); zero(eltya)])) - @test_approx_eq a.diag[1] 0.0 - @test_approx_eq a.diag[2] 0.0 + @test a.diag[1] ≈ 0.0 + @test a.diag[2] ≈ 0.0 end # test sub-normal matrices @@ -293,35 +293,35 @@ for eltya in (Float32, Float64) debug && println("\n\n<<<<<", eltya, ">>>>>") debug && println("\n--- sub-normal constant ---") a = pinv(realmin(eltya)/100) - @test_approx_eq a 0.0 + @test a ≈ 0.0 debug && println("\n\n<<<<>>>>") debug && println("\n--- sub-normal constant ---") a = pinv(realmin(eltya)/100*(1+1im)) - @test_approx_eq a 0.0 + @test a ≈ 0.0 end for eltya in (Float32, Float64) debug && println("\n\n<<<<<", eltya, ">>>>>") debug && println("\n--- sub-normal vector ---") a = pinv([realmin(eltya); realmin(eltya)]/100) - @test_approx_eq a[1] 0.0 - @test_approx_eq a[2] 0.0 + @test a[1] ≈ 0.0 + @test a[2] ≈ 0.0 debug && println("\n\n<<<<>>>>") debug && println("\n--- sub-normal vector ---") a = pinv([realmin(eltya); realmin(eltya)]/100*(1+1im)) - @test_approx_eq a[1] 0.0 - @test_approx_eq a[2] 0.0 + @test a[1] ≈ 0.0 + @test a[2] ≈ 0.0 end for eltya in (Float32, Float64) debug && println("\n\n<<<<<", eltya, ">>>>>") debug && println("\n--- sub-normal Diagonal matrix ---") a = pinv(Diagonal([realmin(eltya); realmin(eltya)]/100)) - @test_approx_eq a.diag[1] 0.0 - @test_approx_eq a.diag[2] 0.0 + @test a.diag[1] ≈ 0.0 + @test a.diag[2] ≈ 0.0 debug && println("\n\n<<<<>>>>") debug && println("\n--- sub-normal Diagonal matrix ---") a = pinv(Diagonal([realmin(eltya); realmin(eltya)]/100*(1+1im))) - @test_approx_eq a.diag[1] 0.0 - @test_approx_eq a.diag[2] 0.0 + @test a.diag[1] ≈ 0.0 + @test a.diag[2] ≈ 0.0 end diff --git a/test/linalg/qr.jl b/test/linalg/qr.jl index 39829db9fdfc6..8df1097b926d6 100644 --- a/test/linalg/qr.jl +++ b/test/linalg/qr.jl @@ -46,14 +46,14 @@ debug && println("QR decomposition (without pivoting)") @inferred qr(a) q, r = qra[:Q], qra[:R] @test_throws KeyError qra[:Z] - @test_approx_eq q'*full(q, thin = false) eye(n) - @test_approx_eq q*full(q, thin = false)' eye(n) - @test_approx_eq q'*eye(n)' full(q, thin = false)' - @test_approx_eq full(q, thin = false)'q eye(n) - @test_approx_eq eye(n)'q' full(q, thin = false)' - @test_approx_eq q*r a + @test q'*full(q, thin=false) ≈ eye(n) + @test q*full(q, thin=false)' ≈ eye(n) + @test q'*eye(n)' ≈ full(q, thin=false)' + @test full(q, thin=false)'q ≈ eye(n) + @test eye(n)'q' ≈ full(q, thin=false)' + @test q*r ≈ a @test_approx_eq_eps a*(qra\b) b 3000ε - @test_approx_eq full(qra) a + @test full(qra) ≈ a @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*full(q, thin=false) eye(n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) @@ -66,9 +66,9 @@ debug && println("Thin QR decomposition (without pivoting)") @inferred qr(a[:,1:n1], Val{false}) q,r = qra[:Q], qra[:R] @test_throws KeyError qra[:Z] - @test_approx_eq q'*full(q, thin=false) eye(n) - @test_approx_eq q'*full(q) eye(n, n1) - @test_approx_eq q*r a[:,1:n1] + @test q'*full(q, thin=false) ≈ eye(n) + @test q'*full(q) ≈ eye(n,n1) + @test q*r ≈ a[:,1:n1] @test_approx_eq_eps q*b[1:n1] full(q)*b[1:n1] 100ε @test_approx_eq_eps q*b full(q, thin=false)*b 100ε @test_throws DimensionMismatch q*b[1:n1 + 1] @@ -86,16 +86,16 @@ debug && println("(Automatic) Fat (pivoted) QR decomposition") q,r = qrpa[:Q], qrpa[:R] @test_throws KeyError qrpa[:Z] p = qrpa[:p] - @test_approx_eq q'*full(q, thin=false) eye(n1) - @test_approx_eq q*full(q, thin=false)' eye(n1) - @test_approx_eq (UpperTriangular(eye(eltya,size(q,2)))*q')*full(q, thin=false) eye(n1) - @test_approx_eq q*r isa(qrpa,QRPivoted) ? a[1:n1,p] : a[1:n1,:] - @test_approx_eq q*r[:,invperm(p)] a[1:n1,:] - @test_approx_eq q*r*qrpa[:P].' a[1:n1,:] + @test q'*full(q, thin=false) ≈ eye(n1) + @test q*full(q, thin=false)' ≈ eye(n1) + @test (UpperTriangular(eye(eltya,size(q,2)))*q')*full(q, thin=false) ≈ eye(n1) + @test q*r ≈ (isa(qrpa,QRPivoted) ? a[1:n1,p] : a[1:n1,:]) + @test q*r[:,invperm(p)] ≈ a[1:n1,:] + @test q*r*qrpa[:P].' ≈ a[1:n1,:] @test_approx_eq_eps a[1:n1,:]*(qrpa\b[1:n1]) b[1:n1] 5000ε - @test_approx_eq full(qrpa) a[1:5,:] - @test_throws DimensionMismatch q*b[1:n1 + 1] - @test_throws DimensionMismatch b[1:n1 + 1]*q' + @test full(qrpa) ≈ a[1:5,:] + @test_throws DimensionMismatch q*b[1:n1+1] + @test_throws DimensionMismatch b[1:n1+1]*q' if eltya != Int @test eye(eltyb,n1)*q ≈ convert(AbstractMatrix{tab},q) end @@ -105,13 +105,13 @@ debug && println("(Automatic) Thin (pivoted) QR decomposition") q,r = qrpa[:Q], qrpa[:R] @test_throws KeyError qrpa[:Z] p = qrpa[:p] - @test_approx_eq q'*full(q, thin=false) eye(n) - @test_approx_eq q*full(q, thin=false)' eye(n) - @test_approx_eq q*r a[:,p] - @test_approx_eq q*r[:,invperm(p)] a[:,1:n1] - @test_approx_eq full(qrpa) a[:,1:5] - @test_throws DimensionMismatch q*b[1:n1 + 1] - @test_throws DimensionMismatch b[1:n1 + 1]*q' + @test q'*full(q, thin=false) ≈ eye(n) + @test q*full(q, thin=false)' ≈ eye(n) + @test q*r ≈ a[:,p] + @test q*r[:,invperm(p)] ≈ a[:,1:n1] + @test full(qrpa) ≈ a[:,1:5] + @test_throws DimensionMismatch q*b[1:n1+1] + @test_throws DimensionMismatch b[1:n1+1]*q' @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*full(q, thin=false) eye(n1,n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) @@ -122,20 +122,20 @@ debug && println("(Automatic) Thin (pivoted) QR decomposition") debug && println("Matmul with QR factorizations") if eltya != Int qrpa = factorize(a[:,1:n1]) - q,r = qrpa[:Q], qrpa[:R] - @test_approx_eq A_mul_B!(full(q,thin=false)',q) eye(n) + q, r = qrpa[:Q], qrpa[:R] + @test A_mul_B!(full(q, thin=false)',q) ≈ eye(n) @test_throws DimensionMismatch A_mul_B!(eye(eltya,n+1),q) - @test_approx_eq A_mul_Bc!(full(q,thin=false),q) eye(n) + @test A_mul_Bc!(full(q, thin=false),q) ≈ eye(n) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) @test_throws DimensionMismatch Base.LinAlg.A_mul_B!(q,zeros(eltya,n1+1)) @test_throws DimensionMismatch Base.LinAlg.Ac_mul_B!(q,zeros(eltya,n1+1)) - qra = qrfact(a[:,1:n1], Val{false}) - q,r = qra[:Q], qra[:R] - @test_approx_eq A_mul_B!(full(q,thin=false)',q) eye(n) + qra = qrfact(a[:,1:n1], Val{false}) + q, r = qra[:Q], qra[:R] + @test A_mul_B!(full(q, thin=false)',q) ≈ eye(n) @test_throws DimensionMismatch A_mul_B!(eye(eltya,n+1),q) - @test_approx_eq A_mul_Bc!(full(q,thin=false),q) eye(n) + @test A_mul_Bc!(full(q, thin=false),q) ≈ eye(n) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) @test_throws DimensionMismatch q * eye(Int8,n+4) @@ -176,7 +176,7 @@ end @test Base.LinAlg.qr!(Int[1]) == (Int[1],1) B = rand(7,2) -@test_approx_eq (1:7)\B collect(1:7)\B +@test (1:7)\B ≈ collect(1:7)\B # Issue 16520 @test_throws DimensionMismatch ones(3,2)\(1:5) diff --git a/test/linalg/schur.jl b/test/linalg/schur.jl index d3dc91567ed2a..5370afd3f82f5 100644 --- a/test/linalg/schur.jl +++ b/test/linalg/schur.jl @@ -34,11 +34,11 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) debug && println("Schur") d,v = eig(a) f = schurfact(a) - @test_approx_eq f[:vectors]*f[:Schur]*f[:vectors]' a - @test_approx_eq sort(real(f[:values])) sort(real(d)) - @test_approx_eq sort(imag(f[:values])) sort(imag(d)) + @test f[:vectors]*f[:Schur]*f[:vectors]' ≈ a + @test sort(real(f[:values])) ≈ sort(real(d)) + @test sort(imag(f[:values])) ≈ sort(imag(d)) @test istriu(f[:Schur]) || eltype(a)<:Real - @test_approx_eq full(f) a + @test full(f) ≈ a @test_throws KeyError f[:A] debug && println("Reorder Schur") @@ -48,8 +48,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) S = schurfact(ordschura) select = bitrand(n) O = ordschur(S, select) - sum(select) != 0 && @test_approx_eq S[:values][find(select)] O[:values][1:sum(select)] - @test_approx_eq O[:vectors]*O[:Schur]*O[:vectors]' ordschura + sum(select) != 0 && @test S[:values][find(select)] ≈ O[:values][1:sum(select)] + @test O[:vectors]*O[:Schur]*O[:vectors]' ≈ ordschura @test_throws KeyError f[:A] Snew = Base.LinAlg.Schur(S.T, S.Z, S.values) SchurNew = ordschur!(copy(Snew), select) @@ -65,8 +65,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) a2_sf = view(a, n1+1:n2, n1+1:n2) end f = schurfact(a1_sf, a2_sf) - @test_approx_eq f[:Q]*f[:S]*f[:Z]' a1_sf - @test_approx_eq f[:Q]*f[:T]*f[:Z]' a2_sf + @test f[:Q]*f[:S]*f[:Z]' ≈ a1_sf + @test f[:Q]*f[:T]*f[:Z]' ≈ a2_sf @test istriu(f[:S]) || eltype(a)<:Real @test istriu(f[:T]) || eltype(a)<:Real @test_throws KeyError f[:A] @@ -77,10 +77,10 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) m = sum(select) S = ordschur(NS, select) # Make sure that the new factorization stil factors matrix - @test_approx_eq S[:Q]*S[:S]*S[:Z]' a1_sf - @test_approx_eq S[:Q]*S[:T]*S[:Z]' a2_sf + @test S[:Q]*S[:S]*S[:Z]' ≈ a1_sf + @test S[:Q]*S[:T]*S[:Z]' ≈ a2_sf # Make sure that we have sorted it correctly - @test_approx_eq NS[:values][find(select)] S[:values][1:m] + @test NS[:values][find(select)] ≈ S[:values][1:m] Snew = Base.LinAlg.GeneralizedSchur(NS.S, NS.T, NS.alpha, NS.beta, NS.Q, NS.Z) SchurNew = ordschur!(copy(Snew), select) diff --git a/test/linalg/special.jl b/test/linalg/special.jl index 8cef4aa854ddc..89ddde5ee4b00 100644 --- a/test/linalg/special.jl +++ b/test/linalg/special.jl @@ -92,16 +92,16 @@ let a=[1.0:n;] for type2 in Spectypes B = convert(type1,A) C = convert(type2,A) - @test_approx_eq full(B + C) full(A + A) - @test_approx_eq full(B - C) full(A - A) + @test full(B + C) ≈ full(A + A) + @test full(B - C) ≈ full(A - A) end end B = SymTridiagonal(a, ones(n-1)) for Spectype in [Diagonal, Bidiagonal, Tridiagonal, Matrix] - @test_approx_eq full(B + convert(Spectype,A)) full(B + A) - @test_approx_eq full(convert(Spectype,A) + B) full(B + A) - @test_approx_eq full(B - convert(Spectype,A)) full(B - A) - @test_approx_eq full(convert(Spectype,A) - B) full(A - B) + @test full(B + convert(Spectype,A)) ≈ full(B + A) + @test full(convert(Spectype,A) + B) ≈ full(B + A) + @test full(B - convert(Spectype,A)) ≈ full(B - A) + @test full(convert(Spectype,A) - B) ≈ full(A - B) end C = rand(n,n) diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index 08d994433e3d4..4c570babc80e9 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -215,12 +215,12 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test 0.5\A1 == 0.5\full(A1) # inversion - @test_approx_eq inv(A1) inv(lufact(full(A1))) + @test inv(A1) ≈ inv(lufact(full(A1))) inv(full(A1)) # issue #11298 @test isa(inv(A1), t1) # make sure the call to LAPACK works right if elty1 <: BlasFloat - @test_approx_eq Base.LinAlg.inv!(copy(A1)) inv(lufact(full(A1))) + @test Base.LinAlg.inv!(copy(A1)) ≈ inv(lufact(full(A1))) end # Determinant @@ -279,15 +279,15 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test full(A1 - A2) == full(A1) - full(A2) # Triangular-Triangualar multiplication and division - @test_approx_eq full(A1*A2) full(A1)*full(A2) - @test_approx_eq full(A1.'A2) full(A1).'full(A2) - @test_approx_eq full(A1'A2) full(A1)'full(A2) - @test_approx_eq full(A1*A2.') full(A1)*full(A2).' - @test_approx_eq full(A1*A2') full(A1)*full(A2)' - @test_approx_eq full(A1.'A2.') full(A1).'full(A2).' - @test_approx_eq full(A1'A2') full(A1)'full(A2)' - @test_approx_eq full(A1/A2) full(A1)/full(A2) - @test_approx_eq full(A1\A2) full(A1)\full(A2) + @test full(A1*A2) ≈ full(A1)*full(A2) + @test full(A1.'A2) ≈ full(A1).'full(A2) + @test full(A1'A2) ≈ full(A1)'full(A2) + @test full(A1*A2.') ≈ full(A1)*full(A2).' + @test full(A1*A2') ≈ full(A1)*full(A2)' + @test full(A1.'A2.') ≈ full(A1).'full(A2).' + @test full(A1'A2') ≈ full(A1)'full(A2)' + @test full(A1/A2) ≈ full(A1)/full(A2) + @test full(A1\A2) ≈ full(A1)\full(A2) @test_throws DimensionMismatch eye(n+1)/A2 @test_throws DimensionMismatch eye(n+1)/A2.' @test_throws DimensionMismatch eye(n+1)/A2' @@ -308,34 +308,34 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa if !(eltyB in (BigFloat, Complex{BigFloat})) # rand does not support BigFloat and Complex{BigFloat} as of Dec 2015 Tri = Tridiagonal(rand(eltyB,n-1),rand(eltyB,n),rand(eltyB,n-1)) - @test_approx_eq Base.LinAlg.A_mul_B!(Tri,copy(A1)) Tri*full(A1) + @test Base.LinAlg.A_mul_B!(Tri,copy(A1)) ≈ Tri*full(A1) end # Triangular-dense Matrix/vector multiplication - @test_approx_eq A1*B[:,1] full(A1)*B[:,1] - @test_approx_eq A1*B full(A1)*B - @test_approx_eq A1.'B[:,1] full(A1).'B[:,1] - @test_approx_eq A1'B[:,1] full(A1)'B[:,1] - @test_approx_eq A1.'B full(A1).'B - @test_approx_eq A1'B full(A1)'B - @test_approx_eq A1*B.' full(A1)*B.' - @test_approx_eq A1*B' full(A1)*B' - @test_approx_eq B*A1 B*full(A1) - @test_approx_eq B[:,1].'A1 B[:,1].'full(A1) - @test_approx_eq B[:,1]'A1 B[:,1]'full(A1) - @test_approx_eq B.'A1 B.'full(A1) - @test_approx_eq B'A1 B'full(A1) - @test_approx_eq B*A1.' B*full(A1).' - @test_approx_eq B*A1' B*full(A1)' - @test_approx_eq B[:,1].'A1.' B[:,1].'full(A1).' - @test_approx_eq B[:,1]'A1' B[:,1]'full(A1)' - @test_approx_eq B.'A1.' B.'full(A1).' - @test_approx_eq B'A1' B'full(A1)' + @test A1*B[:,1] ≈ full(A1)*B[:,1] + @test A1*B ≈ full(A1)*B + @test A1.'B[:,1] ≈ full(A1).'B[:,1] + @test A1'B[:,1] ≈ full(A1)'B[:,1] + @test A1.'B ≈ full(A1).'B + @test A1'B ≈ full(A1)'B + @test A1*B.' ≈ full(A1)*B.' + @test A1*B' ≈ full(A1)*B' + @test B*A1 ≈ B*full(A1) + @test B[:,1].'A1 ≈ B[:,1].'full(A1) + @test B[:,1]'A1 ≈ B[:,1]'full(A1) + @test B.'A1 ≈ B.'full(A1) + @test B'A1 ≈ B'full(A1) + @test B*A1.' ≈ B*full(A1).' + @test B*A1' ≈ B*full(A1)' + @test B[:,1].'A1.' ≈ B[:,1].'full(A1).' + @test B[:,1]'A1' ≈ B[:,1]'full(A1)' + @test B.'A1.' ≈ B.'full(A1).' + @test B'A1' ≈ B'full(A1)' if eltyB == elty1 - @test_approx_eq A_mul_B!(zeros(B),A1,B) A1*B - @test_approx_eq A_mul_Bc!(zeros(B),A1,B) A1*B' - @test_approx_eq A_mul_Bt!(zeros(B),A1,B) A1*B.' + @test A_mul_B!(zeros(B),A1,B) ≈ A1*B + @test A_mul_Bc!(zeros(B),A1,B) ≈ A1*B' + @test A_mul_Bt!(zeros(B),A1,B) ≈ A1*B.' end #error handling @test_throws DimensionMismatch Base.LinAlg.A_mul_B!(A1, ones(eltyB,n+1)) @@ -346,29 +346,29 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test_throws DimensionMismatch Base.LinAlg.A_mul_Bt!(ones(eltyB,n+1,n+1),A1) # ... and division - @test_approx_eq A1\B[:,1] full(A1)\B[:,1] - @test_approx_eq A1\B full(A1)\B - @test_approx_eq A1.'\B[:,1] full(A1).'\B[:,1] - @test_approx_eq A1'\B[:,1] full(A1)'\B[:,1] - @test_approx_eq A1.'\B full(A1).'\B - @test_approx_eq A1'\B full(A1)'\B - @test_approx_eq A1\B.' full(A1)\B.' - @test_approx_eq A1\B' full(A1)\B' - @test_approx_eq A1.'\B.' full(A1).'\B.' - @test_approx_eq A1'\B' full(A1)'\B' + @test A1\B[:,1] ≈ full(A1)\B[:,1] + @test A1\B ≈ full(A1)\B + @test A1.'\B[:,1] ≈ full(A1).'\B[:,1] + @test A1'\B[:,1] ≈ full(A1)'\B[:,1] + @test A1.'\B ≈ full(A1).'\B + @test A1'\B ≈ full(A1)'\B + @test A1\B.' ≈ full(A1)\B.' + @test A1\B' ≈ full(A1)\B' + @test A1.'\B.' ≈ full(A1).'\B.' + @test A1'\B' ≈ full(A1)'\B' @test_throws DimensionMismatch A1\ones(elty1,n+2) @test_throws DimensionMismatch A1'\ones(elty1,n+2) @test_throws DimensionMismatch A1.'\ones(elty1,n+2) if t1 == UpperTriangular || t1 == LowerTriangular @test_throws Base.LinAlg.SingularException naivesub!(t1(zeros(elty1,n,n)),ones(eltyB,n)) end - @test_approx_eq B/A1 B/full(A1) - @test_approx_eq B/A1.' B/full(A1).' - @test_approx_eq B/A1' B/full(A1)' - @test_approx_eq B.'/A1 B.'/full(A1) - @test_approx_eq B'/A1 B'/full(A1) - @test_approx_eq B.'/A1.' B.'/full(A1).' - @test_approx_eq B'/A1' B'/full(A1)' + @test B/A1 ≈ B/full(A1) + @test B/A1.' ≈ B/full(A1).' + @test B/A1' ≈ B/full(A1)' + @test B.'/A1 ≈ B.'/full(A1) + @test B'/A1 ≈ B'/full(A1) + @test B.'/A1.' ≈ B.'/full(A1).' + @test B'/A1' ≈ B'/full(A1)' # Error bounds !(elty1 in (BigFloat, Complex{BigFloat})) && !(eltyB in (BigFloat, Complex{BigFloat})) && errorbounds(A1, A1\B, B) diff --git a/test/linalg/tridiag.jl b/test/linalg/tridiag.jl index 258569cb1fa9a..53fb78a416fb4 100644 --- a/test/linalg/tridiag.jl +++ b/test/linalg/tridiag.jl @@ -82,21 +82,21 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) end # tridiagonal linear algebra - @test_approx_eq T*vv F*vv + @test T*vv ≈ F*vv invFv = F\vv - @test_approx_eq T\vv invFv - # @test_approx_eq Base.solve(T,v) invFv - # @test_approx_eq Base.solve(T, B) F\B + @test T\vv ≈ invFv + # @test Base.solve(T,v) ≈ invFv + # @test Base.solve(T, B) ≈ F\B Tlu = factorize(T) x = Tlu\vv - @test_approx_eq x invFv + @test x ≈ invFv end - @test_approx_eq det(T) det(F) + @test det(T) ≈ det(F) - @test_approx_eq T * Base.LinAlg.UnitUpperTriangular(eye(n)) F*eye(n) - @test_approx_eq T * Base.LinAlg.UnitLowerTriangular(eye(n)) F*eye(n) - @test_approx_eq T * UpperTriangular(eye(n)) F*eye(n) - @test_approx_eq T * LowerTriangular(eye(n)) F*eye(n) + @test T*Base.LinAlg.UnitUpperTriangular(eye(n)) ≈ F*eye(n) + @test T*Base.LinAlg.UnitLowerTriangular(eye(n)) ≈ F*eye(n) + @test T*UpperTriangular(eye(n)) ≈ F*eye(n) + @test T*LowerTriangular(eye(n)) ≈ F*eye(n) # symmetric tridiagonal if elty <: Real @@ -118,8 +118,8 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) end invFsv = Fs\vv x = Ts\vv - @test_approx_eq x invFsv - @test_approx_eq full(full(Tldlt)) Fs + @test x ≈ invFsv + @test full(full(Tldlt)) ≈ Fs end # similar @@ -134,8 +134,8 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) @inferred eig(Ts, 2:4) @inferred eig(Ts, 1.0, 2.0) D, Vecs = eig(Fs) - @test_approx_eq DT D - @test_approx_eq abs(VT'Vecs) eye(elty, n) + @test DT ≈ D + @test abs(VT'Vecs) ≈ eye(elty, n) @test eigvecs(Ts) == eigvecs(Fs) #call to LAPACK.stein here Test.test_approx_eq_modphase(eigvecs(Ts,eigvals(Ts)),eigvecs(Fs)) @@ -154,14 +154,14 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) # The determinant of the identity matrix should always be 1. for i = 1:10 A = eye(elty, i) - @test_approx_eq det(A) one(elty) + @test det(A) ≈ one(elty) end # The determinant of a Householder reflection matrix should always be -1. for i = 1:10 A = eye(elty, 10) A[i, i] = -one(elty) - @test_approx_eq det(A) -one(elty) + @test det(A) ≈ -one(elty) end # The determinant of a rotation matrix should always be 1. @@ -169,7 +169,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) for theta = convert(Vector{elty}, pi ./ [1:4;]) R = [cos(theta) -sin(theta); sin(theta) cos(theta)] - @test_approx_eq convert(elty, det(R)) one(elty) + @test convert(elty, det(R)) ≈ one(elty) end # issue #1490 @@ -289,10 +289,10 @@ let n = 12 #Size of matrix problem to test @test B - A == A - B debug && println("Multiplication with strided vector") - @test_approx_eq A*ones(n) full(A)*ones(n) + @test A*ones(n) ≈ full(A)*ones(n) debug && println("Multiplication with strided matrix") - @test_approx_eq A*ones(n, 2) full(A)*ones(n, 2) + @test A*ones(n, 2) ≈ full(A)*ones(n, 2) debug && println("Eigensystems") if elty <: Real @@ -302,7 +302,7 @@ let n = 12 #Size of matrix problem to test evecs = LAPACK.stein!(a, b, w) (e, v) = eig(SymTridiagonal(a, b)) - @test_approx_eq e w + @test e ≈ w test_approx_eq_vecs(v, evecs) debug && println("stein! call using iblock and isplit") @@ -314,13 +314,13 @@ let n = 12 #Size of matrix problem to test F = eigfact(SymTridiagonal(a, b),1:2) fF = eigfact(Symmetric(full(SymTridiagonal(a, b))),1:2) Test.test_approx_eq_modphase(F[:vectors], fF[:vectors]) - @test_approx_eq F[:values] fF[:values] + @test F[:values] ≈ fF[:values] debug && println("stegr! call with value range") F = eigfact(SymTridiagonal(a, b),0.0,1.0) fF = eigfact(Symmetric(full(SymTridiagonal(a, b))),0.0,1.0) Test.test_approx_eq_modphase(F[:vectors], fF[:vectors]) - @test_approx_eq F[:values] fF[:values] + @test F[:values] ≈ fF[:values] end debug && println("Binary operations") @@ -335,12 +335,12 @@ let n = 12 #Size of matrix problem to test fB = map(elty <: Complex ? Complex128 : Float64, full(B)) for op in (+, -, *) - @test_approx_eq full(op(A, B)) op(fA, fB) + @test full(op(A, B)) ≈ op(fA, fB) end α = rand(elty) - @test_approx_eq full(α*A) α*full(A) - @test_approx_eq full(A*α) full(A)*α - @test_approx_eq full(A/α) full(A)/α + @test full(α*A) ≈ α*full(A) + @test full(A*α) ≈ full(A)*α + @test full(A/α) ≈ full(A)/α debug && println("A_mul_B!") @test_throws DimensionMismatch A_mul_B!(zeros(elty,n,n),B,ones(elty,n+1,n)) @@ -411,22 +411,22 @@ let n = 12 #Size of matrix problem to test end debug && println("Multiplication with strided vector") - @test_approx_eq A*ones(n) full(A)*ones(n) + @test A*ones(n) ≈ full(A)*ones(n) debug && println("Multiplication with strided matrix") - @test_approx_eq A*ones(n, 2) full(A)*ones(n, 2) + @test A*ones(n, 2) ≈ full(A)*ones(n, 2) B = Tridiagonal(a, b, c) fB = map(elty <: Complex ? Complex128 : Float64, full(B)) for op in (+, -, *) - @test_approx_eq full(op(A, B)) op(fA, fB) + @test full(op(A, B)) ≈ op(fA, fB) end α = rand(elty) - @test_approx_eq full(α*A) α*full(A) - @test_approx_eq full(A*α) full(A)*α - @test_approx_eq full(A/α) full(A)/α + @test full(α*A) ≈ α*full(A) + @test full(A*α) ≈ full(A)*α + @test full(A/α) ≈ full(A)/α @test_throws ArgumentError convert(SymTridiagonal{elty},A) diff --git a/test/llvmcall.jl b/test/llvmcall.jl index 849f69f1cd09d..5f76228c91031 100644 --- a/test/llvmcall.jl +++ b/test/llvmcall.jl @@ -84,7 +84,7 @@ function declared_floor(x::Float64) ret double %2"""), Float64, Tuple{Float64}, x) end -@test_approx_eq declared_floor(4.2) 4. +@test declared_floor(4.2) ≈ 4. function doubly_declared_floor(x::Float64) llvmcall( @@ -93,7 +93,7 @@ function doubly_declared_floor(x::Float64) ret double %2"""), Float64, Tuple{Float64}, x+1)-1 end -@test_approx_eq doubly_declared_floor(4.2) 4. +@test doubly_declared_floor(4.2) ≈ 4. function doubly_declared2_trunc(x::Float64) a = llvmcall( @@ -108,7 +108,7 @@ function doubly_declared2_trunc(x::Float64) Float64, Tuple{Float64}, x+1)-1 a + b end -@test_approx_eq doubly_declared2_trunc(4.2) 8. +@test doubly_declared2_trunc(4.2) ≈ 8. # Test for single line function declared_ceil(x::Float64) @@ -118,7 +118,7 @@ function declared_ceil(x::Float64) ret double %2"""), Float64, Tuple{Float64}, x) end -@test_approx_eq declared_ceil(4.2) 5.0 +@test declared_ceil(4.2) ≈ 5.0 # Test for multiple lines function ceilfloor(x::Float64) @@ -130,7 +130,7 @@ function ceilfloor(x::Float64) ret double %3"""), Float64, Tuple{Float64}, x) end -@test_approx_eq ceilfloor(7.4) 8.0 +@test ceilfloor(7.4) ≈ 8.0 # Test for proper declaration extraction function confuse_declname_parsing() diff --git a/test/math.jl b/test/math.jl index b273af6debba3..09d3cf814d52f 100644 --- a/test/math.jl +++ b/test/math.jl @@ -76,34 +76,34 @@ for T in (Float32, Float64) y = T(1//2) yi = 4 # Test random values - @test_approx_eq x^y big(x)^big(y) - @test_approx_eq x^yi big(x)^yi - @test_approx_eq acos(x) acos(big(x)) - @test_approx_eq acosh(1+x) acosh(big(1+x)) - @test_approx_eq asin(x) asin(big(x)) - @test_approx_eq asinh(x) asinh(big(x)) - @test_approx_eq atan(x) atan(big(x)) - @test_approx_eq atan2(x,y) atan2(big(x),big(y)) - @test_approx_eq atanh(x) atanh(big(x)) - @test_approx_eq cbrt(x) cbrt(big(x)) - @test_approx_eq cos(x) cos(big(x)) - @test_approx_eq cosh(x) cosh(big(x)) - @test_approx_eq exp(x) exp(big(x)) - @test_approx_eq exp10(x) exp10(big(x)) - @test_approx_eq exp2(x) exp2(big(x)) - @test_approx_eq expm1(x) expm1(big(x)) - @test_approx_eq hypot(x,y) hypot(big(x),big(y)) - @test_approx_eq hypot(x,x,y) hypot(hypot(big(x),big(x)),big(y)) - @test_approx_eq hypot(x,x,y,y) hypot(hypot(big(x),big(x)),hypot(big(y),big(y))) - @test_approx_eq log(x) log(big(x)) - @test_approx_eq log10(x) log10(big(x)) - @test_approx_eq log1p(x) log1p(big(x)) - @test_approx_eq log2(x) log2(big(x)) - @test_approx_eq sin(x) sin(big(x)) - @test_approx_eq sinh(x) sinh(big(x)) - @test_approx_eq sqrt(x) sqrt(big(x)) - @test_approx_eq tan(x) tan(big(x)) - @test_approx_eq tanh(x) tanh(big(x)) + @test x^y ≈ big(x)^big(y) + @test x^yi ≈ big(x)^yi + @test acos(x) ≈ acos(big(x)) + @test acosh(1+x) ≈ acosh(big(1+x)) + @test asin(x) ≈ asin(big(x)) + @test asinh(x) ≈ asinh(big(x)) + @test atan(x) ≈ atan(big(x)) + @test atan2(x,y) ≈ atan2(big(x),big(y)) + @test atanh(x) ≈ atanh(big(x)) + @test cbrt(x) ≈ cbrt(big(x)) + @test cos(x) ≈ cos(big(x)) + @test cosh(x) ≈ cosh(big(x)) + @test exp(x) ≈ exp(big(x)) + @test exp10(x) ≈ exp10(big(x)) + @test exp2(x) ≈ exp2(big(x)) + @test expm1(x) ≈ expm1(big(x)) + @test hypot(x,y) ≈ hypot(big(x),big(y)) + @test hypot(x,x,y) ≈ hypot(hypot(big(x),big(x)),big(y)) + @test hypot(x,x,y,y) ≈ hypot(hypot(big(x),big(x)),hypot(big(y),big(y))) + @test log(x) ≈ log(big(x)) + @test log10(x) ≈ log10(big(x)) + @test log1p(x) ≈ log1p(big(x)) + @test log2(x) ≈ log2(big(x)) + @test sin(x) ≈ sin(big(x)) + @test sinh(x) ≈ sinh(big(x)) + @test sqrt(x) ≈ sqrt(big(x)) + @test tan(x) ≈ tan(big(x)) + @test tanh(x) ≈ tanh(big(x)) # Test special values @test isequal(T(1//4)^T(1//2), T(1//2)) @test isequal(T(1//4)^2, T(1//16)) @@ -142,39 +142,39 @@ for T in (Float32, Float64) @test isequal(tan(T(0)), T(0)) @test_approx_eq_eps tan(T(pi)/4) T(1) eps(T) # Test inverses - @test_approx_eq acos(cos(x)) x - @test_approx_eq acosh(cosh(x)) x - @test_approx_eq asin(sin(x)) x - @test_approx_eq cbrt(x)^3 x - @test_approx_eq cbrt(x^3) x - @test_approx_eq asinh(sinh(x)) x - @test_approx_eq atan(tan(x)) x - @test_approx_eq atan2(x,y) atan(x/y) - @test_approx_eq atanh(tanh(x)) x - @test_approx_eq cos(acos(x)) x - @test_approx_eq cosh(acosh(1+x)) 1+x - @test_approx_eq exp(log(x)) x - @test_approx_eq exp10(log10(x)) x - @test_approx_eq exp2(log2(x)) x - @test_approx_eq expm1(log1p(x)) x - @test_approx_eq log(exp(x)) x - @test_approx_eq log10(exp10(x)) x - @test_approx_eq log1p(expm1(x)) x - @test_approx_eq log2(exp2(x)) x - @test_approx_eq sin(asin(x)) x - @test_approx_eq sinh(asinh(x)) x - @test_approx_eq sqrt(x)^2 x - @test_approx_eq sqrt(x^2) x - @test_approx_eq tan(atan(x)) x - @test_approx_eq tanh(atanh(x)) x + @test acos(cos(x)) ≈ x + @test acosh(cosh(x)) ≈ x + @test asin(sin(x)) ≈ x + @test cbrt(x)^3 ≈ x + @test cbrt(x^3) ≈ x + @test asinh(sinh(x)) ≈ x + @test atan(tan(x)) ≈ x + @test atan2(x,y) ≈ atan(x/y) + @test atanh(tanh(x)) ≈ x + @test cos(acos(x)) ≈ x + @test cosh(acosh(1+x)) ≈ 1+x + @test exp(log(x)) ≈ x + @test exp10(log10(x)) ≈ x + @test exp2(log2(x)) ≈ x + @test expm1(log1p(x)) ≈ x + @test log(exp(x)) ≈ x + @test log10(exp10(x)) ≈ x + @test log1p(expm1(x)) ≈ x + @test log2(exp2(x)) ≈ x + @test sin(asin(x)) ≈ x + @test sinh(asinh(x)) ≈ x + @test sqrt(x)^2 ≈ x + @test sqrt(x^2) ≈ x + @test tan(atan(x)) ≈ x + @test tanh(atanh(x)) ≈ x # Test some properties - @test_approx_eq cosh(x) (exp(x)+exp(-x))/2 - @test_approx_eq cosh(x)^2-sinh(x)^2 1 - @test_approx_eq hypot(x,y) sqrt(x^2+y^2) - @test_approx_eq sin(x)^2+cos(x)^2 1 - @test_approx_eq sinh(x) (exp(x)-exp(-x))/2 - @test_approx_eq tan(x) sin(x)/cos(x) - @test_approx_eq tanh(x) sinh(x)/cosh(x) + @test cosh(x) ≈ (exp(x)+exp(-x))/2 + @test cosh(x)^2-sinh(x)^2 ≈ 1 + @test hypot(x,y) ≈ sqrt(x^2+y^2) + @test sin(x)^2+cos(x)^2 ≈ 1 + @test sinh(x) ≈ (exp(x)-exp(-x))/2 + @test tan(x) ≈ sin(x)/cos(x) + @test tanh(x) ≈ sinh(x)/cosh(x) #Edge cases @test isinf(log(zero(T))) @@ -189,17 +189,17 @@ for T in (Float32, Float64) @test hypot(T(Inf), T(NaN)) === T(Inf) @test isnan(hypot(T(x), T(NaN))) end -@test_approx_eq exp10(5) exp10(5.0) -@test_approx_eq exp2(Float16(2.)) exp2(2.) +@test exp10(5) ≈ exp10(5.0) +@test exp2(Float16(2.)) ≈ exp2(2.) @test log(e) == 1 for T in (Int, Float64, BigFloat) - @test_approx_eq deg2rad(T(180)) 1pi - @test_approx_eq deg2rad(T[45, 60]) [pi/T(4), pi/T(3)] - @test_approx_eq rad2deg([pi/T(4), pi/T(3)]) [45, 60] - @test_approx_eq rad2deg(T(1)*pi) 180 - @test_approx_eq rad2deg(T(1)) rad2deg(true) - @test_approx_eq deg2rad(T(1)) deg2rad(true) + @test deg2rad(T(180)) ≈ 1pi + @test deg2rad(T[45, 60]) ≈ [pi/T(4), pi/T(3)] + @test rad2deg([pi/T(4), pi/T(3)]) ≈ [45, 60] + @test rad2deg(T(1)*pi) ≈ 180 + @test rad2deg(T(1)) ≈ rad2deg(true) + @test deg2rad(T(1)) ≈ deg2rad(true) end # degree-based trig functions @@ -274,23 +274,23 @@ for T = (Float32,Float64,BigFloat) end # error functions -@test_approx_eq erf(Float16(1)) 0.84270079294971486934 -@test_approx_eq erf(1) 0.84270079294971486934 -@test_approx_eq erfc(1) 0.15729920705028513066 -@test_approx_eq erfc(Float16(1)) 0.15729920705028513066 -@test_approx_eq erfcx(1) 0.42758357615580700442 -@test_approx_eq erfcx(Float32(1)) 0.42758357615580700442 -@test_approx_eq erfcx(Complex64(1)) 0.42758357615580700442 -@test_approx_eq erfi(1) 1.6504257587975428760 -@test_approx_eq erfinv(0.84270079294971486934) 1 -@test_approx_eq erfcinv(0.15729920705028513066) 1 -@test_approx_eq dawson(1) 0.53807950691276841914 - -@test_approx_eq erf(1+2im) -0.53664356577856503399-5.0491437034470346695im -@test_approx_eq erfc(1+2im) 1.5366435657785650340+5.0491437034470346695im -@test_approx_eq erfcx(1+2im) 0.14023958136627794370-0.22221344017989910261im -@test_approx_eq erfi(1+2im) -0.011259006028815025076+1.0036063427256517509im -@test_approx_eq dawson(1+2im) -13.388927316482919244-11.828715103889593303im +@test erf(Float16(1)) ≈ 0.84270079294971486934 +@test erf(1) ≈ 0.84270079294971486934 +@test erfc(1) ≈ 0.15729920705028513066 +@test erfc(Float16(1)) ≈ 0.15729920705028513066 +@test erfcx(1) ≈ 0.42758357615580700442 +@test erfcx(Float32(1)) ≈ 0.42758357615580700442 +@test erfcx(Complex64(1)) ≈ 0.42758357615580700442 +@test erfi(1) ≈ 1.6504257587975428760 +@test erfinv(0.84270079294971486934) ≈ 1 +@test erfcinv(0.15729920705028513066) ≈ 1 +@test dawson(1) ≈ 0.53807950691276841914 + +@test erf(1+2im) ≈ -0.53664356577856503399-5.0491437034470346695im +@test erfc(1+2im) ≈ 1.5366435657785650340+5.0491437034470346695im +@test erfcx(1+2im) ≈ 0.14023958136627794370-0.22221344017989910261im +@test erfi(1+2im) ≈ -0.011259006028815025076+1.0036063427256517509im +@test dawson(1+2im) ≈ -13.388927316482919244-11.828715103889593303im for elty in [Float32,Float64] for x in logspace(-200, -0.01) @@ -316,40 +316,40 @@ end @test erfcinv(one(Int)) == erfcinv(1.0) # airy -@test_approx_eq airy(1.8) airyai(1.8) -@test_approx_eq airyprime(1.8) -0.0685247801186109345638 -@test_approx_eq airyaiprime(1.8) airyprime(1.8) -@test_approx_eq airybi(1.8) 2.595869356743906290060 -@test_approx_eq airybiprime(1.8) 2.98554005084659907283 +@test airy(1.8) ≈ airyai(1.8) +@test airyprime(1.8) ≈ -0.0685247801186109345638 +@test airyaiprime(1.8) ≈ airyprime(1.8) +@test airybi(1.8) ≈ 2.595869356743906290060 +@test airybiprime(1.8) ≈ 2.98554005084659907283 @test_throws Base.Math.AmosException airy(200im) @test_throws Base.Math.AmosException airybi(200) @test_throws ArgumentError airy(5,one(Complex128)) z = 1.8 + 1.0im for elty in [Complex64,Complex128] - @test_approx_eq airy(convert(elty,1.8)) 0.0470362168668458052247 + @test airy(convert(elty,1.8)) ≈ 0.0470362168668458052247 z = convert(elty,z) - @test_approx_eq airyx(z) airyx(0,z) - @test_approx_eq airyx(0, z) airy(0, z) * exp(2/3 * z * sqrt(z)) - @test_approx_eq airyx(1, z) airy(1, z) * exp(2/3 * z * sqrt(z)) - @test_approx_eq airyx(2, z) airy(2, z) * exp(-abs(real(2/3 * z * sqrt(z)))) - @test_approx_eq airyx(3, z) airy(3, z) * exp(-abs(real(2/3 * z * sqrt(z)))) + @test airyx(z) ≈ airyx(0,z) + @test airyx(0, z) ≈ airy(0, z) * exp(2/3 * z * sqrt(z)) + @test airyx(1, z) ≈ airy(1, z) * exp(2/3 * z * sqrt(z)) + @test airyx(2, z) ≈ airy(2, z) * exp(-abs(real(2/3 * z * sqrt(z)))) + @test airyx(3, z) ≈ airy(3, z) * exp(-abs(real(2/3 * z * sqrt(z)))) @test_throws ArgumentError airyx(5,z) end @test_throws MethodError airy(complex(big(1.0))) # bessely0, bessely1, besselj0, besselj1 -@test_approx_eq besselj0(Float32(2.0)) besselj0(Float64(2.0)) -@test_approx_eq besselj1(Float32(2.0)) besselj1(Float64(2.0)) -@test_approx_eq bessely0(Float32(2.0)) bessely0(Float64(2.0)) -@test_approx_eq bessely1(Float32(2.0)) bessely1(Float64(2.0)) -@test_approx_eq besselj0(2) besselj0(2.0) -@test_approx_eq besselj1(2) besselj1(2.0) -@test_approx_eq bessely0(2) bessely0(2.0) -@test_approx_eq bessely1(2) bessely1(2.0) -@test_approx_eq besselj0(2.0 + im) besselj(0, 2.0 + im) -@test_approx_eq besselj1(2.0 + im) besselj(1, 2.0 + im) -@test_approx_eq bessely0(2.0 + im) bessely(0, 2.0 + im) -@test_approx_eq bessely1(2.0 + im) bessely(1, 2.0 + im) +@test besselj0(Float32(2.0)) ≈ besselj0(Float64(2.0)) +@test besselj1(Float32(2.0)) ≈ besselj1(Float64(2.0)) +@test bessely0(Float32(2.0)) ≈ bessely0(Float64(2.0)) +@test bessely1(Float32(2.0)) ≈ bessely1(Float64(2.0)) +@test besselj0(2) ≈ besselj0(2.0) +@test besselj1(2) ≈ besselj1(2.0) +@test bessely0(2) ≈ bessely0(2.0) +@test bessely1(2) ≈ bessely1(2.0) +@test besselj0(2.0 + im) ≈ besselj(0, 2.0 + im) +@test besselj1(2.0 + im) ≈ besselj(1, 2.0 + im) +@test bessely0(2.0 + im) ≈ bessely(0, 2.0 + im) +@test bessely1(2.0 + im) ≈ bessely(1, 2.0 + im) @test_throws MethodError besselj(1.2,big(1.0)) @test_throws MethodError besselj(1,complex(big(1.0))) @@ -358,10 +358,10 @@ end # besselh true_h133 = 0.30906272225525164362 - 0.53854161610503161800im -@test_approx_eq besselh(3,1,3) true_h133 -@test_approx_eq besselh(-3,1,3) -true_h133 -@test_approx_eq besselh(3,2,3) conj(true_h133) -@test_approx_eq besselh(-3,2,3) -conj(true_h133) +@test besselh(3,1,3) ≈ true_h133 +@test besselh(-3,1,3) ≈ -true_h133 +@test besselh(3,2,3) ≈ conj(true_h133) +@test besselh(-3,2,3) ≈ -conj(true_h133) @test_throws Base.Math.AmosException besselh(1,0) @test_throws MethodError besselh(1,big(1.0)) @@ -371,11 +371,11 @@ true_h133 = 0.30906272225525164362 - 0.53854161610503161800im # besseli true_i33 = 0.95975362949600785698 -@test_approx_eq besseli(3,3) true_i33 -@test_approx_eq besseli(-3,3) true_i33 -@test_approx_eq besseli(3,-3) -true_i33 -@test_approx_eq besseli(-3,-3) -true_i33 -@test_approx_eq besseli(Float32(-3),Complex64(-3,0)) -true_i33 +@test besseli(3,3) ≈ true_i33 +@test besseli(-3,3) ≈ true_i33 +@test besseli(3,-3) ≈ -true_i33 +@test besseli(-3,-3) ≈ -true_i33 +@test besseli(Float32(-3),Complex64(-3,0)) ≈ -true_i33 @test_throws Base.Math.AmosException besseli(1,1000) @test_throws DomainError besseli(0.4,-1.0) @@ -414,24 +414,24 @@ j43 = besselj(4,3.) @test besselj(4,complex(3f0)) ≈ j43 @test besselj(4,complex(3)) ≈ j43 -@test_approx_eq j33 0.30906272225525164362 -@test_approx_eq j43 0.13203418392461221033 +@test j33 ≈ 0.30906272225525164362 +@test j43 ≈ 0.13203418392461221033 @test_throws DomainError besselj(0.1, -0.4) -@test_approx_eq besselj(0.1, complex(-0.4)) 0.820421842809028916 + 0.266571215948350899im -@test_approx_eq besselj(3.2, 1.3+0.6im) 0.01135309305831220201 + 0.03927719044393515275im -@test_approx_eq besselj(1, 3im) 3.953370217402609396im -@test_approx_eq besselj(1.0,3im) besselj(1,3im) +@test besselj(0.1, complex(-0.4)) ≈ 0.820421842809028916 + 0.266571215948350899im +@test besselj(3.2, 1.3+0.6im) ≈ 0.01135309305831220201 + 0.03927719044393515275im +@test besselj(1, 3im) ≈ 3.953370217402609396im +@test besselj(1.0,3im) ≈ besselj(1,3im) @test_throws Base.Math.AmosException besselj(20,1000im) @test_throws MethodError besselj(big(1.0),3im) # besselk true_k33 = 0.12217037575718356792 -@test_approx_eq besselk(3,3) true_k33 -@test_approx_eq besselk(-3,3) true_k33 +@test besselk(3,3) ≈ true_k33 +@test besselk(-3,3) ≈ true_k33 true_k3m3 = -0.1221703757571835679 - 3.0151549516807985776im @test_throws DomainError besselk(3,-3) -@test_approx_eq besselk(3,complex(-3)) true_k3m3 -@test_approx_eq besselk(-3,complex(-3)) true_k3m3 +@test besselk(3,complex(-3)) ≈ true_k3m3 +@test besselk(-3,complex(-3)) ≈ true_k3m3 @test_throws Base.Math.AmosException besselk(200,0.01) # issue #6564 @test besselk(1.0,0.0) == Inf @@ -446,11 +446,11 @@ true_k3m3 = -0.1221703757571835679 - 3.0151549516807985776im y33 = bessely(3,3.) @test bessely(3,3) == y33 @test bessely(3.,3.) == y33 -@test_approx_eq bessely(3,Float32(3.)) y33 -@test_approx_eq bessely(-3,3) -y33 -@test_approx_eq y33 -0.53854161610503161800 +@test bessely(3,Float32(3.)) ≈ y33 +@test bessely(-3,3) ≈ -y33 +@test y33 ≈ -0.53854161610503161800 @test_throws DomainError bessely(3,-3) -@test_approx_eq bessely(3,complex(-3)) 0.53854161610503161800 - 0.61812544451050328724im +@test bessely(3,complex(-3)) ≈ 0.53854161610503161800 - 0.61812544451050328724im @test_throws Base.Math.AmosException bessely(200.5,0.1) @test_throws DomainError bessely(0.4,-1.0) @test_throws DomainError bessely(0.4,Float32(-1.0)) @@ -467,8 +467,8 @@ y33 = bessely(3,3.) #besselhx for elty in [Complex64,Complex128] z = convert(elty, 1.0 + 1.9im) - @test_approx_eq besselhx(1.0, 1, z) convert(elty,-0.5949634147786144 - 0.18451272807835967im) - @test_approx_eq besselhx(Float32(1.0), 1, z) convert(elty,-0.5949634147786144 - 0.18451272807835967im) + @test besselhx(1.0, 1, z) ≈ convert(elty,-0.5949634147786144 - 0.18451272807835967im) + @test besselhx(Float32(1.0), 1, z) ≈ convert(elty,-0.5949634147786144 - 0.18451272807835967im) end @test_throws MethodError besselh(1,1,big(1.0)) @@ -478,19 +478,19 @@ end # issue #6653 for f in (besselj,bessely,besseli,besselk,hankelh1,hankelh2) - @test_approx_eq f(0,1) f(0,Complex128(1)) - @test_approx_eq f(0,1) f(0,Complex64(1)) + @test f(0,1) ≈ f(0,Complex128(1)) + @test f(0,1) ≈ f(0,Complex64(1)) end # scaled bessel[ijky] and hankelh[12] for x in (1.0, 0.0, -1.0), y in (1.0, 0.0, -1.0), nu in (1.0, 0.0, -1.0) z = Complex128(x + y * im) - z == zero(z) || @test_approx_eq hankelh1x(nu, z) hankelh1(nu, z) * exp(-z * im) - z == zero(z) || @test_approx_eq hankelh2x(nu, z) hankelh2(nu, z) * exp(z * im) - (nu < 0 && z == zero(z)) || @test_approx_eq besselix(nu, z) besseli(nu, z) * exp(-abs(real(z))) - (nu < 0 && z == zero(z)) || @test_approx_eq besseljx(nu, z) besselj(nu, z) * exp(-abs(imag(z))) - z == zero(z) || @test_approx_eq besselkx(nu, z) besselk(nu, z) * exp(z) - z == zero(z) || @test_approx_eq besselyx(nu, z) bessely(nu, z) * exp(-abs(imag(z))) + z == zero(z) || @test hankelh1x(nu, z) ≈ hankelh1(nu, z) * exp(-z * im) + z == zero(z) || @test hankelh2x(nu, z) ≈ hankelh2(nu, z) * exp(z * im) + (nu < 0 && z == zero(z)) || @test besselix(nu, z) ≈ besseli(nu, z) * exp(-abs(real(z))) + (nu < 0 && z == zero(z)) || @test besseljx(nu, z) ≈ besselj(nu, z) * exp(-abs(imag(z))) + z == zero(z) || @test besselkx(nu, z) ≈ besselk(nu, z) * exp(z) + z == zero(z) || @test besselyx(nu, z) ≈ bessely(nu, z) * exp(-abs(imag(z))) end @test_throws Base.Math.AmosException hankelh1x(1, 0) @test_throws Base.Math.AmosException hankelh2x(1, 0) @@ -504,12 +504,12 @@ end @test_throws DomainError besselyx(0.4,-1.0) # beta, lbeta -@test_approx_eq beta(3/2,7/2) 5π/128 -@test_approx_eq beta(3,5) 1/105 -@test_approx_eq lbeta(5,4) log(beta(5,4)) -@test_approx_eq beta(5,4) beta(4,5) +@test beta(3/2,7/2) ≈ 5π/128 +@test beta(3,5) ≈ 1/105 +@test lbeta(5,4) ≈ log(beta(5,4)) +@test beta(5,4) ≈ beta(4,5) @test beta(-1/2, 3) ≈ beta(-1/2 + 0im, 3 + 0im) ≈ -16/3 -@test_approx_eq lbeta(-1/2, 3) log(16/3) +@test lbeta(-1/2, 3) ≈ log(16/3) @test beta(Float32(5),Float32(4)) == beta(Float32(4),Float32(5)) @test beta(3,5) ≈ beta(3+0im,5+0im) @test(beta(3.2+0.1im,5.3+0.3im) ≈ exp(lbeta(3.2+0.1im,5.3+0.3im)) ≈ @@ -520,16 +520,16 @@ end if Base.Math.libm == "libopenlibm" @test gamma(Float64[1:25;]) == gamma(1:25) else - @test_approx_eq gamma(Float64[1:25;]) gamma(1:25) + @test gamma(Float64[1:25;]) ≈ gamma(1:25) end for elty in (Float32, Float64) - @test_approx_eq gamma(convert(elty,1/2)) convert(elty,sqrt(π)) - @test_approx_eq gamma(convert(elty,-1/2)) convert(elty,-2sqrt(π)) - @test_approx_eq lgamma(convert(elty,-1/2)) convert(elty,log(abs(gamma(-1/2)))) + @test gamma(convert(elty,1/2)) ≈ convert(elty,sqrt(π)) + @test gamma(convert(elty,-1/2)) ≈ convert(elty,-2sqrt(π)) + @test lgamma(convert(elty,-1/2)) ≈ convert(elty,log(abs(gamma(-1/2)))) end -@test_approx_eq lgamma(1.4+3.7im) -3.7094025330996841898 + 2.4568090502768651184im -@test_approx_eq lgamma(1.4+3.7im) log(gamma(1.4+3.7im)) -@test_approx_eq lgamma(-4.2+0im) lgamma(-4.2)-pi*im +@test lgamma(1.4+3.7im) ≈ -3.7094025330996841898 + 2.4568090502768651184im +@test lgamma(1.4+3.7im) ≈ log(gamma(1.4+3.7im)) +@test lgamma(-4.2+0im) ≈ lgamma(-4.2)-pi*im @test factorial(3.0) == gamma(4.0) == factorial(3) for x in (3.2, 2+1im, 3//2, 3.2+0.1im) @test factorial(x) == gamma(1+x) @@ -540,38 +540,38 @@ end # digamma for elty in (Float32, Float64) - @test_approx_eq digamma(convert(elty, 9)) convert(elty, 2.140641477955609996536345) - @test_approx_eq digamma(convert(elty, 2.5)) convert(elty, 0.7031566406452431872257) - @test_approx_eq digamma(convert(elty, 0.1)) convert(elty, -10.42375494041107679516822) - @test_approx_eq digamma(convert(elty, 7e-4)) convert(elty, -1429.147493371120205005198) - @test_approx_eq digamma(convert(elty, 7e-5)) convert(elty, -14286.29138623969227538398) - @test_approx_eq digamma(convert(elty, 7e-6)) convert(elty, -142857.7200612932791081972) - @test_approx_eq digamma(convert(elty, 2e-6)) convert(elty, -500000.5772123750382073831) - @test_approx_eq digamma(convert(elty, 1e-6)) convert(elty, -1000000.577214019968668068) - @test_approx_eq digamma(convert(elty, 7e-7)) convert(elty, -1428572.005785942019703646) - @test_approx_eq digamma(convert(elty, -0.5)) convert(elty, .03648997397857652055902367) - @test_approx_eq digamma(convert(elty, -1.1)) convert(elty, 10.15416395914385769902271) - - @test_approx_eq digamma(convert(elty, 0.1)) convert(elty, -10.42375494041108) - @test_approx_eq digamma(convert(elty, 1/2)) convert(elty, -γ - log(4)) - @test_approx_eq digamma(convert(elty, 1)) convert(elty, -γ) - @test_approx_eq digamma(convert(elty, 2)) convert(elty, 1 - γ) - @test_approx_eq digamma(convert(elty, 3)) convert(elty, 3/2 - γ) - @test_approx_eq digamma(convert(elty, 4)) convert(elty, 11/6 - γ) - @test_approx_eq digamma(convert(elty, 5)) convert(elty, 25/12 - γ) - @test_approx_eq digamma(convert(elty, 10)) convert(elty, 7129/2520 - γ) + @test digamma(convert(elty, 9)) ≈ convert(elty, 2.140641477955609996536345) + @test digamma(convert(elty, 2.5)) ≈ convert(elty, 0.7031566406452431872257) + @test digamma(convert(elty, 0.1)) ≈ convert(elty, -10.42375494041107679516822) + @test digamma(convert(elty, 7e-4)) ≈ convert(elty, -1429.147493371120205005198) + @test digamma(convert(elty, 7e-5)) ≈ convert(elty, -14286.29138623969227538398) + @test digamma(convert(elty, 7e-6)) ≈ convert(elty, -142857.7200612932791081972) + @test digamma(convert(elty, 2e-6)) ≈ convert(elty, -500000.5772123750382073831) + @test digamma(convert(elty, 1e-6)) ≈ convert(elty, -1000000.577214019968668068) + @test digamma(convert(elty, 7e-7)) ≈ convert(elty, -1428572.005785942019703646) + @test digamma(convert(elty, -0.5)) ≈ convert(elty, .03648997397857652055902367) + @test digamma(convert(elty, -1.1)) ≈ convert(elty, 10.15416395914385769902271) + + @test digamma(convert(elty, 0.1)) ≈ convert(elty, -10.42375494041108) + @test digamma(convert(elty, 1/2)) ≈ convert(elty, -γ - log(4)) + @test digamma(convert(elty, 1)) ≈ convert(elty, -γ) + @test digamma(convert(elty, 2)) ≈ convert(elty, 1 - γ) + @test digamma(convert(elty, 3)) ≈ convert(elty, 3/2 - γ) + @test digamma(convert(elty, 4)) ≈ convert(elty, 11/6 - γ) + @test digamma(convert(elty, 5)) ≈ convert(elty, 25/12 - γ) + @test digamma(convert(elty, 10)) ≈ convert(elty, 7129/2520 - γ) end # trigamma for elty in (Float32, Float64) - @test_approx_eq trigamma(convert(elty, 0.1)) convert(elty, 101.433299150792758817) - @test_approx_eq trigamma(convert(elty, 1/2)) convert(elty, π^2/2) - @test_approx_eq trigamma(convert(elty, 1)) convert(elty, π^2/6) - @test_approx_eq trigamma(convert(elty, 2)) convert(elty, π^2/6 - 1) - @test_approx_eq trigamma(convert(elty, 3)) convert(elty, π^2/6 - 5/4) - @test_approx_eq trigamma(convert(elty, 4)) convert(elty, π^2/6 - 49/36) - @test_approx_eq trigamma(convert(elty, 5)) convert(elty, π^2/6 - 205/144) - @test_approx_eq trigamma(convert(elty, 10)) convert(elty, π^2/6 - 9778141/6350400) + @test trigamma(convert(elty, 0.1)) ≈ convert(elty, 101.433299150792758817) + @test trigamma(convert(elty, 1/2)) ≈ convert(elty, π^2/2) + @test trigamma(convert(elty, 1)) ≈ convert(elty, π^2/6) + @test trigamma(convert(elty, 2)) ≈ convert(elty, π^2/6 - 1) + @test trigamma(convert(elty, 3)) ≈ convert(elty, π^2/6 - 5/4) + @test trigamma(convert(elty, 4)) ≈ convert(elty, π^2/6 - 49/36) + @test trigamma(convert(elty, 5)) ≈ convert(elty, π^2/6 - 205/144) + @test trigamma(convert(elty, 10)) ≈ convert(elty, π^2/6 - 9778141/6350400) end # invdigamma @@ -582,36 +582,37 @@ for elty in (Float32, Float64) end @test abs(invdigamma(2)) == abs(invdigamma(2.)) -@test_approx_eq polygamma(20, 7.) -4.644616027240543262561198814998587152547 -@test_approx_eq polygamma(20, Float16(7.)) -4.644616027240543262561198814998587152547 +@test polygamma(20, 7.) ≈ -4.644616027240543262561198814998587152547 +@test polygamma(20, Float16(7.)) ≈ -4.644616027240543262561198814998587152547 # eta, zeta -@test_approx_eq eta(1) log(2) -@test_approx_eq eta(2) pi^2/12 -@test_approx_eq eta(Float32(2)) eta(2) -@test_approx_eq eta(Complex64(2)) eta(2) -@test_approx_eq zeta(0) -0.5 -@test_approx_eq zeta(2) pi^2/6 -@test_approx_eq zeta(Complex64(2)) zeta(2) -@test_approx_eq zeta(4) pi^4/90 -@test_approx_eq zeta(one(Float32)) Float32(zeta(one(Float64))) -@test_approx_eq zeta(1,Float16(2.)) zeta(1,2.) -@test_approx_eq zeta(1.,Float16(2.)) zeta(1,2.) -@test_approx_eq zeta(Float16(1.),Float16(2.)) zeta(1,2.) +@test eta(1) ≈ log(2) +@test eta(2) ≈ pi^2/12 +@test eta(Float32(2)) ≈ eta(2) +@test eta(Complex64(2)) ≈ eta(2) +@test zeta(0) ≈ -0.5 +@test zeta(2) ≈ pi^2/6 +@test zeta(Complex64(2)) ≈ zeta(2) +@test zeta(4) ≈ pi^4/90 +@test zeta(1,Float16(2.)) ≈ zeta(1,2.) +@test zeta(1.,Float16(2.)) ≈ zeta(1,2.) +@test zeta(Float16(1.),Float16(2.)) ≈ zeta(1,2.) @test isnan(zeta(NaN)) +@test isnan(zeta(1.0e0)) +@test isnan(zeta(1.0f0)) @test isnan(zeta(complex(0,Inf))) @test isnan(zeta(complex(-Inf,0))) # quadgk -@test_approx_eq quadgk(cos, 0,0.7,1)[1] sin(1) -@test_approx_eq quadgk(x -> exp(im*x), 0,0.7,1)[1] (exp(1im)-1)/im -@test_approx_eq quadgk(x -> exp(im*x), 0,1im)[1] -1im*expm1(-1) +@test quadgk(cos, 0,0.7,1)[1] ≈ sin(1) +@test quadgk(x -> exp(im*x), 0,0.7,1)[1] ≈ (exp(1im)-1)/im +@test quadgk(x -> exp(im*x), 0,1im)[1] ≈ -1im*expm1(-1) @test_approx_eq_eps quadgk(cos, 0,BigFloat(1),order=40)[1] sin(BigFloat(1)) 1000*eps(BigFloat) -@test_approx_eq quadgk(x -> exp(-x), 0,0.7,Inf)[1] 1.0 -@test_approx_eq quadgk(x -> exp(x), -Inf,0)[1] 1.0 -@test_approx_eq quadgk(x -> exp(-x^2), -Inf,Inf)[1] sqrt(pi) -@test_approx_eq quadgk(x -> [exp(-x), exp(-2x)], 0, Inf)[1] [1,0.5] -@test_approx_eq quadgk(cos, 0,0.7,1, norm=abs)[1] sin(1) +@test quadgk(x -> exp(-x), 0,0.7,Inf)[1] ≈ 1.0 +@test quadgk(x -> exp(x), -Inf,0)[1] ≈ 1.0 +@test quadgk(x -> exp(-x^2), -Inf,Inf)[1] ≈ sqrt(pi) +@test quadgk(x -> [exp(-x), exp(-2x)], 0, Inf)[1] ≈ [1,0.5] +@test quadgk(cos, 0,0.7,1, norm=abs)[1] ≈ sin(1) # Ensure subnormal flags functions don't segfault @test any(set_zero_subnormals(true) .== [false,true]) @@ -663,7 +664,7 @@ end @test isa([trigamma(x) for x in [1.0]], Vector{Float64}) @test isa([polygamma(3,x) for x in [1.0]], Vector{Float64}) @test zeta(2 + 1im, -1.1) ≅ zeta(2 + 1im, -1.1+0im) ≅ -64.580137707692178058665068045847533319237536295165484548 + 73.992688148809018073371913557697318846844796582012921247im -@test_approx_eq polygamma(3,5) polygamma(3,5.) +@test polygamma(3,5) ≈ polygamma(3,5.) @test zeta(-3.0, 7.0) ≅ -52919/120 @test zeta(-3.0, -7.0) ≅ 94081/120 @@ -713,26 +714,26 @@ c = 3 @test 1e-13 > errc(zeta(2 + 99.69im, 1.3), 0.41617652544777996034143623540420694985469543821307918291931-0.74199610821536326325073784018327392143031681111201859489991im) for z in (1.234, 1.234 + 5.678im, [1.234, 5.678]) - @test_approx_eq cis(z) exp(im*z) + @test cis(z) ≈ exp(im*z) end # modf for elty in (Float16, Float32, Float64) - @test_approx_eq modf( convert(elty,1.2) )[1] convert(elty,0.2) - @test_approx_eq modf( convert(elty,1.2) )[2] convert(elty,1.0) - @test_approx_eq modf( convert(elty,1.0) )[1] convert(elty,0.0) - @test_approx_eq modf( convert(elty,1.0) )[2] convert(elty,1.0) + @test modf(convert(elty,1.2))[1] ≈ convert(elty,0.2) + @test modf(convert(elty,1.2))[2] ≈ convert(elty,1.0) + @test modf(convert(elty,1.0))[1] ≈ convert(elty,0.0) + @test modf(convert(elty,1.0))[2] ≈ convert(elty,1.0) end # frexp for elty in (Float32, Float64) - @test frexp( convert(elty,0.5) ) == (convert(elty,0.5),0) - @test frexp( convert(elty,4.0) ) == (convert(elty,0.5),3) - @test_approx_eq frexp( convert(elty,10.5) )[1] convert(elty,0.65625) - @test frexp( convert(elty,10.5) )[2] == 4 - @test_approx_eq frexp( [ convert(elty,4.0) convert(elty,10.5) ] )[1][1] convert(elty,0.5) - @test_approx_eq frexp( [ convert(elty,4.0) convert(elty,10.5) ] )[1][2] convert(elty,0.65625) - @test frexp( [ convert(elty,4.0) convert(elty,10.5) ] )[2] == [ 3 4 ] + @test frexp(convert(elty,0.5)) == (convert(elty,0.5),0) + @test frexp(convert(elty,4.0)) == (convert(elty,0.5),3) + @test frexp(convert(elty,10.5))[1] ≈ convert(elty,0.65625) + @test frexp(convert(elty,10.5) )[2] == 4 + @test frexp([convert(elty,4.0) convert(elty,10.5)])[1][1] ≈ convert(elty,0.5) + @test frexp([convert(elty,4.0) convert(elty,10.5)])[1][2] ≈ convert(elty,0.65625) + @test frexp([convert(elty,4.0) convert(elty,10.5)])[2] == [3 4] end # log/log1p diff --git a/test/mod2pi.jl b/test/mod2pi.jl index f73aecdb4ee69..05e8229159e86 100644 --- a/test/mod2pi.jl +++ b/test/mod2pi.jl @@ -181,16 +181,16 @@ function testModPi() sort!(errsOld) totalErrNew = sum(errsNew) totalErrOld = sum(errsOld) - @test_approx_eq totalErrNew 0.0 + @test totalErrNew ≈ 0.0 end testModPi() # 2pi -@test_approx_eq mod2pi(10) mod(10,2pi) -@test_approx_eq mod2pi(-10) mod(-10,2pi) -@test_approx_eq mod2pi(355) 3.1416227979431572 -@test_approx_eq mod2pi(Int32(355)) 3.1416227979431572 -@test_approx_eq mod2pi(355.0) 3.1416227979431572 -@test_approx_eq mod2pi(355.0f0) 3.1416228f0 +@test mod2pi(10) ≈ mod(10,2pi) +@test mod2pi(-10) ≈ mod(-10,2pi) +@test mod2pi(355) ≈ 3.1416227979431572 +@test mod2pi(Int32(355)) ≈ 3.1416227979431572 +@test mod2pi(355.0) ≈ 3.1416227979431572 +@test mod2pi(355.0f0) ≈ 3.1416228f0 @test mod2pi(Int64(2)^60) == mod2pi(2.0^60) @test_throws ArgumentError mod2pi(Int64(2)^60-1) diff --git a/test/mpfr.jl b/test/mpfr.jl index 40f5759ec370e..0c62de5765317 100644 --- a/test/mpfr.jl +++ b/test/mpfr.jl @@ -8,21 +8,21 @@ setprecision(53) do end x = BigFloat(12) y = BigFloat(x) -@test_approx_eq x y +@test x ≈ y y = BigFloat(0xc) -@test_approx_eq x y +@test x ≈ y y = BigFloat(12.) -@test_approx_eq x y +@test x ≈ y y = BigFloat(BigInt(12)) -@test_approx_eq x y +@test x ≈ y y = BigFloat(BigFloat(12)) -@test_approx_eq x y +@test x ≈ y y = parse(BigFloat,"12") -@test_approx_eq x y +@test x ≈ y y = BigFloat(Float32(12.)) -@test_approx_eq x y +@test x ≈ y y = BigFloat(12//1) -@test_approx_eq x y +@test x ≈ y # + x = BigFloat(12) @@ -422,12 +422,12 @@ end # bessel functions setprecision(53) do - @test_approx_eq besselj(4, BigFloat(2)) besselj(4, 2.) - @test_approx_eq besselj0(BigFloat(2)) besselj0(2.) - @test_approx_eq besselj1(BigFloat(2)) besselj1(2.) - @test_approx_eq bessely(4, BigFloat(2)) bessely(4, 2.) - @test_approx_eq bessely0(BigFloat(2)) bessely0(2.) - @test_approx_eq bessely1(BigFloat(2)) bessely1(2.) + @test besselj(4, BigFloat(2)) ≈ besselj(4, 2.) + @test besselj0(BigFloat(2)) ≈ besselj0(2.) + @test besselj1(BigFloat(2)) ≈ besselj1(2.) + @test bessely(4, BigFloat(2)) ≈ bessely(4, 2.) + @test bessely0(BigFloat(2)) ≈ bessely0(2.) + @test bessely1(BigFloat(2)) ≈ bessely1(2.) end # trigonometric functions @@ -436,7 +436,7 @@ setprecision(53) do :cosh,:sinh,:tanh,:sech,:csch,:coth,:asinh), j in (-1., -0.5, -0.25, .25, .5, 1.) @eval begin - @test_approx_eq ($f)(BigFloat($j)) ($f)($j) + @test ($f)(BigFloat($j)) ≈ ($f)($j) end end for f in (:acos,:asin,:acosh,:atanh), @@ -449,11 +449,11 @@ setprecision(53) do :sech,:csch,:coth,:acosh,:asinh), j in (1., 1.5, 1.9) @eval begin - @test_approx_eq ($f)(BigFloat($j)) ($f)($j) + @test ($f)(BigFloat($j)) ≈ ($f)($j) end end for j in (.25, .5) - @test_approx_eq atanh(BigFloat(j)) atanh(j) + @test atanh(BigFloat(j)) ≈ atanh(j) end end diff --git a/test/numbers.jl b/test/numbers.jl index 550cc6981639a..4ccbd59e7affd 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -1128,7 +1128,7 @@ end @test Complex(1,2) + 1//2 == Complex(3//2,2//1) @test Complex(1,2) + 1//2 * 0.5 == Complex(1.25,2.0) @test (Complex(1,2) + 1//2) * 0.5 == Complex(0.75,1.0) -@test_approx_eq (Complex(1,2)/Complex(2.5,3.0))*Complex(2.5,3.0) Complex(1,2) +@test (Complex(1,2)/Complex(2.5,3.0))*Complex(2.5,3.0) ≈ Complex(1,2) @test 0.7 < real(sqrt(Complex(0,1))) < 0.707107 for T in Base.BitSigned_types @@ -2644,12 +2644,12 @@ let T = Rational yf = Complex{Float64}(1//2 + 1//5*im) yi = 4 - @test_approx_eq x^y big(xf)^big(yf) - @test_approx_eq x^yi big(xf)^yi - @test_approx_eq x^true big(xf)^true - @test_approx_eq x^false big(xf)^false - @test_approx_eq x^1 big(xf)^1 - @test_approx_eq xf^Rational(2, 1) xf*xf + @test x^y ≈ big(xf)^big(yf) + @test_broken x^yi ≈ big(xf)^yi + @test_broken x^true ≈ big(xf)^true + @test x^false == big(xf)^false + @test_broken x^1 ≈ big(xf)^1 + @test xf^Rational(2, 1) ≈ xf*xf @test Complex(1., 1.)^Rational(2,1) == Complex(1., 1.)*Complex(1.,1.) == Complex(0., 2.) end diff --git a/test/ranges.jl b/test/ranges.jl index 8b04717e6df21..2c1fd4cb41f52 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -451,8 +451,8 @@ end r = -0.004532318104333742:1.2597349521122731e-5:0.008065031416788989 @test length(r[1:end-1]) == length(r) - 1 @test isa(r[1:2:end],Range) && length(r[1:2:end]) == div(length(r)+1, 2) -@test_approx_eq r[3:5][2] r[4] -@test_approx_eq r[5:-2:1][2] r[3] +@test r[3:5][2] ≈ r[4] +@test r[5:-2:1][2] ≈ r[3] @test_throws BoundsError r[0:10] @test_throws BoundsError r[1:10000] @@ -489,7 +489,7 @@ end for f in (mean, median) for n = 2:5 @test f(2:n) == f([2:n;]) - @test_approx_eq f(2:0.1:n) f([2:0.1:n;]) + @test f(2:0.1:n) ≈ f([2:0.1:n;]) end end diff --git a/test/reduce.jl b/test/reduce.jl index 81be7086fa23d..4edf431e14966 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -7,8 +7,8 @@ @test Base.mapfoldl(abs2, -, 2:5) == -46 @test Base.mapfoldl(abs2, -, 10, 2:5) == -44 -@test_approx_eq Base.mapfoldl(abs2, /, 2:5 ) 1/900 -@test_approx_eq Base.mapfoldl(abs2, /, 10, 2:5 ) 1/1440 +@test Base.mapfoldl(abs2, /, 2:5) ≈ 1/900 +@test Base.mapfoldl(abs2, /, 10, 2:5) ≈ 1/1440 @test Base.mapfoldl((x)-> x $ true, &, true, [true false true false false]) == false @test Base.mapfoldl((x)-> x $ true, &, [true false true false false]) == false @@ -54,8 +54,8 @@ fz = float(z) @test sum(sin, 3) == sin(3.0) @test sum(sin, [3]) == sin(3.0) a = sum(sin, z) -@test_approx_eq a sum(sin, fz) -@test_approx_eq a sum(sin(fz)) +@test a ≈ sum(sin, fz) +@test a ≈ sum(sin(fz)) z = [-4, -3, 2, 5] fz = float(z) @@ -65,15 +65,15 @@ b = complex(randn(32), randn(32)) @test sumabs([Int8(-2)]) === Int32(2) @test sumabs(z) === 14 @test sumabs(fz) === 14.0 -@test_approx_eq sumabs(a) sum(abs(a)) -@test_approx_eq sumabs(b) sum(abs(b)) +@test sumabs(a) ≈ sum(abs(a)) +@test sumabs(b) ≈ sum(abs(b)) @test sumabs2(Float64[]) === 0.0 @test sumabs2([Int8(-2)]) === Int32(4) @test sumabs2(z) === 54 @test sumabs2(fz) === 54.0 -@test_approx_eq sumabs2(a) sum(abs2(a)) -@test_approx_eq sumabs2(b) sum(abs2(b)) +@test sumabs2(a) ≈ sum(abs2(a)) +@test sumabs2(b) ≈ sum(abs2(b)) # check variants of summation for type-stability and other issues (#6069) sum2(itr) = invoke(sum, Tuple{Any}, itr) diff --git a/test/reducedim.jl b/test/reducedim.jl index 9d2b66a99bbd3..eb3c218dd00ef 100644 --- a/test/reducedim.jl +++ b/test/reducedim.jl @@ -21,61 +21,61 @@ for region in Any[ (1, 2, 3), (1, 3, 4), (2, 3, 4), (1, 2, 3, 4)] # println("region = $region") r = fill(NaN, Base.reduced_dims(size(Areduc), region)) - @test_approx_eq sum!(r, Areduc) safe_sum(Areduc, region) - @test_approx_eq prod!(r, Areduc) safe_prod(Areduc, region) - @test_approx_eq maximum!(r, Areduc) safe_maximum(Areduc, region) - @test_approx_eq minimum!(r, Areduc) safe_minimum(Areduc, region) - @test_approx_eq sumabs!(r, Areduc) safe_sumabs(Areduc, region) - @test_approx_eq sumabs2!(r, Areduc) safe_sumabs2(Areduc, region) - @test_approx_eq maxabs!(r, Areduc) safe_maxabs(Areduc, region) - @test_approx_eq minabs!(r, Areduc) safe_minabs(Areduc, region) + @test sum!(r, Areduc) ≈ safe_sum(Areduc, region) + @test prod!(r, Areduc) ≈ safe_prod(Areduc, region) + @test maximum!(r, Areduc) ≈ safe_maximum(Areduc, region) + @test minimum!(r, Areduc) ≈ safe_minimum(Areduc, region) + @test sumabs!(r, Areduc) ≈ safe_sumabs(Areduc, region) + @test sumabs2!(r, Areduc) ≈ safe_sumabs2(Areduc, region) + @test maxabs!(r, Areduc) ≈ safe_maxabs(Areduc, region) + @test minabs!(r, Areduc) ≈ safe_minabs(Areduc, region) # With init=false r2 = similar(r) fill!(r, 1) - @test_approx_eq sum!(r, Areduc, init=false) safe_sum(Areduc, region)+1 + @test sum!(r, Areduc, init=false) ≈ safe_sum(Areduc, region)+1 fill!(r, 2.2) - @test_approx_eq prod!(r, Areduc, init=false) safe_prod(Areduc, region)*2.2 + @test prod!(r, Areduc, init=false) ≈ safe_prod(Areduc, region)*2.2 fill!(r, 1.8) - @test_approx_eq maximum!(r, Areduc, init=false) fill!(r2, 1.8) + @test maximum!(r, Areduc, init=false) ≈ fill!(r2, 1.8) fill!(r, -0.2) - @test_approx_eq minimum!(r, Areduc, init=false) fill!(r2, -0.2) + @test minimum!(r, Areduc, init=false) ≈ fill!(r2, -0.2) fill!(r, 8.1) - @test_approx_eq sumabs!(r, Areduc, init=false) safe_sumabs(Areduc, region)+8.1 + @test sumabs!(r, Areduc, init=false) ≈ safe_sumabs(Areduc, region)+8.1 fill!(r, 8.1) - @test_approx_eq sumabs2!(r, Areduc, init=false) safe_sumabs2(Areduc, region)+8.1 + @test sumabs2!(r, Areduc, init=false) ≈ safe_sumabs2(Areduc, region)+8.1 fill!(r, 1.5) - @test_approx_eq maxabs!(r, Areduc, init=false) fill!(r2, 1.5) + @test maxabs!(r, Areduc, init=false) ≈ fill!(r2, 1.5) fill!(r, -1.5) - @test_approx_eq minabs!(r, Areduc, init=false) fill!(r2, -1.5) - - @test_approx_eq sum(Areduc, region) safe_sum(Areduc, region) - @test_approx_eq prod(Areduc, region) safe_prod(Areduc, region) - @test_approx_eq maximum(Areduc, region) safe_maximum(Areduc, region) - @test_approx_eq minimum(Areduc, region) safe_minimum(Areduc, region) - @test_approx_eq sumabs(Areduc, region) safe_sumabs(Areduc, region) - @test_approx_eq sumabs2(Areduc, region) safe_sumabs2(Areduc, region) - @test_approx_eq maxabs(Areduc, region) safe_maxabs(Areduc, region) - @test_approx_eq minabs(Areduc, region) safe_minabs(Areduc, region) + @test minabs!(r, Areduc, init=false) ≈ fill!(r2, -1.5) + + @test sum(Areduc, region) ≈ safe_sum(Areduc, region) + @test prod(Areduc, region) ≈ safe_prod(Areduc, region) + @test maximum(Areduc, region) ≈ safe_maximum(Areduc, region) + @test minimum(Areduc, region) ≈ safe_minimum(Areduc, region) + @test sumabs(Areduc, region) ≈ safe_sumabs(Areduc, region) + @test sumabs2(Areduc, region) ≈ safe_sumabs2(Areduc, region) + @test maxabs(Areduc, region) ≈ safe_maxabs(Areduc, region) + @test minabs(Areduc, region) ≈ safe_minabs(Areduc, region) end # Test reduction along first dimension; this is special-cased for # size(A, 1) >= 16 Breduc = rand(64, 3) r = fill(NaN, Base.reduced_dims(size(Breduc), 1)) -@test_approx_eq sum!(r, Breduc) safe_sum(Breduc, 1) -@test_approx_eq sumabs!(r, Breduc) safe_sumabs(Breduc, 1) -@test_approx_eq sumabs2!(r, Breduc) safe_sumabs2(Breduc, 1) -@test_approx_eq sum(Breduc, 1) safe_sum(Breduc, 1) -@test_approx_eq sumabs(Breduc, 1) safe_sumabs(Breduc, 1) -@test_approx_eq sumabs2(Breduc, 1) safe_sumabs2(Breduc, 1) +@test sum!(r, Breduc) ≈ safe_sum(Breduc, 1) +@test sumabs!(r, Breduc) ≈ safe_sumabs(Breduc, 1) +@test sumabs2!(r, Breduc) ≈ safe_sumabs2(Breduc, 1) +@test sum(Breduc, 1) ≈ safe_sum(Breduc, 1) +@test sumabs(Breduc, 1) ≈ safe_sumabs(Breduc, 1) +@test sumabs2(Breduc, 1) ≈ safe_sumabs2(Breduc, 1) fill!(r, 4.2) -@test_approx_eq sum!(r, Breduc, init=false) safe_sum(Breduc, 1)+4.2 +@test sum!(r, Breduc, init=false) ≈ safe_sum(Breduc, 1)+4.2 fill!(r, -6.3) -@test_approx_eq sumabs!(r, Breduc, init=false) safe_sumabs(Breduc, 1)-6.3 +@test sumabs!(r, Breduc, init=false) ≈ safe_sumabs(Breduc, 1)-6.3 fill!(r, -1.1) -@test_approx_eq sumabs2!(r, Breduc, init=false) safe_sumabs2(Breduc, 1)-1.1 +@test sumabs2!(r, Breduc, init=false) ≈ safe_sumabs2(Breduc, 1)-1.1 # Small arrays with init=false A = reshape(1:15, 3, 5) @@ -110,7 +110,7 @@ A = reshape(1:6, 3, 2) @test reducedim((a,b) -> a|b, [true false; false false], 1, false) == [true false] R = reducedim((a,b) -> a+b, [1 2; 3 4], 2, 0.0) @test eltype(R) == Float64 -@test_approx_eq R [3,7] +@test R ≈ [3,7] @test reducedim((a,b) -> a+b, [1 2; 3 4], 1, 0) == [4 6] # inferred return types diff --git a/test/sparsedir/cholmod.jl b/test/sparsedir/cholmod.jl index 6f4ca625c1f05..eedc08eb6c577 100644 --- a/test/sparsedir/cholmod.jl +++ b/test/sparsedir/cholmod.jl @@ -95,8 +95,8 @@ A = CHOLMOD.Sparse(48, 48, -833333.333333,-1.25e6,2.5e8,2.39928529451e6,9.61679848804e8,275828.470683, -5.57173510779e7,1.09411960038e7,2.08333333333e6,1.0e8,-2.5e6, 140838.195984,-1.09779731332e8,5.31278103775e8], 1) -@test_approx_eq CHOLMOD.norm_sparse(A, 0) 3.570948074697437e9 -@test_approx_eq CHOLMOD.norm_sparse(A, 1) 3.570948074697437e9 +@test CHOLMOD.norm_sparse(A, 0) ≈ 3.570948074697437e9 +@test CHOLMOD.norm_sparse(A, 1) ≈ 3.570948074697437e9 @test_throws ArgumentError CHOLMOD.norm_sparse(A, 2) @test CHOLMOD.isvalid(A) @@ -105,14 +105,14 @@ chma = ldltfact(A) # LDL' form @test CHOLMOD.isvalid(chma) @test unsafe_load(chma.p).is_ll == 0 # check that it is in fact an LDLt x = chma\B -@test_approx_eq x ones(size(x)) +@test x ≈ ones(size(x)) @test nnz(ldltfact(A, perm=1:size(A,1))) > nnz(chma) chma = cholfact(A) # LL' form @test CHOLMOD.isvalid(chma) @test unsafe_load(chma.p).is_ll == 1 # check that it is in fact an LLt x = chma\B -@test_approx_eq x ones(size(x)) +@test x ≈ ones(size(x)) @test nnz(chma) == 489 @test nnz(cholfact(A, perm=1:size(A,1))) > nnz(chma) @@ -155,21 +155,21 @@ let # Issue 9160 cmB = CHOLMOD.Sparse(B) # Ac_mul_B - @test_approx_eq sparse(cmA'*cmB) A'*B + @test sparse(cmA'*cmB) ≈ A'*B # A_mul_Bc - @test_approx_eq sparse(cmA*cmB') A*B' + @test sparse(cmA*cmB') ≈ A*B' # A_mul_Ac - @test_approx_eq sparse(cmA*cmA') A*A' + @test sparse(cmA*cmA') ≈ A*A' # Ac_mul_A - @test_approx_eq sparse(cmA'*cmA) A'*A + @test sparse(cmA'*cmA) ≈ A'*A # A_mul_Ac for symmetric A A = 0.5*(A + A') cmA = CHOLMOD.Sparse(A) - @test_approx_eq sparse(cmA*cmA') A*A' + @test sparse(cmA*cmA') ≈ A*A' end end end @@ -278,12 +278,12 @@ for elty in (Float64, Complex{Float64}) @test_throws BoundsError ADense[6, 1] @test_throws BoundsError ADense[1, 6] @test copy(ADense) == ADense - @test_approx_eq CHOLMOD.norm_dense(ADense, 1) norm(A, 1) - @test_approx_eq CHOLMOD.norm_dense(ADense, 0) norm(A, Inf) + @test CHOLMOD.norm_dense(ADense, 1) ≈ norm(A, 1) + @test CHOLMOD.norm_dense(ADense, 0) ≈ norm(A, Inf) @test_throws ArgumentError CHOLMOD.norm_dense(ADense, 2) @test_throws ArgumentError CHOLMOD.norm_dense(ADense, 3) - @test_approx_eq CHOLMOD.norm_dense(bDense, 2) norm(b) + @test CHOLMOD.norm_dense(bDense, 2) ≈ norm(b) @test CHOLMOD.check_dense(bDense) AA = CHOLMOD.eye(3) @@ -338,18 +338,18 @@ for elty in (Float64, Complex{Float64}) @test copy(A1Sparse) == A1Sparse @test size(A1Sparse, 3) == 1 if elty <: Real # multiplication only defined for real matrices in CHOLMOD - @test_approx_eq A1Sparse*A2Sparse A1*A2 + @test A1Sparse*A2Sparse ≈ A1*A2 @test_throws DimensionMismatch CHOLMOD.Sparse(A1[:,1:4])*A2Sparse - @test_approx_eq A1Sparse'A2Sparse A1'A2 - @test_approx_eq A1Sparse*A2Sparse' A1*A2' + @test A1Sparse'A2Sparse ≈ A1'A2 + @test A1Sparse*A2Sparse' ≈ A1*A2' - @test_approx_eq A1Sparse*A1Sparse A1*A1 - @test_approx_eq A1Sparse'A1Sparse A1'A1 - @test_approx_eq A1Sparse*A1Sparse' A1*A1' + @test A1Sparse*A1Sparse ≈ A1*A1 + @test A1Sparse'A1Sparse ≈ A1'A1 + @test A1Sparse*A1Sparse' ≈ A1*A1' - @test_approx_eq A1pdSparse*A1pdSparse A1pd*A1pd - @test_approx_eq A1pdSparse'A1pdSparse A1pd'A1pd - @test_approx_eq A1pdSparse*A1pdSparse' A1pd*A1pd' + @test A1pdSparse*A1pdSparse ≈ A1pd*A1pd + @test A1pdSparse'A1pdSparse ≈ A1pd'A1pd + @test A1pdSparse*A1pdSparse' ≈ A1pd*A1pd' @test_throws DimensionMismatch A1Sparse*CHOLMOD.eye(4, 5, elty) end @@ -369,19 +369,19 @@ for elty in (Float64, Complex{Float64}) show(tmp, F) @test tmp.size > 0 @test isa(CHOLMOD.Sparse(F), CHOLMOD.Sparse{elty}) - @test_approx_eq F\CHOLMOD.Sparse(sparse(ones(elty, 5))) A1pd\ones(5) + @test F\CHOLMOD.Sparse(sparse(ones(elty, 5))) ≈ A1pd\ones(5) @test_throws DimensionMismatch F\CHOLMOD.Dense(ones(elty, 4)) @test_throws DimensionMismatch F\CHOLMOD.Sparse(sparse(ones(elty, 4))) - @test_approx_eq F'\ones(elty, 5) full(A1pd)'\ones(5) - @test_approx_eq F'\sparse(ones(elty, 5)) full(A1pd)'\ones(5) - @test_approx_eq logdet(F) logdet(full(A1pd)) + @test F'\ones(elty, 5) ≈ full(A1pd)'\ones(5) + @test F'\sparse(ones(elty, 5)) ≈ full(A1pd)'\ones(5) + @test logdet(F) ≈ logdet(full(A1pd)) @test det(F) == exp(logdet(F)) let # to test supernodal, we must use a larger matrix Ftmp = sprandn(100,100,0.1) Ftmp = Ftmp'Ftmp + I - @test_approx_eq logdet(cholfact(Ftmp)) logdet(full(Ftmp)) + @test logdet(cholfact(Ftmp)) ≈ logdet(full(Ftmp)) end - @test_approx_eq logdet(ldltfact(A1pd)) logdet(full(A1pd)) + @test logdet(ldltfact(A1pd)) ≈ logdet(full(A1pd)) @test isposdef(A1pd) @test !isposdef(A1) @test !isposdef(A1 + A1' |> t -> t - 2eigmax(full(t))*I) @@ -414,22 +414,22 @@ for elty in (Float64, Complex{Float64}) CHOLMOD.change_factor!(elty, false, false, true, true, F) @test unsafe_load(F.p).is_ll == 0 CHOLMOD.change_factor!(elty, true, false, true, true, F) - @test_approx_eq CHOLMOD.Sparse(cholfact!(copy(F), A1pd)) CHOLMOD.Sparse(F) # surprisingly, this can cause small ulp size changes so we cannot test exact equality + @test CHOLMOD.Sparse(cholfact!(copy(F), A1pd)) ≈ CHOLMOD.Sparse(F) # surprisingly, this can cause small ulp size changes so we cannot test exact equality @test size(F, 2) == 5 @test size(F, 3) == 1 @test_throws ArgumentError size(F, 0) F = cholfact(A1pdSparse, shift=2) @test isa(CHOLMOD.Sparse(F), CHOLMOD.Sparse{elty}) - @test_approx_eq CHOLMOD.Sparse(cholfact!(copy(F), A1pd, shift=2.0)) CHOLMOD.Sparse(F) # surprisingly, this can cause small ulp size changes so we cannot test exact equality + @test CHOLMOD.Sparse(cholfact!(copy(F), A1pd, shift=2.0)) ≈ CHOLMOD.Sparse(F) # surprisingly, this can cause small ulp size changes so we cannot test exact equality F = ldltfact(A1pd) @test isa(CHOLMOD.Sparse(F), CHOLMOD.Sparse{elty}) - @test_approx_eq CHOLMOD.Sparse(ldltfact!(copy(F), A1pd)) CHOLMOD.Sparse(F) # surprisingly, this can cause small ulp size changes so we cannot test exact equality + @test CHOLMOD.Sparse(ldltfact!(copy(F), A1pd)) ≈ CHOLMOD.Sparse(F) # surprisingly, this can cause small ulp size changes so we cannot test exact equality F = ldltfact(A1pdSparse, shift=2) @test isa(CHOLMOD.Sparse(F), CHOLMOD.Sparse{elty}) - @test_approx_eq CHOLMOD.Sparse(ldltfact!(copy(F), A1pd, shift=2.0)) CHOLMOD.Sparse(F) # surprisingly, this can cause small ulp size changes so we cannot test exact equality + @test CHOLMOD.Sparse(ldltfact!(copy(F), A1pd, shift=2.0)) ≈ CHOLMOD.Sparse(F) # surprisingly, this can cause small ulp size changes so we cannot test exact equality @test isa(CHOLMOD.factor_to_sparse!(F), CHOLMOD.Sparse) @test_throws CHOLMOD.CHOLMODException CHOLMOD.factor_to_sparse!(F) @@ -461,9 +461,9 @@ for elty in (Float64, Complex{Float64}) if elty <: Real - @test_approx_eq CHOLMOD.ssmult(A1Sparse, A2Sparse, 0, true, true) A1*A2 - @test_approx_eq CHOLMOD.aat(A1Sparse, [0:size(A1,2)-1;], 1) A1*A1' - @test_approx_eq CHOLMOD.aat(A1Sparse, [0:1;], 1) A1[:,1:2]*A1[:,1:2]' + @test CHOLMOD.ssmult(A1Sparse, A2Sparse, 0, true, true) ≈ A1*A2 + @test CHOLMOD.aat(A1Sparse, [0:size(A1,2)-1;], 1) ≈ A1*A1' + @test CHOLMOD.aat(A1Sparse, [0:1;], 1) ≈ A1[:,1:2]*A1[:,1:2]' @test CHOLMOD.copy(A1Sparse, 0, 1) == A1Sparse end @@ -480,19 +480,19 @@ D_f = float([4 0 0; 0 1 0; 0 0 9]) # cholfact, no permutation Fs = cholfact(As, perm=[1:3;]) @test Fs[:p] == [1:3;] -@test_approx_eq sparse(Fs[:L]) Lf -@test_approx_eq sparse(Fs) As +@test sparse(Fs[:L]) ≈ Lf +@test sparse(Fs) ≈ As b = rand(3) -@test_approx_eq Fs\b Af\b -@test_approx_eq Fs[:UP]\(Fs[:PtL]\b) Af\b -@test_approx_eq Fs[:L]\b Lf\b -@test_approx_eq Fs[:U]\b Lf'\b -@test_approx_eq Fs[:L]'\b Lf'\b -@test_approx_eq Fs[:U]'\b Lf\b -@test_approx_eq Fs[:PtL]\b Lf\b -@test_approx_eq Fs[:UP]\b Lf'\b -@test_approx_eq Fs[:PtL]'\b Lf'\b -@test_approx_eq Fs[:UP]'\b Lf\b +@test Fs\b ≈ Af\b +@test Fs[:UP]\(Fs[:PtL]\b) ≈ Af\b +@test Fs[:L]\b ≈ Lf\b +@test Fs[:U]\b ≈ Lf'\b +@test Fs[:L]'\b ≈ Lf'\b +@test Fs[:U]'\b ≈ Lf\b +@test Fs[:PtL]\b ≈ Lf\b +@test Fs[:UP]\b ≈ Lf'\b +@test Fs[:PtL]'\b ≈ Lf'\b +@test Fs[:UP]'\b ≈ Lf\b @test_throws CHOLMOD.CHOLMODException Fs[:D] @test_throws CHOLMOD.CHOLMODException Fs[:LD] @test_throws CHOLMOD.CHOLMODException Fs[:DU] @@ -506,19 +506,19 @@ Fs = cholfact(As, perm=p) @test Fs[:p] == p Afp = Af[p,p] Lfp = cholfact(Afp)[:L] -@test_approx_eq sparse(Fs[:L]) Lfp -@test_approx_eq sparse(Fs) As +@test sparse(Fs[:L]) ≈ Lfp +@test sparse(Fs) ≈ As b = rand(3) -@test_approx_eq Fs\b Af\b -@test_approx_eq Fs[:UP]\(Fs[:PtL]\b) Af\b -@test_approx_eq Fs[:L]\b Lfp\b -@test_approx_eq Fs[:U]'\b Lfp\b -@test_approx_eq Fs[:U]\b Lfp'\b -@test_approx_eq Fs[:L]'\b Lfp'\b -@test_approx_eq Fs[:PtL]\b Lfp\b[p] -@test_approx_eq Fs[:UP]\b (Lfp'\b)[p_inv] -@test_approx_eq Fs[:PtL]'\b (Lfp'\b)[p_inv] -@test_approx_eq Fs[:UP]'\b Lfp\b[p] +@test Fs\b ≈ Af\b +@test Fs[:UP]\(Fs[:PtL]\b) ≈ Af\b +@test Fs[:L]\b ≈ Lfp\b +@test Fs[:U]'\b ≈ Lfp\b +@test Fs[:U]\b ≈ Lfp'\b +@test Fs[:L]'\b ≈ Lfp'\b +@test Fs[:PtL]\b ≈ Lfp\b[p] +@test Fs[:UP]\b ≈ (Lfp'\b)[p_inv] +@test Fs[:PtL]'\b ≈ (Lfp'\b)[p_inv] +@test Fs[:UP]'\b ≈ Lfp\b[p] @test_throws CHOLMOD.CHOLMODException Fs[:PL] @test_throws CHOLMOD.CHOLMODException Fs[:UPt] @test_throws CHOLMOD.CHOLMODException Fs[:D] @@ -530,60 +530,60 @@ b = rand(3) # ldltfact, no permutation Fs = ldltfact(As, perm=[1:3;]) @test Fs[:p] == [1:3;] -@test_approx_eq sparse(Fs[:LD]) LDf -@test_approx_eq sparse(Fs) As +@test sparse(Fs[:LD]) ≈ LDf +@test sparse(Fs) ≈ As b = rand(3) -@test_approx_eq Fs\b Af\b -@test_approx_eq Fs[:UP]\(Fs[:PtLD]\b) Af\b -@test_approx_eq Fs[:DUP]\(Fs[:PtL]\b) Af\b -@test_approx_eq Fs[:L]\b L_f\b -@test_approx_eq Fs[:U]\b L_f'\b -@test_approx_eq Fs[:L]'\b L_f'\b -@test_approx_eq Fs[:U]'\b L_f\b -@test_approx_eq Fs[:PtL]\b L_f\b -@test_approx_eq Fs[:UP]\b L_f'\b -@test_approx_eq Fs[:PtL]'\b L_f'\b -@test_approx_eq Fs[:UP]'\b L_f\b -@test_approx_eq Fs[:D]\b D_f\b -@test_approx_eq Fs[:D]'\b D_f\b -@test_approx_eq Fs[:LD]\b D_f\(L_f\b) -@test_approx_eq Fs[:DU]'\b D_f\(L_f\b) -@test_approx_eq Fs[:LD]'\b L_f'\(D_f\b) -@test_approx_eq Fs[:DU]\b L_f'\(D_f\b) -@test_approx_eq Fs[:PtLD]\b D_f\(L_f\b) -@test_approx_eq Fs[:DUP]'\b D_f\(L_f\b) -@test_approx_eq Fs[:PtLD]'\b L_f'\(D_f\b) -@test_approx_eq Fs[:DUP]\b L_f'\(D_f\b) +@test Fs\b ≈ Af\b +@test Fs[:UP]\(Fs[:PtLD]\b) ≈ Af\b +@test Fs[:DUP]\(Fs[:PtL]\b) ≈ Af\b +@test Fs[:L]\b ≈ L_f\b +@test Fs[:U]\b ≈ L_f'\b +@test Fs[:L]'\b ≈ L_f'\b +@test Fs[:U]'\b ≈ L_f\b +@test Fs[:PtL]\b ≈ L_f\b +@test Fs[:UP]\b ≈ L_f'\b +@test Fs[:PtL]'\b ≈ L_f'\b +@test Fs[:UP]'\b ≈ L_f\b +@test Fs[:D]\b ≈ D_f\b +@test Fs[:D]'\b ≈ D_f\b +@test Fs[:LD]\b ≈ D_f\(L_f\b) +@test Fs[:DU]'\b ≈ D_f\(L_f\b) +@test Fs[:LD]'\b ≈ L_f'\(D_f\b) +@test Fs[:DU]\b ≈ L_f'\(D_f\b) +@test Fs[:PtLD]\b ≈ D_f\(L_f\b) +@test Fs[:DUP]'\b ≈ D_f\(L_f\b) +@test Fs[:PtLD]'\b ≈ L_f'\(D_f\b) +@test Fs[:DUP]\b ≈ L_f'\(D_f\b) # ldltfact, with permutation Fs = ldltfact(As, perm=p) @test Fs[:p] == p -@test_approx_eq sparse(Fs) As +@test sparse(Fs) ≈ As b = rand(3) Asp = As[p,p] LDp = sparse(ldltfact(Asp, perm=[1,2,3])[:LD]) # LDp = sparse(Fs[:LD]) Lp, dp = Base.SparseArrays.CHOLMOD.getLd!(copy(LDp)) Dp = spdiagm(dp) -@test_approx_eq Fs\b Af\b -@test_approx_eq Fs[:UP]\(Fs[:PtLD]\b) Af\b -@test_approx_eq Fs[:DUP]\(Fs[:PtL]\b) Af\b -@test_approx_eq Fs[:L]\b Lp\b -@test_approx_eq Fs[:U]\b Lp'\b -@test_approx_eq Fs[:L]'\b Lp'\b -@test_approx_eq Fs[:U]'\b Lp\b -@test_approx_eq Fs[:PtL]\b Lp\b[p] -@test_approx_eq Fs[:UP]\b (Lp'\b)[p_inv] -@test_approx_eq Fs[:PtL]'\b (Lp'\b)[p_inv] -@test_approx_eq Fs[:UP]'\b Lp\b[p] -@test_approx_eq Fs[:LD]\b Dp\(Lp\b) -@test_approx_eq Fs[:DU]'\b Dp\(Lp\b) -@test_approx_eq Fs[:LD]'\b Lp'\(Dp\b) -@test_approx_eq Fs[:DU]\b Lp'\(Dp\b) -@test_approx_eq Fs[:PtLD]\b Dp\(Lp\b[p]) -@test_approx_eq Fs[:DUP]'\b Dp\(Lp\b[p]) -@test_approx_eq Fs[:PtLD]'\b (Lp'\(Dp\b))[p_inv] -@test_approx_eq Fs[:DUP]\b (Lp'\(Dp\b))[p_inv] +@test Fs\b ≈ Af\b +@test Fs[:UP]\(Fs[:PtLD]\b) ≈ Af\b +@test Fs[:DUP]\(Fs[:PtL]\b) ≈ Af\b +@test Fs[:L]\b ≈ Lp\b +@test Fs[:U]\b ≈ Lp'\b +@test Fs[:L]'\b ≈ Lp'\b +@test Fs[:U]'\b ≈ Lp\b +@test Fs[:PtL]\b ≈ Lp\b[p] +@test Fs[:UP]\b ≈ (Lp'\b)[p_inv] +@test Fs[:PtL]'\b ≈ (Lp'\b)[p_inv] +@test Fs[:UP]'\b ≈ Lp\b[p] +@test Fs[:LD]\b ≈ Dp\(Lp\b) +@test Fs[:DU]'\b ≈ Dp\(Lp\b) +@test Fs[:LD]'\b ≈ Lp'\(Dp\b) +@test Fs[:DU]\b ≈ Lp'\(Dp\b) +@test Fs[:PtLD]\b ≈ Dp\(Lp\b[p]) +@test Fs[:DUP]'\b ≈ Dp\(Lp\b[p]) +@test Fs[:PtLD]'\b ≈ (Lp'\(Dp\b))[p_inv] +@test Fs[:DUP]\b ≈ (Lp'\(Dp\b))[p_inv] @test_throws CHOLMOD.CHOLMODException Fs[:DUPt] @test_throws CHOLMOD.CHOLMODException Fs[:PLD] @@ -636,11 +636,11 @@ Fnew = deserialize(b) # test \ for Factor and StridedVecOrMat let x = rand(5) A = cholfact(sparse(diagm(x.\1))) - @test_approx_eq A\view(ones(10),1:2:10) x - @test_approx_eq A\view(eye(5,5),:,:) diagm(x) + @test A\view(ones(10),1:2:10) ≈ x + @test A\view(eye(5,5),:,:) ≈ diagm(x) end # Real factorization and complex rhs A = sprandn(5,5,0.4) |> t -> t't + I B = complex(randn(5,2), randn(5,2)) -@test_approx_eq cholfact(A)\B A\B +@test cholfact(A)\B ≈ A\B diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 64cc456147452..88835d4f274fa 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -103,10 +103,10 @@ end # sparse matrix * BitArray A = sprand(5,5,0.2) B = trues(5) -@test_approx_eq A*B full(A)*B +@test A*B ≈ full(A)*B B = trues(5,5) -@test_approx_eq A*B full(A)*B -@test_approx_eq B*A B*full(A) +@test A*B ≈ full(A)*B +@test B*A ≈ B*full(A) # complex matrix-vector multiplication and left-division if Base.USE_GPL_LIBS @@ -362,10 +362,10 @@ pA = sparse(rand(3, 7)) for arr in (se33, sA, pA) for f in (sum, prod, minimum, maximum, var) farr = full(arr) - @test_approx_eq f(arr) f(farr) - @test_approx_eq f(arr, 1) f(farr, 1) - @test_approx_eq f(arr, 2) f(farr, 2) - @test_approx_eq f(arr, (1, 2)) [f(farr)] + @test f(arr) ≈ f(farr) + @test f(arr, 1) ≈ f(farr, 1) + @test f(arr, 2) ≈ f(farr, 2) + @test f(arr, (1, 2)) ≈ [f(farr)] @test isequal(f(arr, 3), f(farr, 3)) end end @@ -373,15 +373,15 @@ end for f in (sum, prod, minimum, maximum) # Test with a map function that maps to non-zero for arr in (se33, sA, pA) - @test_approx_eq f(x->x+1, arr) f(arr+1) + @test f(x->x+1, arr) ≈ f(arr+1) end # case where f(0) would throw - @test_approx_eq f(x->sqrt(x-1), pA+1) f(sqrt(pA)) + @test f(x->sqrt(x-1), pA+1) ≈ f(sqrt(pA)) # these actually throw due to #10533 - # @test_approx_eq f(x->sqrt(x-1), pA+1, 1) f(sqrt(pA), 1) - # @test_approx_eq f(x->sqrt(x-1), pA+1, 2) f(sqrt(pA), 2) - # @test_approx_eq f(x->sqrt(x-1), pA+1, 3) f(pA) + # @test f(x->sqrt(x-1), pA+1, 1) ≈ f(sqrt(pA), 1) + # @test f(x->sqrt(x-1), pA+1, 2) ≈ f(sqrt(pA), 2) + # @test f(x->sqrt(x-1), pA+1, 3) ≈ f(pA) end # empty cases @@ -909,8 +909,8 @@ end # explicit zeros if Base.USE_GPL_LIBS a = SparseMatrixCSC(2, 2, [1, 3, 5], [1, 2, 1, 2], [1.0, 0.0, 0.0, 1.0]) -@test_approx_eq lufact(a)\[2.0, 3.0] [2.0, 3.0] -@test_approx_eq cholfact(a)\[2.0, 3.0] [2.0, 3.0] +@test lufact(a)\[2.0, 3.0] ≈ [2.0, 3.0] +@test cholfact(a)\[2.0, 3.0] ≈ [2.0, 3.0] end # issue #9917 @@ -1288,15 +1288,15 @@ end Ac = sprandn(10,10,.1) + im* sprandn(10,10,.1) Ar = sprandn(10,10,.1) Ai = ceil(Int,Ar*100) -@test_approx_eq norm(Ac,1) norm(full(Ac),1) -@test_approx_eq norm(Ac,Inf) norm(full(Ac),Inf) -@test_approx_eq vecnorm(Ac) vecnorm(full(Ac)) -@test_approx_eq norm(Ar,1) norm(full(Ar),1) -@test_approx_eq norm(Ar,Inf) norm(full(Ar),Inf) -@test_approx_eq vecnorm(Ar) vecnorm(full(Ar)) -@test_approx_eq norm(Ai,1) norm(full(Ai),1) -@test_approx_eq norm(Ai,Inf) norm(full(Ai),Inf) -@test_approx_eq vecnorm(Ai) vecnorm(full(Ai)) +@test norm(Ac,1) ≈ norm(full(Ac),1) +@test norm(Ac,Inf) ≈ norm(full(Ac),Inf) +@test vecnorm(Ac) ≈ vecnorm(full(Ac)) +@test norm(Ar,1) ≈ norm(full(Ar),1) +@test norm(Ar,Inf) ≈ norm(full(Ar),Inf) +@test vecnorm(Ar) ≈ vecnorm(full(Ar)) +@test norm(Ai,1) ≈ norm(full(Ai),1) +@test norm(Ai,Inf) ≈ norm(full(Ai),Inf) +@test vecnorm(Ai) ≈ vecnorm(full(Ai)) # test sparse matrix cond A = sparse(reshape([1.0],1,1)) @@ -1453,4 +1453,4 @@ let m = 5 end # Test temporary fix for issue #16548 in PR #16979. Brittle. Expect to remove with `\` revisions. -@test which(\, (SparseMatrixCSC, AbstractVecOrMat)).module == Base.SparseArrays \ No newline at end of file +@test which(\, (SparseMatrixCSC, AbstractVecOrMat)).module == Base.SparseArrays diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index 7d9f2680b97b4..ca94ed7974832 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -687,10 +687,10 @@ let x = sprand(16, 0.5), x2 = sprand(16, 0.4) let dv = dot(xf, xf2) @test dot(x, x) == sumabs2(x) @test dot(x2, x2) == sumabs2(x2) - @test_approx_eq dot(x, x2) dv - @test_approx_eq dot(x2, x) dv - @test_approx_eq dot(full(x), x2) dv - @test_approx_eq dot(x, full(x2)) dv + @test dot(x, x2) ≈ dv + @test dot(x2, x) ≈ dv + @test dot(full(x), x2) ≈ dv + @test dot(x, full(x2)) ≈ dv end end @@ -698,8 +698,8 @@ let x = complex(sprand(32, 0.6), sprand(32, 0.6)), y = complex(sprand(32, 0.6), sprand(32, 0.6)) xf = full(x)::Vector{Complex128} yf = full(y)::Vector{Complex128} - @test_approx_eq dot(x, x) dot(xf, xf) - @test_approx_eq dot(x, y) dot(xf, yf) + @test dot(x, x) ≈ dot(xf, xf) + @test dot(x, y) ≈ dot(xf, yf) end @@ -711,26 +711,26 @@ let A = randn(9, 16), x = sprand(16, 0.7) xf = full(x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) - rr = α * A * xf + β * y + rr = α*A*xf + β*y @test is(A_mul_B!(α, A, x, β, y), y) - @test_approx_eq y rr + @test y ≈ rr end - y = A * x + y = A*x @test isa(y, Vector{Float64}) - @test_approx_eq A * x A * xf + @test A*x ≈ A*xf end let A = randn(16, 9), x = sprand(16, 0.7) xf = full(x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) - rr = α * A'xf + β * y + rr = α*A'xf + β*y @test is(At_mul_B!(α, A, x, β, y), y) - @test_approx_eq y rr + @test y ≈ rr end y = At_mul_B(A, x) @test isa(y, Vector{Float64}) - @test_approx_eq y At_mul_B(A, xf) + @test y ≈ At_mul_B(A, xf) end ## sparse A * sparse x -> dense y @@ -740,13 +740,13 @@ let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7) xf = full(x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) - rr = α * Af * xf + β * y + rr = α*Af*xf + β*y @test is(A_mul_B!(α, A, x, β, y), y) - @test_approx_eq y rr + @test y ≈ rr end y = SparseArrays.densemv(A, x) @test isa(y, Vector{Float64}) - @test_approx_eq y Af * xf + @test y ≈ Af*xf end let A = sprandn(16, 9, 0.5), x = sprand(16, 0.7) @@ -754,13 +754,13 @@ let A = sprandn(16, 9, 0.5), x = sprand(16, 0.7) xf = full(x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) - rr = α * Af'xf + β * y + rr = α*Af'xf + β*y @test is(At_mul_B!(α, A, x, β, y), y) - @test_approx_eq y rr + @test y ≈ rr end y = SparseArrays.densemv(A, x; trans='T') @test isa(y, Vector{Float64}) - @test_approx_eq y At_mul_B(Af, xf) + @test y ≈ At_mul_B(Af, xf) end let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), @@ -769,9 +769,9 @@ let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), Af = full(A) xf = full(x) x2f = full(x2) - @test_approx_eq SparseArrays.densemv(A, x; trans='N') Af * xf - @test_approx_eq SparseArrays.densemv(A, x2; trans='T') Af.' * x2f - @test_approx_eq SparseArrays.densemv(A, x2; trans='C') Af'x2f + @test SparseArrays.densemv(A, x; trans='N') ≈ Af * xf + @test SparseArrays.densemv(A, x2; trans='T') ≈ Af.' * x2f + @test SparseArrays.densemv(A, x2; trans='C') ≈ Af'x2f end ## sparse A * sparse x -> sparse y @@ -781,15 +781,15 @@ let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7), x2 = sprand(9, 0.7) xf = full(x) x2f = full(x2) - y = A * x + y = A*x @test isa(y, SparseVector{Float64,Int}) @test all(nonzeros(y) .!= 0.0) - @test_approx_eq full(y) Af * xf + @test full(y) ≈ Af * xf y = At_mul_B(A, x2) @test isa(y, SparseVector{Float64,Int}) @test all(nonzeros(y) .!= 0.0) - @test_approx_eq full(y) Af'x2f + @test full(y) ≈ Af'x2f end let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), @@ -799,17 +799,17 @@ let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), xf = full(x) x2f = full(x2) - y = A * x + y = A*x @test isa(y, SparseVector{Complex128,Int}) - @test_approx_eq full(y) Af * xf + @test full(y) ≈ Af * xf y = At_mul_B(A, x2) @test isa(y, SparseVector{Complex128,Int}) - @test_approx_eq full(y) Af.' * x2f + @test full(y) ≈ Af.' * x2f y = Ac_mul_B(A, x2) @test isa(y, SparseVector{Complex128,Int}) - @test_approx_eq full(y) Af'x2f + @test full(y) ≈ Af'x2f end # left-division operations involving triangular matrices and sparse vectors (#14005) diff --git a/test/sparsedir/spqr.jl b/test/sparsedir/spqr.jl index e42d74ea75972..5411dd503f7a7 100644 --- a/test/sparsedir/spqr.jl +++ b/test/sparsedir/spqr.jl @@ -25,10 +25,10 @@ for eltyA in (Float64, Complex{Float64}) end @inferred A\B - @test_approx_eq A\B[:,1] full(A)\B[:,1] - @test_approx_eq A\B full(A)\B + @test A\B[:,1] ≈ full(A)\B[:,1] + @test A\B ≈ full(A)\B @test_throws DimensionMismatch A\B[1:m-1,:] - @test_approx_eq A[1:9,:]*(A[1:9,:]\ones(eltyB, 9)) ones(9) # Underdetermined system + @test A[1:9,:]*(A[1:9,:]\ones(eltyB, 9)) ≈ ones(9) # Underdetermined system if eltyA == eltyB # promotions not defined for unexported methods @test qrfact(sparse(eye(eltyA, 5)))\ones(eltyA, 5) == ones(5) @@ -46,7 +46,7 @@ for eltyA in (Float64, Complex{Float64}) @test_throws DimensionMismatch SPQR.solve(SPQR.RTX_EQUALS_B, F, CHOLMOD.Dense(B)) @test_throws DimensionMismatch SPQR.qmult(SPQR.QX, F, CHOLMOD.Dense(B')) @test_throws DimensionMismatch SPQR.qmult(SPQR.XQ, F, CHOLMOD.Dense(B)) - @test_approx_eq A\B SPQR.backslash(SPQR.ORDERING_DEFAULT, SPQR.DEFAULT_TOL, CHOLMOD.Sparse(A), CHOLMOD.Dense(B)) + @test A\B ≈ SPQR.backslash(SPQR.ORDERING_DEFAULT, SPQR.DEFAULT_TOL, CHOLMOD.Sparse(A), CHOLMOD.Dense(B)) @test_throws DimensionMismatch SPQR.backslash(SPQR.ORDERING_DEFAULT, SPQR.DEFAULT_TOL, CHOLMOD.Sparse(A), CHOLMOD.Dense(B[1:m-1,:])) end end diff --git a/test/statistics.jl b/test/statistics.jl index 0ea874926257f..d0c5032de39d1 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -85,10 +85,10 @@ X = [2 3 1 -1; 7 4 5 -4] @test isnan(var(Int[]; mean=2)) @test isnan(var(Int[]; mean=2, corrected=false)) # reduction across dimensions -@test_approx_eq var(Int[], 1) [NaN] -@test_approx_eq var(Int[], 1; corrected=false) [NaN] -@test_approx_eq var(Int[], 1; mean=[2]) [NaN] -@test_approx_eq var(Int[], 1; mean=[2], corrected=false) [NaN] +@test isequal(var(Int[], 1), [NaN]) +@test isequal(var(Int[], 1; corrected=false), [NaN]) +@test isequal(var(Int[], 1; mean=[2]), [NaN]) +@test isequal(var(Int[], 1; mean=[2], corrected=false), [NaN]) # edge case: one-element vector # iterable @@ -102,10 +102,10 @@ X = [2 3 1 -1; 7 4 5 -4] @test var([1]; mean=2) === Inf @test var([1]; mean=2, corrected=false) === 1.0 # reduction across dimensions -@test_approx_eq @inferred(var([1], 1)) [NaN] -@test_approx_eq var([1], 1; corrected=false) [0.0] -@test_approx_eq var([1], 1; mean=[2]) [Inf] -@test_approx_eq var([1], 1; mean=[2], corrected=false) [1.0] +@test isequal(@inferred(var([1], 1)), [NaN]) +@test var([1], 1; corrected=false) ≈ [0.0] +@test var([1], 1; mean=[2]) ≈ [Inf] +@test var([1], 1; mean=[2], corrected=false) ≈ [1.0] @test var(1:8) == 6. @test varm(1:8,1) == varm(collect(1:8),1) @@ -113,40 +113,40 @@ X = [2 3 1 -1; 7 4 5 -4] @test isnan(var(1:1)) @test isnan(var(1:-1)) -@test_approx_eq varm([1,2,3], 2) 1. -@test_approx_eq var([1,2,3]) 1. -@test_approx_eq var([1,2,3]; corrected=false) 2.0/3 -@test_approx_eq var([1,2,3]; mean=0) 7. -@test_approx_eq var([1,2,3]; mean=0, corrected=false) 14.0/3 - -@test_approx_eq varm((1,2,3), 2) 1. -@test_approx_eq var((1,2,3)) 1. -@test_approx_eq var((1,2,3); corrected=false) 2.0/3 -@test_approx_eq var((1,2,3); mean=0) 7. -@test_approx_eq var((1,2,3); mean=0, corrected=false) 14.0/3 +@test varm([1,2,3], 2) ≈ 1. +@test var([1,2,3]) ≈ 1. +@test var([1,2,3]; corrected=false) ≈ 2.0/3 +@test var([1,2,3]; mean=0) ≈ 7. +@test var([1,2,3]; mean=0, corrected=false) ≈ 14.0/3 + +@test varm((1,2,3), 2) ≈ 1. +@test var((1,2,3)) ≈ 1. +@test var((1,2,3); corrected=false) ≈ 2.0/3 +@test var((1,2,3); mean=0) ≈ 7. +@test var((1,2,3); mean=0, corrected=false) ≈ 14.0/3 @test_throws ArgumentError var((1,2,3); mean=()) -@test_approx_eq var([1 2 3 4 5; 6 7 8 9 10], 2) [2.5 2.5]' -@test_approx_eq var([1 2 3 4 5; 6 7 8 9 10], 2; corrected=false) [2.0 2.0]' +@test var([1 2 3 4 5; 6 7 8 9 10], 2) ≈ [2.5 2.5]' +@test var([1 2 3 4 5; 6 7 8 9 10], 2; corrected=false) ≈ [2.0 2.0]' -@test_approx_eq stdm([1,2,3], 2) 1. -@test_approx_eq std([1,2,3]) 1. -@test_approx_eq std([1,2,3]; corrected=false) sqrt(2.0/3) -@test_approx_eq std([1,2,3]; mean=0) sqrt(7.0) -@test_approx_eq std([1,2,3]; mean=0, corrected=false) sqrt(14.0/3) +@test stdm([1,2,3], 2) ≈ 1. +@test std([1,2,3]) ≈ 1. +@test std([1,2,3]; corrected=false) ≈ sqrt(2.0/3) +@test std([1,2,3]; mean=0) ≈ sqrt(7.0) +@test std([1,2,3]; mean=0, corrected=false) ≈ sqrt(14.0/3) -@test_approx_eq stdm((1,2,3), 2) 1. -@test_approx_eq std((1,2,3)) 1. -@test_approx_eq std((1,2,3); corrected=false) sqrt(2.0/3) -@test_approx_eq std((1,2,3); mean=0) sqrt(7.0) -@test_approx_eq std((1,2,3); mean=0, corrected=false) sqrt(14.0/3) +@test stdm((1,2,3), 2) ≈ 1. +@test std((1,2,3)) ≈ 1. +@test std((1,2,3); corrected=false) ≈ sqrt(2.0/3) +@test std((1,2,3); mean=0) ≈ sqrt(7.0) +@test std((1,2,3); mean=0, corrected=false) ≈ sqrt(14.0/3) -@test_approx_eq std([1 2 3 4 5; 6 7 8 9 10], 2) sqrt([2.5 2.5]') -@test_approx_eq std([1 2 3 4 5; 6 7 8 9 10], 2; corrected=false) sqrt([2.0 2.0]') +@test std([1 2 3 4 5; 6 7 8 9 10], 2) ≈ sqrt([2.5 2.5]') +@test std([1 2 3 4 5; 6 7 8 9 10], 2; corrected=false) ≈ sqrt([2.0 2.0]') A = Complex128[exp(i*im) for i in 1:10^4] -@test_approx_eq varm(A,0.) sum(map(abs2,A))/(length(A)-1) -@test_approx_eq varm(A,mean(A)) var(A) +@test varm(A,0.) ≈ sum(map(abs2,A))/(length(A)-1) +@test varm(A,mean(A)) ≈ var(A) # test covariance @@ -189,21 +189,21 @@ for vd in [1, 2], zm in [true, false], cr in [true, false] c = zm ? Base.covm(x1, 0, cr) : cov(x1, cr) @test isa(c, Float64) - @test_approx_eq c Cxx[1,1] + @test c ≈ Cxx[1,1] @inferred cov(x1, cr) @test cov(X) == Base.covm(X, mean(X, 1)) C = zm ? Base.covm(X, 0, vd, cr) : cov(X, vd, cr) @test size(C) == (k, k) - @test_approx_eq C Cxx + @test C ≈ Cxx @inferred cov(X, vd, cr) @test cov(x1, y1) == Base.covm(x1, mean(x1), y1, mean(y1)) c = zm ? Base.covm(x1, 0, y1, 0, cr) : cov(x1, y1, cr) @test isa(c, Float64) - @test_approx_eq c Cxy[1,1] + @test c ≈ Cxy[1,1] @inferred cov(x1, y1, cr) if vd == 1 @@ -212,7 +212,7 @@ for vd in [1, 2], zm in [true, false], cr in [true, false] C = zm ? Base.covm(x1, 0, Y, 0, vd, cr) : cov(x1, Y, vd, cr) @test size(C) == (1, k) - @test_approx_eq C Cxy[1,:] + @test vec(C) ≈ Cxy[1,:] @inferred cov(x1, Y, vd, cr) if vd == 1 @@ -221,14 +221,14 @@ for vd in [1, 2], zm in [true, false], cr in [true, false] C = zm ? Base.covm(X, 0, y1, 0, vd, cr) : cov(X, y1, vd, cr) @test size(C) == (k, 1) - @test_approx_eq C Cxy[:,1] + @test vec(C) ≈ Cxy[:,1] @inferred cov(X, y1, vd, cr) @test cov(X, Y) == Base.covm(X, mean(X, 1), Y, mean(Y, 1)) C = zm ? Base.covm(X, 0, Y, 0, vd, cr) : cov(X, Y, vd, cr) @test size(C) == (k, k) - @test_approx_eq C Cxy + @test C ≈ Cxy @inferred cov(X, Y, vd, cr) end @@ -270,19 +270,19 @@ for vd in [1, 2], zm in [true, false] c = zm ? Base.corm(x1, 0) : cor(x1) @test isa(c, Float64) - @test_approx_eq c Cxx[1,1] + @test c ≈ Cxx[1,1] @inferred cor(x1) @test cor(X) == Base.corm(X, mean(X, 1)) C = zm ? Base.corm(X, 0, vd) : cor(X, vd) @test size(C) == (k, k) - @test_approx_eq C Cxx + @test C ≈ Cxx @inferred cor(X, vd) @test cor(x1, y1) == Base.corm(x1, mean(x1), y1, mean(y1)) c = zm ? Base.corm(x1, 0, y1, 0) : cor(x1, y1) @test isa(c, Float64) - @test_approx_eq c Cxy[1,1] + @test c ≈ Cxy[1,1] @inferred cor(x1, y1) if vd == 1 @@ -290,7 +290,7 @@ for vd in [1, 2], zm in [true, false] end C = zm ? Base.corm(x1, 0, Y, 0, vd) : cor(x1, Y, vd) @test size(C) == (1, k) - @test_approx_eq C Cxy[1,:] + @test vec(C) ≈ Cxy[1,:] @inferred cor(x1, Y, vd) if vd == 1 @@ -298,13 +298,13 @@ for vd in [1, 2], zm in [true, false] end C = zm ? Base.corm(X, 0, y1, 0, vd) : cor(X, y1, vd) @test size(C) == (k, 1) - @test_approx_eq C Cxy[:,1] + @test vec(C) ≈ Cxy[:,1] @inferred cor(X, y1, vd) @test cor(X, Y) == Base.corm(X, mean(X, 1), Y, mean(Y, 1)) C = zm ? Base.corm(X, 0, Y, 0, vd) : cor(X, Y, vd) @test size(C) == (k, k) - @test_approx_eq C Cxy + @test C ≈ Cxy @inferred cor(X, Y, vd) end From 23308a1717fe920230edca73c24969609b6f7344 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 25 Jun 2016 17:06:15 -0400 Subject: [PATCH 0128/1117] Optimize TLS access on MAC It doesn't have static TLS model and the TLS variables are emulated with `pthread_key_t`. Use `pthread` directly since it is actually faster. --- src/julia_threads.h | 2 ++ src/threading.c | 34 ++++++++++++++++++++++++++++++++++ ui/repl.c | 4 ++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/julia_threads.h b/src/julia_threads.h index 61af77a7e7cb3..c20cc97cee83f 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -366,7 +366,9 @@ STATIC_INLINE int8_t jl_gc_state_set(jl_tls_states_t *ptls, } #else // ifndef JULIA_ENABLE_THREADING typedef jl_tls_states_t *(*jl_get_ptls_states_func)(void); +#ifndef _OS_DARWIN_ JL_DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f); +#endif // Make sure jl_gc_state() is always a rvalue #define jl_gc_state(ptls) ((int8_t)ptls->gc_state) STATIC_INLINE int8_t jl_gc_state_set(jl_tls_states_t *ptls, diff --git a/src/threading.c b/src/threading.c index 3e9c8a4544db1..61738e167b73d 100644 --- a/src/threading.c +++ b/src/threading.c @@ -30,6 +30,39 @@ extern "C" { #include "threading.h" #ifdef JULIA_ENABLE_THREADING +# ifdef _OS_DARWIN_ +// Mac doesn't seem to have static TLS model so the runtime TLS getter +// registration will only add overhead to TLS access. The `__thread` variables +// are emulated with `pthread_key_t` so it is actually faster to use it directly. +static pthread_key_t jl_tls_key; + +__attribute__((constructor)) void jl_mac_init_tls(void) +{ + pthread_key_create(&jl_tls_key, NULL); +} + +JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) +{ + void *ptls = pthread_getspecific(jl_tls_key); + if (__unlikely(!ptls)) { + ptls = calloc(1, sizeof(jl_tls_states_t)); + pthread_setspecific(jl_tls_key, ptls); + } + return (jl_tls_states_t*)ptls; +} + +// This is only used after the tls is already initialized on the thread +static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_fast(void) +{ + return (jl_tls_states_t*)pthread_getspecific(jl_tls_key); +} + +jl_get_ptls_states_func jl_get_ptls_states_getter(void) +{ + // for codegen + return &jl_get_ptls_states_fast; +} +# else // fallback provided for embedding static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_fallback(void) { @@ -78,6 +111,7 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) // for codegen return jl_tls_states_cb; } +# endif #else JL_DLLEXPORT jl_tls_states_t jl_tls_states; JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) diff --git a/ui/repl.c b/ui/repl.c index 1539eac058005..61658937a8eb7 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -34,7 +34,7 @@ extern "C" { #endif -#ifdef JULIA_ENABLE_THREADING +#if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_static(void) { # if !defined(_COMPILER_MICROSOFT_) @@ -660,7 +660,7 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) argv[i] = (wchar_t*)arg; } #endif -#ifdef JULIA_ENABLE_THREADING +#if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) // We need to make sure this function is called before any reference to // TLS variables. Since the compiler is free to move calls to // `jl_get_ptls_states()` around, we should avoid referencing TLS From 0dca9c730861b38ff5a406e548f5fde3de74abd4 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Mon, 27 Jun 2016 21:52:55 -0400 Subject: [PATCH 0129/1117] Try to implement our own TLS on windows too since it does not seem to have a static TLS model either...... --- src/julia_threads.h | 2 +- src/threading.c | 45 ++++++++++++++++++++++++++++++++++++++++----- ui/repl.c | 8 ++------ 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/julia_threads.h b/src/julia_threads.h index c20cc97cee83f..89cd65ceca0c8 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -366,7 +366,7 @@ STATIC_INLINE int8_t jl_gc_state_set(jl_tls_states_t *ptls, } #else // ifndef JULIA_ENABLE_THREADING typedef jl_tls_states_t *(*jl_get_ptls_states_func)(void); -#ifndef _OS_DARWIN_ +#if !defined(_OS_DARWIN_) && !defined(_OS_WINDOWS_) JL_DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f); #endif // Make sure jl_gc_state() is always a rvalue diff --git a/src/threading.c b/src/threading.c index 61738e167b73d..32cc82ebcd138 100644 --- a/src/threading.c +++ b/src/threading.c @@ -30,7 +30,7 @@ extern "C" { #include "threading.h" #ifdef JULIA_ENABLE_THREADING -# ifdef _OS_DARWIN_ +# if defined(_OS_DARWIN_) // Mac doesn't seem to have static TLS model so the runtime TLS getter // registration will only add overhead to TLS access. The `__thread` variables // are emulated with `pthread_key_t` so it is actually faster to use it directly. @@ -62,15 +62,50 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) // for codegen return &jl_get_ptls_states_fast; } +# elif defined(_OS_WINDOWS_) +// Apparently windows doesn't have a static TLS model (or one that can be +// reliably used from a shared library) either..... Use `TLSAlloc` instead. + +static DWORD jl_tls_key; + +// Put this here for now. We can move this out later if we find more use for it. +BOOLEAN WINAPI DllMain(IN HINSTANCE hDllHandle, IN DWORD nReason, + IN LPVOID Reserved) +{ + switch (nReason) { + case DLL_PROCESS_ATTACH: + jl_tls_key = TlsAlloc(); + assert(jl_tls_key != TLS_OUT_OF_INDEXES); + // Fall through + case DLL_THREAD_ATTACH: + TlsSetValue(jl_tls_key, calloc(1, sizeof(jl_tls_states_t))); + break; + case DLL_THREAD_DETACH: + free(TlsGetValue(jl_tls_key)); + TlsSetValue(jl_tls_key, NULL); + break; + case DLL_PROCESS_DETACH: + free(TlsGetValue(jl_tls_key)); + TlsFree(jl_tls_key); + break; + } +} + +JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) +{ + return TlsGetValue(jl_tls_key); +} + +jl_get_ptls_states_func jl_get_ptls_states_getter(void) +{ + // for codegen + return &jl_get_ptls_states; +} # else // fallback provided for embedding static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_fallback(void) { -# if !defined(_COMPILER_MICROSOFT_) static __thread jl_tls_states_t tls_states; -# else - static __declspec(thread) jl_tls_states_t tls_states; -# endif return &tls_states; } static jl_tls_states_t *jl_get_ptls_states_init(void); diff --git a/ui/repl.c b/ui/repl.c index 61658937a8eb7..5f5d4f477c67c 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -34,14 +34,10 @@ extern "C" { #endif -#if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) +#if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) && !defined(_OS_WINDOWS_) static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_static(void) { -# if !defined(_COMPILER_MICROSOFT_) static __attribute__((tls_model("local-exec"))) __thread jl_tls_states_t tls_states; -# else - static __declspec(thread) jl_tls_states_t tls_states; -# endif return &tls_states; } #endif @@ -660,7 +656,7 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) argv[i] = (wchar_t*)arg; } #endif -#if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) +#if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) && !defined(_OS_WINDOWS_) // We need to make sure this function is called before any reference to // TLS variables. Since the compiler is free to move calls to // `jl_get_ptls_states()` around, we should avoid referencing TLS From 047c7f13a5051d471be40a06882a8b85e6a0b557 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 27 Jun 2016 10:32:35 -0400 Subject: [PATCH 0130/1117] fix win64 linfo lookup during stackwalk use the same trick as linux to look at unwind info when the symbol lookup fails --- src/debuginfo.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 29d21a61f97b3..e55325fa193f6 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -859,6 +859,9 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, if (saddr) *saddr = (void*)(uintptr_t)pSymbol->Address; } + else if (saddr) { + *saddr = NULL; + } // If we didn't find the filename before in the debug // info, use the dll name @@ -1113,6 +1116,14 @@ static int jl_getDylibFunctionInfo(jl_frame_t **frames, size_t pointer, int skip if (!saddr && unw_get_proc_info_by_ip(unw_local_addr_space, pointer, &pip, NULL) == 0) saddr = (void*)pip.start_ip; +#endif +#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) + if (!saddr) { + DWORD64 ImageBase; + PRUNTIME_FUNCTION fn = RtlLookupFunctionEntry(pointer, &ImageBase, NULL); + if (fn) + saddr = (void*)(ImageBase + fn->BeginAddress); + } #endif if (saddr) { for (size_t i = 0; i < sysimg_fvars_n; i++) { From cdfe3e2bf4709615019d3204485abd37baadddc1 Mon Sep 17 00:00:00 2001 From: Benjamin Maier Date: Tue, 28 Jun 2016 13:48:46 +0200 Subject: [PATCH 0131/1117] updated llvm python dependency to ver2.7; fixed #17157 --- deps/tools/find_python2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/tools/find_python2 b/deps/tools/find_python2 index 0e07bb0c5c577..88d1410e47f18 100755 --- a/deps/tools/find_python2 +++ b/deps/tools/find_python2 @@ -1,13 +1,13 @@ #!/bin/sh # -# LLVM requires 2.5 <= python2 < 3 +# LLVM requires 2.7 <= python2 < 3 # valid_python() { - $1 -c 'import sys; sys.exit(not (sys.version_info >= (2,5) and sys.version_info < (3,0)))' 2> /dev/null + $1 -c 'import sys; sys.exit(not (sys.version_info >= (2,7) and sys.version_info < (3,0)))' 2> /dev/null } -for python in python python2 python2.7 python27 python2.6 python26 python2.5 python25; do +for python in python python2 python2.7 python27; do if valid_python $python; then echo `which $python` break From 62e7dd8e7cd8dad9a47e63988843d8caf8a6c428 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Tue, 28 Jun 2016 11:52:11 -0400 Subject: [PATCH 0132/1117] Fix rational exponent tests (#17160) fixes #17156. --- test/numbers.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/numbers.jl b/test/numbers.jl index 4ccbd59e7affd..3f33b47cd8a9b 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2640,15 +2640,15 @@ end let T = Rational x = Complex{T}(1//3 + 1//4*im) y = Complex{T}(1//2 + 1//5*im) - xf = Complex{Float64}(1//3 + 1//4*im) - yf = Complex{Float64}(1//2 + 1//5*im) + xf = Complex{BigFloat}(1//3 + 1//4*im) + yf = Complex{BigFloat}(1//2 + 1//5*im) yi = 4 - @test x^y ≈ big(xf)^big(yf) - @test_broken x^yi ≈ big(xf)^yi - @test_broken x^true ≈ big(xf)^true - @test x^false == big(xf)^false - @test_broken x^1 ≈ big(xf)^1 + @test x^y ≈ xf^yf + @test x^yi ≈ xf^yi + @test x^true ≈ xf^true + @test x^false == xf^false + @test x^1 ≈ xf^1 @test xf^Rational(2, 1) ≈ xf*xf @test Complex(1., 1.)^Rational(2,1) == Complex(1., 1.)*Complex(1.,1.) == Complex(0., 2.) end From 5a4232cc76c85517a2c5e04d14d333c07ca24277 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 28 Jun 2016 10:24:56 -0600 Subject: [PATCH 0133/1117] Cleanup of tryparse_internal methods (#17078) * Cleanup of tryparse_internal methods, which were probably written before we had default positional arguments. As they were, they weren't allowing Bool parsing to pass through to the right tryparse_internal(Bool, ...) method. Fixes #17065 * Clarify Bool string signature and fix failing tests in datafmt * Fix failing BigInt tests by making deserialize(x::BigInt) call the appropriate parse() dispatch method --- base/datafmt.jl | 4 ++-- base/gmp.jl | 4 ++++ base/parse.jl | 44 ++++++++++++++++++++++---------------------- base/serialize.jl | 2 +- test/parse.jl | 9 +++++++++ 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index 86762e2e2fcac..95712b55ea869 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -328,7 +328,7 @@ function dlm_fill(T::DataType, offarr::Vector{Vector{Int}}, dims::NTuple{2,Integ end function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Bool,2}, row::Int, col::Int) - n = tryparse_internal(Bool, sbuff, startpos, endpos, false) + n = tryparse_internal(Bool, sbuff, startpos, endpos, 0, false) isnull(n) || (cells[row, col] = get(n)) isnull(n) end @@ -360,7 +360,7 @@ function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Any,2}, isnull(ni64) || (cells[row, col] = get(ni64); return false) # check Bool - nb = tryparse_internal(Bool, sbuff, startpos, endpos, false) + nb = tryparse_internal(Bool, sbuff, startpos, endpos, 0, false) isnull(nb) || (cells[row, col] = get(nb); return false) # check float64 diff --git a/base/gmp.jl b/base/gmp.jl index a71d54d56c8c9..3c221fbb71527 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -83,6 +83,10 @@ function tryparse_internal(::Type{BigInt}, s::AbstractString, startpos::Int, end bstr = startpos == start(s) && endpos == endof(s) ? String(s) : String(SubString(s,startpos,endpos)) sgn, base, i = Base.parseint_preamble(true,base,bstr,start(bstr),endof(bstr)) + if !(2 <= base <= 62) + raise && throw(ArgumentError("invalid base: base must be 2 ≤ base ≤ 62, got $base")) + return _n + end if i == 0 raise && throw(ArgumentError("premature end of integer: $(repr(bstr))")) return _n diff --git a/base/parse.jl b/base/parse.jl index 1da9b742457e3..af17b09998f13 100644 --- a/base/parse.jl +++ b/base/parse.jl @@ -54,23 +54,18 @@ function parseint_preamble(signed::Bool, base::Int, s::AbstractString, startpos: return sgn, base, j end -function tryparse_internal(::Type{Bool}, sbuff::String, startpos::Int, endpos::Int, raise::Bool) - len = endpos-startpos+1 - p = pointer(sbuff)+startpos-1 - (len == 4) && (0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), p, "true", 4)) && (return Nullable(true)) - (len == 5) && (0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), p, "false", 5)) && (return Nullable(false)) - raise && throw(ArgumentError("invalid Bool representation: $(repr(SubString(s,startpos,endpos)))")) - Nullable{Bool}() -end - safe_add{T<:Integer}(n1::T, n2::T) = ((n2 > 0) ? (n1 > (typemax(T) - n2)) : (n1 < (typemin(T) - n2))) ? Nullable{T}() : Nullable{T}(n1 + n2) safe_mul{T<:Integer}(n1::T, n2::T) = ((n2 > 0) ? ((n1 > div(typemax(T),n2)) || (n1 < div(typemin(T),n2))) : (n2 < -1) ? ((n1 > div(typemin(T),n2)) || (n1 < div(typemax(T),n2))) : ((n2 == -1) && n1 == typemin(T))) ? Nullable{T}() : Nullable{T}(n1 * n2) -function tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, startpos::Int, endpos::Int, base::Int, a::Int, raise::Bool) +function tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, startpos::Int, endpos::Int, base::Integer, raise::Bool) _n = Nullable{T}() sgn, base, i = parseint_preamble(T<:Signed, base, s, startpos, endpos) + if !(2 <= base <= 62) + raise && throw(ArgumentError("invalid base: base must be 2 ≤ base ≤ 62, got $base")) + return _n + end if i == 0 raise && throw(ArgumentError("premature end of integer: $(repr(SubString(s,startpos,endpos)))")) return _n @@ -84,6 +79,7 @@ function tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, startpos::I base = convert(T,base) m::T = div(typemax(T)-base+1,base) n::T = 0 + a::Int = base <= 36 ? 10 : 36 while n <= m d::T = '0' <= c <= '9' ? c-'0' : 'A' <= c <= 'Z' ? c-'A'+10 : @@ -131,19 +127,23 @@ function tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, startpos::I end return Nullable{T}(n) end -tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, base::Int, raise::Bool) = - tryparse_internal(T,s,start(s),endof(s),base,raise) -tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, startpos::Int, endpos::Int, base::Int, raise::Bool) = - tryparse_internal(T, s, startpos, endpos, base, base <= 36 ? 10 : 36, raise) -tryparse{T<:Integer}(::Type{T}, s::AbstractString, base::Int) = - 2 <= base <= 62 ? tryparse_internal(T,s,Int(base),false) : throw(ArgumentError("invalid base: base must be 2 ≤ base ≤ 62, got $base")) -tryparse{T<:Integer}(::Type{T}, s::AbstractString) = tryparse_internal(T,s,0,false) - -function parse{T<:Integer}(::Type{T}, s::AbstractString, base::Integer) - (2 <= base <= 62) || throw(ArgumentError("invalid base: base must be 2 ≤ base ≤ 62, got $base")) - get(tryparse_internal(T, s, base, true)) + +function tryparse_internal(::Type{Bool}, sbuff::AbstractString, startpos::Int, endpos::Int, base::Integer, raise::Bool) + len = endpos-startpos+1 + p = pointer(sbuff)+startpos-1 + (len == 4) && (0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), p, "true", 4)) && (return Nullable(true)) + (len == 5) && (0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), p, "false", 5)) && (return Nullable(false)) + raise && throw(ArgumentError("invalid Bool representation: $(repr(SubString(s,startpos,endpos)))")) + Nullable{Bool}() +end + +function tryparse{T<:Integer}(::Type{T}, s::AbstractString, base::Integer=0) + return tryparse_internal(T, s, start(s), endof(s), base, false) +end + +function parse{T<:Integer}(::Type{T}, s::AbstractString, base::Integer=0) + return get(tryparse_internal(T, s, start(s), endof(s), base, true)) end -parse{T<:Integer}(::Type{T}, s::AbstractString) = get(tryparse_internal(T, s, 0, true)) ## string to float functions ## diff --git a/base/serialize.jl b/base/serialize.jl index f3b2589872a5a..fe7cc270e869a 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -893,7 +893,7 @@ end deserialize(s::AbstractSerializer, ::Type{BigFloat}) = parse(BigFloat, deserialize(s)) -deserialize(s::AbstractSerializer, ::Type{BigInt}) = get(GMP.tryparse_internal(BigInt, deserialize(s), 62, true)) +deserialize(s::AbstractSerializer, ::Type{BigInt}) = parse(BigInt, deserialize(s), 62) function deserialize(s::AbstractSerializer, t::Type{Regex}) pattern = deserialize(s) diff --git a/test/parse.jl b/test/parse.jl index 927802591c6ef..3fc6b8869215f 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -528,3 +528,12 @@ let err = try end @test err.line == 7 end + +# issue #17065 +@test parse(Int, "2") === 2 +@test parse(Bool, "true") === true +@test parse(Bool, "false") === false +@test get(tryparse(Bool, "true")) === get(Nullable{Bool}(true)) +@test get(tryparse(Bool, "false")) === get(Nullable{Bool}(false)) +@test_throws ArgumentError parse(Int, "2", 1) +@test_throws ArgumentError parse(Int, "2", 63) From 82b1cd92ccf76f13c8f7ea566eb741b1eaad68e7 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Sat, 25 Jun 2016 20:15:44 -0400 Subject: [PATCH 0134/1117] More Julia over julia consistency "Julia" refers to the language while "`julia`" (lowercase in backticks) specifically refers to the executable. --- CONTRIBUTING.md | 1 + base/docs/helpdb/Base.jl | 16 ++++++++-------- doc/devdocs/ast.rst | 2 +- doc/devdocs/backtraces.rst | 20 ++++++++++---------- doc/devdocs/cartesian.rst | 2 +- doc/devdocs/debuggingtips.rst | 12 ++++++------ doc/devdocs/meta.rst | 2 +- doc/devdocs/types.rst | 2 +- doc/manual/getting-started.rst | 2 +- doc/manual/interacting-with-julia.rst | 2 +- doc/manual/parallel-computing.rst | 2 +- doc/stdlib/base.rst | 6 +++--- doc/stdlib/c.rst | 4 ++-- doc/stdlib/io-network.rst | 2 +- doc/stdlib/parallel.rst | 4 ++-- 15 files changed, 40 insertions(+), 39 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e2b891de99b65..acca272e32723 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -178,6 +178,7 @@ Make sure that [Travis](http://www.travis-ci.org) greenlights the pull request w - use lower case with underscores for method names - it is generally preferred to use ASCII operators and identifiers over Unicode equivalents whenever possible + - in docstring refer to the language as "Julia" and the executable as "`julia`" #### General Formatting Guidelines For C code contributions diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index ff4e5c0290f13..418b4b728fdf6 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -193,7 +193,7 @@ typeintersect """ pointer(array [, index]) -Get the native address of an array or string element. Be careful to ensure that a julia +Get the native address of an array or string element. Be careful to ensure that a Julia reference to `a` exists as long as this pointer will be used. This function is "unsafe" like `unsafe_convert`. @@ -593,7 +593,7 @@ valtype edit(path::AbstractString, [line]) Edit a file or directory optionally providing a line number to edit the file at. Returns to -the julia prompt when you quit the editor. +the `julia` prompt when you quit the editor. """ edit(path::AbstractString, line=?) @@ -3083,7 +3083,7 @@ addprocs() addprocs(machines; keyword_args...) -> List of process identifiers ``` -Add processes on remote machines via SSH. Requires julia to be installed in the same +Add processes on remote machines via SSH. Requires `julia` to be installed in the same location on each node, or to be available via a shared file system. `machines` is a vector of machine specifications. Worker are started for each specification. @@ -3113,7 +3113,7 @@ Keyword arguments: * `dir`: specifies the working directory on the workers. Defaults to the host's current directory (as found by `pwd()`) -* `exename`: name of the julia executable. Defaults to `"\$JULIA_HOME/julia"` or +* `exename`: name of the `julia` executable. Defaults to `"\$JULIA_HOME/julia"` or `"\$JULIA_HOME/julia-debug"` as the case may be. * `exeflags`: additional flags passed to the worker processes. @@ -3466,7 +3466,7 @@ Convert `x` to a value of type `T` In cases where `convert` would need to take a Julia object and turn it into a `Ptr`, this function should be used to define and perform that conversion. -Be careful to ensure that a julia reference to `x` exists as long as the result of this +Be careful to ensure that a Julia reference to `x` exists as long as the result of this function will be used. Accordingly, the argument `x` to this function should never be an expression, only a variable name or field reference. For example, `x=a.b.c` is acceptable, but `x=[a,b,c]` is not. @@ -6571,7 +6571,7 @@ error less(file::AbstractString, [line]) Show a file using the default pager, optionally providing a starting line number. Returns to -the julia prompt when you quit the pager. +the `julia` prompt when you quit the pager. """ less(f::AbstractString, ?) @@ -7470,7 +7470,7 @@ listenany """ getpid() -> Int32 -Get julia's process ID. +Get Julia's process ID. """ getpid @@ -8929,7 +8929,7 @@ redirect_stdout """ redirect_stdout(stream) -Replace `STDOUT` by stream for all C and julia level output to `STDOUT`. Note that `stream` +Replace `STDOUT` by stream for all C and Julia level output to `STDOUT`. Note that `stream` must be a TTY, a `Pipe` or a `TCPSocket`. """ redirect_stdout(stream) diff --git a/doc/devdocs/ast.rst b/doc/devdocs/ast.rst index cc4eb6aeb7961..d2699c0428d62 100644 --- a/doc/devdocs/ast.rst +++ b/doc/devdocs/ast.rst @@ -229,7 +229,7 @@ There is generally a different expression head for each visually distinct syntactic form. Examples will be given in s-expression syntax. Each parenthesized list corresponds to an Expr, where the first element is the head. -For example ``(call f x)`` corresponds to ``Expr(:call, :f, :x)`` in julia. +For example ``(call f x)`` corresponds to ``Expr(:call, :f, :x)`` in Julia. Calls ~~~~~ diff --git a/doc/devdocs/backtraces.rst b/doc/devdocs/backtraces.rst index f14b60dc966fe..6dbe87dd3fe4e 100644 --- a/doc/devdocs/backtraces.rst +++ b/doc/devdocs/backtraces.rst @@ -10,14 +10,14 @@ If you've been directed to this page, find the symptom that best matches what yo * `Segfaults when running a script`_ -* `Errors during julia startup`_ +* `Errors during Julia startup`_ .. _version info: Version/Environment info ------------------------ -No matter the error, we will always need to know what version of julia you are running. When julia first starts up, a header is printed out with a version number and date. If your version is ``0.2.0`` or higher, please include the output of ``versioninfo()`` in any report you create:: +No matter the error, we will always need to know what version of Julia you are running. When Julia first starts up, a header is printed out with a version number and date. If your version is ``0.2.0`` or higher, please include the output of ``versioninfo()`` in any report you create:: julia> versioninfo() Julia Version 0.3.3-pre+25 @@ -37,9 +37,9 @@ No matter the error, we will always need to know what version of julia you are r Segfaults during bootstrap (sysimg.jl) -------------------------------------- -Segfaults toward the end of the ``make`` process of building julia are a common symptom of something going wrong while julia is preparsing the corpus of code in the ``base/`` folder. Many factors can contribute toward this process dying unexpectedly, however it is as often as not due to an error in the C-code portion of julia, and as such must typically be debugged with a debug build inside of ``gdb``. Explicitly: +Segfaults toward the end of the ``make`` process of building Julia are a common symptom of something going wrong while Julia is preparsing the corpus of code in the ``base/`` folder. Many factors can contribute toward this process dying unexpectedly, however it is as often as not due to an error in the C-code portion of Julia, and as such must typically be debugged with a debug build inside of ``gdb``. Explicitly: -Create a debug build of julia:: +Create a debug build of Julia:: $ cd $ make debug @@ -49,7 +49,7 @@ Note that this process will likely fail with the same error as a normal ``make`` $ cd base/ $ gdb -x ../contrib/debug_bootstrap.gdb -This will start ``gdb``, attempt to run the bootstrap process using the debug build of julia, and print out a backtrace if (when) it segfaults. You may need to hit ```` a few times to get the full backtrace. Create a gist_ with the backtrace, the `version info`_, and any other pertinent information you can think of and open a new issue_ on Github with a link to the gist. +This will start ``gdb``, attempt to run the bootstrap process using the debug build of Julia, and print out a backtrace if (when) it segfaults. You may need to hit ```` a few times to get the full backtrace. Create a gist_ with the backtrace, the `version info`_, and any other pertinent information you can think of and open a new issue_ on Github with a link to the gist. .. _Segfaults when running a script: @@ -57,7 +57,7 @@ This will start ``gdb``, attempt to run the bootstrap process using the debug bu Segfaults when running a script ------------------------------- -The procedure is very similar to `Segfaults during bootstrap (sysimg.jl)`_. Create a debug build of Julia, and run your script inside of a debugged julia process:: +The procedure is very similar to `Segfaults during bootstrap (sysimg.jl)`_. Create a debug build of Julia, and run your script inside of a debugged Julia process:: $ cd $ make debug @@ -73,12 +73,12 @@ Note that ``gdb`` will sit there, waiting for instructions. Type ``r`` to run t Create a gist_ with the backtrace, the `version info`_, and any other pertinent information you can think of and open a new issue_ on Github with a link to the gist. -.. _Errors during julia startup: +.. _Errors during Julia startup: -Errors during julia startup +Errors during Julia startup --------------------------- -Occasionally errors occur during julia's startup process (especially when using binary distributions, as opposed to compiling from source) such as the following:: +Occasionally errors occur during Julia's startup process (especially when using binary distributions, as opposed to compiling from source) such as the following:: $ julia exec: error -5 @@ -101,7 +101,7 @@ Glossary A few terms have been used as shorthand in this guide: -* ```` refers to the root directory of the julia source tree; e.g. it should contain folders such as ``base``, ``deps``, ``src``, ``test``, etc..... +* ```` refers to the root directory of the Julia source tree; e.g. it should contain folders such as ``base``, ``deps``, ``src``, ``test``, etc..... .. _gist: https://gist.github.com .. _issue: https://github.com/JuliaLang/julia/issues?state=open diff --git a/doc/devdocs/cartesian.rst b/doc/devdocs/cartesian.rst index 3b43d0cf832ee..79c1100827967 100644 --- a/doc/devdocs/cartesian.rst +++ b/doc/devdocs/cartesian.rst @@ -77,7 +77,7 @@ The first argument to both of these macros is the number of expressions, which must be an integer. When you're writing a function that you intend to work in multiple dimensions, this may not be something you want to hard-code. If you're writing code that -you need to work with older julia versions, currently you +you need to work with older Julia versions, currently you should use the ``@ngenerate`` macro described in `an older version of this documentation `_. Starting in Julia 0.4-pre, the recommended approach is to use diff --git a/doc/devdocs/debuggingtips.rst b/doc/devdocs/debuggingtips.rst index 5ebb31b9cbe89..153e8003399e4 100644 --- a/doc/devdocs/debuggingtips.rst +++ b/doc/devdocs/debuggingtips.rst @@ -12,11 +12,11 @@ Within ``gdb``, any ``jl_value_t*`` object ``obj`` can be displayed using (gdb) call jl_(obj) -The object will be displayed in the julia session, not in the gdb session. +The object will be displayed in the ``julia`` session, not in the gdb session. This is a useful way to discover the types and values of objects being manipulated by Julia's C code. -Similarly, if you're debugging some of julia's internals (e.g., +Similarly, if you're debugging some of Julia's internals (e.g., ``inference.jl``), you can print ``obj`` using :: @@ -44,11 +44,11 @@ Useful Julia functions for Inspecting those variables ----------------------------------------------------- - ``jl_gdblookup($rip)`` :: For looking up the current function and line. (use ``$eip`` on i686 platforms) -- ``jlbacktrace()`` :: For dumping the current julia backtrace stack to stderr. Only usable after ``record_backtrace()`` has been called. +- ``jlbacktrace()`` :: For dumping the current Julia backtrace stack to stderr. Only usable after ``record_backtrace()`` has been called. - ``jl_dump_llvm_value(Value*)`` :: For invoking ``Value->dump()`` in gdb, where it doesn't work natively. For example, ``f->linfo->functionObject``, ``f->linfo->specFunctionObject``, and ``to_function(f->linfo)``. - ``Type->dump()`` :: only works in lldb. Note: add something like ``;1`` to prevent lldb from printing its prompt over the output - ``jl_eval_string("expr")`` :: for invoking side-effects to modify the current state or to lookup symbols -- ``jl_typeof(jl_value_t*)`` :: for extracting the type tag of a julia value (in gdb, call ``macro define jl_typeof jl_typeof`` first, or pick something short like ``ty`` for the first arg to define a shorthand) +- ``jl_typeof(jl_value_t*)`` :: for extracting the type tag of a Julia value (in gdb, call ``macro define jl_typeof jl_typeof`` first, or pick something short like ``ty`` for the first arg to define a shorthand) Inserting breakpoints for inspection from gdb @@ -101,7 +101,7 @@ Dealing with signals Julia requires a few signal to function property. The profiler uses ``SIGUSR2`` for sampling and the garbage collector uses ``SIGSEGV`` for threads synchronization. If you are debugging some code that uses the profiler or -multiple julia threads, you may want to let the debugger ignore these signals +multiple threads, you may want to let the debugger ignore these signals since they can be triggered very often during normal operations. The command to do this in GDB is (replace ``SIGSEGV`` with ``SIGUSRS`` or other signals you want to ignore):: @@ -116,7 +116,7 @@ If you are debugging a segfault with threaded code, you can set a breakpoint on ``jl_critical_error`` (``sigdie_handler`` should also work on Linux and BSD) in order to only catch the actual segfault rather than the GC synchronization points. -Debugging during julia's build process (bootstrap) +Debugging during Julia's build process (bootstrap) -------------------------------------------------- Errors that occur during ``make`` need special handling. Julia is built in two stages, constructing diff --git a/doc/devdocs/meta.rst b/doc/devdocs/meta.rst index 9ed897362e43f..4983b195e61e3 100644 --- a/doc/devdocs/meta.rst +++ b/doc/devdocs/meta.rst @@ -8,7 +8,7 @@ Talking to the compiler (the ``:meta`` mechanism) In some circumstances, one might wish to provide hints or instructions that a given block of code has special properties: you might always want to inline it, or you might want to turn on special compiler -optimization passes. Starting with version 0.4, julia has a +optimization passes. Starting with version 0.4, Julia has a convention that these instructions can be placed inside a ``:meta`` expression, which is typically (but not necessarily) the first expression in the body of a function. diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index 9b66337475400..a75f1830f1fbd 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -63,7 +63,7 @@ tuple-type describing the types of the arguments, and ``sig`` is a tuple-type specifying the types in the function's signature.) For this algorithm to work, it's important that methods be sorted by their specificity, and that the search begins with the most specific -methods. Consequently, julia also implements a partial order on +methods. Consequently, Julia also implements a partial order on types; this is achieved by functionality that is similar to ``<:``, but with differences that will be discussed below. diff --git a/doc/manual/getting-started.rst b/doc/manual/getting-started.rst index a8114ffc683a1..d159c8731e658 100644 --- a/doc/manual/getting-started.rst +++ b/doc/manual/getting-started.rst @@ -45,7 +45,7 @@ To evaluate expressions written in a source file ``file.jl``, write ``include("file.jl")``. To run code in a file non-interactively, you can give it as the first -argument to the julia command:: +argument to the ``julia`` command:: $ julia script.jl arg1 arg2... diff --git a/doc/manual/interacting-with-julia.rst b/doc/manual/interacting-with-julia.rst index 991042bf1acbd..56c746eedb0de 100644 --- a/doc/manual/interacting-with-julia.rst +++ b/doc/manual/interacting-with-julia.rst @@ -4,7 +4,7 @@ Interacting With Julia ************************ -Julia comes with a full-featured interactive command-line REPL (read-eval-print loop) built into the ``julia`` executable. In addition to allowing quick and easy evaluation of Julia statements, it has a searchable history, tab-completion, many helpful keybindings, and dedicated help and shell modes. The REPL can be started by simply calling julia with no arguments or double-clicking on the executable:: +Julia comes with a full-featured interactive command-line REPL (read-eval-print loop) built into the ``julia`` executable. In addition to allowing quick and easy evaluation of Julia statements, it has a searchable history, tab-completion, many helpful keybindings, and dedicated help and shell modes. The REPL can be started by simply calling ``julia`` with no arguments or double-clicking on the executable:: $ julia _ diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 2acb528b7ee80..10270911bcd5a 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -860,7 +860,7 @@ If ``io`` is not specified, ``host`` and ``port`` are used to connect. For example, a cluster manager may launch a single worker per node, and use that to launch additional workers. ``count`` with an integer value ``n`` will launch a total of ``n`` workers, while a value of ``:auto`` will launch as many workers as cores on that machine. -``exename`` is the name of the Julia executable including the full path. +``exename`` is the name of the ``julia`` executable including the full path. ``exeflags`` should be set to the required command line arguments for new workers. ``tunnel``, ``bind_addr``, ``sshflags`` and ``max_parallel`` are used when a ssh tunnel is diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 7e3e92c70930f..5b6f24913c90d 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -68,7 +68,7 @@ Getting Around .. Docstring generated from Julia source - Edit a file or directory optionally providing a line number to edit the file at. Returns to the julia prompt when you quit the editor. + Edit a file or directory optionally providing a line number to edit the file at. Returns to the ``julia`` prompt when you quit the editor. .. function:: edit(function, [types]) @@ -86,7 +86,7 @@ Getting Around .. Docstring generated from Julia source - Show a file using the default pager, optionally providing a starting line number. Returns to the julia prompt when you quit the pager. + Show a file using the default pager, optionally providing a starting line number. Returns to the ``julia`` prompt when you quit the pager. .. function:: less(function, [types]) @@ -891,7 +891,7 @@ System .. Docstring generated from Julia source - Get julia's process ID. + Get Julia's process ID. .. function:: time() diff --git a/doc/stdlib/c.rst b/doc/stdlib/c.rst index 61e5e96d9f573..d7aea1c66f3d2 100644 --- a/doc/stdlib/c.rst +++ b/doc/stdlib/c.rst @@ -46,7 +46,7 @@ In cases where ``convert`` would need to take a Julia object and turn it into a ``Ptr``\ , this function should be used to define and perform that conversion. - Be careful to ensure that a julia reference to ``x`` exists as long as the result of this function will be used. Accordingly, the argument ``x`` to this function should never be an expression, only a variable name or field reference. For example, ``x=a.b.c`` is acceptable, but ``x=[a,b,c]`` is not. + Be careful to ensure that a Julia reference to ``x`` exists as long as the result of this function will be used. Accordingly, the argument ``x`` to this function should never be an expression, only a variable name or field reference. For example, ``x=a.b.c`` is acceptable, but ``x=[a,b,c]`` is not. The ``unsafe`` prefix on this function indicates that using the result of this function after the ``x`` argument to this function is no longer accessible to the program may cause undefined behavior, including program corruption or segfaults, at any later time. @@ -108,7 +108,7 @@ .. Docstring generated from Julia source - Get the native address of an array or string element. Be careful to ensure that a julia reference to ``a`` exists as long as this pointer will be used. This function is "unsafe" like ``unsafe_convert``\ . + Get the native address of an array or string element. Be careful to ensure that a Julia reference to ``a`` exists as long as this pointer will be used. This function is "unsafe" like ``unsafe_convert``\ . Calling ``Ref(array[, index])`` is generally preferable to this function. diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index c86f6f7a79098..3e2885e1635d1 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -342,7 +342,7 @@ General I/O .. Docstring generated from Julia source - Replace ``STDOUT`` by stream for all C and julia level output to ``STDOUT``\ . Note that ``stream`` must be a TTY, a ``Pipe`` or a ``TCPSocket``\ . + Replace ``STDOUT`` by stream for all C and Julia level output to ``STDOUT``\ . Note that ``stream`` must be a TTY, a ``Pipe`` or a ``TCPSocket``\ . .. function:: redirect_stderr([stream]) diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index bba11c5ce0102..65792e164ab4a 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -161,7 +161,7 @@ General Parallel Computing Support .. Docstring generated from Julia source - Add processes on remote machines via SSH. Requires julia to be installed in the same location on each node, or to be available via a shared file system. + Add processes on remote machines via SSH. Requires ``julia`` to be installed in the same location on each node, or to be available via a shared file system. ``machines`` is a vector of machine specifications. Worker are started for each specification. @@ -185,7 +185,7 @@ General Parallel Computing Support * ``dir``\ : specifies the working directory on the workers. Defaults to the host's current directory (as found by ``pwd()``\ ) - * ``exename``\ : name of the julia executable. Defaults to ``"$JULIA_HOME/julia"`` or ``"$JULIA_HOME/julia-debug"`` as the case may be. + * ``exename``\ : name of the ``julia`` executable. Defaults to ``"$JULIA_HOME/julia"`` or ``"$JULIA_HOME/julia-debug"`` as the case may be. * ``exeflags``\ : additional flags passed to the worker processes. From 11465e0192481c250571474f49672b1336786109 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 28 Jun 2016 21:07:33 -0400 Subject: [PATCH 0135/1117] fix #15760, bad warning for type with same name as deprecated binding The cause was `is_exported_from_stdlib` attempting to look up the name from the top level (Main), even if it never could have resolved to Base. --- base/show.jl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/base/show.jl b/base/show.jl index 1a7c481d7651b..b43b2e1396bb3 100644 --- a/base/show.jl +++ b/base/show.jl @@ -127,15 +127,16 @@ end # Check if a particular symbol is exported from a standard library module function is_exported_from_stdlib(name::Symbol, mod::Module) - if (mod === Base || mod === Core) && isexported(mod, name) - return true - end - parent = module_parent(mod) - if parent !== mod && isdefined(mod, name) && isdefined(parent, name) && - getfield(mod, name) === getfield(parent, name) - return is_exported_from_stdlib(name, parent) + !isdefined(mod, name) && return false + orig = getfield(mod, name) + while !(mod === Base || mod === Core) + parent = module_parent(mod) + if mod === Main || mod === parent || parent === Main + return false + end + mod = parent end - return false + return isexported(mod, name) && isdefined(mod, name) && getfield(mod, name) === orig end function show(io::IO, f::Function) From 3583241415192df05f7e2a99d1b92079bf98f9d7 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 28 Jun 2016 18:40:01 -0400 Subject: [PATCH 0136/1117] fix #4883, result type of `broadcast` for arbitrary functions --- base/broadcast.jl | 75 ++++++++++++++++++++++++++++++++++++++++++++++- test/broadcast.jl | 4 +++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 0c0165fac300e..ab8fee3c97753 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -140,7 +140,80 @@ end B end -@inline broadcast(f, As...) = broadcast!(f, allocate_for(Array{promote_eltype_op(f, As...)}, As, broadcast_shape(As...)), As...) +# broadcast with computed element type + +@generated function _broadcast!{M,AT,nargs}(f, B::AbstractArray, indexmaps::M, As::AT, ::Type{Val{nargs}}, iter, st, count) + quote + $(Expr(:meta, :noinline)) + # destructure the indexmaps and As tuples + @nexprs $nargs i->(A_i = As[i]) + @nexprs $nargs i->(imap_i = indexmaps[i]) + while !done(iter, st) + I, st = next(iter, st) + # reverse-broadcast the indices + @nexprs $nargs i->(I_i = newindex(I, imap_i)) + # extract array values + @nexprs $nargs i->(@inbounds val_i = A_i[I_i]) + # call the function + V = @ncall $nargs f val + S = typeof(V) + # store the result + if S <: eltype(B) + @inbounds B[I] = V + else + R = typejoin(eltype(B), S) + new = similar(B, R) + for II in take(iter, count) + new[II] = B[II] + end + new[I] = V + return _broadcast!(f, new, indexmaps, As, Val{nargs}, iter, st, count+1) + end + count += 1 + end + return B + end +end + +function broadcast_t(f, ::Type{Any}, As...) + shp = broadcast_shape(As...) + iter = CartesianRange(shp) + if isempty(iter) + return allocate_for(Array{Union{}}, As, shp) + end + nargs = length(As) + sz = size(iter) + indexmaps = map(x->newindexer(sz, x), As) + st = start(iter) + I, st = next(iter, st) + val = f([ As[i][newindex(I, indexmaps[i])] for i=1:nargs ]...) + B = allocate_for(Array{typeof(val)}, As, shp) + B[I] = val + return _broadcast!(f, B, indexmaps, As, Val{nargs}, iter, st, 1) +end + +@inline broadcast_t(f, T, As...) = broadcast!(f, allocate_for(Array{T}, As, broadcast_shape(As...)), As...) + +@inline broadcast(f, As...) = broadcast_t(f, promote_eltype_op(f, As...), As...) + +# alternate, more compact implementation; unfortunately slower. +# also the `collect` machinery doesn't yet support arbitrary index bases. +#= +@generated function _broadcast{nargs}(f, indexmaps, As, ::Type{Val{nargs}}, iter) + quote + collect((@ncall $nargs f i->As[i][newindex(I, indexmaps[i])]) for I in iter) + end +end + +function broadcast(f, As...) + shp = broadcast_shape(As...) + iter = CartesianRange(shp) + sz = size(iter) + indexmaps = map(x->newindexer(sz, x), As) + naT = Val{nfields(As)} + _broadcast(f, indexmaps, As, naT, iter) +end +=# @inline bitbroadcast(f, As...) = broadcast!(f, allocate_for(BitArray, As, broadcast_shape(As...)), As...) diff --git a/test/broadcast.jl b/test/broadcast.jl index 05a8e42e8d60b..ee6dc7f9ff0d4 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -196,3 +196,7 @@ end let a = broadcast(Float32, [3, 4, 5]) @test eltype(a) == Float32 end + +# issue #4883 +@test isa(broadcast(tuple, [1 2 3], ["a", "b", "c"]), Matrix{Tuple{Int,String}}) +@test isa(broadcast((x,y)->(x==1?1.0:x,y), [1 2 3], ["a", "b", "c"]), Matrix{Tuple{Real,String}}) From 6522d89fb9abef9745ce3f9221c26ce69d4a0f34 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 28 Jun 2016 23:22:57 -0400 Subject: [PATCH 0137/1117] win: cleanup build warnings --- src/threading.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/threading.c b/src/threading.c index 32cc82ebcd138..1ab2c38b64054 100644 --- a/src/threading.c +++ b/src/threading.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "julia.h" #include "julia_internal.h" @@ -89,11 +90,12 @@ BOOLEAN WINAPI DllMain(IN HINSTANCE hDllHandle, IN DWORD nReason, TlsFree(jl_tls_key); break; } + return 1; // success } JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) { - return TlsGetValue(jl_tls_key); + return (jl_tls_states_t*)TlsGetValue(jl_tls_key); } jl_get_ptls_states_func jl_get_ptls_states_getter(void) @@ -545,7 +547,7 @@ JL_DLLEXPORT void jl_threading_profile(void) if (!fork_ns) return; printf("\nti profile:\n"); - printf("prep: %g (%llu)\n", NS_TO_SECS(prep_ns), (unsigned long long)prep_ns); + printf("prep: %g (%" PRIu64 ")\n", NS_TO_SECS(prep_ns), prep_ns); uint64_t min, max, avg; ti_timings(fork_ns, &min, &max, &avg); From ffcf66de44f180b25dafbb332061a500323eb805 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 28 Jun 2016 22:52:30 -0400 Subject: [PATCH 0138/1117] win64: add Julia personality --- src/codegen.cpp | 13 +++++++++++-- src/debuginfo.cpp | 9 +++------ src/jitlayers.cpp | 13 ++++++++++--- src/julia_internal.h | 2 ++ src/signals-win.c | 3 +-- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index f6ad5461e14c9..b4ceb26064caa 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3,13 +3,14 @@ #include "llvm-version.h" #include "platform.h" #include "options.h" -#if defined(_OS_WINDOWS_) && !defined(LLVM38) -// trick pre-llvm38 into skipping the generation of _chkstk calls +#if defined(_OS_WINDOWS_) && !defined(LLVM39) +// trick pre-llvm39 into skipping the generation of _chkstk calls // since it has some codegen issues associated with them: // (a) assumed to be within 32-bit offset // (b) bad asm is generated for certain code patterns: // see https://github.com/JuliaLang/julia/pull/11644#issuecomment-112276813 // also, use ELF because RuntimeDyld COFF I686 support didn't exist +// also, use ELF because RuntimeDyld COFF X86_64 doesn't seem to work (fails to generate function pointers)? #define FORCE_ELF #endif #if defined(_CPU_X86_) @@ -413,6 +414,9 @@ static Function *jlgetnthfieldchecked_func; //static Function *jlsetnthfield_func; #ifdef _OS_WINDOWS_ static Function *resetstkoflw_func; +#if defined(_CPU_X86_64_) +static Function *juliapersonality_func; +#endif #endif static Function *diff_gc_total_bytes_func; static Function *jlarray_data_owner_func; @@ -5410,6 +5414,11 @@ static void init_julia_llvm_env(Module *m) resetstkoflw_func = Function::Create(FunctionType::get(T_int32, false), Function::ExternalLinkage, "_resetstkoflw", m); add_named_global(resetstkoflw_func, &_resetstkoflw); +#if defined(_CPU_X86_64_) + juliapersonality_func = Function::Create(FunctionType::get(T_int32, true), + Function::ExternalLinkage, "__julia_personality", m); + add_named_global(juliapersonality_func, &__julia_personality); +#endif #ifndef FORCE_ELF #if defined(_CPU_X86_64_) #if defined(_COMPILER_MINGW_) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index e55325fa193f6..ce39d8ec45de8 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -128,9 +128,6 @@ void jl_add_linfo_in_flight(StringRef name, jl_lambda_info_t *linfo, const DataL } #if defined(_OS_WINDOWS_) -#if defined(_CPU_X86_64_) -extern "C" EXCEPTION_DISPOSITION _seh_exception_handler(PEXCEPTION_RECORD ExceptionRecord,void *EstablisherFrame, PCONTEXT ContextRecord, void *DispatcherContext); -#endif static void create_PRUNTIME_FUNCTION(uint8_t *Code, size_t Size, StringRef fnname, uint8_t *Section, size_t Allocated, uint8_t *UnwindData) { @@ -143,7 +140,7 @@ static void create_PRUNTIME_FUNCTION(uint8_t *Code, size_t Size, StringRef fnnam if (!catchjmp[0]) { catchjmp[0] = 0x48; catchjmp[1] = 0xb8; // mov RAX, QWORD PTR [...] - *(uint64_t*)(&catchjmp[2]) = (uint64_t)&_seh_exception_handler; + *(uint64_t*)(&catchjmp[2]) = (uint64_t)&__julia_personality; catchjmp[10] = 0xff; catchjmp[11] = 0xe0; // jmp RAX UnwindData[0] = 0x09; // version info, UNW_FLAG_EHANDLER @@ -421,9 +418,9 @@ class JuliaJITEventListener: public JITEventListener assert(SectionAddrCheck); assert(SectionLoadOffset != 1); catchjmp[SectionLoadOffset] = 0x48; - catchjmp[SectionLoadOffset + 1] = 0xb8; // mov RAX, QWORD PTR [&_seh_exception_handle] + catchjmp[SectionLoadOffset + 1] = 0xb8; // mov RAX, QWORD PTR [&__julia_personality] *(uint64_t*)(&catchjmp[SectionLoadOffset + 2]) = - (uint64_t)&_seh_exception_handler; + (uint64_t)&__julia_personality; catchjmp[SectionLoadOffset + 10] = 0xff; catchjmp[SectionLoadOffset + 11] = 0xe0; // jmp RAX UnwindData[SectionLoadOffset] = 0x09; // version info, UNW_FLAG_EHANDLER diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 3476b821e7ee5..097565f955a8f 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -532,28 +532,35 @@ ExecutionEngine *jl_ExecutionEngine; #endif // MSVC's link.exe requires each function declaration to have a Comdat section -// rather than litter the code with conditionals, +// So rather than litter the code with conditionals, // all global values that get emitted call this function // and it decides whether the definition needs a Comdat section and adds the appropriate declaration // TODO: consider moving this into jl_add_to_shadow or jl_dump_shadow? the JIT doesn't care, so most calls are now no-ops template // for GlobalObject's static T *addComdat(T *G) { -#if defined(_OS_WINDOWS_) && defined(_COMPILER_MICROSOFT_) +#if defined(_OS_WINDOWS_) && defined(LLVM35) if (imaging_mode && !G->isDeclaration()) { -#ifdef LLVM35 // Add comdat information to make MSVC link.exe happy + // it's valid to emit this for ld.exe too, + // but makes it very slow to link for no benefit +#if defined(_COMPILER_MICROSOFT_) if (G->getParent() == shadow_output) { Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); // ELF only supports Comdat::Any jl_Comdat->setSelectionKind(Comdat::NoDuplicates); G->setComdat(jl_Comdat); } +#endif // add __declspec(dllexport) to everything marked for export if (G->getLinkage() == GlobalValue::ExternalLinkage) G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); else G->setDLLStorageClass(GlobalValue::DefaultStorageClass); +#if defined(_CPU_X86_64_) + // Add unwind exception personalities to functions to handle async exceptions + if (Function *F = dyn_cast(G)) + F->setPersonalityFn(juliapersonality_func); #endif } #endif diff --git a/src/julia_internal.h b/src/julia_internal.h index 40c184c35bba4..d70b2c73fe6c3 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -343,6 +343,8 @@ typedef struct { uint64_t jl_getUnwindInfo(uint64_t dwBase); #ifdef _OS_WINDOWS_ #include +JL_DLLEXPORT EXCEPTION_DISPOSITION __julia_personality( + PEXCEPTION_RECORD ExceptionRecord, void *EstablisherFrame, PCONTEXT ContextRecord, void *DispatcherContext); extern HANDLE hMainThread; typedef CONTEXT bt_context_t; #if defined(_CPU_X86_64_) diff --git a/src/signals-win.c b/src/signals-win.c index 4950e0c6d016b..8569658c93664 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -278,8 +278,7 @@ static LONG WINAPI exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo) } #if defined(_CPU_X86_64_) -EXCEPTION_DISPOSITION _seh_exception_handler(PEXCEPTION_RECORD ExceptionRecord, void *EstablisherFrame, PCONTEXT ContextRecord, void *DispatcherContext) -{ +JL_DLLEXPORT EXCEPTION_DISPOSITION __julia_personality(PEXCEPTION_RECORD ExceptionRecord, void *EstablisherFrame, PCONTEXT ContextRecord, void *DispatcherContext) { EXCEPTION_POINTERS ExceptionInfo; ExceptionInfo.ExceptionRecord = ExceptionRecord; ExceptionInfo.ContextRecord = ContextRecord; From 71b4d5ece8be172db398384eae85da6ffc5ebdf0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 28 Jun 2016 23:29:07 -0400 Subject: [PATCH 0139/1117] set precompiled=yes by default on windows also fix the test for this to actually test for this, instead of injecting a non-default build configuration in order to test for something that bootstrapping is already testing fix #16953 --- .travis.yml | 7 +++---- appveyor.yml | 2 +- contrib/windows/msys_build.sh | 1 - src/init.c | 5 ----- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 91a4a35d643fd..cc87e6ca68b39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,7 +56,6 @@ notifications: - http://julia.mit.edu:8000/travis-hook before_install: - make check-whitespace - - JULIA_SYSIMG_BUILD_FLAGS="--output-ji ../usr/lib/julia/sys.ji" - if [ `uname` = "Linux" ]; then contrib/travis_fastfail.sh || exit 1; mkdir -p $HOME/bin; @@ -93,7 +92,7 @@ script: - make -C moreutils mispipe - make $BUILDOPTS -C base version_git.jl.phony - moreutils/mispipe "make $BUILDOPTS NO_GIT=1 -C deps" bar > deps.log || cat deps.log - - make $BUILDOPTS NO_GIT=1 JULIA_SYSIMG_BUILD_FLAGS="$JULIA_SYSIMG_BUILD_FLAGS" prefix=/tmp/julia install | moreutils/ts -s "%.s" + - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia install | moreutils/ts -s "%.s" - make $BUILDOPTS NO_GIT=1 build-stats - du -sk /tmp/julia/* - if [ `uname` = "Darwin" ]; then @@ -102,8 +101,8 @@ script: done; fi - cd .. && mv julia julia2 - - cp /tmp/julia/lib/julia/sys.ji local.ji && /tmp/julia/bin/julia -J local.ji -e 'true' && - /tmp/julia/bin/julia-debug -J local.ji -e 'true' && rm local.ji + - /tmp/julia/bin/julia --precompiled=no -e 'true' && + /tmp/julia/bin/julia-debug --precompiled=no -e 'true' - /tmp/julia/bin/julia -e 'versioninfo()' - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN && diff --git a/appveyor.yml b/appveyor.yml index be1fa0d55b214..b417243521f8b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -53,6 +53,6 @@ build_script: test_script: - usr\bin\julia -e "versioninfo()" - - copy usr\lib\julia\sys.ji local.ji && usr\bin\julia -J local.ji -e "true" && del local.ji + - usr\bin\julia --precompiled=no -e "true" - cd test && ..\usr\bin\julia --check-bounds=yes runtests.jl all && ..\usr\bin\julia --check-bounds=yes runtests.jl libgit2-online pkg diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index fdb29f7711eed..aefa0b2864868 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -166,7 +166,6 @@ for lib in SUITESPARSE ARPACK BLAS LAPACK FFTW \ done echo 'override LIBLAPACK = $(LIBBLAS)' >> Make.user echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user -echo 'JULIA_SYSIMG_BUILD_FLAGS=--output-ji ../usr/lib/julia/sys.ji' >> Make.user # Remaining dependencies: # libuv since its static lib is no longer included in the binaries diff --git a/src/init.c b/src/init.c index 6b794c68cf2bf..bf0a45448c2fe 100644 --- a/src/init.c +++ b/src/init.c @@ -73,12 +73,7 @@ jl_options_t jl_options = { 0, // quiet JL_OPTIONS_FAST_MATH_DEFAULT, 0, // worker JL_OPTIONS_HANDLE_SIGNALS_ON, -#ifdef _OS_WINDOWS_ -// TODO remove this when using LLVM 3.5+ - JL_OPTIONS_USE_PRECOMPILED_NO, -#else JL_OPTIONS_USE_PRECOMPILED_YES, -#endif JL_OPTIONS_USE_COMPILECACHE_YES, NULL, // bindto NULL, // outputbc From a20566be5b67632cea079ed09169c9ad9a3278c3 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 28 Jun 2016 23:33:22 -0400 Subject: [PATCH 0140/1117] Re-key llvm-win64-reloc-dwarf.patch against LLVM 3.7 should fix #17177, mac build failure due to patch pickiness --- deps/patches/llvm-win64-reloc-dwarf.patch | 121 ++++++++++++---------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/deps/patches/llvm-win64-reloc-dwarf.patch b/deps/patches/llvm-win64-reloc-dwarf.patch index 45bd2604e8488..7cd5564950b07 100644 --- a/deps/patches/llvm-win64-reloc-dwarf.patch +++ b/deps/patches/llvm-win64-reloc-dwarf.patch @@ -1,18 +1,20 @@ -diff -rpu llvm-3.8.0/include/llvm/MC/MCStreamer.h llvm-3.8.0-reloc/include/llvm/MC/MCStreamer.h ---- llvm-3.8.0/include/llvm/MC/MCStreamer.h 2016-01-12 08:38:15.000000000 -0500 -+++ llvm-3.8.0-reloc/include/llvm/MC/MCStreamer.h 2016-05-23 18:54:01.830143100 -0400 -@@ -456,7 +456,7 @@ public: +diff --git a/include/llvm/MC/MCStreamer.h b/include/llvm/MC/MCStreamer.h +index 6b9b8a1..0113664 100644 +--- a/include/llvm/MC/MCStreamer.h ++++ b/include/llvm/MC/MCStreamer.h +@@ -452,7 +452,7 @@ public: /// \brief Emits a COFF section relative relocation. /// /// \param Symbol - Symbol the section relative relocation should point to. - virtual void EmitCOFFSecRel32(MCSymbol const *Symbol); + virtual void EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset); - + /// \brief Emit an ELF .size directive. /// -diff -rpu llvm-3.8.0/include/llvm/MC/MCWinCOFFStreamer.h llvm-3.8.0-reloc/include/llvm/MC/MCWinCOFFStreamer.h ---- llvm-3.8.0/include/llvm/MC/MCWinCOFFStreamer.h 2015-11-17 05:00:43.000000000 -0500 -+++ llvm-3.8.0-reloc/include/llvm/MC/MCWinCOFFStreamer.h 2016-05-23 18:54:01.845722400 -0400 +diff --git a/include/llvm/MC/MCWinCOFFStreamer.h b/include/llvm/MC/MCWinCOFFStreamer.h +index 6fbc754..500b1c0 100644 +--- a/include/llvm/MC/MCWinCOFFStreamer.h ++++ b/include/llvm/MC/MCWinCOFFStreamer.h @@ -52,7 +52,7 @@ public: void EndCOFFSymbolDef() override; void EmitCOFFSafeSEH(MCSymbol const *Symbol) override; @@ -22,10 +24,11 @@ diff -rpu llvm-3.8.0/include/llvm/MC/MCWinCOFFStreamer.h llvm-3.8.0-reloc/includ void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size, unsigned ByteAlignment) override; void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size, -diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinter.cpp llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/AsmPrinter.cpp ---- llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinter.cpp 2016-01-12 20:18:13.000000000 -0500 -+++ llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/AsmPrinter.cpp 2016-05-23 18:54:01.845722400 -0400 -@@ -1729,7 +1729,7 @@ void AsmPrinter::EmitLabelPlusOffset(con +diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +index 125047e..3483412 100644 +--- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp ++++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +@@ -1598,7 +1598,7 @@ void AsmPrinter::EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset, unsigned Size, bool IsSectionRelative) const { if (MAI->needsDwarfSectionOffsetDirective() && IsSectionRelative) { @@ -33,11 +36,12 @@ diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinter.cpp llvm-3.8.0-reloc/lib/ + OutStreamer->EmitCOFFSecRel32(Label, Offset); return; } - -diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp ---- llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp 2015-11-17 19:34:10.000000000 -0500 -+++ llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp 2016-05-23 20:17:42.105695800 -0400 -@@ -150,7 +150,7 @@ void AsmPrinter::emitDwarfSymbolReferenc + +diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp +index ad180b6..6bd7dfb 100644 +--- a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp ++++ b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp +@@ -162,7 +162,7 @@ void AsmPrinter::emitDwarfSymbolReference(const MCSymbol *Label, if (!ForceOffset) { // On COFF targets, we have to emit the special .secrel32 directive. if (MAI->needsDwarfSectionOffsetDirective()) { @@ -45,11 +49,12 @@ diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp llvm-3.8.0-reloc + OutStreamer->EmitCOFFSecRel32(Label, /*offset*/0); return; } - -diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/DIE.cpp llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/DIE.cpp ---- llvm-3.8.0/lib/CodeGen/AsmPrinter/DIE.cpp 2016-01-07 09:28:20.000000000 -0500 -+++ llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/DIE.cpp 2016-05-23 19:08:35.434799900 -0400 -@@ -487,7 +487,7 @@ void DIEEntry::EmitValue(const AsmPrinte + +diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp +index 46dbc76..70b2014 100644 +--- a/lib/CodeGen/AsmPrinter/DIE.cpp ++++ b/lib/CodeGen/AsmPrinter/DIE.cpp +@@ -457,7 +457,7 @@ void DIEEntry::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const { Addr += CU->getDebugInfoOffset(); if (AP->MAI->doesDwarfUseRelocationsAcrossSections()) AP->EmitLabelPlusOffset(CU->getSectionSym(), Addr, @@ -58,10 +63,11 @@ diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/DIE.cpp llvm-3.8.0-reloc/lib/CodeGen else AP->OutStreamer->EmitIntValue(Addr, DIEEntry::getRefAddrSize(AP)); } else -diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp ---- llvm-3.8.0/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp 2016-01-12 20:05:23.000000000 -0500 -+++ llvm-3.8.0-reloc/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp 2016-05-23 21:19:31.454460900 -0400 -@@ -231,7 +231,7 @@ void WinCodeViewLineTables::emitDebugInf +diff --git a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp +index 6610ac7..3b2188a 100644 +--- a/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp ++++ b/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp +@@ -221,7 +221,7 @@ void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) { // code is located and what's its size: EmitLabelDiff(*Asm->OutStreamer, Fn, FI.End); Asm->OutStreamer->EmitFill(12, 0); @@ -70,19 +76,20 @@ diff -rpu llvm-3.8.0/lib/CodeGen/AsmPrinter/WinCodeViewLineTables.cpp llvm-3.8.0 Asm->OutStreamer->EmitCOFFSectionIndex(Fn); Asm->EmitInt8(0); // Emit the function display name as a null-terminated string. -@@ -272,7 +272,7 @@ void WinCodeViewLineTables::emitDebugInf +@@ -262,7 +262,7 @@ void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) { Asm->OutStreamer->EmitLabel(LineTableBegin); - + // Identify the function this subsection is for. - Asm->OutStreamer->EmitCOFFSecRel32(Fn); + Asm->OutStreamer->EmitCOFFSecRel32(Fn, /*offset*/0); Asm->OutStreamer->EmitCOFFSectionIndex(Fn); // Insert flags after a 16-bit section index. Asm->EmitInt16(COFF::DEBUG_LINE_TABLES_HAVE_COLUMN_RECORDS); -diff -rpu llvm-3.8.0/lib/MC/MCAsmStreamer.cpp llvm-3.8.0-reloc/lib/MC/MCAsmStreamer.cpp ---- llvm-3.8.0/lib/MC/MCAsmStreamer.cpp 2015-11-12 08:33:00.000000000 -0500 -+++ llvm-3.8.0-reloc/lib/MC/MCAsmStreamer.cpp 2016-05-23 18:54:01.859727900 -0400 -@@ -143,7 +143,7 @@ public: +diff --git a/lib/MC/MCAsmStreamer.cpp b/lib/MC/MCAsmStreamer.cpp +index 227c937..c2f3e6e 100644 +--- a/lib/MC/MCAsmStreamer.cpp ++++ b/lib/MC/MCAsmStreamer.cpp +@@ -138,7 +138,7 @@ public: void EndCOFFSymbolDef() override; void EmitCOFFSafeSEH(MCSymbol const *Symbol) override; void EmitCOFFSectionIndex(MCSymbol const *Symbol) override; @@ -91,55 +98,58 @@ diff -rpu llvm-3.8.0/lib/MC/MCAsmStreamer.cpp llvm-3.8.0-reloc/lib/MC/MCAsmStrea void emitELFSize(MCSymbolELF *Symbol, const MCExpr *Value) override; void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size, unsigned ByteAlignment) override; -@@ -525,7 +525,7 @@ void MCAsmStreamer::EmitCOFFSectionIndex +@@ -514,7 +514,7 @@ void MCAsmStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) { EmitEOL(); } - + -void MCAsmStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol) { +void MCAsmStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) { OS << "\t.secrel32\t"; Symbol->print(OS, MAI); EmitEOL(); -diff -rpu llvm-3.8.0/lib/MC/MCParser/COFFAsmParser.cpp llvm-3.8.0-reloc/lib/MC/MCParser/COFFAsmParser.cpp ---- llvm-3.8.0/lib/MC/MCParser/COFFAsmParser.cpp 2015-11-18 01:02:15.000000000 -0500 -+++ llvm-3.8.0-reloc/lib/MC/MCParser/COFFAsmParser.cpp 2016-05-23 18:54:15.937859600 -0400 -@@ -450,7 +450,7 @@ bool COFFAsmParser::ParseDirectiveSecRel +diff --git a/lib/MC/MCParser/COFFAsmParser.cpp b/lib/MC/MCParser/COFFAsmParser.cpp +index f09bce0..39a9ad7 100644 +--- a/lib/MC/MCParser/COFFAsmParser.cpp ++++ b/lib/MC/MCParser/COFFAsmParser.cpp +@@ -451,7 +451,7 @@ bool COFFAsmParser::ParseDirectiveSecRel32(StringRef, SMLoc) { MCSymbol *Symbol = getContext().getOrCreateSymbol(SymbolID); - + Lex(); - getStreamer().EmitCOFFSecRel32(Symbol); + getStreamer().EmitCOFFSecRel32(Symbol, /*Offset=*/0); return false; } - -diff -rpu llvm-3.8.0/lib/MC/MCStreamer.cpp llvm-3.8.0-reloc/lib/MC/MCStreamer.cpp ---- llvm-3.8.0/lib/MC/MCStreamer.cpp 2015-11-04 18:59:18.000000000 -0500 -+++ llvm-3.8.0-reloc/lib/MC/MCStreamer.cpp 2016-05-23 18:54:15.953281800 -0400 -@@ -119,7 +119,7 @@ void MCStreamer::EmitSymbolValue(const M + +diff --git a/lib/MC/MCStreamer.cpp b/lib/MC/MCStreamer.cpp +index 7fbbbd9..c9e8958 100644 +--- a/lib/MC/MCStreamer.cpp ++++ b/lib/MC/MCStreamer.cpp +@@ -120,7 +120,7 @@ void MCStreamer::EmitSymbolValue(const MCSymbol *Sym, unsigned Size, if (!IsSectionRelative) EmitValueImpl(MCSymbolRefExpr::create(Sym, getContext()), Size); else - EmitCOFFSecRel32(Sym); + EmitCOFFSecRel32(Sym, /*Offset=*/0); } - + void MCStreamer::EmitGPRel64Value(const MCExpr *Value) { -@@ -570,7 +570,7 @@ void MCStreamer::EmitCOFFSafeSEH(MCSymbo +@@ -562,7 +562,7 @@ void MCStreamer::EmitCOFFSafeSEH(MCSymbol const *Symbol) { void MCStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) { } - + -void MCStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol) { +void MCStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) { } - + /// EmitRawText - If this file is backed by an assembly streamer, this dumps -diff -rpu llvm-3.8.0/lib/MC/WinCOFFStreamer.cpp llvm-3.8.0-reloc/lib/MC/WinCOFFStreamer.cpp ---- llvm-3.8.0/lib/MC/WinCOFFStreamer.cpp 2015-11-17 05:00:43.000000000 -0500 -+++ llvm-3.8.0-reloc/lib/MC/WinCOFFStreamer.cpp 2016-06-25 22:00:26.530421900 -0400 -@@ -199,14 +199,21 @@ void MCWinCOFFStreamer::EmitCOFFSectionI +diff --git a/lib/MC/WinCOFFStreamer.cpp b/lib/MC/WinCOFFStreamer.cpp +index 36dd691..d6c91db 100644 +--- a/lib/MC/WinCOFFStreamer.cpp ++++ b/lib/MC/WinCOFFStreamer.cpp +@@ -192,11 +192,18 @@ void MCWinCOFFStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) { DF->getContents().resize(DF->getContents().size() + 2, 0); } - + -void MCWinCOFFStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol) { +void MCWinCOFFStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) { MCDataFragment *DF = getOrCreateDataFragment(); @@ -157,7 +167,4 @@ diff -rpu llvm-3.8.0/lib/MC/WinCOFFStreamer.cpp llvm-3.8.0-reloc/lib/MC/WinCOFFS + // Emit 4 bytes (zeros) to the object file. DF->getContents().resize(DF->getContents().size() + 4, 0); } - - void MCWinCOFFStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size, - unsigned ByteAlignment) { - assert((!Symbol->isInSection() || + From c6f62879bf9732f8740fea96e39d27db161a6b06 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 25 Jun 2016 12:39:48 -0400 Subject: [PATCH 0141/1117] remove io :multiline attribute this attribute doesn't nest properly, and functions just end up branching almost the entire code based on this parameter, so it's cleaner as a separate method fix #17087 fix #16910 fix #17090 --- base/Enums.jl | 23 +++---- base/REPL.jl | 4 +- base/deprecated.jl | 13 ++++ base/dict.jl | 132 ++++++----------------------------- base/linalg/bidiag.jl | 15 ++-- base/multimedia.jl | 12 +--- base/range.jl | 30 +++----- base/replutil.jl | 133 ++++++++++++++++++++++++++++++++++++ base/show.jl | 45 +++--------- base/sparse/cholmod.jl | 5 ++ base/sparse/sparsematrix.jl | 16 +++-- base/sparse/sparsevector.jl | 14 ++-- base/task.jl | 6 -- test/dict.jl | 12 ++-- test/offsetarray.jl | 4 +- test/ranges.jl | 2 +- test/replutil.jl | 4 +- test/show.jl | 9 +-- test/sparsedir/cholmod.jl | 7 +- 19 files changed, 250 insertions(+), 236 deletions(-) diff --git a/base/Enums.jl b/base/Enums.jl index 39581d411f758..ba1073cd36f72 100644 --- a/base/Enums.jl +++ b/base/Enums.jl @@ -85,14 +85,14 @@ macro enum(T,syms...) let insts = ntuple(i->$(esc(typename))($values[i]), $(length(vals))) Base.instances(::Type{$(esc(typename))}) = insts end - function Base.print(io::IO,x::$(esc(typename))) + function Base.print(io::IO, x::$(esc(typename))) for (sym, i) in $vals if i == Int32(x) print(io, sym); break end end end - function Base.show(io::IO,x::$(esc(typename))) + function Base.show(io::IO, x::$(esc(typename))) if get(io, :compact, false) print(io, x) else @@ -101,16 +101,15 @@ macro enum(T,syms...) print(io, " = ", Int(x)) end end - function Base.show(io::IO,t::Type{$(esc(typename))}) - if get(io, :multiline, false) - print(io, "Enum ") - Base.show_datatype(io, t) - print(io, ":") - for (sym, i) in $vals - print(io, "\n", sym, " = ", i) - end - else - Base.show_datatype(io, t) + function Base.show(io::IO, t::Type{$(esc(typename))}) + Base.show_datatype(io, t) + end + function Base.show(io::IO, ::MIME"text/plain", t::Type{$(esc(typename))}) + print(io, "Enum ") + Base.show_datatype(io, t) + print(io, ":") + for (sym, i) in $vals + print(io, "\n", sym, " = ", i) end end end diff --git a/base/REPL.jl b/base/REPL.jl index 977fb156bdfc4..deebc606df6a5 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -108,10 +108,10 @@ end ==(a::REPLDisplay, b::REPLDisplay) = a.repl === b.repl -function display(d::REPLDisplay, ::MIME"text/plain", x) +function display(d::REPLDisplay, mime::MIME"text/plain", x) io = outstream(d.repl) Base.have_color && write(io, answer_color(d.repl)) - show(IOContext(io, multiline=true, limit=true), MIME("text/plain"), x) + show(IOContext(io, :limit => true), mime, x) println(io) end display(d::REPLDisplay, x) = display(d, MIME("text/plain"), x) diff --git a/base/deprecated.jl b/base/deprecated.jl index b22c1a3cc6523..8e59e53e1269e 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -788,6 +788,19 @@ function symperm{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}) "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) end +# needs to be a macro so that we can use ::@mime(s) in type declarations +eval(Multimedia, quote +export @MIME +macro MIME(s) + Base.warn_once("@MIME(\"\") is deprecated, use MIME\"\" instead.") + if isa(s,AbstractString) + :(MIME{$(Expr(:quote, Symbol(s)))}) + else + :(MIME{Symbol($s)}) + end +end +end) + # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/dict.jl b/base/dict.jl index 7c473f211185f..00e56434d2745 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -49,91 +49,34 @@ function _truncate_at_width_or_chars(str, width, chars="", truncmark="…") end function show{K,V}(io::IO, t::Associative{K,V}) - recur_io = IOContext(io, SHOWN_SET=t, multiline=false) + recur_io = IOContext(io, :SHOWN_SET => t) limit::Bool = get(io, :limit, false) - compact = !get(io, :multiline, false) if !haskey(io, :compact) - recur_io = IOContext(recur_io, compact=true) + recur_io = IOContext(recur_io, :compact => true) end - if compact - # show in a Julia-syntax-like form: Dict(k=>v, ...) - if isempty(t) - print(io, typeof(t), "()") - else - if isleaftype(K) && isleaftype(V) - print(io, typeof(t).name) - else - print(io, typeof(t)) - end - print(io, '(') - if !show_circular(io, t) - first = true - n = 0 - for pair in t - first || print(io, ',') - first = false - show(recur_io, pair) - n+=1 - limit && n >= 10 && (print(io, "…"); break) - end - end - print(io, ')') - end - return - end - - # Otherwise show more descriptively, with one line per key/value pair - print(io, summary(t)) - isempty(t) && return - print(io, ":\n ") - show_circular(io, t) && return - if limit - sz = displaysize(io) - rows, cols = sz[1] - 3, sz[2] - rows < 2 && (print(io, " …"); return) - cols < 12 && (cols = 12) # Minimum widths of 2 for key, 4 for value - cols -= 6 # Subtract the widths of prefix " " separator " => " - rows -= 2 # Subtract the summary and final ⋮ continuation lines - - # determine max key width to align the output, caching the strings - ks = Array{AbstractString}(min(rows, length(t))) - vs = Array{AbstractString}(min(rows, length(t))) - keylen = 0 - vallen = 0 - for (i, (k, v)) in enumerate(t) - i > rows && break - ks[i] = sprint(0, show, k, env=recur_io) - vs[i] = sprint(0, show, v, env=recur_io) - keylen = clamp(length(ks[i]), keylen, cols) - vallen = clamp(length(vs[i]), vallen, cols) - end - if keylen > max(div(cols, 2), cols - vallen) - keylen = max(cld(cols, 3), cols - vallen) - end - else - rows = cols = 0 - end - - first = true - for (i, (k, v)) in enumerate(t) - first || print(io, "\n ") - first = false - limit && i > rows && (print(io, rpad("⋮", keylen), " => ⋮"); break) - if limit - key = rpad(_truncate_at_width_or_chars(ks[i], keylen, "\r\n"), keylen) + # show in a Julia-syntax-like form: Dict(k=>v, ...) + if isempty(t) + print(io, typeof(t), "()") + else + if isleaftype(K) && isleaftype(V) + print(io, typeof(t).name) else - key = sprint(0, show, k, env=recur_io) + print(io, typeof(t)) end - print(recur_io, key) - print(io, " => ") - - if limit - val = _truncate_at_width_or_chars(vs[i], cols - keylen, "\r\n") - print(io, val) - else - show(recur_io, v) + print(io, '(') + if !show_circular(io, t) + first = true + n = 0 + for pair in t + first || print(io, ',') + first = false + show(recur_io, pair) + n+=1 + limit && n >= 10 && (print(io, "…"); break) + end end + print(io, ')') end end @@ -147,38 +90,7 @@ end summary{T<:Union{KeyIterator,ValueIterator}}(iter::T) = string(T.name, " for a ", summary(iter.dict)) -function show(io::IO, iter::Union{KeyIterator,ValueIterator}) - if !get(io, :multiline, false) - return show(io, collect(iter)) - end - print(io, summary(iter)) - isempty(iter) && return - print(io, ". ", isa(iter,KeyIterator) ? "Keys" : "Values", ":") - limit::Bool = get(io, :limit, false) - if limit - sz = displaysize(io) - rows, cols = sz[1] - 3, sz[2] - rows < 2 && (print(io, " …"); return) - cols < 4 && (cols = 4) - cols -= 2 # For prefix " " - rows -= 2 # For summary and final ⋮ continuation lines - else - rows = cols = 0 - end - - for (i, v) in enumerate(iter) - print(io, "\n ") - limit && i >= rows && (print(io, "⋮"); break) - - if limit - str = sprint(0, show, v, env=io) - str = _truncate_at_width_or_chars(str, cols, "\r\n") - print(io, str) - else - show(io, v) - end - end -end +show(io::IO, iter::Union{KeyIterator,ValueIterator}) = show(io, collect(iter)) length(v::Union{KeyIterator,ValueIterator}) = length(v.dict) isempty(v::Union{KeyIterator,ValueIterator}) = isempty(v.dict) diff --git a/base/linalg/bidiag.jl b/base/linalg/bidiag.jl index 1b6452b2a4373..dfa3aae1d44c7 100644 --- a/base/linalg/bidiag.jl +++ b/base/linalg/bidiag.jl @@ -168,15 +168,12 @@ svdfact(M::Bidiagonal; thin::Bool=true) = svdfact!(copy(M),thin=thin) #################### function show(io::IO, M::Bidiagonal) - if get(io, :multiline, false) - Base.showarray(io, M) - else - println(io, summary(M), ":") - print(io, " diag:") - print_matrix(io, (M.dv)') - print(io, M.isupper?"\n super:":"\n sub:") - print_matrix(io, (M.ev)') - end + # TODO: make this readable and one-line + println(io, summary(M), ":") + print(io, " diag:") + print_matrix(io, (M.dv)') + print(io, M.isupper?"\n super:":"\n sub:") + print_matrix(io, (M.ev)') end size(M::Bidiagonal) = (length(M.dv), length(M.dv)) diff --git a/base/multimedia.jl b/base/multimedia.jl index 416b4df051257..45b827e66d859 100644 --- a/base/multimedia.jl +++ b/base/multimedia.jl @@ -3,7 +3,7 @@ module Multimedia export Display, display, pushdisplay, popdisplay, displayable, redisplay, - MIME, @MIME, @MIME_str, reprmime, stringmime, istextmime, + MIME, @MIME_str, reprmime, stringmime, istextmime, mimewritable, TextDisplay ########################################################################### @@ -18,16 +18,6 @@ MIME(s) = MIME{Symbol(s)}() show{mime}(io::IO, ::MIME{mime}) = print(io, "MIME type ", string(mime)) print{mime}(io::IO, ::MIME{mime}) = print(io, mime) -# needs to be a macro so that we can use ::@mime(s) in type declarations -macro MIME(s) - Base.warn_once("@MIME(\"\") is deprecated, use MIME\"\" instead.") - if isa(s,AbstractString) - :(MIME{$(Expr(:quote, Symbol(s)))}) - else - :(MIME{Symbol($s)}) - end -end - macro MIME_str(s) :(MIME{$(Expr(:quote, Symbol(s)))}) end diff --git a/base/range.jl b/base/range.jl index 9dfe9a394060d..58ec756d1b078 100644 --- a/base/range.jl +++ b/base/range.jl @@ -236,25 +236,13 @@ linspace(start::Real, stop::Real, len::Real=50) = linspace(promote(AbstractFloat(start), AbstractFloat(stop))..., len) function show(io::IO, r::LinSpace) - if get(io, :multiline, false) - # show for linspace, e.g. - # linspace(1,3,7) - # 7-element LinSpace{Float64}: - # 1.0,1.33333,1.66667,2.0,2.33333,2.66667,3.0 - print(io, summary(r)) - if !isempty(r) - println(io, ":") - print_range(io, r) - end - else - print(io, "linspace(") - show(io, first(r)) - print(io, ',') - show(io, last(r)) - print(io, ',') - show(io, length(r)) - print(io, ')') - end + print(io, "linspace(") + show(io, first(r)) + print(io, ',') + show(io, last(r)) + print(io, ',') + show(io, length(r)) + print(io, ')') end """ @@ -497,9 +485,7 @@ function getindex{T}(r::LinSpace{T}, s::OrdinalRange) return linspace(vfirst, vlast, sl) end -function show(io::IO, r::Range) - print(io, repr(first(r)), ':', repr(step(r)), ':', repr(last(r))) -end +show(io::IO, r::Range) = print(io, repr(first(r)), ':', repr(step(r)), ':', repr(last(r))) show(io::IO, r::UnitRange) = print(io, repr(first(r)), ':', repr(last(r))) =={T<:Range}(r::T, s::T) = (first(r) == first(s)) & (step(r) == step(s)) & (last(r) == last(s)) diff --git a/base/replutil.jl b/base/replutil.jl index 90260b515363c..229aa877f657c 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -3,6 +3,139 @@ # fallback text/plain representation of any type: show(io::IO, ::MIME"text/plain", x) = show(io, x) +# multiline show functions for types defined before multimedia.jl: +function show(io::IO, ::MIME"text/plain", iter::Union{KeyIterator,ValueIterator}) + print(io, summary(iter)) + isempty(iter) && return + print(io, ". ", isa(iter,KeyIterator) ? "Keys" : "Values", ":") + limit::Bool = get(io, :limit, false) + if limit + sz = displaysize(io) + rows, cols = sz[1] - 3, sz[2] + rows < 2 && (print(io, " …"); return) + cols < 4 && (cols = 4) + cols -= 2 # For prefix " " + rows -= 2 # For summary and final ⋮ continuation lines + else + rows = cols = 0 + end + + for (i, v) in enumerate(iter) + print(io, "\n ") + limit && i >= rows && (print(io, "⋮"); break) + + if limit + str = sprint(0, show, v, env=io) + str = _truncate_at_width_or_chars(str, cols, "\r\n") + print(io, str) + else + show(io, v) + end + end +end + +function show{K,V}(io::IO, ::MIME"text/plain", t::Associative{K,V}) + # show more descriptively, with one line per key/value pair + recur_io = IOContext(io, :SHOWN_SET => t) + limit::Bool = get(io, :limit, false) + if !haskey(io, :compact) + recur_io = IOContext(recur_io, compact=true) + end + + print(io, summary(t)) + isempty(t) && return + print(io, ":\n ") + show_circular(io, t) && return + if limit + sz = displaysize(io) + rows, cols = sz[1] - 3, sz[2] + rows < 2 && (print(io, " …"); return) + cols < 12 && (cols = 12) # Minimum widths of 2 for key, 4 for value + cols -= 6 # Subtract the widths of prefix " " separator " => " + rows -= 2 # Subtract the summary and final ⋮ continuation lines + + # determine max key width to align the output, caching the strings + ks = Array{AbstractString}(min(rows, length(t))) + vs = Array{AbstractString}(min(rows, length(t))) + keylen = 0 + vallen = 0 + for (i, (k, v)) in enumerate(t) + i > rows && break + ks[i] = sprint(0, show, k, env=recur_io) + vs[i] = sprint(0, show, v, env=recur_io) + keylen = clamp(length(ks[i]), keylen, cols) + vallen = clamp(length(vs[i]), vallen, cols) + end + if keylen > max(div(cols, 2), cols - vallen) + keylen = max(cld(cols, 3), cols - vallen) + end + else + rows = cols = 0 + end + + first = true + for (i, (k, v)) in enumerate(t) + first || print(io, "\n ") + first = false + limit && i > rows && (print(io, rpad("⋮", keylen), " => ⋮"); break) + + if limit + key = rpad(_truncate_at_width_or_chars(ks[i], keylen, "\r\n"), keylen) + else + key = sprint(0, show, k, env=recur_io) + end + print(recur_io, key) + print(io, " => ") + + if limit + val = _truncate_at_width_or_chars(vs[i], cols - keylen, "\r\n") + print(io, val) + else + show(recur_io, v) + end + end +end + +function show(io::IO, ::MIME"text/plain", f::Function) + ft = typeof(f) + mt = ft.name.mt + if isa(f, Core.Builtin) + print(io, mt.name, " (built-in function)") + else + name = mt.name + isself = isdefined(ft.name.module, name) && + ft == typeof(getfield(ft.name.module, name)) + n = length(mt) + m = n==1 ? "method" : "methods" + ns = isself ? string(name) : string("(::", name, ")") + what = startswith(ns, '@') ? "macro" : "generic function" + print(io, ns, " (", what, " with $n $m)") + end +end + +function show(io::IO, ::MIME"text/plain", r::LinSpace) + # show for linspace, e.g. + # linspace(1,3,7) + # 7-element LinSpace{Float64}: + # 1.0,1.33333,1.66667,2.0,2.33333,2.66667,3.0 + print(io, summary(r)) + if !isempty(r) + println(io, ":") + print_range(io, r) + end +end + +function show(io::IO, ::MIME"text/plain", t::Task) + show(io, t) + if t.state == :failed + println(io) + showerror(io, CapturedException(t.result, t.backtrace)) + end +end + +show(io::IO, ::MIME"text/plain", X::AbstractArray) = showarray(io, X, false) +show(io::IO, ::MIME"text/plain", r::Range) = show(io, r) # always use the compact form for printing ranges + # showing exception objects as descriptive error messages diff --git a/base/show.jl b/base/show.jl index b43b2e1396bb3..e0ae1ea79a902 100644 --- a/base/show.jl +++ b/base/show.jl @@ -101,7 +101,7 @@ function show_default(io::IO, x::ANY) nf = nfields(t) if nf != 0 || t.size==0 if !show_circular(io, x) - recur_io = IOContext(IOContext(io, :SHOWN_SET=>x), :multiline=>false) + recur_io = IOContext(io, :SHOWN_SET => x) for i=1:nf f = fieldname(t, i) if !isdefined(x, f) @@ -142,25 +142,10 @@ end function show(io::IO, f::Function) ft = typeof(f) mt = ft.name.mt - if get(io, :multiline, false) - if isa(f, Core.Builtin) - print(io, mt.name, " (built-in function)") - else - name = mt.name - isself = isdefined(ft.name.module, name) && - ft == typeof(getfield(ft.name.module, name)) - n = length(mt) - m = n==1 ? "method" : "methods" - ns = isself ? string(name) : string("(::", name, ")") - what = startswith(ns, '@') ? "macro" : "generic function" - print(io, ns, " (", what, " with $n $m)") - end + if !isdefined(mt, :module) || is_exported_from_stdlib(mt.name, mt.module) || mt.module === Main + print(io, mt.name) else - if !isdefined(mt, :module) || is_exported_from_stdlib(mt.name, mt.module) || mt.module === Main - print(io, mt.name) - else - print(io, mt.module, ".", mt.name) - end + print(io, mt.module, ".", mt.name) end end @@ -302,9 +287,9 @@ function show_delim_array(io::IO, itr::Union{AbstractArray,SimpleVector}, op, de delim_one, i1=first(linearindices(itr)), l=last(linearindices(itr))) print(io, op) if !show_circular(io, itr) - recur_io = IOContext(io, SHOWN_SET=itr, multiline=false) + recur_io = IOContext(io, :SHOWN_SET => itr) if !haskey(io, :compact) - recur_io = IOContext(recur_io, compact=true) + recur_io = IOContext(recur_io, :compact => true) end newline = true first = true @@ -1535,10 +1520,9 @@ function print_matrix_repr(io, X::AbstractArray) print(io, "]") end -show(io::IO, X::AbstractArray) = showarray(io, X) +show(io::IO, X::AbstractArray) = showarray(io, X, true) -function showarray(io::IO, X::AbstractArray; header = true) - repr = !get(io, :multiline, false) +function showarray(io::IO, X::AbstractArray, repr::Bool = true; header = true) if repr && ndims(X) == 1 return show_vector(io, X, "[", "]") end @@ -1591,18 +1575,9 @@ end showcompact(x) = showcompact(STDOUT, x) function showcompact(io::IO, x) if get(io, :compact, false) - if !get(io, :multiline, false) - show(io, x) - else - show(IOContext(io, :multiline => false), x) - end + show(io, x) else - if !get(io, :multiline, false) - show(IOContext(io, :compact => true), x) - else - show(IOContext(IOContext(io, :compact => true), - :multiline => false), x) - end + show(IOContext(io, :compact => true), x) end end diff --git a/base/sparse/cholmod.jl b/base/sparse/cholmod.jl index 87c0d30c59141..6ae290c3449ff 100644 --- a/base/sparse/cholmod.jl +++ b/base/sparse/cholmod.jl @@ -1093,6 +1093,10 @@ function showfactor(io::IO, F::Factor) @printf(io, "nnz: %13d\n", nnz(F)) end +# getindex not defined for these, so don't use the normal array printer +show(io::IO, ::MIME"text/plain", FC::FactorComponent) = show(io, FC) +show(io::IO, ::MIME"text/plain", F::Factor) = show(io, F) + isvalid(A::Dense) = check_dense(A) isvalid(A::Sparse) = check_sparse(A) isvalid(A::Factor) = check_factor(A) @@ -1115,6 +1119,7 @@ function size(F::Factor, i::Integer) end return 1 end +size(F::Factor) = (size(F, 1), size(F, 2)) linearindexing(::Dense) = LinearFast() diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 489c290c543c8..a2f2391a2d6e5 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -76,9 +76,17 @@ convenient iterating over a sparse matrix : """ nzrange(S::SparseMatrixCSC, col::Integer) = S.colptr[col]:(S.colptr[col+1]-1) +function Base.show(io::IO, ::MIME"text/plain", S::SparseMatrixCSC) + print(io, S.m, "×", S.n, " sparse matrix with ", nnz(S), " ", eltype(S), " nonzero entries") + if nnz(S) != 0 + print(io, ":") + show(io, S) + end +end + function Base.show(io::IO, S::SparseMatrixCSC) - if get(io, :multiline, false) || (nnz(S) == 0) - print(io, S.m, "×", S.n, " sparse matrix with ", nnz(S), " ", eltype(S), " nonzero entries", nnz(S) == 0 ? "" : ":") + if nnz(S) == 0 + return show(io, MIME("text/plain"), S) end limit::Bool = get(io, :limit, false) @@ -91,9 +99,9 @@ function Base.show(io::IO, S::SparseMatrixCSC) pad = ndigits(max(S.m,S.n)) k = 0 sep = "\n\t" - io = IOContext(io, multiline=false) + io = IOContext(io) if !haskey(io, :compact) - io = IOContext(io, compact=true) + io = IOContext(io, :compact => true) end for col = 1:S.n, k = S.colptr[col] : (S.colptr[col+1]-1) if k < half_screen_rows || k > nnz(S)-half_screen_rows diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index 4bbad1ca948f7..13a35fa33f494 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -607,23 +607,25 @@ getindex(x::AbstractSparseVector, ::Colon) = copy(x) ### show and friends +function show(io::IO, ::MIME"text/plain", x::AbstractSparseVector) + println(io, summary(x)) + show(io, x) +end + function show(io::IO, x::AbstractSparseVector) + # TODO: make this a one-line form n = length(x) nzind = nonzeroinds(x) nzval = nonzeros(x) xnnz = length(nzind) - if get(io, :multiline, false) - println(io, summary(x)) - end - limit::Bool = get(io, :limit, false) half_screen_rows = limit ? div(displaysize(io)[1] - 8, 2) : typemax(Int) pad = ndigits(n) sep = "\n\t" - io = IOContext(io, multiline=false) + io = IOContext(io) if !haskey(io, :compact) - io = IOContext(io, compact=true) + io = IOContext(io, :compact => true) end for k = 1:length(nzind) if k < half_screen_rows || k > xnnz - half_screen_rows diff --git a/base/task.jl b/base/task.jl index 09859decd2cae..00d546f705b04 100644 --- a/base/task.jl +++ b/base/task.jl @@ -48,12 +48,6 @@ end function show(io::IO, t::Task) print(io, "Task ($(t.state)) @0x$(hex(convert(UInt, pointer_from_objref(t)), Sys.WORD_SIZE>>2))") - if get(io, :multiline, false) - if t.state == :failed - println(io) - showerror(io, CapturedException(t.result, t.backtrace)) - end - end end macro task(ex) diff --git a/test/dict.jl b/test/dict.jl index e9aa4241cbb6c..c0cde55fcee63 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -268,8 +268,8 @@ for d in (Dict("\n" => "\n", "1" => "\n", "\n" => "2"), for cols in (12, 40, 80), rows in (2, 10, 24) # Ensure output is limited as requested s = IOBuffer() - io = Base.IOContext(s, limit=true, displaysize=(rows, cols), multiline=true) - Base.show(io, d) + io = Base.IOContext(s, limit=true, displaysize=(rows, cols)) + Base.show(io, MIME("text/plain"), d) out = split(takebuf_string(s),'\n') for line in out[2:end] @test strwidth(line) <= cols @@ -278,8 +278,8 @@ for d in (Dict("\n" => "\n", "1" => "\n", "\n" => "2"), for f in (keys, values) s = IOBuffer() - io = Base.IOContext(s, limit=true, displaysize=(rows, cols), multiline=true) - Base.show(io, f(d)) + io = Base.IOContext(s, limit=true, displaysize=(rows, cols)) + Base.show(io, MIME("text/plain"), f(d)) out = split(takebuf_string(s),'\n') for line in out[2:end] @test strwidth(line) <= cols @@ -311,9 +311,9 @@ end type Alpha end Base.show(io::IO, ::Alpha) = print(io,"α") let sbuff = IOBuffer(), - io = Base.IOContext(sbuff, limit=true, displaysize=(10, 20), multiline=true) + io = Base.IOContext(sbuff, limit=true, displaysize=(10, 20)) - Base.show(io, Dict(Alpha()=>1)) + Base.show(io, MIME("text/plain"), Dict(Alpha()=>1)) @test !contains(String(sbuff), "…") @test endswith(String(sbuff), "α => 1") end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index caf19f96b77f6..790edde60d350 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -178,7 +178,7 @@ str = takebuf_string(io) show(io, S) str = takebuf_string(io) @test str == "[1 3; 2 4]" -show(IOContext(io, multiline=true), A) +show(io, MIME("text/plain"), A) strs = split(strip(takebuf_string(io)), '\n') @test strs[2] == " 1 3" @test strs[3] == " 2 4" @@ -188,7 +188,7 @@ str = takebuf_string(io) show(io, parent(v)) @test str == takebuf_string(io) function cmp_showf(printfunc, io, A) - ioc = IOContext(io, limit=true, multiline=true, compact=true) + ioc = IOContext(io, limit=true, compact=true) printfunc(ioc, A) str1 = takebuf_string(io) printfunc(ioc, parent(A)) diff --git a/test/ranges.jl b/test/ranges.jl index 2c1fd4cb41f52..bbe8a4ad931f4 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -569,7 +569,7 @@ end # stringmime/show should display the range or linspace nicely # to test print_range in range.jl -replstrmime(x) = sprint((io,x) -> show(IOContext(io, multiline=true, limit=true), MIME("text/plain"), x), x) +replstrmime(x) = sprint((io,x) -> show(IOContext(io, limit=true), MIME("text/plain"), x), x) @test replstrmime(1:4) == "1:4" @test stringmime("text/plain", 1:4) == "1:4" @test stringmime("text/plain", linspace(1,5,7)) == "7-element LinSpace{Float64}:\n 1.0,1.66667,2.33333,3.0,3.66667,4.33333,5.0" diff --git a/test/replutil.jl b/test/replutil.jl index a8e0d2acead96..a272c23a4dba1 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -392,10 +392,10 @@ end # Issue #14684: `display` should prints associative types in full. let d = Dict(1 => 2, 3 => 45) - buf = IOContext(IOBuffer(), multiline=true) + buf = IOBuffer() td = TextDisplay(buf) display(td, d) - result = String(td.io.io) + result = String(td.io) @test contains(result, summary(d)) diff --git a/test/show.jl b/test/show.jl index 0d0dfc61eaf25..989e311b3a565 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -replstr(x) = sprint((io,x) -> show(IOContext(io, multiline=true, limit=true), MIME("text/plain"), x), x) +replstr(x) = sprint((io,x) -> show(IOContext(io, limit=true), MIME("text/plain"), x), x) @test replstr(Array{Any}(2)) == "2-element Array{Any,1}:\n #undef\n #undef" @test replstr(Array{Any}(2,2)) == "2×2 Array{Any,2}:\n #undef #undef\n #undef #undef" @@ -490,18 +490,13 @@ let io = IOBuffer() x = [1, 2] showcompact(io, x) @test takebuf_string(io) == "[1,2]" - showcompact(IOContext(io, :multiline=>true), x) - @test takebuf_string(io) == "[1,2]" showcompact(IOContext(io, :compact=>true), x) @test takebuf_string(io) == "[1,2]" - showcompact(IOContext(IOContext(io, :compact=>true), :multiline=>true), x) - @test takebuf_string(io) == "[1,2]" end # PR 17117 # test show array let s = IOBuffer(Array{UInt8}(0), true, true) - io = IOContext(s, multiline=true) - Base.showarray(io, [1,2,3], header = false) + Base.showarray(s, [1,2,3], false, header = false) @test String(resize!(s.data, s.size)) == " 1\n 2\n 3" end diff --git a/test/sparsedir/cholmod.jl b/test/sparsedir/cholmod.jl index eedc08eb6c577..3dc70563d8c45 100644 --- a/test/sparsedir/cholmod.jl +++ b/test/sparsedir/cholmod.jl @@ -591,7 +591,12 @@ Dp = spdiagm(dp) sparse(cholfact(sparse(Float64[ 10 1 1 1; 1 10 0 0; 1 0 10 0; 1 0 0 10]))); gc() # Issue 11747 - Wrong show method defined for FactorComponent -Base.show(IOBuffer(), MIME"text/plain"(), cholfact(sparse(Float64[ 10 1 1 1; 1 10 0 0; 1 0 10 0; 1 0 0 10]))[:L]) +let v = cholfact(sparse(Float64[ 10 1 1 1; 1 10 0 0; 1 0 10 0; 1 0 0 10]))[:L] + for s in (sprint(show, MIME("text/plain"), v), sprint(show, v)) + @test contains(s, "method: simplicial") + @test !contains(s, "#undef") + end +end # Element promotion and type inference @inferred cholfact(As)\ones(Int, size(As, 1)) From b2ab59cb0ec643ebff48c2f586e03b1350cd04d9 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 28 Jun 2016 23:57:53 -0400 Subject: [PATCH 0142/1117] fix #17147, segfault in expanding Vararg types --- src/jltypes.c | 4 ++-- test/core.jl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index 887ad84ede3ac..ac426eb252edf 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2341,7 +2341,7 @@ static jl_value_t *inst_tuple_w_(jl_value_t *t, jl_value_t **env, size_t n, // Instantiate NTuple{3,Int} // Note this does not instantiate Tuple{Vararg{Int,3}}; that's done in // jl_apply_tuple_type_v_ - if (jl_is_va_tuple(tt) && ntp == 1 && n == 2) { + if (jl_is_va_tuple(tt) && ntp == 1) { // If this is a Tuple{Vararg{T,N}} with known N, expand it to // a fixed-length tuple jl_value_t *T=NULL, *N=NULL; @@ -2787,7 +2787,7 @@ jl_datatype_t *jl_fix_vararg_bound(jl_datatype_t *tt, int nfix) JL_GC_PUSH2(&env[0], &env[1]); env[0] = jl_tparam1(jl_tparam(tt, ntp-1)); env[1] = jl_box_long(nfix); - jl_datatype_t *ret = (jl_datatype_t*)jl_instantiate_type_with((jl_value_t*)tt, env, 2); + jl_datatype_t *ret = (jl_datatype_t*)jl_instantiate_type_with((jl_value_t*)tt, env, 1); JL_GC_POP(); return ret; } diff --git a/test/core.jl b/test/core.jl index b270850d2716f..4ea39697cdf1c 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4451,3 +4451,8 @@ end let g = f16340(1) @test isa(typeof(g).name.mt.defs.tvars, TypeVar) end + +# issue #17147 +f17147(::Tuple) = 1 +f17147{N}(::Vararg{Tuple,N}) = 2 +@test f17147((), ()) == 2 From d9694e46dd58ccfbd36859248a778923cdd14ba5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 29 Jun 2016 01:20:51 -0400 Subject: [PATCH 0143/1117] declassify call-site devirtualization as an inlining optimization this ensures tests like bitarray are still non-terminal even with --precompile=no --inline=no (with apologies to #265) --- base/inference.jl | 74 ++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index cd0906d203505..51cd4bd644022 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -72,11 +72,13 @@ type InferenceState fixedpoint::Bool typegotoredo::Bool inworkq::Bool + # optimization optimize::Bool + inlining::Bool needtree::Bool inferred::Bool - function InferenceState(linfo::LambdaInfo, optimize::Bool, needtree::Bool) + function InferenceState(linfo::LambdaInfo, optimize::Bool, inlining::Bool, needtree::Bool) @assert isa(linfo.code,Array{Any,1}) nslots = length(linfo.slotnames) nl = label_counter(linfo.code)+1 @@ -161,7 +163,7 @@ type InferenceState ssavalue_uses, ssavalue_init, ObjectIdDict(), #Dict{InferenceState, Vector{LineNum}}(), Vector{Tuple{InferenceState, Vector{LineNum}}}(), - false, false, false, optimize, needtree, false) + false, false, false, optimize, inlining, needtree, false) push!(active, frame) nactive[] += 1 return frame @@ -1408,6 +1410,8 @@ function unshare_linfo!(li::LambdaInfo) return li end +inlining_enabled() = (JLOptions().can_inline == 1) + #### entry points for inferring a LambdaInfo given a type signature #### function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtree::Bool, optimize::Bool, cached::Bool, caller) local code = nothing @@ -1510,7 +1514,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr else # inference not started yet, make a new frame for a new lambda linfo.inInference = true - frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), optimize, needtree) + frame = InferenceState(unshare_linfo!(linfo::LambdaInfo), optimize, inlining_enabled(), needtree) end frame = frame::InferenceState @@ -1571,7 +1575,7 @@ function typeinf_ext(linfo::LambdaInfo) else # toplevel lambda - infer directly linfo.inInference = true - frame = InferenceState(linfo, true, true) + frame = InferenceState(linfo, true, inlining_enabled(), true) typeinf_loop(frame) @assert frame.inferred # TODO: deal with this better return linfo @@ -1921,10 +1925,8 @@ function finish(me::InferenceState) # if we start to create `SSAValue` in type inference when not # optimizing and use unoptimized IR in codegen. gotoifnot_elim_pass!(me.linfo, me) - if JLOptions().can_inline == 1 - inlining_pass!(me.linfo, me) - inbounds_meta_elim_pass!(me.linfo.code) - end + inlining_pass!(me.linfo, me) + inbounds_meta_elim_pass!(me.linfo.code) alloc_elim_pass!(me.linfo, me) getfield_elim_pass!(me.linfo, me) reindex_labels!(me.linfo, me) @@ -2322,28 +2324,30 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end end topmod = _topmod(sv) - if istopfunction(topmod, f, :isbits) && length(atypes)==2 && isType(atypes[2]) && - effect_free(argexprs[2],sv,true) && isleaftype(atypes[2].parameters[1]) - return (isbits(atypes[2].parameters[1]),()) - end # special-case inliners for known pure functions that compute types - if isType(e.typ) && !has_typevars(e.typ.parameters[1],true) - if (is(f, apply_type) || is(f, fieldtype) || is(f, typeof) || - istopfunction(topmod, f, :typejoin) || - istopfunction(topmod, f, :promote_type)) - # XXX: compute effect_free for the actual arguments - if length(argexprs) < 2 || effect_free(argexprs[2], sv, true) - return (e.typ.parameters[1],()) - else - return (e.typ.parameters[1], Any[argexprs[2]]) + if sv.inlining + if isType(e.typ) && !has_typevars(e.typ.parameters[1],true) + if (is(f, apply_type) || is(f, fieldtype) || is(f, typeof) || + istopfunction(topmod, f, :typejoin) || + istopfunction(topmod, f, :promote_type)) + # XXX: compute effect_free for the actual arguments + if length(argexprs) < 2 || effect_free(argexprs[2], sv, true) + return (e.typ.parameters[1],()) + else + return (e.typ.parameters[1], Any[argexprs[2]]) + end end end - end - if is(f, Core.kwfunc) && length(argexprs) == 2 && isa(e.typ, Const) - if effect_free(argexprs[2], sv, true) - return (e.typ.val, ()) - else - return (e.typ.val, Any[argexprs[2]]) + if istopfunction(topmod, f, :isbits) && length(atypes)==2 && isType(atypes[2]) && + effect_free(argexprs[2],sv,true) && isleaftype(atypes[2].parameters[1]) + return (isbits(atypes[2].parameters[1]),()) + end + if is(f, Core.kwfunc) && length(argexprs) == 2 && isa(e.typ, Const) + if effect_free(argexprs[2], sv, true) + return (e.typ.val, ()) + else + return (e.typ.val, Any[argexprs[2]]) + end end end if isa(f, IntrinsicFunction) || ft ⊑ IntrinsicFunction || @@ -2352,11 +2356,6 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end atype_unlimited = argtypes_to_type(atypes) - if length(atype_unlimited.parameters) - 1 > MAX_TUPLETYPE_LEN - atype = limit_tuple_type(atype_unlimited) - else - atype = atype_unlimited - end function invoke_NF() # converts a :call to :invoke cache_linfo = ccall(:jl_get_spec_lambda, Any, (Any,), atype_unlimited) @@ -2366,6 +2365,15 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end return NF end + if !sv.inlining + return invoke_NF() + end + + if length(atype_unlimited.parameters) - 1 > MAX_TUPLETYPE_LEN + atype = limit_tuple_type(atype_unlimited) + else + atype = atype_unlimited + end meth = _methods_by_ftype(atype, 1) if meth === false || length(meth) != 1 return invoke_NF() @@ -2943,7 +2951,7 @@ function inlining_pass(e::Expr, sv, linfo) end end - if isdefined(Main, :Base) && + if sv.inlining && isdefined(Main, :Base) && ((isdefined(Main.Base, :^) && is(f, Main.Base.:^)) || (isdefined(Main.Base, :.^) && is(f, Main.Base.:.^))) if length(e.args) == 3 && isa(e.args[3],Union{Int32,Int64}) From 336ad8699f71a512047fae7abaa6d58d62534202 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 25 Jun 2016 15:42:26 -0400 Subject: [PATCH 0144/1117] Cell array syntax has been deleted --- doc/stdlib/punctuation.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/stdlib/punctuation.rst b/doc/stdlib/punctuation.rst index 22a43234b277e..45e4dbde658c9 100644 --- a/doc/stdlib/punctuation.rst +++ b/doc/stdlib/punctuation.rst @@ -30,7 +30,6 @@ symbol meaning ``[;]`` also vertical concatenation ``[ ]`` with space-separated expressions, horizontal concatenation ``T{ }`` parametric type instantiation -``{ }`` construct a heterogeneous array (deprecated in 0.4 in favor of ``Any[]``) ``;`` statement separator ``,`` separate function arguments or tuple components ``?`` 3-argument conditional operator (conditional ? if_true : if_false) From f72bb027729f3059a4dfc80c731251f3ef3501ec Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 25 Jun 2016 16:05:56 -0400 Subject: [PATCH 0145/1117] Actually check the return value of a cholmod test for #11745 --- test/sparsedir/cholmod.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/sparsedir/cholmod.jl b/test/sparsedir/cholmod.jl index 3dc70563d8c45..f7b4461911a70 100644 --- a/test/sparsedir/cholmod.jl +++ b/test/sparsedir/cholmod.jl @@ -588,7 +588,10 @@ Dp = spdiagm(dp) @test_throws CHOLMOD.CHOLMODException Fs[:PLD] # Issue 11745 - row and column pointers were not sorted in sparse(Factor) -sparse(cholfact(sparse(Float64[ 10 1 1 1; 1 10 0 0; 1 0 10 0; 1 0 0 10]))); gc() +let A = Float64[10 1 1 1; 1 10 0 0; 1 0 10 0; 1 0 0 10] + @test sparse(cholfact(sparse(A))) ≈ A +end +gc() # Issue 11747 - Wrong show method defined for FactorComponent let v = cholfact(sparse(Float64[ 10 1 1 1; 1 10 0 0; 1 0 10 0; 1 0 0 10]))[:L] From 2a7f205fc1bbda2fc1c2472e623b1d926fa96a6b Mon Sep 17 00:00:00 2001 From: Jutho Date: Wed, 29 Jun 2016 16:41:55 +0200 Subject: [PATCH 0146/1117] make complex givensAlgorithm type stable --- base/linalg/givens.jl | 2 +- test/linalg/givens.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/base/linalg/givens.jl b/base/linalg/givens.jl index 48055c7b91697..f543ed5477521 100644 --- a/base/linalg/givens.jl +++ b/base/linalg/givens.jl @@ -170,7 +170,7 @@ function givensAlgorithm{T<:AbstractFloat}(f::Complex{T}, g::Complex{T}) if f == 0 cs = zero(T) - r = hypot(real(g), imag(g)) + r = complex(hypot(real(g), imag(g))) # do complex/real division explicitly with two real divisions d = hypot(real(gs), imag(gs)) sn = complex(real(gs)/d, -imag(gs)/d) diff --git a/test/linalg/givens.jl b/test/linalg/givens.jl index 97bb27246bc4d..91eeb414bc016 100644 --- a/test/linalg/givens.jl +++ b/test/linalg/givens.jl @@ -61,10 +61,12 @@ for elty in (Float32, Float64, Complex64, Complex128) G, r = givens(x[2], x[4], 2, 4) @test (G*x)[2] ≈ r @test abs((G*x)[4]) < eps(real(elty)) + @inferred givens(x[2], x[4], 2, 4) G, r = givens(x, 2, 4) @test (G*x)[2] ≈ r @test abs((G*x)[4]) < eps(real(elty)) + @inferred givens(x, 2, 4) G, r = givens(x, 4, 2) @test (G*x)[4] ≈ r From 6cf69cbc16cb072b3efc669ae7395caa0c83e1ff Mon Sep 17 00:00:00 2001 From: Carlo Baldassi Date: Wed, 29 Jun 2016 18:28:15 +0200 Subject: [PATCH 0147/1117] Fix several issues with Pkg.pin() * Actually check out the branch after creation (fix #17176) * Activate test in test/pkg.jl * Avoid trying to update a pinned package (and emit info message) * Allow to pin a package several times without producing errors * Fix the cases in which resolve() is called after a pin() --- base/pkg/entry.jl | 38 ++++++++++++++++++++++++++++++++------ base/pkg/read.jl | 14 ++++++++++++++ test/pkg.jl | 2 +- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 9e1e90d23e6b1..bb45ece3adda2 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -295,18 +295,42 @@ function pin(pkg::AbstractString, head::AbstractString) should_resolve = true with(GitRepo, pkg) do repo id = if isempty(head) # get HEAD commit - LibGit2.head_oid(repo) - else # no need to resolve, branch will be from HEAD should_resolve = false + LibGit2.head_oid(repo) + else LibGit2.revparseid(repo, head) end commit = LibGit2.get(LibGit2.GitCommit, repo, id) - branch = "pinned.$(string(id)[1:8]).tmp" - info("Creating $pkg branch $branch") try - ref = LibGit2.create_branch(repo, branch, commit) - finalize(ref) + # note: changing the following naming scheme requires a corresponding change in Read.ispinned() + branch = "pinned.$(string(id)[1:8]).tmp" + if LibGit2.isattached(repo) && LibGit2.branch(repo) == branch + info("Package $pkg is already pinned" * (isempty(head) ? "" : " to the selected commit")) + should_resolve = false + return + end + ref = LibGit2.lookup_branch(repo, branch) + try + if ref !== nothing + if LibGit2.revparseid(repo, branch) != id + throw(PkgError("Package $pkg: existing branch $branch has been edited and doesn't correspond to its original commit")) + end + info("Package $pkg: checking out existing branch $branch") + else + info("Creating $pkg branch $branch") + ref = LibGit2.create_branch(repo, branch, commit) + end + + # checkout selected branch + with(LibGit2.peel(LibGit2.GitTree, ref)) do btree + LibGit2.checkout_tree(repo, btree) + end + # switch head to the branch + LibGit2.head!(repo, ref) + finally + finalize(ref) + end finally finalize(commit) end @@ -384,6 +408,8 @@ function update(branch::AbstractString) if LibGit2.isattached(repo) if LibGit2.isdirty(repo) warn("Package $pkg: skipping update (dirty)...") + elseif Read.ispinned(repo) + info("Package $pkg: skipping update (pinned)...") else prev_sha = string(LibGit2.head_oid(repo)) success = true diff --git a/base/pkg/read.jl b/base/pkg/read.jl index c675dc82689dc..842d083d5aa3a 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -100,6 +100,20 @@ function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=availa return res end +function ispinned(pkg::AbstractString) + ispath(pkg,".git") || return false + LibGit2.with(LibGit2.GitRepo, pkg) do repo + return ispinned(repo) + end +end + +function ispinned(prepo::LibGit2.GitRepo) + LibGit2.isattached(prepo) || return false + br = LibGit2.branch(prepo) + # note: regex is based on the naming scheme used in Entry.pin() + return ismatch(r"^pinned\.[0-9a-f]{8}\.tmp$", br) +end + function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=available(pkg)) ispath(pkg,".git") || return typemin(VersionNumber) diff --git a/test/pkg.jl b/test/pkg.jl index 65331f5bc259e..20ffad9c9352a 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -184,7 +184,7 @@ temp_pkg_dir() do Pkg.pin("Example", v"0.4.0") @test Pkg.update() == nothing - Pkg.installed()["Example"] == v"0.4.0" + @test Pkg.installed()["Example"] == v"0.4.0" # bug identified in #16850, Base.url \ vs / for non-Base methods include(Pkg.dir("Example","src","Example.jl")) From b89dc1a6ff0c2e7163e76c61b03529dc491a25c8 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Wed, 29 Jun 2016 13:30:11 -0400 Subject: [PATCH 0148/1117] This fixes 2 issues with the Cholesky update/downdate tests 1. It was possible to get random matrices which would not be positive definite when down-dated (e.g. if I ran the block with `srand(1)`. I have no idea how this didn't become an issue more often. 2. Apparently `v*v'` isn't Hermitian when using the system BLAS on Power. --- test/linalg/cholesky.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index d988371528608..00e4696b4f12b 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -175,9 +175,12 @@ end let A = complex(randn(10,5), randn(10, 5)), v = complex(randn(5), randn(5)) for uplo in (:U, :L) AcA = A'A + BcB = AcA + v*v' + BcB = (BcB + BcB')/2 F = cholfact(AcA, uplo) - @test LinAlg.lowrankupdate(F, v)[uplo] ≈ cholfact(AcA + v*v')[uplo] - @test LinAlg.lowrankdowndate(F, v)[uplo] ≈ cholfact(AcA - v*v')[uplo] + G = cholfact(BcB, uplo) + @test LinAlg.lowrankupdate(F, v)[uplo] ≈ G[uplo] + @test LinAlg.lowrankdowndate(G, v)[uplo] ≈ F[uplo] end end From 38ed9f58abb10c59e15198cf3721b5b202d98df1 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 29 Jun 2016 18:41:42 +0200 Subject: [PATCH 0149/1117] Move the find_region assertion to the use site. This tripped up on ASAN->MEMDEBUG, even though gc_setmark_pool_ already properly reverted to gc_setmark_big in the case of MEMDEBUG. --- src/gc.c | 1 + src/gc.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/gc.c b/src/gc.c index 8ac80fea4a8f9..5d1ea30e3dd03 100644 --- a/src/gc.c +++ b/src/gc.c @@ -473,6 +473,7 @@ static inline int gc_setmark_pool_(jl_taggedvalue_t *o, int mark_mode, #ifdef MEMDEBUG return gc_setmark_big(o, mark_mode); #endif + assert(r != NULL); if (gc_verifying) { o->bits.gc = mark_mode; return mark_mode; diff --git a/src/gc.h b/src/gc.h index 73421acd5a69c..8e8531faf78e9 100644 --- a/src/gc.h +++ b/src/gc.h @@ -242,12 +242,12 @@ STATIC_INLINE region_t *find_region(void *ptr, int maybe) } } (void)maybe; - assert(maybe && "find_region failed"); return NULL; } STATIC_INLINE jl_gc_pagemeta_t *page_metadata_(void *data, region_t *r) { + assert(r != NULL); int pg_idx = page_index(r, (char*)data - GC_PAGE_OFFSET); return &r->meta[pg_idx]; } From 33cb5585a250fda23fdaf79ddc4f77b7ad405cf2 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 29 Jun 2016 20:03:38 +0200 Subject: [PATCH 0150/1117] Kill of the maybe argument to find_region. NULL regions are now checked at each use site. --- src/gc-debug.c | 6 +++--- src/gc.c | 6 +++--- src/gc.h | 5 ++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index 91670730e66b4..d9442090c3f16 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -14,9 +14,9 @@ jl_gc_pagemeta_t *jl_gc_page_metadata(void *data) return page_metadata(data); } -region_t *jl_gc_find_region(void *ptr, int maybe) +region_t *jl_gc_find_region(void *ptr) { - return find_region(ptr, maybe); + return find_region(ptr); } // Find the memory block in the pool that owns the byte pointed to by p. @@ -26,7 +26,7 @@ region_t *jl_gc_find_region(void *ptr, int maybe) // the end of the page. JL_DLLEXPORT jl_taggedvalue_t *jl_gc_find_taggedvalue_pool(char *p, size_t *osize_p) { - region_t *r = find_region(p, 1); + region_t *r = find_region(p); // Not in the pool if (!r) return NULL; diff --git a/src/gc.c b/src/gc.c index 5d1ea30e3dd03..4d74afd7005af 100644 --- a/src/gc.c +++ b/src/gc.c @@ -434,7 +434,7 @@ static inline int gc_setmark_big(jl_taggedvalue_t *o, int mark_mode) o->bits.gc = mark_mode; return 0; } - assert(find_region(o, 1) == NULL); + assert(find_region(o) == NULL); bigval_t *hdr = bigval_header(o); int bits = o->bits.gc; if (mark_reset_age && !gc_marked(bits)) { @@ -513,7 +513,7 @@ static inline int gc_setmark_pool_(jl_taggedvalue_t *o, int mark_mode, static inline int gc_setmark_pool(jl_taggedvalue_t *o, int mark_mode) { - return gc_setmark_pool_(o, mark_mode, find_region(o, 0)); + return gc_setmark_pool_(o, mark_mode, find_region(o)); } static inline int gc_setmark(jl_value_t *v, int sz) @@ -533,7 +533,7 @@ inline void gc_setmark_buf(void *o, int mark_mode, size_t minsz) // where the size estimate is a little off so we do a pool lookup to make // sure. if (minsz <= GC_MAX_SZCLASS) { - region_t *r = find_region(buf, 1); + region_t *r = find_region(buf); if (r) { gc_setmark_pool_(buf, mark_mode, r); return; diff --git a/src/gc.h b/src/gc.h index 8e8531faf78e9..37181247df897 100644 --- a/src/gc.h +++ b/src/gc.h @@ -230,7 +230,7 @@ STATIC_INLINE void *gc_ptr_clear_tag(void *v, uintptr_t mask) NOINLINE uintptr_t gc_get_stack_ptr(void); -STATIC_INLINE region_t *find_region(void *ptr, int maybe) +STATIC_INLINE region_t *find_region(void *ptr) { // on 64bit systems we could probably use a single region and remove this loop for (int i = 0; i < REGION_COUNT && regions[i].pages; i++) { @@ -241,7 +241,6 @@ STATIC_INLINE region_t *find_region(void *ptr, int maybe) return region; } } - (void)maybe; return NULL; } @@ -254,7 +253,7 @@ STATIC_INLINE jl_gc_pagemeta_t *page_metadata_(void *data, region_t *r) STATIC_INLINE jl_gc_pagemeta_t *page_metadata(void *data) { - return page_metadata_(data, find_region(data, 0)); + return page_metadata_(data, find_region(data)); } STATIC_INLINE void gc_big_object_unlink(const bigval_t *hdr) From e540e4fb1a30020422034d55cee3bc9990e2dbd2 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 29 Jun 2016 22:12:42 +0200 Subject: [PATCH 0151/1117] Prevent OOB access on llvm::FunctionType's parameter types. When compiling a function with a trailing ghost argument, this code first fetched the parameter type before continuing out of the loop. --- src/codegen.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index b4ceb26064caa..8535df24ab5b7 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2642,7 +2642,6 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval idx++; } for(size_t i=0; i < nargs+1; i++) { - Type *at = cft->getParamType(idx); jl_value_t *jt = jl_nth_slot_type(li->specTypes,i); bool isboxed; Type *et = julia_type_to_llvm(jt, &isboxed); @@ -2651,6 +2650,8 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval if (i>0) emit_expr(args[i], ctx); continue; } + assert(idx < nfargs); + Type *at = cft->getParamType(idx); if (isboxed) { assert(at == T_pjlvalue && et == T_pjlvalue); jl_cgval_t origval = i==0 ? theF : emit_expr(args[i], ctx); From bce4c36f604fa32ea5caccf670c7fb95126365ac Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 17 Jun 2016 00:14:46 -0400 Subject: [PATCH 0152/1117] some updates to code metadata handling - detect pure and inline meta once, in the front end - fixes #16712 - remove `skip_meta` since it probably assumes too much about where meta nodes occur - add jl_ prefix to has_meta - remove location/pure/inline meta early, in the front end (the first line node isn't needed, since this info is in the Method object) - remove null_sym; this Expr head doesn't occur in the IR --- base/inference.jl | 29 +++++++++++++++----------- src/alloc.c | 48 ++++++++++++++++++++++++++++++++------------ src/ast.c | 11 +--------- src/codegen.cpp | 44 ++++++++++++++++++---------------------- src/interpreter.c | 3 --- src/jltypes.c | 2 +- src/julia.h | 4 ++-- src/julia_internal.h | 3 +-- src/toplevel.c | 2 +- test/core.jl | 2 +- test/reflection.jl | 5 +---- 11 files changed, 80 insertions(+), 73 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 51cd4bd644022..82504c1627ea8 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1929,6 +1929,8 @@ function finish(me::InferenceState) inbounds_meta_elim_pass!(me.linfo.code) alloc_elim_pass!(me.linfo, me) getfield_elim_pass!(me.linfo, me) + # remove placeholders + filter!(x->x!==nothing, me.linfo.code) reindex_labels!(me.linfo, me) end widen_all_consts!(me.linfo) @@ -1937,7 +1939,9 @@ function finish(me::InferenceState) me.linfo.pure = ispure # determine and cache inlineability - me.linfo.inlineable = isinlineable(me.linfo) + if !me.linfo.inlineable + me.linfo.inlineable = isinlineable(me.linfo) + end if !me.needtree me.needtree = me.linfo.inlineable || ccall(:jl_is_cacheable_sig, Int32, (Any, Any, Any), @@ -2744,13 +2748,16 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end end - if !isempty(stmts) - if all(stmt -> isa(stmt,Expr) && stmt.head === :line || isa(stmt, LineNumberNode), stmts) - empty!(stmts) - else - unshift!(stmts, Expr(:meta, :push_loc, linfo.def.file, linfo.def.name)) - push!(stmts, Expr(:meta, :pop_loc)) - end + if !isempty(stmts) + if all(stmt -> (isa(stmt,Expr) && stmt.head === :line) || isa(stmt, LineNumberNode) || stmt === nothing, + stmts) + empty!(stmts) + else + isa(stmts[1], LineNumberNode) && shift!(stmts) + unshift!(stmts, Expr(:meta, :push_loc, linfo.def.file, linfo.def.name, linfo.def.line)) + isa(stmts[end], LineNumberNode) && pop!(stmts) + push!(stmts, Expr(:meta, :pop_loc)) + end end if !isempty(stmts) && !propagate_inbounds # avoid redundant inbounds annotations @@ -2787,16 +2794,14 @@ const inline_incompletematch_allowed = false inline_worthy(body::ANY, cost::Integer) = true # should the expression be part of the inline cost model -function inline_ignore(ex) +function inline_ignore(ex::ANY) isa(ex, LineNumberNode) || + ex === nothing || isa(ex, Expr) && ((ex::Expr).head === :line || (ex::Expr).head === :meta) end function inline_worthy(body::Expr, cost::Integer=1000) # precondition: 0 < cost; nominal cost = 1000 - if popmeta!(body, :inline)[1] - return true - end if popmeta!(body, :noinline)[1] return false end diff --git a/src/alloc.c b/src/alloc.c index 79056b762c694..97677d3926c8b 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -84,7 +84,7 @@ jl_sym_t *line_sym; jl_sym_t *jl_incomplete_sym; jl_sym_t *goto_sym; jl_sym_t *goto_ifnot_sym; jl_sym_t *label_sym; jl_sym_t *return_sym; jl_sym_t *lambda_sym; jl_sym_t *assign_sym; -jl_sym_t *null_sym; jl_sym_t *body_sym; +jl_sym_t *body_sym; jl_sym_t *method_sym; jl_sym_t *core_sym; jl_sym_t *enter_sym; jl_sym_t *leave_sym; jl_sym_t *exc_sym; jl_sym_t *error_sym; @@ -103,7 +103,7 @@ jl_sym_t *pure_sym; jl_sym_t *simdloop_sym; jl_sym_t *meta_sym; jl_sym_t *compiler_temp_sym; jl_sym_t *inert_sym; jl_sym_t *vararg_sym; jl_sym_t *unused_sym; jl_sym_t *static_parameter_sym; -jl_sym_t *polly_sym; +jl_sym_t *polly_sym; jl_sym_t *inline_sym; typedef struct { int64_t a; @@ -316,8 +316,28 @@ static void jl_lambda_info_set_ast(jl_lambda_info_t *li, jl_expr_t *ast) assert(jl_is_expr(bodyex)); jl_array_t *body = bodyex->args; li->code = (jl_value_t*)body; jl_gc_wb(li, li->code); - if (has_meta(body, pure_sym)) - li->pure = 1; + size_t j, n = jl_array_len(body); + jl_value_t **bd = (jl_value_t**)jl_array_data((jl_array_t*)li->code); + for(j=0; j < n; j++) { + jl_value_t *st = bd[j]; + if (jl_is_expr(st) && ((jl_expr_t*)st)->head == meta_sym) { + size_t k, ins = 0, na = jl_expr_nargs(st); + jl_array_t *meta = ((jl_expr_t*)st)->args; + for(k=0; k < na; k++) { + jl_value_t *ma = jl_array_ptr_ref(meta, k); + if (ma == (jl_value_t*)pure_sym) + li->pure = 1; + else if (ma == (jl_value_t*)inline_sym) + li->inlineable = 1; + else + jl_array_ptr_set(meta, ins++, ma); + } + if (ins == 0) + bd[j] = jl_nothing; + else + jl_array_del_end(meta, na-ins); + } + } jl_array_t *vinfo = (jl_array_t*)jl_exprarg(ast, 1); jl_array_t *vis = (jl_array_t*)jl_array_ptr_ref(vinfo, 0); size_t nslots = jl_array_len(vis); @@ -530,15 +550,17 @@ JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t JL_DLLEXPORT void jl_method_init_properties(jl_method_t *m) { jl_lambda_info_t *li = m->lambda_template; - jl_value_t *body1 = skip_meta((jl_array_t*)li->code); - if (jl_is_linenode(body1)) { - m->line = jl_linenode_line(body1); - } - else if (jl_is_expr(body1) && ((jl_expr_t*)body1)->head == line_sym) { - m->file = (jl_sym_t*)jl_exprarg(body1, 1); - m->line = jl_unbox_long(jl_exprarg(body1, 0)); + size_t j, n = jl_array_len((jl_array_t*)li->code); + jl_value_t **body = (jl_value_t**)jl_array_data((jl_array_t*)li->code); + for(j=0; j < n; j++) { + jl_value_t *st = body[j]; + if (jl_is_expr(st) && ((jl_expr_t*)st)->head == line_sym) { + m->line = jl_unbox_long(jl_exprarg(st, 0)); + m->file = (jl_sym_t*)jl_exprarg(st, 1); + body[j] = jl_nothing; + break; + } } - int i; uint8_t called=0; for(i=1; i < li->nargs && i <= 8; i++) { @@ -563,7 +585,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(void) m->module = jl_current_module; m->lambda_template = NULL; m->name = NULL; - m->file = null_sym; + m->file = empty_sym; m->line = 0; m->called = 0xff; m->invokes.unknown = NULL; diff --git a/src/ast.c b/src/ast.c index e620d1a94eb4b..3ce3fec653321 100644 --- a/src/ast.c +++ b/src/ast.c @@ -874,16 +874,7 @@ JL_DLLEXPORT int jl_operator_precedence(char *sym) return res; } -jl_value_t *skip_meta(jl_array_t *body) -{ - jl_value_t *body1 = jl_array_ptr_ref(body,0); - if (jl_is_expr(body1) && ((jl_expr_t*)body1)->head == meta_sym - && jl_array_len(body) > 1) - body1 = jl_array_ptr_ref(body,1); - return body1; -} - -int has_meta(jl_array_t *body, jl_sym_t *sym) +int jl_has_meta(jl_array_t *body, jl_sym_t *sym) { size_t i, l = jl_array_len(body); for (i = 0; i < l; i++) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 8535df24ab5b7..7bba28bea50c9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3250,9 +3250,6 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) literal_pointer_val(bnd)); } } - else if (head == null_sym) { - return ghostValue(jl_void_type); - } else if (head == static_typeof_sym) { jl_value_t *extype = expr_type((jl_value_t*)ex, ctx); if (jl_is_type_type(extype)) { @@ -3374,8 +3371,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) jl_value_t *arg = args[0]; if (jl_is_quotenode(arg)) { jl_value_t *arg1 = jl_fieldref(arg,0); - if (!((jl_is_expr(arg1) && ((jl_expr_t*)arg1)->head!=null_sym) || - jl_typeis(arg1,jl_array_any_type) || jl_is_quotenode(arg1))) { + if (!(jl_is_expr(arg1) || jl_typeis(arg1,jl_array_any_type) || jl_is_quotenode(arg1))) { // elide call to jl_copy_ast when possible return emit_expr(arg, ctx); } @@ -4160,7 +4156,7 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func #endif #ifdef USE_POLLY - if (!has_meta(code, polly_sym)) { + if (!jl_has_meta(code, polly_sym)) { f->addFnAttr(polly::PollySkipFnAttr); } #endif @@ -4177,24 +4173,14 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func (jl_options.code_coverage == JL_LOG_USER && in_user_code); bool do_malloc_log = jl_options.malloc_log == JL_LOG_ALL || (jl_options.malloc_log == JL_LOG_USER && in_user_code); - jl_value_t *stmt = skip_meta(stmts); StringRef filename = ""; StringRef dbgFuncName = ctx.name; int lno = -1; - // look for initial (line num filename [funcname]) node, [funcname] for kwarg methods. - if (jl_is_linenode(stmt)) { - lno = jl_linenode_line(stmt); - } - else if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == line_sym && - jl_array_dim0(((jl_expr_t*)stmt)->args) > 0) { - jl_value_t *a1 = jl_exprarg(stmt,0); - if (jl_is_long(a1)) - lno = jl_unbox_long(a1); - } - if (lno == -1 && lam->def) + if (lam->def) { lno = lam->def->line; - if (lam->def && lam->def->file != empty_sym) - filename = jl_symbol_name(lam->def->file); + if (lam->def->file != empty_sym) + filename = jl_symbol_name(lam->def->file); + } ctx.file = filename; int toplineno = lno; @@ -4694,10 +4680,20 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func DI_sp_stack.push_back(SP); DI_loc_stack.push_back(builder.getCurrentDebugLocation()); std::string inl_name; - if (jl_array_len(stmt_e->args) > 2) - inl_name = jl_symbol_name((jl_sym_t*)jl_exprarg(stmt_e, 2)); - else + int inlined_func_lineno = 0; + if (jl_array_len(stmt_e->args) > 2) { + size_t ii; + for(ii=2; ii < jl_array_len(stmt_e->args); ii++) { + jl_value_t *arg = jl_exprarg(stmt_e, ii); + if (jl_is_symbol(arg)) + inl_name = jl_symbol_name((jl_sym_t*)arg); + else if (jl_is_long(arg)) + inlined_func_lineno = jl_unbox_long(arg); + } + } + else { inl_name = "macro expansion"; + } SP = dbuilder.createFunction(new_file, inl_name + ";", inl_name, @@ -4710,7 +4706,7 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func 0, true, nullptr); - builder.SetCurrentDebugLocation(DebugLoc::get(0, 0, (MDNode*)SP, builder.getCurrentDebugLocation())); + builder.SetCurrentDebugLocation(DebugLoc::get(inlined_func_lineno, 0, (MDNode*)SP, builder.getCurrentDebugLocation())); } else if (meta_arg == (jl_value_t*)jl_symbol("pop_loc")) { SP = DI_sp_stack.back(); diff --git a/src/interpreter.c b/src/interpreter.c index 8d83eec54bd65..2aa62dfd765b1 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -202,9 +202,6 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) JL_GC_POP(); return v; } - else if (ex->head == null_sym) { - return (jl_value_t*)jl_nothing; - } else if (ex->head == static_parameter_sym) { ssize_t n = jl_unbox_long(args[0]); assert(n > 0); diff --git a/src/jltypes.c b/src/jltypes.c index ac426eb252edf..4afd3baa4fd27 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3942,7 +3942,6 @@ void jl_init_types(void) using_sym = jl_symbol("using"); importall_sym = jl_symbol("importall"); assign_sym = jl_symbol("="); - null_sym = jl_symbol("null"); body_sym = jl_symbol("body"); colons_sym = jl_symbol("::"); method_sym = jl_symbol("method"); @@ -3978,6 +3977,7 @@ void jl_init_types(void) static_parameter_sym = jl_symbol("static_parameter"); compiler_temp_sym = jl_symbol("#temp#"); polly_sym = jl_symbol("polly"); + inline_sym = jl_symbol("inline"); tttvar = jl_new_typevar(jl_symbol("T"), (jl_value_t*)jl_bottom_type, diff --git a/src/julia.h b/src/julia.h index ad021d135319c..7ecd28ba3e25e 100644 --- a/src/julia.h +++ b/src/julia.h @@ -560,7 +560,7 @@ extern jl_sym_t *importall_sym; extern jl_sym_t *using_sym; extern jl_sym_t *goto_sym; extern jl_sym_t *goto_ifnot_sym; extern jl_sym_t *label_sym; extern jl_sym_t *return_sym; extern jl_sym_t *lambda_sym; extern jl_sym_t *assign_sym; -extern jl_sym_t *null_sym; extern jl_sym_t *body_sym; +extern jl_sym_t *body_sym; extern jl_sym_t *method_sym; extern jl_sym_t *slot_sym; extern jl_sym_t *enter_sym; extern jl_sym_t *leave_sym; extern jl_sym_t *exc_sym; extern jl_sym_t *new_sym; @@ -575,7 +575,7 @@ extern jl_sym_t *copyast_sym; extern jl_sym_t *fastmath_sym; extern jl_sym_t *pure_sym; extern jl_sym_t *simdloop_sym; extern jl_sym_t *meta_sym; extern jl_sym_t *list_sym; extern jl_sym_t *inert_sym; extern jl_sym_t *static_parameter_sym; -extern jl_sym_t *polly_sym; +extern jl_sym_t *polly_sym; extern jl_sym_t *inline_sym; // gc ------------------------------------------------------------------------- diff --git a/src/julia_internal.h b/src/julia_internal.h index d70b2c73fe6c3..231504ced755b 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -326,8 +326,7 @@ uint32_t jl_module_next_counter(jl_module_t *m); void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig); jl_tupletype_t *arg_type_tuple(jl_value_t **args, size_t nargs); -jl_value_t *skip_meta(jl_array_t *body); -int has_meta(jl_array_t *body, jl_sym_t *sym); +int jl_has_meta(jl_array_t *body, jl_sym_t *sym); // backtraces typedef struct { diff --git a/src/toplevel.c b/src/toplevel.c index ea048c1c8eff5..415e9c142753f 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -428,7 +428,7 @@ jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast, int expanded) } jl_expr_t *ex = (jl_expr_t*)e; - if (ex->head == null_sym || ex->head == error_sym || ex->head == jl_incomplete_sym) { + if (ex->head == error_sym || ex->head == jl_incomplete_sym) { // expression types simple enough not to need expansion return jl_interpret_toplevel_expr(e); } diff --git a/test/core.jl b/test/core.jl index 4ea39697cdf1c..8a9007e4d00bd 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3342,7 +3342,7 @@ typealias PossiblyInvalidUnion{T} Union{T,Int} @test split(string(gensym("abc")),'#')[3] == "abc" # meta nodes for optional positional arguments -@test Base.uncompressed_ast(expand(:(@inline f(p::Int=2) = 3)).args[2].args[3])[1].args[1] === :inline +@test expand(:(@inline f(p::Int=2) = 3)).args[2].args[3].inlineable # issue #13007 call13007{T,N}(::Type{Array{T,N}}) = 0 diff --git a/test/reflection.jl b/test/reflection.jl index 7860e121de176..d09b836e65934 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -364,10 +364,7 @@ function test_typed_ast_printing(f::ANY, types::ANY, must_used_vars) for str in (sprint(io->code_warntype(io, f, types)), sprint(io->show(io, li))) # Test to make sure the clearing of file path below works - # If we don't store the full path in line number node/ast printing - # anymore, the test and the string replace below should be fixed. - @test contains(str, @__FILE__) - str = replace(str, @__FILE__, "") + @test string(li.def.file) == @__FILE__ for var in must_used_vars @test contains(str, string(var)) end From 5ccae12e4b99d2e2a0afc0852d95e7141845380f Mon Sep 17 00:00:00 2001 From: Carlo Baldassi Date: Wed, 29 Jun 2016 22:38:52 +0200 Subject: [PATCH 0153/1117] More Pkg.pin() tests --- test/pkg.jl | 111 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 5 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 20ffad9c9352a..4556e95372499 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -16,6 +16,44 @@ function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) end end +macro grab_outputs(ex) + quote + local err::String, out::String + local ferrname = "" + local foutname = "" + local ret + try + OLD_STDERR = STDERR + ferrname = tempname() + ferr = open(ferrname, "w") + try + OLD_STDOUT = STDOUT + foutname = tempname() + fout = open(foutname, "w") + try + redirect_stderr(ferr) + try + redirect_stdout(fout) + ret = $(esc(ex)) + finally + redirect_stdout(OLD_STDOUT) + close(fout) + end + finally + redirect_stderr(OLD_STDERR) + close(ferr) + end + out = readstring(foutname) + err = readstring(ferrname) + finally + isfile(foutname) && rm(foutname) + end + finally + isfile(ferrname) && rm(ferrname) + end + ret, out, err + end +end # Test basic operations: adding or removing a package, status, free #Also test for the existence of REQUIRE and META_Branch @@ -178,14 +216,77 @@ temp_pkg_dir() do end end + # Various pin/free/re-pin/change-pin patterns (issue #17176) begin - Pkg.pin("Example") - Pkg.free("Example") + ret, out, err = @grab_outputs Pkg.free("Example") + @test ret == nothing && out == "" + @test contains(err, "INFO: Freeing Example") + + ret, out, err = @grab_outputs Pkg.pin("Example") + @test ret == nothing && out == "" + @test ismatch(r"INFO: Creating Example branch pinned\.[0-9a-f]{8}\.tmp", err) + @test !contains(err, "INFO: No packages to install, update or remove") + branchid = replace(err, r".*pinned\.([0-9a-f]{8})\.tmp.*"s, s"\1") + vers = Pkg.installed("Example") + + ret, out, err = @grab_outputs Pkg.free("Example") + @test ret == nothing && out == "" + @test contains(err, "INFO: Freeing Example") + + ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") + @test ret == nothing && out == "" + @test contains(err, "INFO: Creating Example branch pinned.b1990792.tmp") + @test contains(err, "INFO: No packages to install, update or remove") + @test Pkg.installed("Example") == v"0.4.0" - Pkg.pin("Example", v"0.4.0") - @test Pkg.update() == nothing - @test Pkg.installed()["Example"] == v"0.4.0" + ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") + @test ret == nothing && out == "" + @test contains(err, "INFO: Package Example is already pinned to the selected commit") + @test !contains(err, "INFO: No packages to install, update or remove") + @test Pkg.installed("Example") == v"0.4.0" + ret, out, err = @grab_outputs Pkg.pin("Example") + @test ret == nothing && out == "" + @test contains(err, "INFO: Package Example is already pinned") + @test !contains(err, "INFO: No packages to install, update or remove") + @test Pkg.installed("Example") == v"0.4.0" + + ret, out, err = @grab_outputs Pkg.update() + @test ret == nothing && out == "" + @test contains(err, "INFO: Package Example: skipping update (pinned)...") + @test Pkg.installed("Example") == v"0.4.0" + + ret, out, err = @grab_outputs Pkg.pin("Example", v"0.3.1") + @test ret == nothing && out == "" + @test contains(err, "INFO: Creating Example branch pinned.d1ef7b00.tmp") + @test contains(err, "INFO: No packages to install, update or remove") + @test Pkg.installed("Example") == v"0.3.1" + + ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") + @test ret == nothing && out == "" + @test contains(err, "INFO: Package Example: checking out existing branch pinned.b1990792.tmp") + @test contains(err, "INFO: No packages to install, update or remove") + @test Pkg.installed("Example") == v"0.4.0" + + ret, out, err = @grab_outputs Pkg.free("Example") + @test ret == nothing && out == "" + @test contains(err, "INFO: Freeing Example") + @test contains(err, "INFO: No packages to install, update or remove") + @test Pkg.installed("Example") == vers + + ret, out, err = @grab_outputs Pkg.pin("Example") + @test ret == nothing && out == "" + @test contains(err, "INFO: Package Example: checking out existing branch pinned.$branchid.tmp") + @test !contains(err, "INFO: No packages to install, update or remove") + @test Pkg.installed("Example") == vers + + ret, out, err = @grab_outputs Pkg.free("Example") + @test ret == nothing && out == "" + @test contains(err, "INFO: Freeing Example") + @test Pkg.installed("Example") == vers + end + + begin # bug identified in #16850, Base.url \ vs / for non-Base methods include(Pkg.dir("Example","src","Example.jl")) meth = first(methods(Example.domath)) From a7bb4a4c641b5954d99c991db72a5084cbe892e8 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 29 Jun 2016 16:02:43 +0200 Subject: [PATCH 0154/1117] Fix wrong check in typemap. --- src/typemap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/typemap.c b/src/typemap.c index 922f004f4dcc9..6e0764265219e 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -798,11 +798,11 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v if (ml) return ml; } } - if (jl_typeof(cache->linear) != (jl_value_t*)jl_nothing) { + if (cache->linear != (jl_typemap_entry_t*)jl_nothing) { jl_typemap_entry_t *ml = jl_typemap_entry_assoc_exact(cache->linear, args, n); if (ml) return ml; } - if (jl_typeof(cache->any.unknown) != (jl_value_t*)jl_nothing) + if (cache->any.unknown != jl_nothing) return jl_typemap_assoc_exact(cache->any, args, n, offs+1); return NULL; } From 5f9d43fc3834bcf3a1ef5699ae0a90e4f79bb4ad Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Tue, 28 Jun 2016 20:43:33 -0400 Subject: [PATCH 0155/1117] Optimize TLS access in generated code on Linux * Detect if we are using a static TLS model * Emit the assembly directly in codegen to access static TLS variables --- src/codegen.cpp | 30 +++++++++++- src/julia_internal.h | 1 + src/threading.c | 111 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 7bba28bea50c9..25e8b84efda20 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -105,7 +105,8 @@ #include #include #endif -#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) +#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) || \ + (defined(LLVM37) && defined(JULIA_ENABLE_THREADING)) # include #endif #if defined(USE_POLLY) @@ -3451,6 +3452,31 @@ static void finalize_gc_frame(Function *F) ptlsStates->setCalledFunction(getter); ptlsStates->setAttributes(jltls_states_func->getAttributes()); } + else if (jl_tls_offset != -1) { +#ifdef LLVM37 + // Replace the function call with inline assembly if we know + // how to generate it. + const char *asm_str = nullptr; +# if defined(_CPU_X86_64_) + asm_str = "movq %fs:0, $0"; +# elif defined(_CPU_X86_) + asm_str = "movl %gs:0, $0"; +# elif defined(_CPU_AARCH64_) + asm_str = "mrs $0, tpidr_el0"; +# endif + assert(asm_str && "Cannot emit thread pointer for this architecture."); + static auto offset = ConstantInt::getSigned(T_size, jl_tls_offset); + static auto tp = InlineAsm::get(FunctionType::get(T_pint8, false), + asm_str, "=r", false); + Value *tls = CallInst::Create(tp, "thread_ptr", ptlsStates); + tls = GetElementPtrInst::Create(T_int8, tls, {offset}, + "ptls_i8", ptlsStates); + tls = new BitCastInst(tls, PointerType::get(T_ppjlvalue, 0), + "ptls", ptlsStates); + ptlsStates->replaceAllUsesWith(tls); + ptlsStates->eraseFromParent(); +#endif + } #else ptlsStates->replaceAllUsesWith(prepare_global(jltls_states_var, M)); ptlsStates->eraseFromParent(); @@ -5138,6 +5164,8 @@ static void init_julia_llvm_env(Module *m) // In non-imaging mode, (i.e. the code will not be saved to disk), we // use the address of the actual getter function directly // (`jl_tls_states_cb` returned by `jl_get_ptls_states_getter()`) + // (Alternatively if we know how to generate the tls address directly + // we will inline the assembly, see `finalize_gc_frame(Function*)`) // In imaging mode, we emit the function address as a load of a static // variable to be filled (in `dump.c`) at initialization time of the sysimg. // This way we can by pass the extra indirection in `jl_get_ptls_states` diff --git a/src/julia_internal.h b/src/julia_internal.h index 231504ced755b..2ff0aa59c5141 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -249,6 +249,7 @@ void _julia_init(JL_IMAGE_SEARCH rel); void jl_set_base_ctx(char *__stk); +extern size_t jl_tls_offset; void jl_init_threading(void); void jl_start_threads(void); void jl_shutdown_threading(void); diff --git a/src/threading.c b/src/threading.c index 1ab2c38b64054..36ead899055ba 100644 --- a/src/threading.c +++ b/src/threading.c @@ -23,6 +23,21 @@ #include "julia.h" #include "julia_internal.h" +#ifdef _OS_LINUX_ +# if defined(_CPU_X86_64_) || defined(_CPU_X86_) +# define JL_ELF_TLS_VARIANT 2 +# define JL_ELF_TLS_INIT_SIZE 0 +# endif +# if defined(_CPU_AARCH64_) +# define JL_ELF_TLS_VARIANT 1 +# define JL_ELF_TLS_INIT_SIZE 16 +# endif +#endif + +#ifdef JL_ELF_TLS_VARIANT +# include +#endif + #ifdef __cplusplus extern "C" { #endif @@ -337,11 +352,107 @@ void ti_threadfun(void *arg) void ti_reset_timings(void); #endif +size_t jl_tls_offset = -1; + +#ifdef JL_ELF_TLS_VARIANT +// Optimize TLS access in codegen if the TLS buffer is using a IE or LE model. +// To detect such case, we find the size of the TLS segment in the main +// executable and the TIB pointer and then see if the TLS pointer on the +// current thread is in the right range. +// This can in principle be extended to the case where the TLS buffer is +// in the shared library but is part of the static buffer but that seems harder +// to detect. +# if JL_ELF_TLS_VARIANT == 1 +// In Variant 1, the static TLS buffer comes after a fixed size TIB. +// The alignment needs to be applied to the original size. +static inline size_t jl_add_tls_size(size_t orig_size, size_t size, size_t align) +{ + return LLT_ALIGN(orig_size, align) + size; +} +static inline ssize_t jl_check_tls_bound(void *tp, void *ptls, size_t tls_size) +{ + ssize_t offset = (char*)ptls - (char*)tp; + if (offset < JL_ELF_TLS_INIT_SIZE || + (size_t)offset + sizeof(jl_tls_states_t) > tls_size) + return -1; + return offset; +} +# elif JL_ELF_TLS_VARIANT == 2 +// In Variant 2, the static TLS buffer comes before a unknown size TIB. +// The alignment needs to be applied to the new size. +static inline size_t jl_add_tls_size(size_t orig_size, size_t size, size_t align) +{ + return LLT_ALIGN(orig_size + size, align); +} +static inline ssize_t jl_check_tls_bound(void *tp, void *ptls, size_t tls_size) +{ + ssize_t offset = (char*)tp - (char*)ptls; + if (offset < sizeof(jl_tls_states_t) || offset > tls_size) + return -1; + return -offset; +} +# else +# error "Unknown static TLS variant" +# endif + +// Find the size of the TLS segment in the main executable +typedef struct { + size_t total_size; +} check_tls_cb_t; + +static int check_tls_cb(struct dl_phdr_info *info, size_t size, void *_data) +{ + check_tls_cb_t *data = (check_tls_cb_t*)_data; + const ElfW(Phdr) *phdr = info->dlpi_phdr; + unsigned phnum = info->dlpi_phnum; + size_t total_size = JL_ELF_TLS_INIT_SIZE; + + for (unsigned i = 0; i < phnum; i++) { + const ElfW(Phdr) *seg = &phdr[i]; + if (seg->p_type != PT_TLS) + continue; + // There should be only one TLS segment + // Variant II + total_size = jl_add_tls_size(total_size, seg->p_memsz, seg->p_align); + } + data->total_size = total_size; + // only run once (on the main executable) + return 1; +} + +static void jl_check_tls(void) +{ + jl_tls_states_t *ptls = jl_get_ptls_states(); + check_tls_cb_t data = {0}; + dl_iterate_phdr(check_tls_cb, &data); + if (data.total_size == 0) + return; + void *tp; // Thread pointer +#if defined(_CPU_X86_64_) + asm("movq %%fs:0, %0" : "=r"(tp)); +#elif defined(_CPU_X86_) + asm("movl %%gs:0, %0" : "=r"(tp)); +#elif defined(_CPU_AARCH64_) + asm("mrs %0, tpidr_el0" : "=r"(tp)); +#else +# error "Cannot emit thread pointer for this architecture." +#endif + size_t offset = jl_check_tls_bound(tp, ptls, data.total_size); + if (offset == -1) + return; + jl_tls_offset = offset; +} +#endif + // interface to Julia; sets up to make the runtime thread-safe void jl_init_threading(void) { char *cp; +#ifdef JL_ELF_TLS_VARIANT + jl_check_tls(); +#endif + // how many threads available, usable int max_threads = jl_cpu_cores(); jl_n_threads = JULIA_NUM_THREADS; From bc01951bd4e3ee18225739d3c3a3089634f8877d Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Wed, 29 Jun 2016 22:24:18 -0400 Subject: [PATCH 0156/1117] Clean up Complex Rational division. Fixes #15920 and #16282. --- base/rational.jl | 16 ++++------------ test/numbers.jl | 6 ++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/base/rational.jl b/base/rational.jl index 1dcf17b80bf66..c79b0a7b9bf64 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -35,17 +35,9 @@ function //(x::Rational, y::Rational) checked_mul(xn,yd)//checked_mul(xd,yn) end -//(x::Complex, y::Real ) = complex(real(x)//y,imag(x)//y) -function //(x::Number, y::Complex) - xr = complex(Rational(real(x)),Rational(imag(x))) - yr = complex(Rational(real(y)),Rational(imag(y))) - xr // yr -end -function //{Ra<:Rational,Rb<:Rational}(x::Complex{Ra}, y::Complex{Rb}) - xy = x*y' - yy = real(y*y') - complex(real(xy)//yy, imag(xy)//yy) -end +//(x::Complex, y::Real) = complex(real(x)//y,imag(x)//y) +//(x::Number, y::Complex) = x*y'//abs2(y) + //(X::AbstractArray, y::Number) = X .// y .//(X::AbstractArray, y::Number) = reshape([ x // y for x in X ], size(X)) @@ -196,7 +188,7 @@ function *(x::Rational, y::Rational) checked_mul(xn,yn) // checked_mul(xd,yd) end /(x::Rational, y::Rational) = x//y -/(x::Rational, z::Complex ) = inv(z/x) +/{T<:Union{Integer,Rational}}(x::Rational, y::Complex{T}) = x//y fma(x::Rational, y::Rational, z::Rational) = x*y+z diff --git a/test/numbers.jl b/test/numbers.jl index 3f33b47cd8a9b..97c368601670b 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2756,3 +2756,9 @@ testmi(typemax(UInt32)-UInt32(1000):typemax(UInt32), map(UInt32, 1:100)) @test indices(1) == () @test indices(1,1) == 1:1 @test_throws BoundsError indices(1,-1) + +# issue #15920 +@test Rational(0, 1) / Complex(3, 2) == 0 + +# issue #16282 +@test_throws MethodError 3 // 4.5im From 85c2f5bd585302a7beb6ee19936bac21f2484f08 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Tue, 27 Oct 2015 18:13:55 -0400 Subject: [PATCH 0157/1117] Add boundaries to wire format --- base/multi.jl | 175 +++++++++++++++++++++++++++++------------- test/parallel_exec.jl | 10 +++ 2 files changed, 130 insertions(+), 55 deletions(-) diff --git a/base/multi.jl b/base/multi.jl index 38aab509dc45f..5b9b5ad789c3b 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -37,19 +37,36 @@ end hash(r::RRID, h::UInt) = hash(r.whence, hash(r.id, h)) ==(r::RRID, s::RRID) = (r.whence==s.whence && r.id==s.id) +## Wire format description +# +# Each message has three parts, which are written in order to the worker's stream. +# 1) A header of type MsgHeader is serialized to the stream (via `serialize`). +# 2) A message of type AbstractMsg is then serialized. +# 3) Finally, a fixed bounday of 10 bytes is written. + +# Message header stored separately from body to be able to send back errors if +# a deserialization error occurs when reading the message body. +type MsgHeader + response_oid::RRID + notify_oid::RRID +end + +# Special oid (0,0) uses to indicate a null ID. +# Used instead of Nullable to decrease wire size of header. +null_id(id) = id == RRID(0, 0) + +MsgHeader(;response_oid::RRID=RRID(0,0), notify_oid::RRID=RRID(0,0)) = + MsgHeader(response_oid, notify_oid) type CallMsg{Mode} <: AbstractMsg f::Function args::Tuple kwargs::Array - response_oid::RRID end type CallWaitMsg <: AbstractMsg f::Function args::Tuple kwargs::Array - response_oid::RRID - notify_oid::RRID end type RemoteDoMsg <: AbstractMsg f::Function @@ -57,10 +74,10 @@ type RemoteDoMsg <: AbstractMsg kwargs::Array end type ResultMsg <: AbstractMsg - response_oid::RRID value::Any end + # Worker initialization messages type IdentifySocketMsg <: AbstractMsg from_pid::Int @@ -70,34 +87,32 @@ end type JoinPGRPMsg <: AbstractMsg self_pid::Int other_workers::Array - notify_oid::RRID topology::Symbol worker_pool end type JoinCompleteMsg <: AbstractMsg - notify_oid::RRID cpu_cores::Int ospid::Int end -function send_msg_unknown(s::IO, msg) +function send_msg_unknown(s::IO, header, msg) error("attempt to send to unknown socket") end -function send_msg(s::IO, msg) +function send_msg(s::IO, header, msg) id = worker_id_from_socket(s) if id > -1 - return send_msg(worker_from_id(id), msg) + return send_msg(worker_from_id(id), header, msg) end - send_msg_unknown(s, msg) + send_msg_unknown(s, header, msg) end -function send_msg_now(s::IO, msg::AbstractMsg) +function send_msg_now(s::IO, msghdr, msg::AbstractMsg) id = worker_id_from_socket(s) if id > -1 - return send_msg_now(worker_from_id(id), msg) + return send_msg_now(worker_from_id(id), msghdr, msg) end - send_msg_unknown(s, msg) + send_msg_unknown(s, msghdr, msg) end abstract ClusterManager @@ -197,12 +212,12 @@ function set_worker_state(w, state) notify(w.c_state; all=true) end -function send_msg_now(w::Worker, msg) - send_msg_(w, msg, true) +function send_msg_now(w::Worker, msghdr, msg) + send_msg_(w, msghdr, msg, true) end -function send_msg(w::Worker, msg) - send_msg_(w, msg, false) +function send_msg(w::Worker, msghdr, msg) + send_msg_(w, msghdr, msg, false) end function flush_gc_msgs(w::Worker) @@ -241,14 +256,20 @@ function check_worker_state(w::Worker) end end +# Boundary inserted between messages on the wire, used for recovering +# from deserialization errors. Picked arbitrarily. +# A size of 10 bytes indicates ~ ~1e24 possible boundaries, so chance of collision with message contents is trivial. +const MSG_BOUNDARY = UInt8[0x79, 0x8e, 0x8e, 0xf5, 0x6e, 0x9b, 0x2e, 0x97, 0xd5, 0x7d] -function send_msg_(w::Worker, msg, now::Bool) +function send_msg_(w::Worker, header, msg, now::Bool) check_worker_state(w) io = w.w_stream lock(io.lock) try reset_state(w.w_serializer) + serialize(w.w_serializer, header) serialize(w.w_serializer, msg) # io is wrapped in w_serializer + write(io, MSG_BOUNDARY) if !now && w.gcflag flush_gc_msgs(w) @@ -768,7 +789,6 @@ function showerror(io::IO, re::RemoteException) showerror(io, re.captured) end - function run_work_thunk(thunk, print_error) local result try @@ -811,7 +831,7 @@ end function remotecall(f, w::Worker, args...; kwargs...) rr = Future(w) #println("$(myid()) asking for $rr") - send_msg(w, CallMsg{:call}(f, args, kwargs, remoteref_id(rr))) + send_msg(w, MsgHeader(response_oid=remoteref_id(rr)), CallMsg{:call}(f, args, kwargs)) rr end @@ -829,7 +849,7 @@ function remotecall_fetch(f, w::Worker, args...; kwargs...) oid = RRID() rv = lookup_ref(oid) rv.waitingfor = w.id - send_msg(w, CallMsg{:call_fetch}(f, args, kwargs, oid)) + send_msg(w, MsgHeader(response_oid=oid), CallMsg{:call_fetch}(f, args, kwargs)) v = take!(rv) delete!(PGRP.refs, oid) isa(v, RemoteException) ? throw(v) : v @@ -846,7 +866,7 @@ function remotecall_wait(f, w::Worker, args...; kwargs...) rv = lookup_ref(prid) rv.waitingfor = w.id rr = Future(w) - send_msg(w, CallWaitMsg(f, args, kwargs, remoteref_id(rr), prid)) + send_msg(w, MsgHeader(response_oid=remoteref_id(rr), notify_oid=prid), CallWaitMsg(f, args, kwargs)) v = fetch(rv.c) delete!(PGRP.refs, prid) isa(v, RemoteException) && throw(v) @@ -866,7 +886,7 @@ function remote_do(f, w::LocalProcess, args...; kwargs...) end function remote_do(f, w::Worker, args...; kwargs...) - send_msg(w, RemoteDoMsg(f, args, kwargs)) + send_msg(w, MsgHeader(), RemoteDoMsg(f, args, kwargs)) nothing end @@ -952,13 +972,13 @@ close(rr::RemoteChannel) = call_on_owner(close_ref, rr) function deliver_result(sock::IO, msg, oid, value) #print("$(myid()) sending result $oid\n") - if is(msg,:call_fetch) || isa(value, RemoteException) + if is(msg, :call_fetch) || isa(value, RemoteException) val = value else val = :OK end try - send_msg_now(sock, ResultMsg(oid, val)) + send_msg_now(sock, MsgHeader(response_oid=oid), ResultMsg(val)) catch e # terminate connection in case of serialization error # otherwise the reading end would hang @@ -996,28 +1016,73 @@ function process_messages(r_stream::IO, w_stream::IO, incoming=true) end function message_handler_loop(r_stream::IO, w_stream::IO, incoming::Bool) + wpid=0 # the worker r_stream is connected to. + boundary = similar(MSG_BOUNDARY) try version = process_hdr(r_stream, incoming) serializer = ClusterSerializer(r_stream) + + # The first message will associate wpid with r_stream + msghdr = deserialize(serializer) + msg = deserialize(serializer) + readbytes!(r_stream, boundary, length(MSG_BOUNDARY)) + + handle_msg(msg, msghdr, r_stream, w_stream, version) + wpid = worker_id_from_socket(r_stream) + + @assert wpid > 0 + while true reset_state(serializer) - msg = deserialize(serializer) - # println("got msg: ", msg) - handle_msg(msg, r_stream, w_stream, version) + msghdr = deserialize(serializer) +# println("msghdr: ", msghdr) + + try + msg = deserialize(serializer) + catch e + # Deserialization error; discard bytes in stream until boundary found + boundary_idx = 1 + while true + # This may throw an EOF error if the terminal boundary was not written + # correctly, triggering the higher-scoped catch block below + byte = read(r_stream, UInt8) + if byte == MSG_BOUNDARY[boundary_idx] + boundary_idx += 1 + if boundary_idx > length(MSG_BOUNDARY) + break + end + else + boundary_idx = 1 + end + end + # println("Deserialization error.") + remote_err = RemoteException(myid(), CapturedException(e, catch_backtrace())) + if !null_id(msghdr.response_oid) + ref = lookup_ref(msghdr.response_oid) + put!(ref, remote_err) + end + if !null_id(msghdr.notify_oid) + deliver_result(w_stream, :call_fetch, msghdr.notify_oid, remote_err) + end + continue + end + readbytes!(r_stream, boundary, length(MSG_BOUNDARY)) + + # println("got msg: ", typeof(msg)) + handle_msg(msg, msghdr, r_stream, w_stream, version) end catch e # println(STDERR, "Process($(myid())) - Exception ", e) - iderr = worker_id_from_socket(r_stream) - if (iderr < 1) + if (wpid < 1) println(STDERR, e) println(STDERR, "Process($(myid())) - Unknown remote, closing connection.") else - werr = worker_from_id(iderr) + werr = worker_from_id(wpid) oldstate = werr.state set_worker_state(werr, W_TERMINATED) - # If error occured talking to pid 1, commit harakiri - if iderr == 1 + # If unhandleable error occured talking to pid 1, exit + if wpid == 1 if isopen(w_stream) print(STDERR, "fatal error on ", myid(), ": ") display_error(e, catch_backtrace()) @@ -1028,15 +1093,15 @@ function message_handler_loop(r_stream::IO, w_stream::IO, incoming::Bool) # Will treat any exception as death of node and cleanup # since currently we do not have a mechanism for workers to reconnect # to each other on unhandled errors - deregister_worker(iderr) + deregister_worker(wpid) end isopen(r_stream) && close(r_stream) isopen(w_stream) && close(w_stream) - if (myid() == 1) && (iderr > 1) + if (myid() == 1) && (wpid > 1) if oldstate != W_TERMINATING - println(STDERR, "Worker $iderr terminated.") + println(STDERR, "Worker $wpid terminated.") rethrow(e) end end @@ -1071,44 +1136,44 @@ function process_hdr(s, validate_cookie) return VersionNumber(strip(String(version))) end -function handle_msg(msg::CallMsg{:call}, r_stream, w_stream, version) - schedule_call(msg.response_oid, ()->msg.f(msg.args...; msg.kwargs...)) +function handle_msg(msg::CallMsg{:call}, msghdr, r_stream, w_stream, version) + schedule_call(msghdr.response_oid, ()->msg.f(msg.args...; msg.kwargs...)) end -function handle_msg(msg::CallMsg{:call_fetch}, r_stream, w_stream, version) +function handle_msg(msg::CallMsg{:call_fetch}, msghdr, r_stream, w_stream, version) @schedule begin v = run_work_thunk(()->msg.f(msg.args...; msg.kwargs...), false) - deliver_result(w_stream, :call_fetch, msg.response_oid, v) + deliver_result(w_stream, :call_fetch, msghdr.response_oid, v) end end -function handle_msg(msg::CallWaitMsg, r_stream, w_stream, version) +function handle_msg(msg::CallWaitMsg, msghdr, r_stream, w_stream, version) @schedule begin - rv = schedule_call(msg.response_oid, ()->msg.f(msg.args...; msg.kwargs...)) - deliver_result(w_stream, :call_wait, msg.notify_oid, fetch(rv.c)) + rv = schedule_call(msghdr.response_oid, ()->msg.f(msg.args...; msg.kwargs...)) + deliver_result(w_stream, :call_wait, msghdr.notify_oid, fetch(rv.c)) end end -function handle_msg(msg::RemoteDoMsg, r_stream, w_stream, version) +function handle_msg(msg::RemoteDoMsg, msghdr, r_stream, w_stream, version) @schedule run_work_thunk(()->msg.f(msg.args...; msg.kwargs...), true) end -function handle_msg(msg::ResultMsg, r_stream, w_stream, version) - put!(lookup_ref(msg.response_oid), msg.value) +function handle_msg(msg::ResultMsg, msghdr, r_stream, w_stream, version) + put!(lookup_ref(msghdr.response_oid), msg.value) end -function handle_msg(msg::IdentifySocketMsg, r_stream, w_stream, version) +function handle_msg(msg::IdentifySocketMsg, msghdr, r_stream, w_stream, version) # register a new peer worker connection w=Worker(msg.from_pid, r_stream, w_stream, cluster_manager; version=version) send_connection_hdr(w, false) - send_msg_now(w, IdentifySocketAckMsg()) + send_msg_now(w, MsgHeader(), IdentifySocketAckMsg()) end -function handle_msg(msg::IdentifySocketAckMsg, r_stream, w_stream, version) +function handle_msg(msg::IdentifySocketAckMsg, msghdr, r_stream, w_stream, version) w = map_sock_wrkr[r_stream] w.version = version end -function handle_msg(msg::JoinPGRPMsg, r_stream, w_stream, version) +function handle_msg(msg::JoinPGRPMsg, msghdr, r_stream, w_stream, version) LPROC.id = msg.self_pid controller = Worker(1, r_stream, w_stream, cluster_manager; version=version) register_worker(LPROC) @@ -1129,7 +1194,7 @@ function handle_msg(msg::JoinPGRPMsg, r_stream, w_stream, version) set_default_worker_pool(msg.worker_pool) send_connection_hdr(controller, false) - send_msg_now(controller, JoinCompleteMsg(msg.notify_oid, Sys.CPU_CORES, getpid())) + send_msg_now(controller, MsgHeader(notify_oid=msghdr.notify_oid), JoinCompleteMsg(Sys.CPU_CORES, getpid())) end function connect_to_peer(manager::ClusterManager, rpid::Int, wconfig::WorkerConfig) @@ -1138,7 +1203,7 @@ function connect_to_peer(manager::ClusterManager, rpid::Int, wconfig::WorkerConf w = Worker(rpid, r_s, w_s, manager; config=wconfig) process_messages(w.r_stream, w.w_stream, false) send_connection_hdr(w, true) - send_msg_now(w, IdentifySocketMsg(myid())) + send_msg_now(w, MsgHeader(), IdentifySocketMsg(myid())) catch e display_error(e, catch_backtrace()) println(STDERR, "Error [$e] on $(myid()) while connecting to peer $rpid. Exiting.") @@ -1146,7 +1211,7 @@ function connect_to_peer(manager::ClusterManager, rpid::Int, wconfig::WorkerConf end end -function handle_msg(msg::JoinCompleteMsg, r_stream, w_stream, version) +function handle_msg(msg::JoinCompleteMsg, msghdr, r_stream, w_stream, version) w = map_sock_wrkr[r_stream] environ = get(w.config.environ, Dict()) environ[:cpu_cores] = msg.cpu_cores @@ -1154,7 +1219,7 @@ function handle_msg(msg::JoinCompleteMsg, r_stream, w_stream, version) w.config.ospid = msg.ospid w.version = version - ntfy_channel = lookup_ref(msg.notify_oid) + ntfy_channel = lookup_ref(msghdr.notify_oid) put!(ntfy_channel, w.id) push!(default_worker_pool(), w) @@ -1478,7 +1543,7 @@ function create_worker(manager, wconfig) all_locs = map(x -> isa(x, Worker) ? (get(x.config.connect_at, ()), x.id) : ((), x.id, true), join_list) send_connection_hdr(w, true) - send_msg_now(w, JoinPGRPMsg(w.id, all_locs, ntfy_oid, PGRP.topology, default_worker_pool())) + send_msg_now(w, MsgHeader(notify_oid=ntfy_oid), JoinPGRPMsg(w.id, all_locs, PGRP.topology, default_worker_pool())) @schedule manage(w.manager, w.id, w.config, :register) wait(rr_ntfy_join) diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 978204b210628..4c5eb78c5cd73 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -1025,3 +1025,13 @@ f_myid = ()->myid() @test wrkr1 == remotecall_fetch(f_myid, wrkr1) @test wrkr2 == remotecall_fetch(f_myid, wrkr2) @test wrkr2 == remotecall_fetch((f, p)->remotecall_fetch(f, p), wrkr1, f_myid, wrkr2) + +# Deserialization error recovery test +let + bad_thunk = ()->NonexistantModule.f() + @test_throws RemoteException remotecall_fetch(bad_thunk, 2) + # Test that the stream is still usable + @test remotecall_fetch(()->:test,2) == :test + ref = remotecall(bad_thunk, 2) + @test_throws RemoteException fetch(ref) +end From a194eb26845e33e9412a0913b9d1c133b9928aad Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Sat, 25 Jun 2016 14:47:08 -0400 Subject: [PATCH 0158/1117] Make MsgHeader stack-allocatable --- .tags | 52 +++++++++++++++++++++++++++++++++++++++++++++ .tags1 | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ base/multi.jl | 25 +++++++++++----------- 3 files changed, 122 insertions(+), 13 deletions(-) create mode 100644 .tags create mode 100644 .tags1 diff --git a/.tags b/.tags new file mode 100644 index 0000000000000..80635bed2cc22 --- /dev/null +++ b/.tags @@ -0,0 +1,52 @@ +invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{Char}) = reinterpret(Char, 0xfffffffe)$/;" function line:11 +invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt8}) = 0xfe$/;" function line:12 +invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt16}) = 0xfffe$/;" function line:13 +invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt32}) = 0xfffffffe$/;" function line:14 +countlines /Users/malmaud/julia/base/datafmt.jl /^countlines(f::AbstractString, eol::Char='\\n') = open(io->countlines(io,eol), f)::Int$/;" function line:18 +countlines /Users/malmaud/julia/base/datafmt.jl /^function countlines(io::IO, eol::Char='\\n')$/;" function line:19 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, T::Type; opts...) = readdlm(input, invalid_dlm(Char), T, '\\n'; opts...)$/;" function line:32 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, T::Type; opts...) = readdlm(input, dlm, T, '\\n'; opts...)$/;" function line:33 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input; opts...) = readdlm(input, invalid_dlm(Char), '\\n'; opts...)$/;" function line:35 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char; opts...) = readdlm(input, dlm, '\\n'; opts...)$/;" function line:36 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, eol::Char; opts...) =$/;" function line:38 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, T::Type, eol::Char; opts...) =$/;" function line:40 +readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^readdlm_auto(input::Vector{UInt8}, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) =$/;" function line:43 +readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^readdlm_auto(input::IO, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) =$/;" function line:45 +readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^function readdlm_auto(input::AbstractString, dlm::Char, T::Type, eol::Char, auto::Bool; opts...)$/;" function line:47 +DLMHandler /Users/malmaud/julia/base/datafmt.jl /^abstract DLMHandler$/;" function line:68 +DLMOffsets /Users/malmaud/julia/base/datafmt.jl /^type DLMOffsets <: DLMHandler$/;" function line:70 +DLMOffsets /Users/malmaud/julia/base/datafmt.jl /^ function DLMOffsets(sbuff::String)$/;" function line:76 +store_cell /Users/malmaud/julia/base/datafmt.jl /^function store_cell(dlmoffsets::DLMOffsets, row::Int, col::Int,$/;" function line:84 +result /Users/malmaud/julia/base/datafmt.jl /^function result(dlmoffsets::DLMOffsets)$/;" function line:115 +DLMStore /Users/malmaud/julia/base/datafmt.jl /^type DLMStore{T} <: DLMHandler$/;" function line:121 +DLMStore /Users/malmaud/julia/base/datafmt.jl /^function DLMStore{T}(::Type{T}, dims::NTuple{2,Integer},$/;" function line:135 +_chrinstr /Users/malmaud/julia/base/datafmt.jl /^_chrinstr(sbuff::String, chr::UInt8, startpos::Int, endpos::Int) =$/;" function line:145 +store_cell /Users/malmaud/julia/base/datafmt.jl /^function store_cell{T}(dlmstore::DLMStore{T}, row::Int, col::Int,$/;" function line:149 +result /Users/malmaud/julia/base/datafmt.jl /^function result{T}(dlmstore::DLMStore{T})$/;" function line:218 +readdlm_string /Users/malmaud/julia/base/datafmt.jl /^function readdlm_string(sbuff::String, dlm::Char, T::Type, eol::Char, auto::Bool, optsd::Dict)$/;" function line:248 +val_opts /Users/malmaud/julia/base/datafmt.jl /^function val_opts(opts)$/;" function line:291 +dlm_fill /Users/malmaud/julia/base/datafmt.jl /^function dlm_fill(T::DataType, offarr::Vector{Vector{Int}}, dims::NTuple{2,Integer}, has_header::Bool, sbuff::String, auto::Bool, eol::Char)$/;" function line:303 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Bool,2}, row::Int, col::Int)$/;" function line:329 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:Integer}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:334 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Float64,2}, row::Int, col::Int)$/;" function line:339 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Float32,2}, row::Int, col::Int)$/;" function line:344 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:AbstractString}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:349 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Any,2}, row::Int, col::Int)$/;" function line:353 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:Char}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:372 +colval /Users/malmaud/julia/base/datafmt.jl /^colval(sbuff::String, startpos::Int, endpos::Int, cells::Array, row::Int, col::Int) = true$/;" function line:380 +dlm_parse /Users/malmaud/julia/base/datafmt.jl /^function dlm_parse{T,D}(dbuff::T, eol::D, dlm::D, qchar::D, cchar::D,$/;" function line:382 +readcsv /Users/malmaud/julia/base/datafmt.jl /^readcsv(io; opts...) = readdlm(io, ','; opts...)$/;" function line:549 +readcsv /Users/malmaud/julia/base/datafmt.jl /^readcsv(io, T::Type; opts...) = readdlm(io, ',', T; opts...)$/;" function line:550 +writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^writedlm_cell(io::IO, elt::AbstractFloat, dlm, quotes) = print_shortest(io, elt)$/;" function line:553 +writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^function writedlm_cell{T}(io::IO, elt::AbstractString, dlm::T, quotes::Bool)$/;" function line:554 +writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^writedlm_cell(io::IO, elt, dlm, quotes) = print(io, elt)$/;" function line:561 +writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(io::IO, a::AbstractMatrix, dlm; opts...)$/;" function line:562 +writedlm /Users/malmaud/julia/base/datafmt.jl /^writedlm{T}(io::IO, a::AbstractArray{T,0}, dlm; opts...) = writedlm(io, reshape(a,1), dlm; opts...)$/;" function line:579 +writedlm_row /Users/malmaud/julia/base/datafmt.jl /^function writedlm_row(io::IO, row, dlm, quotes)$/;" function line:582 +writedlm_row /Users/malmaud/julia/base/datafmt.jl /^function writedlm_row(io::IO, row::Union{Number,AbstractString}, dlm, quotes)$/;" function line:595 +writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(io::IO, itr, dlm; opts...)$/;" function line:601 +writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(fname::AbstractString, a, dlm; opts...)$/;" function line:613 +writedlm /Users/malmaud/julia/base/datafmt.jl /^writedlm(io, a; opts...) = writedlm(io, a, '\\t'; opts...)$/;" function line:619 +writecsv /Users/malmaud/julia/base/datafmt.jl /^writecsv(io, a; opts...) = writedlm(io, a, ','; opts...)$/;" function line:620 +show /Users/malmaud/julia/base/datafmt.jl /^show(io::IO, ::MIME"text\/csv", a) = writedlm(io, a, ',')$/;" function line:622 +show /Users/malmaud/julia/base/datafmt.jl /^show(io::IO, ::MIME"text\/tab-separated-values", a) = writedlm(io, a, '\\t')$/;" function line:623 diff --git a/.tags1 b/.tags1 new file mode 100644 index 0000000000000..5f6da3864e339 --- /dev/null +++ b/.tags1 @@ -0,0 +1,58 @@ +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 0 /0=unsorted, 1=sorted, 2=foldcase/ +!_TAG_PROGRAM_AUTHOR Darren Hiebert /dhiebert@users.sourceforge.net/ +!_TAG_PROGRAM_NAME Exuberant Ctags // +!_TAG_PROGRAM_URL http://ctags.sourceforge.net /official site/ +!_TAG_PROGRAM_VERSION 5.8 // +invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{Char}) = reinterpret(Char, 0xfffffffe)$/;" function line:11 +invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt8}) = 0xfe$/;" function line:12 +invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt16}) = 0xfffe$/;" function line:13 +invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt32}) = 0xfffffffe$/;" function line:14 +countlines /Users/malmaud/julia/base/datafmt.jl /^countlines(f::AbstractString, eol::Char='\\n') = open(io->countlines(io,eol), f)::Int$/;" function line:18 +countlines /Users/malmaud/julia/base/datafmt.jl /^function countlines(io::IO, eol::Char='\\n')$/;" function line:19 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, T::Type; opts...) = readdlm(input, invalid_dlm(Char), T, '\\n'; opts...)$/;" function line:32 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, T::Type; opts...) = readdlm(input, dlm, T, '\\n'; opts...)$/;" function line:33 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input; opts...) = readdlm(input, invalid_dlm(Char), '\\n'; opts...)$/;" function line:35 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char; opts...) = readdlm(input, dlm, '\\n'; opts...)$/;" function line:36 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, eol::Char; opts...) =$/;" function line:38 +readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, T::Type, eol::Char; opts...) =$/;" function line:40 +readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^readdlm_auto(input::Vector{UInt8}, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) =$/;" function line:43 +readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^readdlm_auto(input::IO, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) =$/;" function line:45 +readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^function readdlm_auto(input::AbstractString, dlm::Char, T::Type, eol::Char, auto::Bool; opts...)$/;" function line:47 +DLMHandler /Users/malmaud/julia/base/datafmt.jl /^abstract DLMHandler$/;" function line:68 +DLMOffsets /Users/malmaud/julia/base/datafmt.jl /^type DLMOffsets <: DLMHandler$/;" function line:70 +DLMOffsets /Users/malmaud/julia/base/datafmt.jl /^ function DLMOffsets(sbuff::String)$/;" function line:76 +store_cell /Users/malmaud/julia/base/datafmt.jl /^function store_cell(dlmoffsets::DLMOffsets, row::Int, col::Int,$/;" function line:84 +result /Users/malmaud/julia/base/datafmt.jl /^function result(dlmoffsets::DLMOffsets)$/;" function line:115 +DLMStore /Users/malmaud/julia/base/datafmt.jl /^type DLMStore{T} <: DLMHandler$/;" function line:121 +DLMStore /Users/malmaud/julia/base/datafmt.jl /^function DLMStore{T}(::Type{T}, dims::NTuple{2,Integer},$/;" function line:135 +_chrinstr /Users/malmaud/julia/base/datafmt.jl /^_chrinstr(sbuff::String, chr::UInt8, startpos::Int, endpos::Int) =$/;" function line:145 +store_cell /Users/malmaud/julia/base/datafmt.jl /^function store_cell{T}(dlmstore::DLMStore{T}, row::Int, col::Int,$/;" function line:149 +result /Users/malmaud/julia/base/datafmt.jl /^function result{T}(dlmstore::DLMStore{T})$/;" function line:218 +readdlm_string /Users/malmaud/julia/base/datafmt.jl /^function readdlm_string(sbuff::String, dlm::Char, T::Type, eol::Char, auto::Bool, optsd::Dict)$/;" function line:248 +val_opts /Users/malmaud/julia/base/datafmt.jl /^function val_opts(opts)$/;" function line:291 +dlm_fill /Users/malmaud/julia/base/datafmt.jl /^function dlm_fill(T::DataType, offarr::Vector{Vector{Int}}, dims::NTuple{2,Integer}, has_header::Bool, sbuff::String, auto::Bool, eol::Char)$/;" function line:303 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Bool,2}, row::Int, col::Int)$/;" function line:329 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:Integer}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:334 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Float64,2}, row::Int, col::Int)$/;" function line:339 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Float32,2}, row::Int, col::Int)$/;" function line:344 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:AbstractString}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:349 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Any,2}, row::Int, col::Int)$/;" function line:353 +colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:Char}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:372 +colval /Users/malmaud/julia/base/datafmt.jl /^colval(sbuff::String, startpos::Int, endpos::Int, cells::Array, row::Int, col::Int) = true$/;" function line:380 +dlm_parse /Users/malmaud/julia/base/datafmt.jl /^function dlm_parse{T,D}(dbuff::T, eol::D, dlm::D, qchar::D, cchar::D,$/;" function line:382 +readcsv /Users/malmaud/julia/base/datafmt.jl /^readcsv(io; opts...) = readdlm(io, ','; opts...)$/;" function line:549 +readcsv /Users/malmaud/julia/base/datafmt.jl /^readcsv(io, T::Type; opts...) = readdlm(io, ',', T; opts...)$/;" function line:550 +writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^writedlm_cell(io::IO, elt::AbstractFloat, dlm, quotes) = print_shortest(io, elt)$/;" function line:553 +writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^function writedlm_cell{T}(io::IO, elt::AbstractString, dlm::T, quotes::Bool)$/;" function line:554 +writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^writedlm_cell(io::IO, elt, dlm, quotes) = print(io, elt)$/;" function line:561 +writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(io::IO, a::AbstractMatrix, dlm; opts...)$/;" function line:562 +writedlm /Users/malmaud/julia/base/datafmt.jl /^writedlm{T}(io::IO, a::AbstractArray{T,0}, dlm; opts...) = writedlm(io, reshape(a,1), dlm; opts...)$/;" function line:579 +writedlm_row /Users/malmaud/julia/base/datafmt.jl /^function writedlm_row(io::IO, row, dlm, quotes)$/;" function line:582 +writedlm_row /Users/malmaud/julia/base/datafmt.jl /^function writedlm_row(io::IO, row::Union{Number,AbstractString}, dlm, quotes)$/;" function line:595 +writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(io::IO, itr, dlm; opts...)$/;" function line:601 +writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(fname::AbstractString, a, dlm; opts...)$/;" function line:613 +writedlm /Users/malmaud/julia/base/datafmt.jl /^writedlm(io, a; opts...) = writedlm(io, a, '\\t'; opts...)$/;" function line:619 +writecsv /Users/malmaud/julia/base/datafmt.jl /^writecsv(io, a; opts...) = writedlm(io, a, ','; opts...)$/;" function line:620 +show /Users/malmaud/julia/base/datafmt.jl /^show(io::IO, ::MIME"text\/csv", a) = writedlm(io, a, ',')$/;" function line:622 +show /Users/malmaud/julia/base/datafmt.jl /^show(io::IO, ::MIME"text\/tab-separated-values", a) = writedlm(io, a, '\\t')$/;" function line:623 diff --git a/base/multi.jl b/base/multi.jl index 5b9b5ad789c3b..4cf91f7fce205 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -27,9 +27,9 @@ let REF_ID::Int = 1 next_ref_id() = (id = REF_ID; REF_ID += 1; id) end -type RRID - whence - id +immutable RRID + whence::Int + id::Int RRID() = RRID(myid(),next_ref_id()) RRID(whence, id) = new(whence,id) @@ -46,18 +46,17 @@ hash(r::RRID, h::UInt) = hash(r.whence, hash(r.id, h)) # Message header stored separately from body to be able to send back errors if # a deserialization error occurs when reading the message body. -type MsgHeader +immutable MsgHeader response_oid::RRID notify_oid::RRID + MsgHeader(respond_oid=RRID(0,0), notify_oid=RRID(0,0)) = + new(respond_oid, notify_oid) end # Special oid (0,0) uses to indicate a null ID. # Used instead of Nullable to decrease wire size of header. null_id(id) = id == RRID(0, 0) -MsgHeader(;response_oid::RRID=RRID(0,0), notify_oid::RRID=RRID(0,0)) = - MsgHeader(response_oid, notify_oid) - type CallMsg{Mode} <: AbstractMsg f::Function args::Tuple @@ -831,7 +830,7 @@ end function remotecall(f, w::Worker, args...; kwargs...) rr = Future(w) #println("$(myid()) asking for $rr") - send_msg(w, MsgHeader(response_oid=remoteref_id(rr)), CallMsg{:call}(f, args, kwargs)) + send_msg(w, MsgHeader(remoteref_id(rr)), CallMsg{:call}(f, args, kwargs)) rr end @@ -849,7 +848,7 @@ function remotecall_fetch(f, w::Worker, args...; kwargs...) oid = RRID() rv = lookup_ref(oid) rv.waitingfor = w.id - send_msg(w, MsgHeader(response_oid=oid), CallMsg{:call_fetch}(f, args, kwargs)) + send_msg(w, MsgHeader(oid), CallMsg{:call_fetch}(f, args, kwargs)) v = take!(rv) delete!(PGRP.refs, oid) isa(v, RemoteException) ? throw(v) : v @@ -866,7 +865,7 @@ function remotecall_wait(f, w::Worker, args...; kwargs...) rv = lookup_ref(prid) rv.waitingfor = w.id rr = Future(w) - send_msg(w, MsgHeader(response_oid=remoteref_id(rr), notify_oid=prid), CallWaitMsg(f, args, kwargs)) + send_msg(w, MsgHeader(remoteref_id(rr), prid), CallWaitMsg(f, args, kwargs)) v = fetch(rv.c) delete!(PGRP.refs, prid) isa(v, RemoteException) && throw(v) @@ -978,7 +977,7 @@ function deliver_result(sock::IO, msg, oid, value) val = :OK end try - send_msg_now(sock, MsgHeader(response_oid=oid), ResultMsg(val)) + send_msg_now(sock, MsgHeader(oid), ResultMsg(val)) catch e # terminate connection in case of serialization error # otherwise the reading end would hang @@ -1194,7 +1193,7 @@ function handle_msg(msg::JoinPGRPMsg, msghdr, r_stream, w_stream, version) set_default_worker_pool(msg.worker_pool) send_connection_hdr(controller, false) - send_msg_now(controller, MsgHeader(notify_oid=msghdr.notify_oid), JoinCompleteMsg(Sys.CPU_CORES, getpid())) + send_msg_now(controller, MsgHeader(RRID(0,0), msghdr.notify_oid), JoinCompleteMsg(Sys.CPU_CORES, getpid())) end function connect_to_peer(manager::ClusterManager, rpid::Int, wconfig::WorkerConfig) @@ -1543,7 +1542,7 @@ function create_worker(manager, wconfig) all_locs = map(x -> isa(x, Worker) ? (get(x.config.connect_at, ()), x.id) : ((), x.id, true), join_list) send_connection_hdr(w, true) - send_msg_now(w, MsgHeader(notify_oid=ntfy_oid), JoinPGRPMsg(w.id, all_locs, PGRP.topology, default_worker_pool())) + send_msg_now(w, MsgHeader(RRID(0,0), ntfy_oid), JoinPGRPMsg(w.id, all_locs, PGRP.topology, default_worker_pool())) @schedule manage(w.manager, w.id, w.config, :register) wait(rr_ntfy_join) From 4568e75050c6f0d3ef9fe45b85cc2ddc21393440 Mon Sep 17 00:00:00 2001 From: Jon Malmaud Date: Sat, 25 Jun 2016 14:48:49 -0400 Subject: [PATCH 0159/1117] Accidentally added ctags file --- .tags | 52 ---------------------------------------------------- .tags1 | 58 ---------------------------------------------------------- 2 files changed, 110 deletions(-) delete mode 100644 .tags delete mode 100644 .tags1 diff --git a/.tags b/.tags deleted file mode 100644 index 80635bed2cc22..0000000000000 --- a/.tags +++ /dev/null @@ -1,52 +0,0 @@ -invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{Char}) = reinterpret(Char, 0xfffffffe)$/;" function line:11 -invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt8}) = 0xfe$/;" function line:12 -invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt16}) = 0xfffe$/;" function line:13 -invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt32}) = 0xfffffffe$/;" function line:14 -countlines /Users/malmaud/julia/base/datafmt.jl /^countlines(f::AbstractString, eol::Char='\\n') = open(io->countlines(io,eol), f)::Int$/;" function line:18 -countlines /Users/malmaud/julia/base/datafmt.jl /^function countlines(io::IO, eol::Char='\\n')$/;" function line:19 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, T::Type; opts...) = readdlm(input, invalid_dlm(Char), T, '\\n'; opts...)$/;" function line:32 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, T::Type; opts...) = readdlm(input, dlm, T, '\\n'; opts...)$/;" function line:33 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input; opts...) = readdlm(input, invalid_dlm(Char), '\\n'; opts...)$/;" function line:35 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char; opts...) = readdlm(input, dlm, '\\n'; opts...)$/;" function line:36 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, eol::Char; opts...) =$/;" function line:38 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, T::Type, eol::Char; opts...) =$/;" function line:40 -readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^readdlm_auto(input::Vector{UInt8}, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) =$/;" function line:43 -readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^readdlm_auto(input::IO, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) =$/;" function line:45 -readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^function readdlm_auto(input::AbstractString, dlm::Char, T::Type, eol::Char, auto::Bool; opts...)$/;" function line:47 -DLMHandler /Users/malmaud/julia/base/datafmt.jl /^abstract DLMHandler$/;" function line:68 -DLMOffsets /Users/malmaud/julia/base/datafmt.jl /^type DLMOffsets <: DLMHandler$/;" function line:70 -DLMOffsets /Users/malmaud/julia/base/datafmt.jl /^ function DLMOffsets(sbuff::String)$/;" function line:76 -store_cell /Users/malmaud/julia/base/datafmt.jl /^function store_cell(dlmoffsets::DLMOffsets, row::Int, col::Int,$/;" function line:84 -result /Users/malmaud/julia/base/datafmt.jl /^function result(dlmoffsets::DLMOffsets)$/;" function line:115 -DLMStore /Users/malmaud/julia/base/datafmt.jl /^type DLMStore{T} <: DLMHandler$/;" function line:121 -DLMStore /Users/malmaud/julia/base/datafmt.jl /^function DLMStore{T}(::Type{T}, dims::NTuple{2,Integer},$/;" function line:135 -_chrinstr /Users/malmaud/julia/base/datafmt.jl /^_chrinstr(sbuff::String, chr::UInt8, startpos::Int, endpos::Int) =$/;" function line:145 -store_cell /Users/malmaud/julia/base/datafmt.jl /^function store_cell{T}(dlmstore::DLMStore{T}, row::Int, col::Int,$/;" function line:149 -result /Users/malmaud/julia/base/datafmt.jl /^function result{T}(dlmstore::DLMStore{T})$/;" function line:218 -readdlm_string /Users/malmaud/julia/base/datafmt.jl /^function readdlm_string(sbuff::String, dlm::Char, T::Type, eol::Char, auto::Bool, optsd::Dict)$/;" function line:248 -val_opts /Users/malmaud/julia/base/datafmt.jl /^function val_opts(opts)$/;" function line:291 -dlm_fill /Users/malmaud/julia/base/datafmt.jl /^function dlm_fill(T::DataType, offarr::Vector{Vector{Int}}, dims::NTuple{2,Integer}, has_header::Bool, sbuff::String, auto::Bool, eol::Char)$/;" function line:303 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Bool,2}, row::Int, col::Int)$/;" function line:329 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:Integer}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:334 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Float64,2}, row::Int, col::Int)$/;" function line:339 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Float32,2}, row::Int, col::Int)$/;" function line:344 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:AbstractString}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:349 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Any,2}, row::Int, col::Int)$/;" function line:353 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:Char}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:372 -colval /Users/malmaud/julia/base/datafmt.jl /^colval(sbuff::String, startpos::Int, endpos::Int, cells::Array, row::Int, col::Int) = true$/;" function line:380 -dlm_parse /Users/malmaud/julia/base/datafmt.jl /^function dlm_parse{T,D}(dbuff::T, eol::D, dlm::D, qchar::D, cchar::D,$/;" function line:382 -readcsv /Users/malmaud/julia/base/datafmt.jl /^readcsv(io; opts...) = readdlm(io, ','; opts...)$/;" function line:549 -readcsv /Users/malmaud/julia/base/datafmt.jl /^readcsv(io, T::Type; opts...) = readdlm(io, ',', T; opts...)$/;" function line:550 -writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^writedlm_cell(io::IO, elt::AbstractFloat, dlm, quotes) = print_shortest(io, elt)$/;" function line:553 -writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^function writedlm_cell{T}(io::IO, elt::AbstractString, dlm::T, quotes::Bool)$/;" function line:554 -writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^writedlm_cell(io::IO, elt, dlm, quotes) = print(io, elt)$/;" function line:561 -writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(io::IO, a::AbstractMatrix, dlm; opts...)$/;" function line:562 -writedlm /Users/malmaud/julia/base/datafmt.jl /^writedlm{T}(io::IO, a::AbstractArray{T,0}, dlm; opts...) = writedlm(io, reshape(a,1), dlm; opts...)$/;" function line:579 -writedlm_row /Users/malmaud/julia/base/datafmt.jl /^function writedlm_row(io::IO, row, dlm, quotes)$/;" function line:582 -writedlm_row /Users/malmaud/julia/base/datafmt.jl /^function writedlm_row(io::IO, row::Union{Number,AbstractString}, dlm, quotes)$/;" function line:595 -writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(io::IO, itr, dlm; opts...)$/;" function line:601 -writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(fname::AbstractString, a, dlm; opts...)$/;" function line:613 -writedlm /Users/malmaud/julia/base/datafmt.jl /^writedlm(io, a; opts...) = writedlm(io, a, '\\t'; opts...)$/;" function line:619 -writecsv /Users/malmaud/julia/base/datafmt.jl /^writecsv(io, a; opts...) = writedlm(io, a, ','; opts...)$/;" function line:620 -show /Users/malmaud/julia/base/datafmt.jl /^show(io::IO, ::MIME"text\/csv", a) = writedlm(io, a, ',')$/;" function line:622 -show /Users/malmaud/julia/base/datafmt.jl /^show(io::IO, ::MIME"text\/tab-separated-values", a) = writedlm(io, a, '\\t')$/;" function line:623 diff --git a/.tags1 b/.tags1 deleted file mode 100644 index 5f6da3864e339..0000000000000 --- a/.tags1 +++ /dev/null @@ -1,58 +0,0 @@ -!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ -!_TAG_FILE_SORTED 0 /0=unsorted, 1=sorted, 2=foldcase/ -!_TAG_PROGRAM_AUTHOR Darren Hiebert /dhiebert@users.sourceforge.net/ -!_TAG_PROGRAM_NAME Exuberant Ctags // -!_TAG_PROGRAM_URL http://ctags.sourceforge.net /official site/ -!_TAG_PROGRAM_VERSION 5.8 // -invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{Char}) = reinterpret(Char, 0xfffffffe)$/;" function line:11 -invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt8}) = 0xfe$/;" function line:12 -invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt16}) = 0xfffe$/;" function line:13 -invalid_dlm /Users/malmaud/julia/base/datafmt.jl /^invalid_dlm(::Type{UInt32}) = 0xfffffffe$/;" function line:14 -countlines /Users/malmaud/julia/base/datafmt.jl /^countlines(f::AbstractString, eol::Char='\\n') = open(io->countlines(io,eol), f)::Int$/;" function line:18 -countlines /Users/malmaud/julia/base/datafmt.jl /^function countlines(io::IO, eol::Char='\\n')$/;" function line:19 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, T::Type; opts...) = readdlm(input, invalid_dlm(Char), T, '\\n'; opts...)$/;" function line:32 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, T::Type; opts...) = readdlm(input, dlm, T, '\\n'; opts...)$/;" function line:33 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input; opts...) = readdlm(input, invalid_dlm(Char), '\\n'; opts...)$/;" function line:35 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char; opts...) = readdlm(input, dlm, '\\n'; opts...)$/;" function line:36 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, eol::Char; opts...) =$/;" function line:38 -readdlm /Users/malmaud/julia/base/datafmt.jl /^readdlm(input, dlm::Char, T::Type, eol::Char; opts...) =$/;" function line:40 -readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^readdlm_auto(input::Vector{UInt8}, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) =$/;" function line:43 -readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^readdlm_auto(input::IO, dlm::Char, T::Type, eol::Char, auto::Bool; opts...) =$/;" function line:45 -readdlm_auto /Users/malmaud/julia/base/datafmt.jl /^function readdlm_auto(input::AbstractString, dlm::Char, T::Type, eol::Char, auto::Bool; opts...)$/;" function line:47 -DLMHandler /Users/malmaud/julia/base/datafmt.jl /^abstract DLMHandler$/;" function line:68 -DLMOffsets /Users/malmaud/julia/base/datafmt.jl /^type DLMOffsets <: DLMHandler$/;" function line:70 -DLMOffsets /Users/malmaud/julia/base/datafmt.jl /^ function DLMOffsets(sbuff::String)$/;" function line:76 -store_cell /Users/malmaud/julia/base/datafmt.jl /^function store_cell(dlmoffsets::DLMOffsets, row::Int, col::Int,$/;" function line:84 -result /Users/malmaud/julia/base/datafmt.jl /^function result(dlmoffsets::DLMOffsets)$/;" function line:115 -DLMStore /Users/malmaud/julia/base/datafmt.jl /^type DLMStore{T} <: DLMHandler$/;" function line:121 -DLMStore /Users/malmaud/julia/base/datafmt.jl /^function DLMStore{T}(::Type{T}, dims::NTuple{2,Integer},$/;" function line:135 -_chrinstr /Users/malmaud/julia/base/datafmt.jl /^_chrinstr(sbuff::String, chr::UInt8, startpos::Int, endpos::Int) =$/;" function line:145 -store_cell /Users/malmaud/julia/base/datafmt.jl /^function store_cell{T}(dlmstore::DLMStore{T}, row::Int, col::Int,$/;" function line:149 -result /Users/malmaud/julia/base/datafmt.jl /^function result{T}(dlmstore::DLMStore{T})$/;" function line:218 -readdlm_string /Users/malmaud/julia/base/datafmt.jl /^function readdlm_string(sbuff::String, dlm::Char, T::Type, eol::Char, auto::Bool, optsd::Dict)$/;" function line:248 -val_opts /Users/malmaud/julia/base/datafmt.jl /^function val_opts(opts)$/;" function line:291 -dlm_fill /Users/malmaud/julia/base/datafmt.jl /^function dlm_fill(T::DataType, offarr::Vector{Vector{Int}}, dims::NTuple{2,Integer}, has_header::Bool, sbuff::String, auto::Bool, eol::Char)$/;" function line:303 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Bool,2}, row::Int, col::Int)$/;" function line:329 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:Integer}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:334 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Float64,2}, row::Int, col::Int)$/;" function line:339 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Float32,2}, row::Int, col::Int)$/;" function line:344 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:AbstractString}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:349 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval(sbuff::String, startpos::Int, endpos::Int, cells::Array{Any,2}, row::Int, col::Int)$/;" function line:353 -colval /Users/malmaud/julia/base/datafmt.jl /^function colval{T<:Char}(sbuff::String, startpos::Int, endpos::Int, cells::Array{T,2}, row::Int, col::Int)$/;" function line:372 -colval /Users/malmaud/julia/base/datafmt.jl /^colval(sbuff::String, startpos::Int, endpos::Int, cells::Array, row::Int, col::Int) = true$/;" function line:380 -dlm_parse /Users/malmaud/julia/base/datafmt.jl /^function dlm_parse{T,D}(dbuff::T, eol::D, dlm::D, qchar::D, cchar::D,$/;" function line:382 -readcsv /Users/malmaud/julia/base/datafmt.jl /^readcsv(io; opts...) = readdlm(io, ','; opts...)$/;" function line:549 -readcsv /Users/malmaud/julia/base/datafmt.jl /^readcsv(io, T::Type; opts...) = readdlm(io, ',', T; opts...)$/;" function line:550 -writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^writedlm_cell(io::IO, elt::AbstractFloat, dlm, quotes) = print_shortest(io, elt)$/;" function line:553 -writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^function writedlm_cell{T}(io::IO, elt::AbstractString, dlm::T, quotes::Bool)$/;" function line:554 -writedlm_cell /Users/malmaud/julia/base/datafmt.jl /^writedlm_cell(io::IO, elt, dlm, quotes) = print(io, elt)$/;" function line:561 -writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(io::IO, a::AbstractMatrix, dlm; opts...)$/;" function line:562 -writedlm /Users/malmaud/julia/base/datafmt.jl /^writedlm{T}(io::IO, a::AbstractArray{T,0}, dlm; opts...) = writedlm(io, reshape(a,1), dlm; opts...)$/;" function line:579 -writedlm_row /Users/malmaud/julia/base/datafmt.jl /^function writedlm_row(io::IO, row, dlm, quotes)$/;" function line:582 -writedlm_row /Users/malmaud/julia/base/datafmt.jl /^function writedlm_row(io::IO, row::Union{Number,AbstractString}, dlm, quotes)$/;" function line:595 -writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(io::IO, itr, dlm; opts...)$/;" function line:601 -writedlm /Users/malmaud/julia/base/datafmt.jl /^function writedlm(fname::AbstractString, a, dlm; opts...)$/;" function line:613 -writedlm /Users/malmaud/julia/base/datafmt.jl /^writedlm(io, a; opts...) = writedlm(io, a, '\\t'; opts...)$/;" function line:619 -writecsv /Users/malmaud/julia/base/datafmt.jl /^writecsv(io, a; opts...) = writedlm(io, a, ','; opts...)$/;" function line:620 -show /Users/malmaud/julia/base/datafmt.jl /^show(io::IO, ::MIME"text\/csv", a) = writedlm(io, a, ',')$/;" function line:622 -show /Users/malmaud/julia/base/datafmt.jl /^show(io::IO, ::MIME"text\/tab-separated-values", a) = writedlm(io, a, '\\t')$/;" function line:623 From 117f9006995aaefdcbe495b2ffa7ce53db440cc6 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Sat, 25 Jun 2016 15:17:41 -0400 Subject: [PATCH 0160/1117] custom serializers for MsgHeader and AbstractMsg types to improve performance --- base/multi.jl | 140 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 94 insertions(+), 46 deletions(-) diff --git a/base/multi.jl b/base/multi.jl index 4cf91f7fce205..c80c51b140a72 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -57,43 +57,79 @@ end # Used instead of Nullable to decrease wire size of header. null_id(id) = id == RRID(0, 0) -type CallMsg{Mode} <: AbstractMsg +immutable CallMsg{Mode} <: AbstractMsg f::Function args::Tuple kwargs::Array end -type CallWaitMsg <: AbstractMsg +immutable CallWaitMsg <: AbstractMsg f::Function args::Tuple kwargs::Array end -type RemoteDoMsg <: AbstractMsg +immutable RemoteDoMsg <: AbstractMsg f::Function args::Tuple kwargs::Array end -type ResultMsg <: AbstractMsg +immutable ResultMsg <: AbstractMsg value::Any end # Worker initialization messages -type IdentifySocketMsg <: AbstractMsg +immutable IdentifySocketMsg <: AbstractMsg from_pid::Int end -type IdentifySocketAckMsg <: AbstractMsg + +immutable IdentifySocketAckMsg <: AbstractMsg end -type JoinPGRPMsg <: AbstractMsg + +immutable JoinPGRPMsg <: AbstractMsg self_pid::Int other_workers::Array topology::Symbol worker_pool end -type JoinCompleteMsg <: AbstractMsg +immutable JoinCompleteMsg <: AbstractMsg cpu_cores::Int ospid::Int end +# Avoiding serializing AbstractMsg containers results in a speedup +# of approximately 10%. Can be removed once module Serializer +# has been suitably improved. + +# replace CallMsg{Mode} with specific invocations +const msgtypes = filter!(x->x!=CallMsg, subtypes(AbstractMsg)) +push!(msgtypes, CallMsg{:call}, CallMsg{:call_fetch}) + +for (idx, tname) in enumerate(msgtypes) + nflds = length(fieldnames(tname)) + @eval begin + function serialize(s::AbstractSerializer, o::$tname) + write(s.io, UInt8($idx)) + for fld in fieldnames($tname) + serialize(s, getfield(o, fld)) + end + end + + function deserialize_msg(s::AbstractSerializer, ::Type{$tname}) + data=Array(Any, $nflds) + for i in 1:$nflds + data[i] = deserialize(s) + end + return $tname(data...) + end + end +end + +function deserialize_msg(s) + idx = read(s.io, UInt8) + t = msgtypes[idx] + return deserialize_msg(s, t) +end + function send_msg_unknown(s::IO, header, msg) error("attempt to send to unknown socket") end @@ -106,12 +142,12 @@ function send_msg(s::IO, header, msg) send_msg_unknown(s, header, msg) end -function send_msg_now(s::IO, msghdr, msg::AbstractMsg) +function send_msg_now(s::IO, header, msg::AbstractMsg) id = worker_id_from_socket(s) if id > -1 - return send_msg_now(worker_from_id(id), msghdr, msg) + return send_msg_now(worker_from_id(id), header, msg) end - send_msg_unknown(s, msghdr, msg) + send_msg_unknown(s, header, msg) end abstract ClusterManager @@ -211,12 +247,12 @@ function set_worker_state(w, state) notify(w.c_state; all=true) end -function send_msg_now(w::Worker, msghdr, msg) - send_msg_(w, msghdr, msg, true) +function send_msg_now(w::Worker, header, msg) + send_msg_(w, header, msg, true) end -function send_msg(w::Worker, msghdr, msg) - send_msg_(w, msghdr, msg, false) +function send_msg(w::Worker, header, msg) + send_msg_(w, header, msg, false) end function flush_gc_msgs(w::Worker) @@ -257,16 +293,28 @@ end # Boundary inserted between messages on the wire, used for recovering # from deserialization errors. Picked arbitrarily. -# A size of 10 bytes indicates ~ ~1e24 possible boundaries, so chance of collision with message contents is trivial. +# A size of 10 bytes indicates ~ ~1e24 possible boundaries, so chance of collision +# with message contents is negligible. const MSG_BOUNDARY = UInt8[0x79, 0x8e, 0x8e, 0xf5, 0x6e, 0x9b, 0x2e, 0x97, 0xd5, 0x7d] +# Faster serialization/deserialization of MsgHeader and RRID +function serialize_hdr_raw(io, hdr) + write(io, hdr.response_oid.whence, hdr.response_oid.id, hdr.notify_oid.whence, hdr.notify_oid.id) +end + +function deserialize_hdr_raw(io) + data = Array(Int, 4) + read!(io, data) + return MsgHeader(RRID(data[1], data[2]), RRID(data[3], data[4])) +end + function send_msg_(w::Worker, header, msg, now::Bool) check_worker_state(w) io = w.w_stream lock(io.lock) try reset_state(w.w_serializer) - serialize(w.w_serializer, header) + serialize_hdr_raw(io, header) serialize(w.w_serializer, msg) # io is wrapped in w_serializer write(io, MSG_BOUNDARY) @@ -1022,22 +1070,22 @@ function message_handler_loop(r_stream::IO, w_stream::IO, incoming::Bool) serializer = ClusterSerializer(r_stream) # The first message will associate wpid with r_stream - msghdr = deserialize(serializer) - msg = deserialize(serializer) + header = deserialize_hdr_raw(r_stream) + msg = deserialize_msg(serializer) readbytes!(r_stream, boundary, length(MSG_BOUNDARY)) - handle_msg(msg, msghdr, r_stream, w_stream, version) + handle_msg(msg, header, r_stream, w_stream, version) wpid = worker_id_from_socket(r_stream) @assert wpid > 0 while true reset_state(serializer) - msghdr = deserialize(serializer) -# println("msghdr: ", msghdr) + header = deserialize_hdr_raw(r_stream) + # println("header: ", header) try - msg = deserialize(serializer) + msg = deserialize_msg(serializer) catch e # Deserialization error; discard bytes in stream until boundary found boundary_idx = 1 @@ -1054,21 +1102,21 @@ function message_handler_loop(r_stream::IO, w_stream::IO, incoming::Bool) boundary_idx = 1 end end - # println("Deserialization error.") remote_err = RemoteException(myid(), CapturedException(e, catch_backtrace())) - if !null_id(msghdr.response_oid) - ref = lookup_ref(msghdr.response_oid) + # println("Deserialization error. ", remote_err) + if !null_id(header.response_oid) + ref = lookup_ref(header.response_oid) put!(ref, remote_err) end - if !null_id(msghdr.notify_oid) - deliver_result(w_stream, :call_fetch, msghdr.notify_oid, remote_err) + if !null_id(header.notify_oid) + deliver_result(w_stream, :call_fetch, header.notify_oid, remote_err) end continue end readbytes!(r_stream, boundary, length(MSG_BOUNDARY)) - # println("got msg: ", typeof(msg)) - handle_msg(msg, msghdr, r_stream, w_stream, version) + # println("got msg: ", typeof(msg)) + handle_msg(msg, header, r_stream, w_stream, version) end catch e # println(STDERR, "Process($(myid())) - Exception ", e) @@ -1135,44 +1183,44 @@ function process_hdr(s, validate_cookie) return VersionNumber(strip(String(version))) end -function handle_msg(msg::CallMsg{:call}, msghdr, r_stream, w_stream, version) - schedule_call(msghdr.response_oid, ()->msg.f(msg.args...; msg.kwargs...)) +function handle_msg(msg::CallMsg{:call}, header, r_stream, w_stream, version) + schedule_call(header.response_oid, ()->msg.f(msg.args...; msg.kwargs...)) end -function handle_msg(msg::CallMsg{:call_fetch}, msghdr, r_stream, w_stream, version) +function handle_msg(msg::CallMsg{:call_fetch}, header, r_stream, w_stream, version) @schedule begin v = run_work_thunk(()->msg.f(msg.args...; msg.kwargs...), false) - deliver_result(w_stream, :call_fetch, msghdr.response_oid, v) + deliver_result(w_stream, :call_fetch, header.response_oid, v) end end -function handle_msg(msg::CallWaitMsg, msghdr, r_stream, w_stream, version) +function handle_msg(msg::CallWaitMsg, header, r_stream, w_stream, version) @schedule begin - rv = schedule_call(msghdr.response_oid, ()->msg.f(msg.args...; msg.kwargs...)) - deliver_result(w_stream, :call_wait, msghdr.notify_oid, fetch(rv.c)) + rv = schedule_call(header.response_oid, ()->msg.f(msg.args...; msg.kwargs...)) + deliver_result(w_stream, :call_wait, header.notify_oid, fetch(rv.c)) end end -function handle_msg(msg::RemoteDoMsg, msghdr, r_stream, w_stream, version) +function handle_msg(msg::RemoteDoMsg, header, r_stream, w_stream, version) @schedule run_work_thunk(()->msg.f(msg.args...; msg.kwargs...), true) end -function handle_msg(msg::ResultMsg, msghdr, r_stream, w_stream, version) - put!(lookup_ref(msghdr.response_oid), msg.value) +function handle_msg(msg::ResultMsg, header, r_stream, w_stream, version) + put!(lookup_ref(header.response_oid), msg.value) end -function handle_msg(msg::IdentifySocketMsg, msghdr, r_stream, w_stream, version) +function handle_msg(msg::IdentifySocketMsg, header, r_stream, w_stream, version) # register a new peer worker connection w=Worker(msg.from_pid, r_stream, w_stream, cluster_manager; version=version) send_connection_hdr(w, false) send_msg_now(w, MsgHeader(), IdentifySocketAckMsg()) end -function handle_msg(msg::IdentifySocketAckMsg, msghdr, r_stream, w_stream, version) +function handle_msg(msg::IdentifySocketAckMsg, header, r_stream, w_stream, version) w = map_sock_wrkr[r_stream] w.version = version end -function handle_msg(msg::JoinPGRPMsg, msghdr, r_stream, w_stream, version) +function handle_msg(msg::JoinPGRPMsg, header, r_stream, w_stream, version) LPROC.id = msg.self_pid controller = Worker(1, r_stream, w_stream, cluster_manager; version=version) register_worker(LPROC) @@ -1193,7 +1241,7 @@ function handle_msg(msg::JoinPGRPMsg, msghdr, r_stream, w_stream, version) set_default_worker_pool(msg.worker_pool) send_connection_hdr(controller, false) - send_msg_now(controller, MsgHeader(RRID(0,0), msghdr.notify_oid), JoinCompleteMsg(Sys.CPU_CORES, getpid())) + send_msg_now(controller, MsgHeader(RRID(0,0), header.notify_oid), JoinCompleteMsg(Sys.CPU_CORES, getpid())) end function connect_to_peer(manager::ClusterManager, rpid::Int, wconfig::WorkerConfig) @@ -1210,7 +1258,7 @@ function connect_to_peer(manager::ClusterManager, rpid::Int, wconfig::WorkerConf end end -function handle_msg(msg::JoinCompleteMsg, msghdr, r_stream, w_stream, version) +function handle_msg(msg::JoinCompleteMsg, header, r_stream, w_stream, version) w = map_sock_wrkr[r_stream] environ = get(w.config.environ, Dict()) environ[:cpu_cores] = msg.cpu_cores @@ -1218,7 +1266,7 @@ function handle_msg(msg::JoinCompleteMsg, msghdr, r_stream, w_stream, version) w.config.ospid = msg.ospid w.version = version - ntfy_channel = lookup_ref(msghdr.notify_oid) + ntfy_channel = lookup_ref(header.notify_oid) put!(ntfy_channel, w.id) push!(default_worker_pool(), w) From 31d9ebf4dbff08ff39dee2f924846b0eeeca40ec Mon Sep 17 00:00:00 2001 From: Art Date: Thu, 30 Jun 2016 11:38:19 -0400 Subject: [PATCH 0161/1117] respect the key path on all platforms --- base/libgit2/callbacks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index a9a254e264ca7..b24612d4dcd82 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -119,7 +119,7 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, keydefpath # use cached value else keydefpath = if keydefpath === nothing - homedir()*"/.ssh/id_rsa" + joinpath(homedir(),".ssh","id_rsa") end prompt("Private key location for '$schema$username@$host'", default=keydefpath) end From fbb550b2880f43a339ba8711969f2fa8a71a6077 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 6 Jun 2016 22:54:46 -0400 Subject: [PATCH 0162/1117] identify pure functions that return a known constant this allows deleting their code and avoiding native code gen entirely --- base/inference.jl | 125 +++++++++++++++++++++++++++++-------------- src/alloc.c | 2 + src/codegen.cpp | 40 ++++++++++++-- src/dump.c | 13 +++-- src/jltypes.c | 10 ++-- src/julia.h | 3 +- src/julia_internal.h | 4 +- 7 files changed, 141 insertions(+), 56 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 82504c1627ea8..be6e9d13524ba 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1535,7 +1535,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr end end typeinf_loop(frame) - return (frame.linfo, frame.bestguess, frame.inferred) + return (frame.linfo, widenconst(frame.bestguess), frame.inferred) end function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, caller) @@ -1570,6 +1570,10 @@ function typeinf_ext(linfo::LambdaInfo) ccall(:jl_set_lambda_rettype, Void, (Any, Any), linfo, code.rettype) linfo.inferred = true linfo.inInference = false + if code.jlcall_api == 2 + linfo.constval = code.constval + linfo.jlcall_api = 2 + end end return code else @@ -1742,7 +1746,7 @@ function typeinf_frame(frame) rt = abstract_eval(stmt.args[1], s[pc], frame) if tchanged(rt, frame.bestguess) # new (wider) return type for frame - frame.bestguess = widenconst(tmerge(frame.bestguess, rt)) + frame.bestguess = tmerge(frame.bestguess, rt) for (caller, callerW) in frame.backedges # notify backedges of updated type information for caller_pc in callerW @@ -1934,13 +1938,38 @@ function finish(me::InferenceState) reindex_labels!(me.linfo, me) end widen_all_consts!(me.linfo) - # make sure (meta pure) is stripped from full tree - ispure = popmeta!(me.linfo.code, :pure)[1] - me.linfo.pure = ispure + + ispure = me.linfo.pure + + if (isa(me.bestguess,Const) && me.bestguess.val !== nothing) || + (isType(me.bestguess) && !has_typevars(me.bestguess.parameters[1],true)) + if !ispure && length(me.linfo.code) < 10 + ispure = true + for stmt in me.linfo.code + if !statement_effect_free(stmt, me) + ispure = false; break + end + end + if ispure + for fl in me.linfo.slotflags + if (fl & Slot_UsedUndef) != 0 + ispure = false; break + end + end + end + end + if ispure + # use constant calling convention + setfield!(me.linfo, :constval, + isa(me.bestguess,Const) ? me.bestguess.val : me.bestguess.parameters[1]) + me.linfo.jlcall_api = 2 + end + me.linfo.pure = ispure + end # determine and cache inlineability if !me.linfo.inlineable - me.linfo.inlineable = isinlineable(me.linfo) + me.linfo.inlineable = me.linfo.jlcall_api==2 || isinlineable(me.linfo) end if !me.needtree @@ -1959,7 +1988,7 @@ function finish(me::InferenceState) me.linfo.inlineable = false end - ccall(:jl_set_lambda_rettype, Void, (Any, Any), me.linfo, me.bestguess) + ccall(:jl_set_lambda_rettype, Void, (Any, Any), me.linfo, widenconst(me.bestguess)) me.linfo.inferred = true me.linfo.inInference = false # finalize and record the linfo result @@ -2218,34 +2247,39 @@ function is_pure_builtin(f::ANY) return false end +function statement_effect_free(e::ANY, sv) + if isa(e,Expr) + if e.head === :(=) + return !isa(e.args[1],GlobalRef) && effect_free(e.args[2], sv, false) + elseif e.head === :gotoifnot + return effect_free(e.args[1], sv, false) + end + elseif isa(e,LabelNode) || isa(e,GotoNode) + return true + end + return effect_free(e, sv, false) +end + # detect some important side-effect-free calls (allow_volatile=true) # and some affect-free calls (allow_volatile=false) -- affect_free means the call # cannot be affected by previous calls, except assignment nodes function effect_free(e::ANY, sv, allow_volatile::Bool) - if isa(e,Slot) - return true - end - if isa(e,Symbol) - return allow_volatile - end - if isa(e,Number) || isa(e,AbstractString) || isa(e,SSAValue) || - isa(e,QuoteNode) || isa(e,Type) || isa(e,Tuple) - return true - end if isa(e,GlobalRef) - return (isdefined(e.mod, e.name) && - (allow_volatile || isconst(e.mod, e.name))) - end - if isa(e,Expr) + return (isdefined(e.mod, e.name) && (allow_volatile || isconst(e.mod, e.name))) + elseif isa(e,Symbol) + return allow_volatile + elseif isa(e,Expr) e = e::Expr - if e.head === :static_typeof + head = e.head + if head === :static_typeof return true end - if e.head === :static_parameter + if head === :static_parameter || head === :meta || head === :line || + head === :inbounds || head === :boundscheck return true end ea = e.args - if e.head === :call && !isa(e.args[1], SSAValue) && !isa(e.args[1], Slot) + if head === :call && !isa(e.args[1], SSAValue) && !isa(e.args[1], Slot) if is_known_call_p(e, is_pure_builtin, sv) if !allow_volatile if is_known_call(e, arrayref, sv) || is_known_call(e, arraylen, sv) @@ -2278,7 +2312,7 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) else return false end - elseif e.head === :new + elseif head === :new if !allow_volatile a = ea[1] typ = widenconst(exprtype(a,sv)) @@ -2287,8 +2321,10 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) end end # fall-through - elseif e.head === :return + elseif head === :return # fall-through + elseif head === :the_exception + return allow_volatile else return false end @@ -2297,14 +2333,27 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) return false end end - return true + elseif isa(e,LabelNode) || isa(e,GotoNode) + return false end - return false + return true end #### post-inference optimizations #### +function inline_as_constant(val::ANY, argexprs, sv) + # check if any arguments aren't effect_free and need to be kept around + stmts = Any[] + for i = 1:length(argexprs) + arg = argexprs[i] + if !effect_free(arg, sv, false) + push!(stmts, arg) + end + end + return (QuoteNode(val), stmts) +end + # inline functions whose bodies are "inline_worthy" # where the function body doesn't contain any argument more than once. # static parameters are ok if all the static parameter values are leaf types, @@ -2388,22 +2437,13 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference method = meth[3]::Method # check whether call can be inlined to just a quoted constant value if isa(f, widenconst(ft)) && !method.isstaged && method.lambda_template.pure && (isType(e.typ) || isa(e.typ,Const)) - # check if any arguments aren't effect_free and need to be kept around - stmts = Any[] - for i = 1:length(argexprs) - arg = argexprs[i] - if !effect_free(arg, sv, false) - push!(stmts, arg) - end - end - if isType(e.typ) if !has_typevars(e.typ.parameters[1]) - return (QuoteNode(e.typ.parameters[1]), stmts) + return inline_as_constant(e.typ.parameters[1], argexprs, sv) end else assert(isa(e.typ,Const)) - return (QuoteNode(e.typ.val), stmts) + return inline_as_constant(e.typ.val, argexprs, sv) end end @@ -2448,7 +2488,10 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference if linfo === nothing || !inferred return invoke_NF() end - if !linfo.inlineable + if linfo !== nothing && linfo.jlcall_api == 2 + # in this case function can be inlined to a constant + return inline_as_constant(linfo.constval, argexprs, sv) + elseif linfo !== nothing && !linfo.inlineable # TODO #= if incompletematch @@ -2475,7 +2518,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end =# return invoke_NF() - elseif linfo.code === nothing + elseif linfo === nothing || linfo.code === nothing (linfo, ty, inferred) = typeinf(method, metharg, methsp, true) end if linfo === nothing || !inferred || !linfo.inlineable || (ast = linfo.code) === nothing diff --git a/src/alloc.c b/src/alloc.c index 97677d3926c8b..79d64a8386dec 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -407,6 +407,7 @@ JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info_uninit(void) li->inInference = 0; li->inCompile = 0; li->def = NULL; + li->constval = NULL; li->pure = 0; li->inlineable = 0; return li; @@ -525,6 +526,7 @@ static jl_lambda_info_t *jl_copy_lambda(jl_lambda_info_t *linfo) new_linfo->isva = linfo->isva; new_linfo->rettype = linfo->rettype; new_linfo->def = linfo->def; + new_linfo->constval = linfo->constval; return new_linfo; } diff --git a/src/codegen.cpp b/src/codegen.cpp index 25e8b84efda20..7834a743dc1d5 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1003,6 +1003,7 @@ uint64_t jl_get_llvm_fptr(llvm::Function *llvmf) // and forces compilation of the lambda info extern "C" void jl_generate_fptr(jl_lambda_info_t *li) { + if (li->jlcall_api == 2) return; JL_LOCK(&codegen_lock); // objective: assign li->fptr assert(li->functionObjectsDecls.functionObject); @@ -1019,7 +1020,11 @@ extern "C" void jl_generate_fptr(jl_lambda_info_t *li) // or generate object code for it extern "C" void jl_compile_linfo(jl_lambda_info_t *li) { - if (li->functionObjectsDecls.functionObject == NULL) { + if (li->jlcall_api == 2) { + // delete code for functions reduced to a constant + jl_set_lambda_code_null(li); + } + else if (li->functionObjectsDecls.functionObject == NULL) { // objective: assign li->functionObject to_function(li); } @@ -1168,6 +1173,15 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) return specf; } } + if (linfo->jlcall_api == 2) { + // normally we don't generate native code for these functions, so need an exception here + if (linfo->functionObjectsDecls.functionObject == NULL) + to_function(linfo); + jl_set_lambda_code_null(linfo); + } + else { + jl_compile_linfo(linfo); + } Function *llvmf; if (!getwrapper && linfo->functionObjectsDecls.specFunctionObject != NULL) { llvmf = (Function*)linfo->functionObjectsDecls.specFunctionObject; @@ -2693,8 +2707,11 @@ static jl_cgval_t emit_invoke(jl_expr_t *ex, jl_codectx_t *ctx) jl_lambda_info_t *li = (jl_lambda_info_t*)args[0]; assert(jl_is_lambda_info(li)); - jl_cgval_t result; - if (li->functionObjectsDecls.functionObject == NULL) { + if (li->jlcall_api == 2) { + assert(li->constval); + return mark_julia_const(li->constval); + } + else if (li->functionObjectsDecls.functionObject == NULL) { assert(!li->inCompile); if (li->code == jl_nothing && !li->inInference && li->inferred) { // XXX: it was inferred in the past, so it's almost valid to re-infer it now @@ -2705,6 +2722,7 @@ static jl_cgval_t emit_invoke(jl_expr_t *ex, jl_codectx_t *ctx) } } Value *theFptr = (Value*)li->functionObjectsDecls.functionObject; + jl_cgval_t result; if (theFptr && li->jlcall_api == 0) { jl_cgval_t fval = emit_expr(args[1], ctx); result = emit_call_function_object(li, fval, theFptr, &args[1], nargs - 1, (jl_value_t*)ex, ctx); @@ -3549,7 +3567,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t // may throw. jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match", name); } - if (!lam->functionObjectsDecls.functionObject) { + if (!lam->functionObjectsDecls.functionObject && lam->jlcall_api != 2) { jl_errorf("ERROR: cfunction: compiling %s failed", name); } } @@ -3623,6 +3641,13 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } myargs = NULL; } + else if (lam->jlcall_api == 2) { + nargs = 0; // arguments not needed + specsig = false; + jlfunc_sret = false; + myargs = NULL; + theFptr = NULL; + } else { theFptr = (Function*)lam->functionObjectsDecls.functionObject; specsig = false; @@ -3632,7 +3657,6 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t "", /*InsertBefore*/ctx.ptlsStates); } - assert(theFptr); // first emit the arguments for (size_t i = 0; i < nargs; i++) { @@ -3736,6 +3760,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t // Create the call jl_cgval_t retval; if (lam == NULL) { + assert(theFptr); assert(nargs >= 0); #ifdef LLVM37 Value *ret = builder.CreateCall(prepare_call(theFptr), {myargs, @@ -3747,13 +3772,18 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t retval = mark_julia_type(ret, true, astrt, &ctx); } else if (specsig) { + assert(theFptr); bool retboxed; CallInst *call = builder.CreateCall(prepare_call(theFptr), ArrayRef(args)); call->setAttributes(theFptr->getAttributes()); (void)julia_type_to_llvm(astrt, &retboxed); retval = mark_julia_type(jlfunc_sret ? (Value*)builder.CreateLoad(result) : (Value*)call, retboxed, astrt, &ctx); } + else if (lam->jlcall_api == 2) { + retval = mark_julia_const(lam->constval); + } else { + assert(theFptr); assert(nargs >= 0); // for jlcall, we need to pass the function object even if it is a ghost. // here we reconstruct the function instance from its type (first elt of argt) diff --git a/src/dump.c b/src/dump.c index fbd6cb2c54f50..0dee90f2bae8f 100644 --- a/src/dump.c +++ b/src/dump.c @@ -891,7 +891,10 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) else if (jl_is_lambda_info(v)) { writetag(s, jl_lambda_info_type); jl_lambda_info_t *li = (jl_lambda_info_t*)v; - jl_serialize_value(s, li->code); + if (li->jlcall_api == 2) + jl_serialize_value(s, jl_nothing); + else + jl_serialize_value(s, li->code); jl_serialize_value(s, li->slotnames); jl_serialize_value(s, li->slottypes); jl_serialize_value(s, li->slotflags); @@ -906,12 +909,12 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) write_int8(s, li->isva); write_int32(s, li->nargs); jl_serialize_value(s, (jl_value_t*)li->def); + jl_serialize_value(s, li->constval); jl_serialize_fptr(s, li->fptr); // save functionObject pointers write_int32(s, li->functionID); write_int32(s, li->specFunctionID); - if (li->functionID) - write_int8(s, li->jlcall_api); + write_int8(s, li->jlcall_api); } else if (jl_typeis(v, jl_module_type)) { jl_serialize_module(s, (jl_module_t*)v); @@ -1541,6 +1544,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t li->nargs = read_int32(s); li->def = (jl_method_t*)jl_deserialize_value(s, (jl_value_t**)&li->def); if (li->def) jl_gc_wb(li, li->def); + li->constval = jl_deserialize_value(s, &li->constval); + if (li->constval) jl_gc_wb(li, li->constval); li->fptr = NULL; li->functionObjectsDecls.functionObject = NULL; li->functionObjectsDecls.specFunctionObject = NULL; @@ -1553,7 +1558,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t func_llvm = read_int32(s); cfunc_llvm = read_int32(s); jl_delayed_fptrs(li, func_llvm, cfunc_llvm); - li->jlcall_api = func_llvm ? read_int8(s) : 0; + li->jlcall_api = read_int8(s); li->compile_traced = 0; return (jl_value_t*)li; } diff --git a/src/jltypes.c b/src/jltypes.c index 4afd3baa4fd27..f03a7d525ec4e 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3801,7 +3801,7 @@ void jl_init_types(void) jl_lambda_info_type = jl_new_datatype(jl_symbol("LambdaInfo"), jl_any_type, jl_emptysvec, - jl_svec(25, + jl_svec(26, jl_symbol("rettype"), jl_symbol("sparam_syms"), jl_symbol("sparam_vals"), @@ -3813,6 +3813,7 @@ void jl_init_types(void) jl_symbol("slotflags"), jl_symbol("unspecialized_ducttape"), jl_symbol("def"), + jl_symbol("constval"), jl_symbol("nargs"), jl_symbol("isva"), jl_symbol("inferred"), @@ -3825,7 +3826,7 @@ void jl_init_types(void) jl_symbol("fptr"), jl_symbol(""), jl_symbol(""), jl_symbol(""), jl_symbol("")), - jl_svec(25, + jl_svec(26, jl_any_type, jl_simplevector_type, jl_simplevector_type, @@ -3837,6 +3838,7 @@ void jl_init_types(void) jl_array_uint8_type, jl_any_type, jl_method_type, + jl_any_type, jl_int32_type, jl_bool_type, jl_bool_type, @@ -3844,7 +3846,7 @@ void jl_init_types(void) jl_bool_type, jl_bool_type, jl_bool_type, - jl_bool_type, + jl_uint8_type, jl_bool_type, jl_any_type, jl_any_type, jl_any_type, @@ -3900,9 +3902,9 @@ void jl_init_types(void) jl_svecset(jl_simplevector_type->types, 0, jl_long_type); jl_svecset(jl_typename_type->types, 6, jl_long_type); jl_svecset(jl_methtable_type->types, 3, jl_long_type); - jl_svecset(jl_lambda_info_type->types, 20, jl_voidpointer_type); jl_svecset(jl_lambda_info_type->types, 21, jl_voidpointer_type); jl_svecset(jl_lambda_info_type->types, 22, jl_voidpointer_type); + jl_svecset(jl_lambda_info_type->types, 23, jl_voidpointer_type); jl_compute_field_offsets(jl_datatype_type); jl_compute_field_offsets(jl_typename_type); diff --git a/src/julia.h b/src/julia.h index 7ecd28ba3e25e..201212a78073d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -246,6 +246,7 @@ typedef struct _jl_lambda_info_t { jl_array_t *slotflags; // local var bit flags struct _jl_lambda_info_t *unspecialized_ducttape; // if template can't be compiled due to intrinsics, an un-inferred executable copy may get stored here jl_method_t *def; // method this is specialized from, (null if this is a toplevel thunk) + jl_value_t *constval; // value of the function if jlcall_api==2 int32_t nargs; int8_t isva; int8_t inferred; @@ -253,7 +254,7 @@ typedef struct _jl_lambda_info_t { int8_t inlineable; int8_t inInference; // flags to tell if inference is running on this function int8_t inCompile; // flag to tell if codegen is running on this function - int8_t jlcall_api; // the c-abi for fptr; 0 = jl_fptr_t, 1 = jl_fptr_sparam_t + int8_t jlcall_api; // the c-abi for fptr; 0 = jl_fptr_t, 1 = jl_fptr_sparam_t, 2 = constval int8_t compile_traced; // if set will notify callback if this linfo is compiled jl_fptr_t fptr; // jlcall entry point diff --git a/src/julia_internal.h b/src/julia_internal.h index 2ff0aa59c5141..ea10139b4f4ad 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -90,8 +90,10 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val mfptr = jl_compile_for_dispatch(mfptr); if (mfptr->jlcall_api == 0) return mfptr->fptr(args[0], &args[1], nargs-1); - else + else if (mfptr->jlcall_api == 1) return ((jl_fptr_sparam_t)mfptr->fptr)(meth->sparam_vals, args[0], &args[1], nargs-1); + else + return meth->constval; } jl_tupletype_t *jl_argtype_with_function(jl_function_t *f, jl_tupletype_t *types); From 400f2e2c71522b29daa8708780ac070379291d1b Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 20 Jun 2016 19:08:14 -0400 Subject: [PATCH 0163/1117] fix result type of `collect(T, itr)` --- base/array.jl | 24 +++++++++++++++++------- test/functional.jl | 2 ++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/base/array.jl b/base/array.jl index 270ddff15924a..4fc82d644a089 100644 --- a/base/array.jl +++ b/base/array.jl @@ -201,18 +201,28 @@ promote_rule{T,n,S}(::Type{Array{T,n}}, ::Type{Array{S,n}}) = Array{promote_type ## copying iterators to containers -# make a collection similar to `c` and appropriate for collecting `itr` -_similar_for(c::AbstractArray, T, itr, ::SizeUnknown) = similar(c, T, 0) -_similar_for(c::AbstractArray, T, itr, ::HasLength) = similar(c, T, Int(length(itr)::Integer)) -_similar_for(c::AbstractArray, T, itr, ::HasShape) = similar(c, T, convert(Dims,size(itr))) -_similar_for(c, T, itr, isz) = similar(c, T) - """ collect(element_type, collection) Return an array of type `Array{element_type,1}` of all items in a collection. """ -collect{T}(::Type{T}, itr) = collect(Generator(T, itr)) +collect{T}(::Type{T}, itr) = _collect(T, itr, iteratorsize(itr)) + +_collect{T}(::Type{T}, itr, isz::HasLength) = copy!(Array{T,1}(Int(length(itr)::Integer)), itr) +_collect{T}(::Type{T}, itr, isz::HasShape) = copy!(Array{T}(convert(Dims,size(itr))), itr) +function _collect{T}(::Type{T}, itr, isz::SizeUnknown) + a = Array{T,1}(0) + for x in itr + push!(a,x) + end + return a +end + +# make a collection similar to `c` and appropriate for collecting `itr` +_similar_for(c::AbstractArray, T, itr, ::SizeUnknown) = similar(c, T, 0) +_similar_for(c::AbstractArray, T, itr, ::HasLength) = similar(c, T, Int(length(itr)::Integer)) +_similar_for(c::AbstractArray, T, itr, ::HasShape) = similar(c, T, convert(Dims,size(itr))) +_similar_for(c, T, itr, isz) = similar(c, T) """ collect(collection) diff --git a/test/functional.jl b/test/functional.jl index 8ec9ff7b7a5d6..a43fe7b0f3e77 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -66,6 +66,8 @@ end # typed `collect` @test collect(Float64, Filter(isodd, [1,2,3,4]))[1] === 1.0 +@test isa(collect(Any, [1,2]), Vector{Any}) + # enumerate (issue #6284) let b = IOBuffer("1\n2\n3\n"), a = [] for (i,x) in enumerate(eachline(b)) From f912ce5780fa517ca4efddcc5a69c4b16aa78819 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 27 Jun 2016 14:17:52 -0400 Subject: [PATCH 0164/1117] fix `iteratoreltype` of `Flatten` --- base/iterator.jl | 4 +++- test/functional.jl | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/base/iterator.jl b/base/iterator.jl index 05359a4a14093..c512c6d3b92de 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -479,7 +479,9 @@ flatten(itr) = Flatten(itr) eltype{I}(::Type{Flatten{I}}) = eltype(eltype(I)) iteratorsize{I}(::Type{Flatten{I}}) = SizeUnknown() -iteratoreltype{I}(::Type{Flatten{I}}) = iteratoreltype(eltype(I)) +iteratoreltype{I}(::Type{Flatten{I}}) = _flatteneltype(I, iteratoreltype(I)) +_flatteneltype(I, ::HasEltype) = iteratoreltype(eltype(I)) +_flatteneltype(I, et) = EltypeUnknown() function start(f::Flatten) local inner, s2 diff --git a/test/functional.jl b/test/functional.jl index a43fe7b0f3e77..5042a81507bdc 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -376,6 +376,8 @@ import Base.flatten @test eltype(flatten(UnitRange{Int8}[1:2, 3:4])) == Int8 @test_throws ArgumentError collect(flatten(Any[])) +@test Base.iteratoreltype(Base.Flatten((i for i=1:2) for j=1:1)) == Base.EltypeUnknown() + # foreach let a = [] From cc5e46cbc5509a3a2c69f6d26ff86d1679f58f7b Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 27 Jun 2016 17:36:51 -0400 Subject: [PATCH 0165/1117] some small optimizations to Flatten and Filter iterators --- base/iterator.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/iterator.jl b/base/iterator.jl index c512c6d3b92de..843ecc7e4dc0a 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -134,7 +134,7 @@ function advance_filter(pred, itr, st) end s=t end - v, (true,) + v, (true, v, s) end done(f::Filter, s) = s[1] @@ -498,7 +498,7 @@ function start(f::Flatten) return s, inner, s2 end -function next(f::Flatten, state) +@inline function next(f::Flatten, state) s, inner, s2 = state val, s2 = next(inner, s2) while done(inner, s2) && !done(f.it, s) From 3f3114d98a5fe60f0e51ed33ea3eb6d615198da9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 30 Jun 2016 21:15:31 +0200 Subject: [PATCH 0166/1117] Fix OOB in debuginfo. --- src/debuginfo.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index ce39d8ec45de8..0aa7959139c6a 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -693,7 +693,6 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, free(*frames); *frames = new_frames; } - jl_lambda_info_t *outer_linfo = (*frames)[n_frames-1].linfo; for (int i = 0; i < n_frames; i++) { bool inlined_frame = i != n_frames - 1; DILineInfo info; @@ -714,7 +713,7 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, if (inlined_frame) { frame->inlined = 1; frame->fromC = fromC; - if (outer_linfo) { + if (jl_lambda_info_t *outer_linfo = (*frames)[n_frames-1].linfo) { std::size_t semi_pos = func_name.find(';'); if (semi_pos != std::string::npos) { func_name = func_name.substr(0, semi_pos); From 04be57094b85bab06a2199e212bf5433cf9618ee Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Thu, 30 Jun 2016 22:30:47 +0530 Subject: [PATCH 0167/1117] fix callsite remotecall_fetch notification for deserialization errors --- base/multi.jl | 4 ++-- base/serialize.jl | 6 ------ test/parallel_exec.jl | 8 ++++++++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/base/multi.jl b/base/multi.jl index c80c51b140a72..79a45a3eea8b7 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -896,7 +896,7 @@ function remotecall_fetch(f, w::Worker, args...; kwargs...) oid = RRID() rv = lookup_ref(oid) rv.waitingfor = w.id - send_msg(w, MsgHeader(oid), CallMsg{:call_fetch}(f, args, kwargs)) + send_msg(w, MsgHeader(RRID(0,0), oid), CallMsg{:call_fetch}(f, args, kwargs)) v = take!(rv) delete!(PGRP.refs, oid) isa(v, RemoteException) ? throw(v) : v @@ -1189,7 +1189,7 @@ end function handle_msg(msg::CallMsg{:call_fetch}, header, r_stream, w_stream, version) @schedule begin v = run_work_thunk(()->msg.f(msg.args...; msg.kwargs...), false) - deliver_result(w_stream, :call_fetch, header.response_oid, v) + deliver_result(w_stream, :call_fetch, header.notify_oid, v) end end diff --git a/base/serialize.jl b/base/serialize.jl index fe7cc270e869a..3f4be28406318 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -582,17 +582,11 @@ function deserialize(s::AbstractSerializer, ::Type{Module}) if isa(path,Tuple) && path !== () # old version for mname in path - if !isdefined(m,mname) - warn("Module $mname not defined on process $(myid())") # an error seemingly fails - end m = getfield(m,mname)::Module end else mname = path while mname !== () - if !isdefined(m,mname) - warn("Module $mname not defined on process $(myid())") # an error seemingly fails - end m = getfield(m,mname)::Module mname = deserialize(s) end diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 4c5eb78c5cd73..e135dd1f8fa26 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -1027,9 +1027,17 @@ f_myid = ()->myid() @test wrkr2 == remotecall_fetch((f, p)->remotecall_fetch(f, p), wrkr1, f_myid, wrkr2) # Deserialization error recovery test +# locally defined module, but unavailable on workers +module LocalFoo + global foo=1 +end + let + @test_throws RemoteException remotecall_fetch(()->LocalFoo.foo, 2) + bad_thunk = ()->NonexistantModule.f() @test_throws RemoteException remotecall_fetch(bad_thunk, 2) + # Test that the stream is still usable @test remotecall_fetch(()->:test,2) == :test ref = remotecall(bad_thunk, 2) From 1ea56e0ecc4f5b739041ab7c11680c0f5405bb41 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 27 Jun 2016 17:37:18 -0400 Subject: [PATCH 0168/1117] widen Tuple types less in inference; helps `Filter` iterator --- base/inference.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index be6e9d13524ba..6e9c7537a5224 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1269,7 +1269,12 @@ function tmerge(typea::ANY, typeb::ANY) if isa(typea, DataType) && isa(typeb, DataType) && length(typea.parameters) == length(typeb.parameters) && !isvatuple(typea) && !isvatuple(typeb) return typejoin(typea, typeb) end - return Tuple + if isa(typea, Union) || isa(typeb, Union) || (isa(typea,DataType) && length(typea.parameters)>3) || + (isa(typeb,DataType) && length(typeb.parameters)>3) + # widen tuples faster (see #6704), but not too much, to make sure we can infer + # e.g. (t::Union{Tuple{Bool},Tuple{Bool,Int}})[1] + return Tuple + end end u = Union{typea, typeb} if length(u.types) > MAX_TYPEUNION_LEN || type_too_complex(u, 0) From 2ae929ce4b2ed06d98dd5be62f75842faaef7db9 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 28 Jun 2016 00:56:06 -0400 Subject: [PATCH 0169/1117] clarify that vararg Generator ctor is not applicable to 2 arguments --- base/generator.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/generator.jl b/base/generator.jl index 48e00c749b84e..b3ef8523a706e 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -13,7 +13,7 @@ immutable Generator{I,F} iter::I end -Generator(f, c1, c...) = Generator(a->f(a...), zip(c1, c...)) +Generator(f, I1, I2, Is...) = Generator(a->f(a...), zip(I1, I2, Is...)) Generator{T,I}(::Type{T}, iter::I) = Generator{I,Type{T}}(T, iter) From 7b1f67845a45d3a1f3858fb61404558e2a15644c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 28 Jun 2016 00:57:36 -0400 Subject: [PATCH 0170/1117] remove some unnecessary typevars as parameters of Tuple types --- src/jltypes.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index f03a7d525ec4e..064d4d65191b5 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2380,10 +2380,12 @@ static jl_value_t *inst_tuple_w_(jl_value_t *t, jl_value_t **env, size_t n, int i; for(i=0; i < ntp; i++) { jl_value_t *elt = jl_svecref(tp, i); - iparams[i] = (jl_value_t*)inst_type_w_(elt, env, n, stack, 0); + jl_value_t *pi = (jl_value_t*)inst_type_w_(elt, env, n, stack, 0); + if (jl_is_typevar(pi) && !((jl_tvar_t*)pi)->bound) + pi = ((jl_tvar_t*)pi)->ub; + iparams[i] = pi; if (ip_heap) - jl_gc_wb(ip_heap, iparams[i]); - jl_value_t *pi = iparams[i]; + jl_gc_wb(ip_heap, pi); check_tuple_parameter(pi, i, ntp); if (cacheable && !jl_is_leaf_type(pi)) { cacheable = 0; From f601171431cc2497b2ce0b138d2d3f89c46e1cb1 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Fri, 1 Jul 2016 04:03:56 +0530 Subject: [PATCH 0171/1117] Fix unnecessary printing of errors in case of worker-workr connections dropping --- base/multi.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base/multi.jl b/base/multi.jl index 79a45a3eea8b7..548516c14f31a 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -1072,13 +1072,12 @@ function message_handler_loop(r_stream::IO, w_stream::IO, incoming::Bool) # The first message will associate wpid with r_stream header = deserialize_hdr_raw(r_stream) msg = deserialize_msg(serializer) - readbytes!(r_stream, boundary, length(MSG_BOUNDARY)) - handle_msg(msg, header, r_stream, w_stream, version) wpid = worker_id_from_socket(r_stream) - @assert wpid > 0 + readbytes!(r_stream, boundary, length(MSG_BOUNDARY)) + while true reset_state(serializer) header = deserialize_hdr_raw(r_stream) @@ -1119,9 +1118,10 @@ function message_handler_loop(r_stream::IO, w_stream::IO, incoming::Bool) handle_msg(msg, header, r_stream, w_stream, version) end catch e - # println(STDERR, "Process($(myid())) - Exception ", e) + # Check again as it may have been set in a message handler but not propagated to the calling block above + wpid = worker_id_from_socket(r_stream) if (wpid < 1) - println(STDERR, e) + println(STDERR, e, CapturedException(e, catch_backtrace())) println(STDERR, "Process($(myid())) - Unknown remote, closing connection.") else werr = worker_from_id(wpid) From 5387643f8255530b24a192e4ffa16615368fb68a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 30 Jun 2016 21:47:56 -0400 Subject: [PATCH 0172/1117] fix #17098, bad codegen from too many i1 casts This affected code using tuples of Bools. Previously, we always represented Bool as i1 except in storage. Now we represent Bool as i8 most of the time, and truncate to i1 only just before use in branches. --- src/cgutils.cpp | 25 ++++--------------------- src/codegen.cpp | 26 +++++++++++++------------- src/intrinsics.cpp | 16 ++++++++++------ 3 files changed, 27 insertions(+), 40 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 981fe3edd482b..528645da22634 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -113,7 +113,7 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed #endif } if (jl_is_bitstype(jt)) { - uint64_t SizeInBits = jdt == jl_bool_type ? 1 : 8*jdt->size; + uint64_t SizeInBits = 8*jdt->size; #ifdef LLVM37 llvm::DIType *t = dbuilder->createBasicType(jl_symbol_name(jdt->name->name),SizeInBits,8*jdt->alignment,llvm::dwarf::DW_ATE_unsigned); jdt->ditype = t; @@ -301,7 +301,7 @@ JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed) { // this function converts a Julia Type into the equivalent LLVM type if (isboxed) *isboxed = false; - if (jt == (jl_value_t*)jl_bool_type) return T_int1; + if (jt == (jl_value_t*)jl_bool_type) return T_int8; if (jt == (jl_value_t*)jl_bottom_type) return T_void; if (!jl_is_leaf_type(jt)) { if (isboxed) *isboxed = true; @@ -849,11 +849,6 @@ static jl_cgval_t typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, assert(elty != NULL); if (type_is_ghost(elty)) return ghostValue(jltype); - bool isbool=false; - if (elty == T_int1) { - elty = T_int8; - isbool = true; - } Value *data; if (ptr->getType()->getContainedType(0) != elty) data = builder.CreatePointerCast(ptr, PointerType::get(elty, 0)); @@ -878,8 +873,6 @@ static jl_cgval_t typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, null_pointer_check(elt, ctx); } //} - if (isbool) - return mark_julia_type(builder.CreateTrunc(elt, T_int1), false, jltype, ctx); return mark_julia_type(elt, isboxed, jltype, ctx); } @@ -892,9 +885,6 @@ static void typed_store(Value *ptr, Value *idx_0based, const jl_cgval_t &rhs, assert(elty != NULL); if (type_is_ghost(elty)) return; - if (elty == T_int1) { - elty = T_int8; - } Value *r; if (jl_isbits(jltype) && ((jl_datatype_t*)jltype)->size > 0) { r = emit_unbox(elty, rhs, jltype); @@ -1065,9 +1055,6 @@ static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, V if (sizeof(void*) != sizeof(int)) idx0 = builder.CreateTrunc(idx0, T_int32); // llvm3.3 requires this, harmless elsewhere Value *fld = builder.CreateExtractElement(strct.V, idx0); - if (jt == (jl_value_t*)jl_bool_type) { - fld = builder.CreateTrunc(fld, T_int1); - } *ret = mark_julia_type(fld, false, jt, ctx); return true; } @@ -1128,9 +1115,6 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, assert( strct.V->getType()->isSingleValueType() ); fldv = strct.V; } - if (jfty == (jl_value_t*)jl_bool_type) { - fldv = builder.CreateTrunc(fldv, T_int1); - } assert(!jl_field_isptr(jt, idx)); return mark_julia_type(fldv, false, jfty, ctx); } @@ -1480,7 +1464,8 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) Type *t = julia_type_to_llvm(vinfo.typ); assert(!type_is_ghost(t)); // should have been handled by isghost above! - + if (jt == (jl_value_t*)jl_bool_type) + return julia_bool(builder.CreateTrunc(as_value(t,vinfo), T_int1)); if (t == T_int1) return julia_bool(as_value(t,vinfo)); @@ -1703,8 +1688,6 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg // and use memcpy instead dest = builder.CreateConstInBoundsGEP2_32(LLVM37_param(lt) strct, 0, i); } - if (fty == T_int1) - fty = T_int8; fval = emit_unbox(fty, fval_info, jtype, dest); if (init_as_value) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 7834a743dc1d5..4f0589add90b8 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2094,7 +2094,7 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, if (rt1) { rt2 = static_eval(args[2], ctx, true); if (rt2) { - *ret = mark_julia_type(ConstantInt::get(T_int1, jl_egal(rt1, rt2)), false, jl_bool_type, ctx); + *ret = mark_julia_type(ConstantInt::get(T_int8, jl_egal(rt1, rt2)), false, jl_bool_type, ctx); JL_GC_POP(); return true; } @@ -2111,7 +2111,7 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, Value *ans = emit_f_is(v1, v2, ctx); mark_gc_use(v1); mark_gc_use(v2); - *ret = mark_julia_type(ans, false, jl_bool_type, ctx); + *ret = mark_julia_type(builder.CreateZExt(ans,T_int8), false, jl_bool_type, ctx); JL_GC_POP(); return true; } @@ -2174,24 +2174,24 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, jl_value_t *tp0 = jl_tparam0(ty); if (jl_subtype(arg, tp0, 0)) { emit_expr(args[1], ctx); // TODO remove if no side effects - *ret = mark_julia_type(ConstantInt::get(T_int1, 1), false, jl_bool_type, ctx); + *ret = mark_julia_type(ConstantInt::get(T_int8, 1), false, jl_bool_type, ctx); JL_GC_POP(); return true; } if (!jl_subtype(tp0, (jl_value_t*)jl_type_type, 0)) { if (jl_is_leaf_type(arg)) { emit_expr(args[1], ctx); // TODO remove if no side effects - *ret = mark_julia_type(ConstantInt::get(T_int1, 0), false, jl_bool_type, ctx); + *ret = mark_julia_type(ConstantInt::get(T_int8, 0), false, jl_bool_type, ctx); JL_GC_POP(); return true; } if (jl_is_leaf_type(tp0)) { jl_cgval_t arg1 = emit_expr(args[1], ctx); - *ret = mark_julia_type( - builder.CreateICmpEQ(emit_typeof_boxed(arg1,ctx), - literal_pointer_val(tp0)), - false, - jl_bool_type, ctx); + *ret = mark_julia_type(builder.CreateZExt(builder.CreateICmpEQ(emit_typeof_boxed(arg1,ctx), + literal_pointer_val(tp0)), + T_int8), + false, + jl_bool_type, ctx); JL_GC_POP(); return true; } @@ -2206,7 +2206,7 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, jl_is_type_type(rt2) && !jl_is_typevar(jl_tparam0(rt2))) { int issub = jl_subtype(jl_tparam0(rt1), jl_tparam0(rt2), 0); // TODO: emit args[1] and args[2] in case of side effects? - *ret = mark_julia_type(ConstantInt::get(T_int1, issub), false, jl_bool_type, ctx); + *ret = mark_julia_type(ConstantInt::get(T_int8, issub), false, jl_bool_type, ctx); JL_GC_POP(); return true; } @@ -3050,9 +3050,9 @@ static Value *emit_condition(const jl_cgval_t &condV, const std::string &msg, jl_codectx_t *ctx) { if (condV.typ == (jl_value_t*)jl_bool_type) { - Value *cond = emit_unbox(T_int1, condV, (jl_value_t*)jl_bool_type); - assert(cond->getType() == T_int1); - return builder.CreateXor(cond, ConstantInt::get(T_int1,1)); + Value *cond = emit_unbox(T_int8, condV, (jl_value_t*)jl_bool_type); + assert(cond->getType() == T_int8); + return builder.CreateXor(builder.CreateTrunc(cond,T_int1), ConstantInt::get(T_int1,1)); } emit_typecheck(condV, (jl_value_t*)jl_bool_type, msg, ctx); if (condV.isboxed) { diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index a7c9af2a7b84b..626774ff02be3 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -151,9 +151,9 @@ static Constant *julia_const_to_llvm(jl_value_t *e, bool nested=false) return NULL; if (e == jl_true) - return ConstantInt::get(T_int1, 1); + return ConstantInt::get(T_int8, 1); if (e == jl_false) - return ConstantInt::get(T_int1, 0); + return ConstantInt::get(T_int8, 0); if (jl_is_cpointer_type(jt)) return ConstantExpr::getIntToPtr(ConstantInt::get(T_size, jl_unbox_long(e)), julia_type_to_llvm((jl_value_t*)bt)); @@ -412,8 +412,6 @@ static Type *staticeval_bitstype(jl_value_t *bt) static int get_bitstype_nbits(jl_value_t *bt) { assert(jl_is_bitstype(bt)); - if (bt == (jl_value_t*)jl_bool_type) - return 1; return jl_datatype_size(bt)*8; } @@ -1127,10 +1125,16 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, } jl_value_t *newtyp = NULL; // TODO: compare the type validity of x,y,z before emitting the intrinsic - Value *r = emit_untyped_intrinsic(f, x, y, z, nargs, ctx, (jl_datatype_t**)&newtyp); + Value *r; + if (f == not_int && xinfo.typ == (jl_value_t*)jl_bool_type) + r = builder.CreateXor(x, ConstantInt::get(T_int8, 1, true)); + else + r = emit_untyped_intrinsic(f, x, y, z, nargs, ctx, (jl_datatype_t**)&newtyp); if (!newtyp && r->getType() != x->getType()) // cast back to the exact original type (e.g. float vs. int) before remarking as a julia type r = builder.CreateBitCast(r, x->getType()); + if (r->getType() == T_int1) + r = builder.CreateZExt(r, T_int8); return mark_julia_type(r, false, newtyp ? newtyp : xinfo.typ, ctx); } #endif @@ -1139,7 +1143,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, } static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, size_t nargs, - jl_codectx_t *ctx, jl_datatype_t **newtyp) + jl_codectx_t *ctx, jl_datatype_t **newtyp) { Type *t = x->getType(); Value *fy; From 7c43a3de195ffe1c89189951579fb68831a2cb60 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 30 Jun 2016 22:55:29 -0400 Subject: [PATCH 0173/1117] Fix compiler warning in debuginfo.cpp --- src/debuginfo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 0aa7959139c6a..81e6ed19a812a 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -713,7 +713,7 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, if (inlined_frame) { frame->inlined = 1; frame->fromC = fromC; - if (jl_lambda_info_t *outer_linfo = (*frames)[n_frames-1].linfo) { + if ((*frames)[n_frames-1].linfo) { std::size_t semi_pos = func_name.find(';'); if (semi_pos != std::string::npos) { func_name = func_name.substr(0, semi_pos); From 9221ffecd0d6c6c093cd9503a3b3826e20d7ca95 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 30 Jun 2016 23:07:14 -0400 Subject: [PATCH 0174/1117] Fix type of `jl_tls_offset` It is negative on x86/x64 (Variant II) and positive on aarch64 (Variant I). --- src/julia_internal.h | 2 +- src/threading.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/julia_internal.h b/src/julia_internal.h index ea10139b4f4ad..fcd5807371f83 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -251,7 +251,7 @@ void _julia_init(JL_IMAGE_SEARCH rel); void jl_set_base_ctx(char *__stk); -extern size_t jl_tls_offset; +extern ssize_t jl_tls_offset; void jl_init_threading(void); void jl_start_threads(void); void jl_shutdown_threading(void); diff --git a/src/threading.c b/src/threading.c index 36ead899055ba..c24253bf4a9cd 100644 --- a/src/threading.c +++ b/src/threading.c @@ -352,7 +352,7 @@ void ti_threadfun(void *arg) void ti_reset_timings(void); #endif -size_t jl_tls_offset = -1; +ssize_t jl_tls_offset = -1; #ifdef JL_ELF_TLS_VARIANT // Optimize TLS access in codegen if the TLS buffer is using a IE or LE model. @@ -437,7 +437,7 @@ static void jl_check_tls(void) #else # error "Cannot emit thread pointer for this architecture." #endif - size_t offset = jl_check_tls_bound(tp, ptls, data.total_size); + ssize_t offset = jl_check_tls_bound(tp, ptls, data.total_size); if (offset == -1) return; jl_tls_offset = offset; From d15062da932a320143e7181c75dcb0e39066f087 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Wed, 29 Jun 2016 21:59:40 -0400 Subject: [PATCH 0175/1117] Optimize TLS access in C code on Linux * Use indirect function (`ifunc`) to set the address of a symbol at runtime. * Use `weak` symbol to detect (in `ifunc`) whether the static version of TLS is available. --- src/ccall.cpp | 10 +++++- src/threading.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++--- ui/repl.c | 16 ++++----- 3 files changed, 97 insertions(+), 15 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index d63fdec235c8e..d7db816a56046 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1235,7 +1235,15 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) emit_signal_fence(); return ghostValue(jl_void_type); } - if (fptr == (void(*)(void))&jl_get_ptls_states || +#ifdef _OS_LINUX_ + // directly access the address of a ifunc can cause linker issue on + // some configurations (e.g. AArch64 + -Bsymbolic-functions). + static const auto ptls_getter = jl_dlsym_e(jl_dlopen(nullptr, 0), + "jl_get_ptls_states"); +#else + static const auto ptls_getter = &jl_get_ptls_states; +#endif + if (fptr == (void(*)(void))(uintptr_t)ptls_getter || ((!f_lib || (intptr_t)f_lib == 2) && f_name && strcmp(f_name, "jl_get_ptls_states") == 0)) { assert(lrt == T_pint8); diff --git a/src/threading.c b/src/threading.c index c24253bf4a9cd..0dd109c96cdad 100644 --- a/src/threading.c +++ b/src/threading.c @@ -45,6 +45,19 @@ extern "C" { #include "threadgroup.h" #include "threading.h" +// The tls_states buffer: +// +// On platforms that do not use ELF (i.e. where `__thread` is emulated with +// lower level API) (Mac, Windows), we use the platform runtime API to create +// TLS variable directly. +// This is functionally equivalent to using `__thread` but can be +// more efficient since we can have better control over the creation and +// initialization of the TLS buffer. +// +// On platforms that use ELF (Linux, FreeBSD), we use a `__thread` variable +// as the fallback in the shared object. For better efficiency, we also +// create a `__thread` variable in the main executable using a static TLS +// model. #ifdef JULIA_ENABLE_THREADING # if defined(_OS_DARWIN_) // Mac doesn't seem to have static TLS model so the runtime TLS getter @@ -119,12 +132,43 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) return &jl_get_ptls_states; } # else +// We use the faster static version in the main executable to replace +// the slower version in the shared object. The code in different libraries +// or executables, however, have to agree on which version to use. +// The general solution is to add one more indirection in the C entry point +// (see `jl_get_ptls_states_wrapper`). +// +// When `ifunc` is availabe, we can use it to trick the linker to use the +// real address (`jl_get_ptls_states_static`) directly as the symbol address. +// (see `jl_get_ptls_states_resolve`). +// +// However, since the detection of the static version in `ifunc` +// is not guaranteed to be reliable, we still need to fallback to the wrapper +// version as the symbol address if we didn't find the static version in `ifunc`. +#if defined(__GLIBC__) && (defined(_CPU_X86_64_) || defined(_CPU_X86_) || \ + defined(_CPU_AARCH64_) || defined(_CPU_ARM_)) +// Only enable this on architectures that are tested. +// For example, GCC doesn't seem to support the `ifunc` attribute on power yet. +# if __GLIBC_PREREQ(2, 12) +# define JL_TLS_USE_IFUNC +# endif +#endif +// Disable ifunc on clang <= 3.8 since it is not supported +#if defined(JL_TLS_USE_IFUNC) && defined(__clang__) +# if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ <= 8) +# undef JL_TLS_USE_IFUNC +# endif +#endif // fallback provided for embedding static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_fallback(void) { static __thread jl_tls_states_t tls_states; return &tls_states; } +#ifdef JL_TLS_USE_IFUNC +JL_DLLEXPORT JL_CONST_FUNC __attribute__((weak)) +jl_tls_states_t *jl_get_ptls_states_static(void); +#endif static jl_tls_states_t *jl_get_ptls_states_init(void); static jl_get_ptls_states_func jl_tls_states_cb = jl_get_ptls_states_init; static jl_tls_states_t *jl_get_ptls_states_init(void) @@ -137,18 +181,26 @@ static jl_tls_states_t *jl_get_ptls_states_init(void) // This is clearly not thread safe but should be fine since we // make sure the tls states callback is finalized before adding // multiple threads - jl_tls_states_cb = jl_get_ptls_states_fallback; - return jl_get_ptls_states_fallback(); + jl_get_ptls_states_func cb = jl_get_ptls_states_fallback; +#ifdef JL_TLS_USE_IFUNC + if (jl_get_ptls_states_static) + cb = jl_get_ptls_states_static; +#endif + jl_tls_states_cb = cb; + return cb(); } -JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) + +static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_wrapper(void) { return (*jl_tls_states_cb)(); } + JL_DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f) { + if (f == jl_tls_states_cb || !f) + return; // only allow setting this once - if (f && f != jl_get_ptls_states_init && - jl_tls_states_cb == jl_get_ptls_states_init) { + if (jl_tls_states_cb == jl_get_ptls_states_init) { jl_tls_states_cb = f; } else { @@ -156,6 +208,30 @@ JL_DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f) exit(1); } } + +#ifdef JL_TLS_USE_IFUNC +static jl_get_ptls_states_func jl_get_ptls_states_resolve(void) +{ + if (jl_tls_states_cb != jl_get_ptls_states_init) + return jl_tls_states_cb; + // If we can't find the static version, return the wrapper instead + // of the slow version so that we won't resolve to the slow version + // due to issues in the relocation order. + // This may not be necessary once `ifunc` support in glibc is more mature. + if (!jl_get_ptls_states_static) + return jl_get_ptls_states_wrapper; + jl_tls_states_cb = jl_get_ptls_states_static; + return jl_tls_states_cb; +} + +JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) + __attribute__((ifunc ("jl_get_ptls_states_resolve"))); +#else // JL_TLS_USE_IFUNC +JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) +{ + return jl_get_ptls_states_wrapper(); +} +#endif // JL_TLS_USE_IFUNC jl_get_ptls_states_func jl_get_ptls_states_getter(void) { if (jl_tls_states_cb == jl_get_ptls_states_init) diff --git a/ui/repl.c b/ui/repl.c index 5f5d4f477c67c..185c9a8222b0d 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -35,11 +35,17 @@ extern "C" { #endif #if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) && !defined(_OS_WINDOWS_) -static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_static(void) +JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_static(void) { static __attribute__((tls_model("local-exec"))) __thread jl_tls_states_t tls_states; return &tls_states; } +__attribute__((constructor)) void jl_register_ptls_states_getter(void) +{ + // We need to make sure this function is called before any reference to + // TLS variables. + jl_set_ptls_states_getter(jl_get_ptls_states_static); +} #endif static int lisp_prompt = 0; @@ -655,14 +661,6 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) if (!WideCharToMultiByte(CP_UTF8, 0, warg, -1, arg, len, NULL, NULL)) return 1; argv[i] = (wchar_t*)arg; } -#endif -#if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) && !defined(_OS_WINDOWS_) - // We need to make sure this function is called before any reference to - // TLS variables. Since the compiler is free to move calls to - // `jl_get_ptls_states()` around, we should avoid referencing TLS - // variables in this function. (Mark `true_main` as noinline for this - // reason). - jl_set_ptls_states_getter(jl_get_ptls_states_static); #endif libsupport_init(); parse_opts(&argc, (char***)&argv); From 644e1faf45f4ac271e0461335548e5fd688b46fc Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 1 Jul 2016 11:31:50 -0500 Subject: [PATCH 0176/1117] Revert "Speed up iteration with numbers" This reverts commit 526695c40b538951a234fc1168344a481c67a438. Jeff's fix to codegen on Bools, #17225, should make this unnecessary. --- base/number.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/number.jl b/base/number.jl index ab33caf0ebd25..218c883e5f2d4 100644 --- a/base/number.jl +++ b/base/number.jl @@ -50,9 +50,9 @@ angle(z::Real) = atan2(zero(z), z) widemul(x::Number, y::Number) = widen(x)*widen(y) -start(x::Number) = 0 # see #16687 -next(x::Number, state) = (x, state+1) -done(x::Number, state) = state == 1 +start(x::Number) = false +next(x::Number, state) = (x, true) +done(x::Number, state) = state isempty(x::Number) = false in(x::Number, y::Number) = x == y From 1ce5fb71b1aa3f8927c1f3e8c44312fcaad3eb9e Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 1 Jul 2016 01:06:59 +0000 Subject: [PATCH 0177/1117] Add patch to fix SuiteSparse-Clang-Win64 build --- deps/patches/SuiteSparse-winclang.patch | 14 ++++++++++++++ deps/suitesparse.mk | 6 +++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 deps/patches/SuiteSparse-winclang.patch diff --git a/deps/patches/SuiteSparse-winclang.patch b/deps/patches/SuiteSparse-winclang.patch new file mode 100644 index 0000000000000..8bfb64dc27839 --- /dev/null +++ b/deps/patches/SuiteSparse-winclang.patch @@ -0,0 +1,14 @@ +--- SuiteSparse_config/SuiteSparse_config.h 2015-07-15 03:26:41.000000000 +0000 ++++ SuiteSparse_config/SuiteSparse_config.h 2016-07-01 00:55:57.157465600 +0000 +@@ -54,7 +54,11 @@ + #ifdef _WIN64 + + #define SuiteSparse_long __int64 ++#ifdef _MSVC_VER + #define SuiteSparse_long_max _I64_MAX ++#else ++#define SuiteSparse_long_max LLONG_MAX ++#endif + #define SuiteSparse_long_idd "I64d" + + #else diff --git a/deps/suitesparse.mk b/deps/suitesparse.mk index c8974b19fa47c..975572e762466 100644 --- a/deps/suitesparse.mk +++ b/deps/suitesparse.mk @@ -37,6 +37,10 @@ $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile: $(SRCDIR)/srccache/SuiteSpa $(TAR) -C $(dir $@) --strip-components 1 -zxf $< touch -c $@ +$(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SuiteSparse-winclang.patch-applied: $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile + cd $(dir $@) && patch -p0 < $(SRCDIR)/patches/SuiteSparse-winclang.patch + echo 1 > $@ + ifeq ($(USE_ATLAS), 1) $(SUITESPARSE_OBJ_SOURCE): | $(ATLAS_OBJ_TARGET) endif @@ -46,7 +50,7 @@ $(SUITESPARSE_OBJ_SOURCE): | $(OPENBLAS_OBJ_TARGET) else ifeq ($(USE_SYSTEM_LAPACK), 0) $(SUITESPARSE_OBJ_SOURCE): | $(LAPACK_OBJ_TARGET) endif -$(SUITESPARSE_OBJ_SOURCE): $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile +$(SUITESPARSE_OBJ_SOURCE): $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/Makefile $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/SuiteSparse-winclang.patch-applied $(MAKE) -C $(dir $<) library $(SUITESPARSE_MFLAGS) touch -c $@ $(BUILDDIR)/SuiteSparse-$(SUITESPARSE_VER)/checked: $(SUITESPARSE_OBJ_SOURCE) From da1fdfe423b215e51acdcc1230b18d5641b99457 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 1 Jul 2016 11:38:39 -0400 Subject: [PATCH 0178/1117] reduce startup time by avoiding GC and type cache re-insertion --- base/util.jl | 1 + src/dump.c | 85 ++++++++++++++------------------------------ src/gc.c | 19 ++++++++-- src/gc.h | 1 + src/jltypes.c | 15 ++++++++ src/julia_internal.h | 2 ++ 6 files changed, 62 insertions(+), 61 deletions(-) diff --git a/base/util.jl b/base/util.jl index 8c7406d320cd1..3bed25e62ee63 100644 --- a/base/util.jl +++ b/base/util.jl @@ -10,6 +10,7 @@ time_ns() = ccall(:jl_hrtime, UInt64, ()) # This type must be kept in sync with the C struct in src/gc.h immutable GC_Num allocd ::Int64 # GC internal + deferred_alloc::Int64 freed ::Int64 # GC internal malloc ::UInt64 realloc ::UInt64 diff --git a/src/dump.c b/src/dump.c index 0dee90f2bae8f..486f44e260a2f 100644 --- a/src/dump.c +++ b/src/dump.c @@ -119,8 +119,7 @@ static DUMP_MODES mode = (DUMP_MODES) 0; static jl_value_t *jl_idtable_type=NULL; -// queue of types to cache -static jl_array_t *datatype_list=NULL; // (only used in MODE_SYSTEM_IMAGE) +static arraylist_t builtin_types; #define write_uint8(s, n) ios_putc((n), (s)) #define read_uint8(s) ((uint8_t)ios_getc(s)) @@ -333,30 +332,6 @@ static void jl_serialize_gv_syms(ios_t *s, jl_sym_t *v) if (v->right) jl_serialize_gv_syms(s, v->right); } -static void jl_serialize_gv_svec(ios_t *s, jl_svec_t *svec) -{ - size_t i, l = jl_svec_len(svec); - for (i = 0; i < l; i++) { - jl_value_t *v = jl_svecref(svec, i); - if (v) { - void *bp = ptrhash_get(&backref_table, v); - if (bp == HT_NOTFOUND) { - int32_t gv = jl_get_llvm_gv((jl_value_t*)v); - if (gv != 0) { - jl_serialize_value(s, v); - write_int32(s, gv); - } - } - } - } -} - -static void jl_serialize_gv_tn(ios_t *s, jl_typename_t *tn) -{ - jl_serialize_gv_svec(s, tn->cache); - jl_serialize_gv_svec(s, tn->linearcache); -} - static void jl_serialize_gv_others(ios_t *s) { // ensures all objects referenced in the code have @@ -388,15 +363,6 @@ static void jl_serialize_gv_others(ios_t *s) } } jl_serialize_gv_syms(s, jl_get_root_symbol()); - jl_serialize_gv_tn(s, jl_array_type->name); - jl_serialize_gv_tn(s, jl_ref_type->name); - jl_serialize_gv_tn(s, jl_pointer_type->name); - jl_serialize_gv_tn(s, jl_type_type->name); - jl_serialize_gv_tn(s, jl_simplevector_type->name); - jl_serialize_gv_tn(s, jl_abstractarray_type->name); - jl_serialize_gv_tn(s, jl_densearray_type->name); - jl_serialize_gv_tn(s, jl_tuple_typename); - jl_serialize_gv_tn(s, jl_vararg_type->name); jl_serialize_value(s, NULL); // signal the end of this list } @@ -1294,18 +1260,6 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->types = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&dt->types); jl_gc_wb(dt, dt->types); } - if (datatype_list) { - if (dt->name == jl_array_type->name || dt->name == jl_ref_type->name || - dt->name == jl_pointer_type->name || dt->name == jl_type_type->name || - dt->name == jl_simplevector_type->name || dt->name == jl_abstractarray_type->name || - dt->name == jl_densearray_type->name || dt->name == jl_tuple_typename || - dt->name == jl_vararg_type->name) { - // builtin types are not serialized, so their caches aren't - // explicitly saved. so we reconstruct the caches of builtin - // parametric types here. - jl_array_ptr_1d_push(datatype_list, (jl_value_t*)dt); - } - } return (jl_value_t*)dt; } @@ -1952,8 +1906,13 @@ static void jl_save_system_image_to_stream(ios_t *f) jl_serialize_value(f, jl_typeinf_func); jl_serialize_value(f, jl_type_type->name->mt); - // ensure everything in deser_tag is reassociated with its GlobalValue intptr_t i=2; + for(i=0; i < builtin_types.len; i++) { + jl_serialize_value(f, ((jl_datatype_t*)builtin_types.items[i])->name->cache); + jl_serialize_value(f, ((jl_datatype_t*)builtin_types.items[i])->name->linearcache); + } + + // ensure everything in deser_tag is reassociated with its GlobalValue for (i=2; i < 255; i++) { jl_serialize_gv(f, deser_tag[i]); } @@ -2031,8 +1990,6 @@ static void jl_restore_system_image_from_stream(ios_t *f) mode = MODE_SYSTEM_IMAGE; arraylist_new(&backref_list, 250000); - datatype_list = jl_alloc_vec_any(0); - jl_main_module = (jl_module_t*)jl_deserialize_value(f, NULL); jl_top_module = (jl_module_t*)jl_deserialize_value(f, NULL); jl_internal_main_module = jl_main_module; @@ -2041,6 +1998,15 @@ static void jl_restore_system_image_from_stream(ios_t *f) jl_type_type->name->mt = jl_typector_type->name->mt = jl_uniontype_type->name->mt = jl_datatype_type->name->mt = jl_type_type_mt; + intptr_t i; + for(i=0; i < builtin_types.len; i++) { + jl_typename_t *tn = ((jl_datatype_t*)builtin_types.items[i])->name; + tn->cache = (jl_svec_t*)jl_deserialize_value(f, NULL); jl_gc_wb(tn, tn->cache); + tn->linearcache = (jl_svec_t*)jl_deserialize_value(f, NULL); jl_gc_wb(tn, tn->linearcache); + jl_resort_type_cache(tn->cache); + jl_resort_type_cache(tn->linearcache); + } + jl_core_module = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol("Core")); jl_base_module = (jl_module_t*)jl_get_global(jl_main_module, @@ -2048,7 +2014,6 @@ static void jl_restore_system_image_from_stream(ios_t *f) jl_current_module = jl_base_module; // run start_image in Base // ensure everything in deser_tag is reassociated with its GlobalValue - intptr_t i; for (i = 2; i < 255; i++) { jl_deserialize_gv(f, deser_tag[i]); } @@ -2062,13 +2027,6 @@ static void jl_restore_system_image_from_stream(ios_t *f) jl_set_t_uid_ctr(uid_ctr); jl_set_gs_ctr(gs_ctr); - // cache builtin parametric types - for (int i = 0; i < jl_array_len(datatype_list); i++) { - jl_value_t *v = jl_array_ptr_ref(datatype_list, i); - jl_cache_type_((jl_datatype_t*)v); - } - datatype_list = NULL; - jl_get_builtins(); jl_get_builtin_hooks(); if (jl_base_module) { @@ -2080,6 +2038,7 @@ static void jl_restore_system_image_from_stream(ios_t *f) //jl_printf(JL_STDERR, "backref_list.len = %d\n", backref_list.len); arraylist_free(&backref_list); + jl_gc_reset_alloc_count(); jl_gc_enable(en); mode = last_mode; jl_update_all_fptrs(); @@ -2547,6 +2506,16 @@ void jl_init_serializer(void) i += 1; } assert(i <= 256); + + arraylist_new(&builtin_types, 0); + arraylist_push(&builtin_types, jl_array_type); + arraylist_push(&builtin_types, jl_ref_type); + arraylist_push(&builtin_types, jl_pointer_type); + arraylist_push(&builtin_types, jl_type_type); + arraylist_push(&builtin_types, jl_abstractarray_type); + arraylist_push(&builtin_types, jl_densearray_type); + arraylist_push(&builtin_types, jl_tuple_type); + arraylist_push(&builtin_types, jl_vararg_type); } #ifdef __cplusplus diff --git a/src/gc.c b/src/gc.c index 4d74afd7005af..e0ef7a3b5a0f7 100644 --- a/src/gc.c +++ b/src/gc.c @@ -44,7 +44,7 @@ static jl_mutex_t finalizers_lock; * finalizers in unmanaged (GC safe) mode. */ -jl_gc_num_t gc_num = {0,0,0,0,0,0,0,0,0,0,0,0,0}; +jl_gc_num_t gc_num = {0,0,0,0,0,0,0,0,0,0,0,0,0,0}; static size_t last_long_collect_interval; region_t regions[REGION_COUNT]; @@ -705,6 +705,13 @@ void jl_gc_count_allocd(size_t sz) gc_num.allocd += sz; } +void jl_gc_reset_alloc_count(void) +{ + live_bytes += (gc_num.deferred_alloc + (gc_num.allocd + gc_num.interval)); + gc_num.allocd = -(int64_t)gc_num.interval; + gc_num.deferred_alloc = 0; +} + static size_t array_nbytes(jl_array_t *a) { size_t sz = 0; @@ -1658,7 +1665,10 @@ JL_DLLEXPORT int jl_gc_enable(int on) ptls->disable_gc = (on == 0); if (on && !prev) { // disable -> enable - jl_atomic_fetch_add(&jl_gc_disable_counter, -1); + if (jl_atomic_fetch_add(&jl_gc_disable_counter, -1) == 1) { + gc_num.allocd += gc_num.deferred_alloc; + gc_num.deferred_alloc = 0; + } } else if (prev && !on) { // enable -> disable @@ -1878,8 +1888,11 @@ static void _jl_gc_collect(int full, char *stack_hi) JL_DLLEXPORT void jl_gc_collect(int full) { jl_tls_states_t *ptls = jl_get_ptls_states(); - if (jl_gc_disable_counter) + if (jl_gc_disable_counter) { + gc_num.deferred_alloc += (gc_num.allocd + gc_num.interval); + gc_num.allocd = -(int64_t)gc_num.interval; return; + } char *stack_hi = (char*)gc_get_stack_ptr(); gc_debug_print(); diff --git a/src/gc.h b/src/gc.h index 37181247df897..2d6cbf6bf538c 100644 --- a/src/gc.h +++ b/src/gc.h @@ -66,6 +66,7 @@ typedef struct { // This struct must be kept in sync with the Julia type of the same name in base/util.jl typedef struct { int64_t allocd; + int64_t deferred_alloc; int64_t freed; uint64_t malloc; uint64_t realloc; diff --git a/src/jltypes.c b/src/jltypes.c index 064d4d65191b5..39c0478477e10 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1901,6 +1901,21 @@ static int typekey_compare(jl_datatype_t *tt, jl_value_t **key, size_t n) return 0; } +static int dt_compare(const void *ap, const void *bp) +{ + jl_datatype_t *a = *(jl_datatype_t**)ap; + jl_datatype_t *b = *(jl_datatype_t**)bp; + if (a == b) return 0; + if (b == NULL) return -1; + if (a == NULL) return 1; + return typekey_compare(b, jl_svec_data(a->parameters), jl_svec_len(a->parameters)); +} + +void jl_resort_type_cache(jl_svec_t *c) +{ + qsort(jl_svec_data(c), jl_svec_len(c), sizeof(jl_value_t*), dt_compare); +} + static int typekey_eq(jl_datatype_t *tt, jl_value_t **key, size_t n) { size_t j; diff --git a/src/julia_internal.h b/src/julia_internal.h index fcd5807371f83..86be924044c9d 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -129,8 +129,10 @@ STATIC_INLINE void jl_gc_wb_buf(void *parent, void *bufptr, size_t minsz) // par void gc_debug_print_status(void); void gc_debug_critical_error(void); void jl_print_gc_stats(JL_STREAM *s); +void jl_gc_reset_alloc_count(void); int jl_assign_type_uid(void); jl_value_t *jl_cache_type_(jl_datatype_t *type); +void jl_resort_type_cache(jl_svec_t *c); int jl_get_t_uid_ctr(void); void jl_set_t_uid_ctr(int i); uint32_t jl_get_gs_ctr(void); From b25a2118cdb609bfb61b24698b331a33c016dab9 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 29 Jun 2016 01:20:51 -0400 Subject: [PATCH 0179/1117] implement call-site union splitting using invoke expr --- base/inference.jl | 101 ++++++++++++++++++++++++++++++++++++++++++++- base/reflection.jl | 4 +- base/show.jl | 9 ++++ src/cgutils.cpp | 2 +- src/codegen.cpp | 55 ++++++++++++------------ 5 files changed, 139 insertions(+), 32 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 6e9c7537a5224..546335276c219 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -9,6 +9,7 @@ const MAX_TUPLETYPE_LEN = 15 const MAX_TUPLE_DEPTH = 4 const MAX_TUPLE_SPLAT = 16 +const MAX_UNION_SPLITTING = 6 # alloc_elim_pass! relies on `Slot_AssignedOnce | Slot_UsedUndef` being # SSA. This should be true now but can break if we start to track conditional @@ -2359,6 +2360,16 @@ function inline_as_constant(val::ANY, argexprs, sv) return (QuoteNode(val), stmts) end +function countunionsplit(atypes::Vector{Any}) + nu = 1 + for ti in atypes + if isa(ti, Union) + nu *= length((ti::Union).types) + end + end + return nu +end + # inline functions whose bodies are "inline_worthy" # where the function body doesn't contain any argument more than once. # static parameters are ok if all the static parameter values are leaf types, @@ -2416,10 +2427,96 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference atype_unlimited = argtypes_to_type(atypes) function invoke_NF() # converts a :call to :invoke - cache_linfo = ccall(:jl_get_spec_lambda, Any, (Any,), atype_unlimited) - if cache_linfo !== nothing + local nu = countunionsplit(atypes) + nu > MAX_UNION_SPLITTING && return NF + + if nu > 1 + local ex = copy(e) + local linfo_var = add_slot!(enclosing, LambdaInfo, false) + local spec_hit = nothing + local spec_miss = nothing + local error_label = nothing + function splitunion(atypes::Vector{Any}, i::Int) + if i == 0 + local sig = argtypes_to_type(atypes) + local li = ccall(:jl_get_spec_lambda, Any, (Any,), sig) + li === nothing && return false + local stmt = [] + push!(stmt, Expr(:(=), linfo_var, li)) + spec_hit === nothing && (spec_hit = genlabel(sv)) + push!(stmt, GotoNode(spec_hit.label)) + return stmt + else + local ti = atypes[i] + if isa(ti, Union) + local all = false + local stmts = [] + local aei = ex.args[i] + if !effect_free(aei, sv, false) + aei = newvar!(sv, ti) + push!(stmts, Expr(:(=), aei, ex.args[i])) + ex.args[i] = aei + end + for ty in (ti::Union).types + atypes[i] = ty + local match = splitunion(atypes, i - 1) + if match !== false + after = genlabel(sv) + unshift!(match, Expr(:gotoifnot, Expr(:call, GlobalRef(Core, :isa), aei, ty), after.label)) + append!(stmts, match) + push!(stmts, after) + else + all = false + end + end + if all + error_label === nothing && (error_label = genlabel(sv)) + push!(stmts, GotoNode(error_label.label)) + else + spec_miss === nothing && (spec_miss = genlabel(sv)) + push!(stmts, GotoNode(spec_miss.label)) + end + atypes[i] = ti + return isempty(stmts) ? false : stmts + else + return splitunion(atypes, i - 1) + end + end + end + local stmts = splitunion(atypes, length(atypes)) + if stmts !== false && spec_hit !== nothing + stmts = stmts::Array{Any,1} + if error_label !== nothing + push!(stmts, error_label) + push!(stmts, Expr(:call, GlobalRef(_topmod(sv.mod), :error), "error in type inference due to #265")) + end + local ret_var, merge + if spec_miss !== nothing + ret_var = add_slot!(enclosing, ex.typ, false) + merge = genlabel(sv) + push!(stmts, spec_miss) + push!(stmts, Expr(:(=), ret_var, ex)) + push!(stmts, GotoNode(merge.label)) + else + ret_var = newvar!(sv, ex.typ) + end + push!(stmts, spec_hit) + ex = copy(ex) + ex.head = :invoke + unshift!(ex.args, linfo_var) + push!(stmts, Expr(:(=), ret_var, ex)) + if spec_miss !== nothing + push!(stmts, merge) + end + #println(stmts) + return (ret_var, stmts) + end + else + local cache_linfo = ccall(:jl_get_spec_lambda, Any, (Any,), atype_unlimited) + cache_linfo === nothing && return NF e.head = :invoke unshift!(e.args, cache_linfo) + return e end return NF end diff --git a/base/reflection.jl b/base/reflection.jl index a2f5889b42bca..c62df75011d47 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -192,7 +192,7 @@ function _methods_by_ftype(t::ANY, lim) if 1 < nu <= 64 return _methods(Any[tp...], length(tp), lim, []) end - # TODO: the following can return incorrect answers that the above branch would have corrected + # XXX: the following can return incorrect answers that the above branch would have corrected return ccall(:jl_matching_methods, Any, (Any,Cint,Cint), t, lim, 0) end function _methods(t::Array,i,lim::Integer,matching::Array{Any,1}) @@ -206,7 +206,7 @@ function _methods(t::Array,i,lim::Integer,matching::Array{Any,1}) for ty in (ti::Union).types t[i] = ty if _methods(t,i-1,lim,matching) === false - t[i] = ty + t[i] = ti return false end end diff --git a/base/show.jl b/base/show.jl index e0ae1ea79a902..9fbe79d85a7c7 100644 --- a/base/show.jl +++ b/base/show.jl @@ -561,6 +561,15 @@ show_unquoted(io::IO, ex::LabelNode, ::Int, ::Int) = print(io, ex.label, ": show_unquoted(io::IO, ex::GotoNode, ::Int, ::Int) = print(io, "goto ", ex.label) show_unquoted(io::IO, ex::GlobalRef, ::Int, ::Int) = print(io, ex.mod, '.', ex.name) +function show_unquoted(io::IO, ex::LambdaInfo, ::Int, ::Int) + if isdefined(ex, :specTypes) + print(io, "LambdaInfo for ") + show_lambda_types(io, ex.specTypes.parameters) + else + show(io, ex) + end +end + function show_unquoted(io::IO, ex::Slot, ::Int, ::Int) typ = isa(ex,TypedSlot) ? ex.typ : Any slotid = ex.id diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 528645da22634..ebbc9ef049998 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -515,7 +515,7 @@ static jl_cgval_t emit_typeof(const jl_cgval_t &p, jl_codectx_t *ctx) { // given p, compute its type if (!p.constant && p.isboxed && !jl_is_leaf_type(p.typ)) { - return mark_julia_type(emit_typeof(p.V), true, jl_datatype_type, ctx); + return mark_julia_type(emit_typeof(p.V), true, jl_datatype_type, ctx, /*needsroot*/false); } jl_value_t *aty = p.typ; if (jl_is_type_type(aty)) // convert Int::Type{Int} ==> typeof(Int) ==> DataType diff --git a/src/codegen.cpp b/src/codegen.cpp index 4f0589add90b8..90bd9438c48ff 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2704,38 +2704,39 @@ static jl_cgval_t emit_invoke(jl_expr_t *ex, jl_codectx_t *ctx) size_t arglen = jl_array_dim0(ex->args); size_t nargs = arglen - 1; assert(arglen >= 2); - jl_lambda_info_t *li = (jl_lambda_info_t*)args[0]; - assert(jl_is_lambda_info(li)); - if (li->jlcall_api == 2) { - assert(li->constval); - return mark_julia_const(li->constval); - } - else if (li->functionObjectsDecls.functionObject == NULL) { - assert(!li->inCompile); - if (li->code == jl_nothing && !li->inInference && li->inferred) { - // XXX: it was inferred in the past, so it's almost valid to re-infer it now - jl_type_infer(li, 0); + jl_cgval_t lival = emit_expr(args[0], ctx); + if (lival.constant) { + jl_lambda_info_t *li = (jl_lambda_info_t*)lival.constant; + assert(jl_is_lambda_info(li)); + if (li->jlcall_api == 2) { + assert(li->constval); + return mark_julia_const(li->constval); + } + if (li->functionObjectsDecls.functionObject == NULL) { + assert(!li->inCompile); + if (li->code == jl_nothing && !li->inInference && li->inferred) { + // XXX: it was inferred in the past, so it's almost valid to re-infer it now + jl_type_infer(li, 0); + } + if (!li->inInference && li->inferred && li->code != jl_nothing) { + jl_compile_linfo(li); + } } - if (!li->inInference && li->inferred && li->code != jl_nothing) { - jl_compile_linfo(li); + Value *theFptr = (Value*)li->functionObjectsDecls.functionObject; + if (theFptr && li->jlcall_api == 0) { + jl_cgval_t fval = emit_expr(args[1], ctx); + jl_cgval_t result = emit_call_function_object(li, fval, theFptr, &args[1], nargs - 1, (jl_value_t*)ex, ctx); + if (result.typ == jl_bottom_type) + CreateTrap(builder); + return result; } } - Value *theFptr = (Value*)li->functionObjectsDecls.functionObject; - jl_cgval_t result; - if (theFptr && li->jlcall_api == 0) { - jl_cgval_t fval = emit_expr(args[1], ctx); - result = emit_call_function_object(li, fval, theFptr, &args[1], nargs - 1, (jl_value_t*)ex, ctx); - } - else { - result = mark_julia_type(emit_jlcall(prepare_call(jlinvoke_func), literal_pointer_val((jl_value_t*)li), - &args[1], nargs, ctx), - true, expr_type((jl_value_t*)ex, ctx), ctx); - } - - if (result.typ == jl_bottom_type) { + jl_cgval_t result = mark_julia_type(emit_jlcall(prepare_call(jlinvoke_func), boxed(lival, ctx, false), + &args[1], nargs, ctx), + true, expr_type((jl_value_t*)ex, ctx), ctx); + if (result.typ == jl_bottom_type) CreateTrap(builder); - } return result; } From 2bfc51d5e6f2c17d97829fa306ceab44a2861cf6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 30 Jun 2016 02:39:59 -0400 Subject: [PATCH 0180/1117] add jl_gdblookuplinfo lookup helper for debugging --- base/inference.jl | 33 ++++++++++++++++++++------------- src/debuginfo.cpp | 31 +++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 546335276c219..0f2c36e8d7e65 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2424,18 +2424,30 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference return NF end - atype_unlimited = argtypes_to_type(atypes) + local atype_unlimited = argtypes_to_type(atypes) function invoke_NF() # converts a :call to :invoke local nu = countunionsplit(atypes) nu > MAX_UNION_SPLITTING && return NF if nu > 1 - local ex = copy(e) - local linfo_var = add_slot!(enclosing, LambdaInfo, false) local spec_hit = nothing local spec_miss = nothing local error_label = nothing + local linfo_var = add_slot!(enclosing, LambdaInfo, false) + local ex = copy(e) + local stmts = [] + for i = 1:length(atypes); local i + local ti = atypes[i] + if isa(ti, Union) + aei = ex.args[i] + if !effect_free(aei, sv, false) + newvar = newvar!(sv, ti) + push!(stmts, Expr(:(=), newvar, aei)) + ex.args[i] = newvar + end + end + end function splitunion(atypes::Vector{Any}, i::Int) if i == 0 local sig = argtypes_to_type(atypes) @@ -2449,15 +2461,10 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference else local ti = atypes[i] if isa(ti, Union) - local all = false + local all = true local stmts = [] local aei = ex.args[i] - if !effect_free(aei, sv, false) - aei = newvar!(sv, ti) - push!(stmts, Expr(:(=), aei, ex.args[i])) - ex.args[i] = aei - end - for ty in (ti::Union).types + for ty in (ti::Union).types; local ty atypes[i] = ty local match = splitunion(atypes, i - 1) if match !== false @@ -2483,9 +2490,9 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end end end - local stmts = splitunion(atypes, length(atypes)) - if stmts !== false && spec_hit !== nothing - stmts = stmts::Array{Any,1} + local match = splitunion(atypes, length(atypes)) + if match !== false && spec_hit !== nothing + append!(stmts, match) if error_label !== nothing push!(stmts, error_label) push!(stmts, Expr(:call, GlobalRef(_topmod(sv.mod), :error), "error in type inference due to #265")) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 81e6ed19a812a..00e91725497b3 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -480,8 +480,8 @@ class JuliaJITEventListener: public JITEventListener else SectionAddrCheck = SectionLoadAddr; create_PRUNTIME_FUNCTION( - (uint8_t*)(intptr_t)Addr, (size_t)Size, sName, - (uint8_t*)(intptr_t)SectionLoadAddr, (size_t)SectionSize, UnwindData); + (uint8_t*)(uintptr_t)Addr, (size_t)Size, sName, + (uint8_t*)(uintptr_t)SectionLoadAddr, (size_t)SectionSize, UnwindData); #endif StringMap::iterator linfo_it = linfo_in_flight.find(sName); jl_lambda_info_t *linfo = NULL; @@ -556,8 +556,8 @@ class JuliaJITEventListener: public JITEventListener else SectionAddrCheck = SectionLoadAddr; create_PRUNTIME_FUNCTION( - (uint8_t*)(intptr_t)Addr, (size_t)Size, sName, - (uint8_t*)(intptr_t)SectionLoadAddr, (size_t)SectionSize, UnwindData); + (uint8_t*)(uintptr_t)Addr, (size_t)Size, sName, + (uint8_t*)(uintptr_t)SectionLoadAddr, (size_t)SectionSize, UnwindData); #endif StringMap::iterator linfo_it = linfo_in_flight.find(sName); jl_lambda_info_t *linfo = NULL; @@ -1253,7 +1253,7 @@ int jl_getFunctionInfo(jl_frame_t **frames_out, size_t pointer, int skipC, int n // Without MCJIT we use the FuncInfo structure containing address maps std::map &info = jl_jit_events->getMap(); std::map::iterator it = info.lower_bound(pointer); - if (it != info.end() && (intptr_t)(*it).first + (*it).second.lengthAdr >= pointer) { + if (it != info.end() && (uintptr_t)(*it).first + (*it).second.lengthAdr >= pointer) { // We do this to hide the jlcall wrappers when getting julia backtraces, // but it is still good to have them for regular lookup of C frames. if (skipC && (*it).second.lines.empty()) { @@ -1327,6 +1327,21 @@ int jl_getFunctionInfo(jl_frame_t **frames_out, size_t pointer, int skipC, int n return jl_getDylibFunctionInfo(frames_out, pointer, skipC, noInline); } +extern "C" jl_lambda_info_t *jl_gdblookuplinfo(void *p) +{ +#ifndef USE_MCJIT + std::map &info = jl_jit_events->getMap(); + std::map::iterator it = info.lower_bound((size_t)p); + jl_lambda_info_t *li = NULL; + if (it != info.end() && (uintptr_t)(*it).first + (*it).second.lengthAdr >= (uintptr_t)p) + li = (*it).second.linfo; + uv_rwlock_rdunlock(&threadsafe); + return li; +#else + return jl_jit_events->lookupLinfo((size_t)p); +#endif +} + #if defined(LLVM37) && (defined(_OS_LINUX_) || (defined(_OS_DARWIN_) && defined(LLVM_SHLIB))) extern "C" void __register_frame(void*); extern "C" void __deregister_frame(void*); @@ -1742,7 +1757,7 @@ uint64_t jl_getUnwindInfo(uint64_t dwAddr) std::map::iterator it = objmap.lower_bound(dwAddr); uint64_t ipstart = 0; // ip of the start of the section (if found) if (it != objmap.end() && dwAddr < it->first + it->second.SectionSize) { - ipstart = (uint64_t)(intptr_t)(*it).first; + ipstart = (uint64_t)(uintptr_t)(*it).first; } uv_rwlock_rdunlock(&threadsafe); return ipstart; @@ -1755,8 +1770,8 @@ uint64_t jl_getUnwindInfo(uint64_t dwAddr) std::map &info = jl_jit_events->getMap(); std::map::iterator it = info.lower_bound(dwAddr); uint64_t ipstart = 0; // ip of the first instruction in the function (if found) - if (it != info.end() && (intptr_t)(*it).first + (*it).second.lengthAdr > dwAddr) { - ipstart = (uint64_t)(intptr_t)(*it).first; + if (it != info.end() && (uintptr_t)(*it).first + (*it).second.lengthAdr > dwAddr) { + ipstart = (uint64_t)(uintptr_t)(*it).first; } uv_rwlock_rdunlock(&threadsafe); return ipstart; From d5579d37ed9d5d373b8031293df4fb4c6c6fe231 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 30 Jun 2016 12:00:56 -0400 Subject: [PATCH 0181/1117] disable gcframe rooting on return path of jlcall wrapper --- src/codegen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 90bd9438c48ff..b9ac40f96c46a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4002,7 +4002,7 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, Function *f, bool sre bool retboxed; (void)julia_type_to_llvm(jlretty, &retboxed); if (sret) { assert(!retboxed); } - jl_cgval_t retval = sret ? mark_julia_slot(result, jlretty, tbaa_stack) : mark_julia_type(call, retboxed, jlretty, &ctx); + jl_cgval_t retval = sret ? mark_julia_slot(result, jlretty, tbaa_stack) : mark_julia_type(call, retboxed, jlretty, &ctx, /*needsroot*/false); builder.CreateRet(boxed(retval, &ctx, false)); // no gcroot needed since this on the return path return w; From 099f99dbbba2337236ee8978d85c237c999ca26d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 17 Jun 2016 00:37:57 -0400 Subject: [PATCH 0182/1117] add ppc64le vector abi, with tests also add convert functions for VecElement to make them easier to construct --- base/boot.jl | 2 + base/sysimg.jl | 3 + src/abi_ppc64le.cpp | 88 ++++++++++++++++---- src/ccalltest.c | 113 +++++++++++++++++++++++++- src/codegen.cpp | 2 - src/gf.c | 1 - src/julia_internal.h | 5 +- test/ccall.jl | 185 ++++++++++++++++++++++++++++++++++++------- 8 files changed, 349 insertions(+), 50 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index 8056568824fdf..d2cf29d60ebba 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -271,7 +271,9 @@ Void() = nothing immutable VecElement{T} value::T + VecElement(value::T) = new(value) # disable converting constructor in Core end +VecElement{T}(arg::T) = VecElement{T}(arg) Expr(args::ANY...) = _expr(args...) diff --git a/base/sysimg.jl b/base/sysimg.jl index e76daa36856b4..7e588b56d01a3 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -61,6 +61,9 @@ include("operators.jl") include("pointer.jl") include("refpointer.jl") (::Type{T}){T}(arg) = convert(T, arg)::T +(::Type{VecElement{T}}){T}(arg) = VecElement{T}(convert(T, arg)) +convert{T<:VecElement}(::Type{T}, arg) = T(arg) +convert{T<:VecElement}(::Type{T}, arg::T) = arg include("checked.jl") importall .Checked diff --git a/src/abi_ppc64le.cpp b/src/abi_ppc64le.cpp index 40c75247e33b3..6fea221f06f85 100644 --- a/src/abi_ppc64le.cpp +++ b/src/abi_ppc64le.cpp @@ -43,22 +43,50 @@ typedef bool AbiState; AbiState default_abi_state = 0; // count the homogeneous floating agregate size (saturating at max count of 8) -static unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0) +static unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) { size_t i, l = ty->nfields; + // handle homogeneous float aggregates if (l == 0) { - if (*ty0 == NULL) { - if (ty == jl_float64_type || ty == jl_float32_type) - *ty0 = ty; + if (ty != jl_float64_type && ty != jl_float32_type) + return 9; + *hva = false; + if (*ty0 == NULL) + *ty0 = ty; + else if (*hva || ty->size != (*ty0)->size) + return 9; + return 1; + } + + // handle homogeneous vector aggregates + jl_datatype_t *fld0 = (jl_datatype_t*)jl_field_type(ty, 0); + if (!jl_is_datatype(fld0) || ty->name == jl_vecelement_typename) + return 9; + if (fld0->name == jl_vecelement_typename) { + if (!jl_is_bitstype(jl_tparam0(fld0)) || jl_datatype_size(ty) > 16) + return 9; + if (l != 1 && l != 2 && l != 4 && l != 8 && l != 16) + return 9; + *hva = true; + if (*ty0 == NULL) + *ty0 = ty; + else if (!*hva || ty->size != (*ty0)->size) + return 9; + for (i = 1; i < l; i++) { + jl_datatype_t *fld = (jl_datatype_t*)jl_field_type(ty, i); + if (fld != fld0) + return 9; } - return ty == *ty0 ? 1 : 9; + return 1; } + + // recurse through other struct types int n = 0; for (i = 0; i < l; i++) { - jl_value_t *fld = jl_field_type(ty, i); + jl_datatype_t *fld = (jl_datatype_t*)jl_field_type(ty, i); if (!jl_is_datatype(fld)) return 9; - n += isHFA((jl_datatype_t*)fld, ty0); + n += isHFA((jl_datatype_t*)fld, ty0, hva); if (n > 8) return 9; } @@ -70,7 +98,8 @@ bool use_sret(AbiState *state, jl_value_t *ty) // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) jl_datatype_t *dt = (jl_datatype_t*)ty; jl_datatype_t *ty0 = NULL; - if (dt->size > 16 && isHFA(dt, &ty0) > 8) + bool hva = false; + if (dt->size > 16 && isHFA(dt, &ty0, &hva) > 8) return true; return false; } @@ -79,8 +108,10 @@ void needPassByRef(AbiState *state, jl_value_t *ty, bool *byRef, bool *inReg) { if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) return; - size_t size = jl_datatype_size(ty); - if (size > 64) + jl_datatype_t *dt = (jl_datatype_t*)ty; + jl_datatype_t *ty0 = NULL; + bool hva = false; + if (dt->size > 64 && isHFA(dt, &ty0, &hva) > 8) *byRef = true; } @@ -96,12 +127,37 @@ Type *preferred_llvm_type(jl_value_t *ty, bool isret) return NULL; // legalize this into [n x f32/f64] jl_datatype_t *ty0 = NULL; - int hfa = isHFA(dt, &ty0); - if (hfa <= 8) - return ArrayType::get(ty0 == jl_float32_type ? T_float32 : T_float64, hfa); - // rewrite integer-sized (non-HFA) struct to an array of i64 - if (size > 8) - return ArrayType::get(T_int64, (size + 7) / 8); + bool hva = false; + int hfa = isHFA(dt, &ty0, &hva); + if (hfa <= 8) { + if (ty0 == jl_float32_type) { + return ArrayType::get(T_float32, hfa); + } + else if (ty0 == jl_float64_type) { + return ArrayType::get(T_float64, hfa); + } + else { + jl_datatype_t *vecty = (jl_datatype_t*)jl_field_type(ty0, 0); + assert(jl_is_datatype(vecty) && vecty->name == jl_vecelement_typename); + jl_value_t *elemty = jl_tparam0(vecty); + assert(jl_is_bitstype(elemty)); + + Type *ety = julia_type_to_llvm(elemty); + Type *vty = VectorType::get(ety, ty0->nfields); + return ArrayType::get(vty, hfa); + } + } + // rewrite integer-sized (non-HFA) struct to an array + // the bitsize of the integer gives the desired alignment + if (size > 8) { + if (dt->alignment <= 8) { + return ArrayType::get(T_int64, (size + 7) / 8); + } + else { + Type *T_int128 = Type::getIntNTy(jl_LLVMContext, 128); + return ArrayType::get(T_int128, (size + 15) / 16); + } + } return Type::getIntNTy(jl_LLVMContext, size * 8); } diff --git a/src/ccalltest.c b/src/ccalltest.c index ca329f8db7533..e9c46e5d789a3 100644 --- a/src/ccalltest.c +++ b/src/ccalltest.c @@ -595,13 +595,13 @@ JL_DLLEXPORT int64x2_t test_aa64_vec_1(int32x2_t v1, float _v2, int32x2_t v3) return vmovl_s32(v1 * v2 + v3); } -// This is a homogenious short vector aggregate +// This is a homogeneous short vector aggregate typedef struct { int8x8_t v1; float32x2_t v2; } struct_aa64_3; -// This is NOT a homogenious short vector aggregate +// This is NOT a homogeneous short vector aggregate typedef struct { float32x2_t v2; int16x8_t v1; @@ -614,3 +614,112 @@ JL_DLLEXPORT struct_aa64_3 test_aa64_vec_2(struct_aa64_3 v1, struct_aa64_4 v2) } #endif + +#if defined(_CPU_PPC64_) + +typedef int32_t int32x2_t __attribute__ ((vector_size (8))); +typedef float float32x2_t __attribute__ ((vector_size (8))); +typedef int32_t int32x4_t __attribute__ ((vector_size (16))); +typedef float float32x4_t __attribute__ ((vector_size (16))); +typedef double float64x2_t __attribute__ ((vector_size (16))); + +typedef struct { + int64_t m; + float32x4_t v; +} struct_huge1_ppc64; + +typedef struct { + float32x4_t v1; + int32x2_t v2; +} struct_huge2_ppc64; + +typedef struct { + float32x4_t v1; + struct { + float f1; + float f2; + float f3; + float f4; + }; +} struct_huge3_ppc64; + +typedef struct { + float32x2_t v1; + float64x2_t v2; +} struct_huge4_ppc64; + +typedef struct { + float32x4_t v1[9]; +} struct_huge5_ppc64; + +typedef struct { + float32x4_t v1[8]; + float32x4_t v2; +} struct_huge6_ppc64; + +typedef struct { + float32x4_t v1[8]; +} struct_huge1_ppc64_hva; + +typedef struct { + struct { + float32x4_t vf[2]; + } v[2]; +} struct_huge2_ppc64_hva; + +typedef struct { + float32x4_t vf1; + struct { + float32x4_t vf2[2]; + }; +} struct_huge3_ppc64_hva; + +typedef struct { + int32x4_t v1; + float32x4_t v2; +} struct_huge4_ppc64_hva; + +typedef struct { + float32x4_t v1; + float64x2_t v2; +} struct_huge5_ppc64_hva; + +test_huge(1_ppc64, m); +test_huge(2_ppc64, v1[0]); +test_huge(3_ppc64, v1[0]); +test_huge(4_ppc64, v1[0]); +test_huge(5_ppc64, v1[0][0]); +test_huge(6_ppc64, v1[0][0]); +test_huge(1_ppc64_hva, v1[0][0]); +test_huge(2_ppc64_hva, v[0].vf[0][0]); +test_huge(3_ppc64_hva, vf1[0]); +test_huge(4_ppc64_hva, v1[0]); +test_huge(5_ppc64_hva, v1[0]); + +JL_DLLEXPORT int64_t test_ppc64_vec1long( + int64_t d1, int64_t d2, int64_t d3, int64_t d4, int64_t d5, int64_t d6, + int64_t d7, int64_t d8, int64_t d9, struct_huge1_ppc64 vs) +{ + return d1 + d2 + d3 + d4 + d5 + d6 + d7 + d8 + d9 + vs.m + vs.v[0] + vs.v[1] + vs.v[2] + vs.v[3]; +} + +JL_DLLEXPORT int64_t test_ppc64_vec1long_vec( + int64_t d1, int64_t d2, int64_t d3, int64_t d4, int64_t d5, int64_t d6, + int64_t d7, int64_t d8, int64_t d9, float32x4_t vs) +{ + return d1 + d2 + d3 + d4 + d5 + d6 + d7 + d8 + d9 + vs[0] + vs[1] + vs[2] + vs[3]; +} + +JL_DLLEXPORT float32x4_t test_ppc64_vec2(int64_t d1, float32x4_t a, float32x4_t b, float32x4_t c, float32x4_t d, + float32x4_t e, float32x4_t f, float32x4_t g, float32x4_t h, float32x4_t i, + float32x4_t j, float32x4_t k, float32x4_t l, float32x4_t m, float32x4_t n) +{ + float32x4_t r; + r[0] = d1 + a[0] + b[0] + c[0] + d[0] + e[0] + f[0] + g[0] + h[0] + i[0] + j[0] + k[0] + l[0] + m[0] + n[0]; + r[1] = d1 + a[1] + b[1] + c[1] + d[1] + e[1] + f[1] + g[1] + h[1] + i[1] + j[1] + k[1] + l[1] + m[1] + n[1]; + r[2] = d1 + a[2] + b[2] + c[2] + d[2] + e[2] + f[2] + g[2] + h[2] + i[2] + j[2] + k[2] + l[2] + m[2] + n[2]; + r[3] = d1 + a[3] + b[3] + c[3] + d[3] + e[3] + f[3] + g[3] + h[3] + i[3] + j[3] + k[3] + l[3] + m[3] + n[3]; + return r; +} + +#endif diff --git a/src/codegen.cpp b/src/codegen.cpp index b9ac40f96c46a..6277b576a43af 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3762,7 +3762,6 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t jl_cgval_t retval; if (lam == NULL) { assert(theFptr); - assert(nargs >= 0); #ifdef LLVM37 Value *ret = builder.CreateCall(prepare_call(theFptr), {myargs, ConstantInt::get(T_int32, nargs + 1)}); @@ -3785,7 +3784,6 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t } else { assert(theFptr); - assert(nargs >= 0); // for jlcall, we need to pass the function object even if it is a ghost. // here we reconstruct the function instance from its type (first elt of argt) Value *theF = literal_pointer_val((jl_value_t*)ff); diff --git a/src/gf.c b/src/gf.c index 688009ac1fa38..f592733ab2392 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1232,7 +1232,6 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) jl_compile_linfo(li); } } - jl_generate_fptr(li); return li; } diff --git a/src/julia_internal.h b/src/julia_internal.h index 86be924044c9d..801997e027a42 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -86,8 +86,11 @@ jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method); STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_value_t **args, uint32_t nargs) { jl_lambda_info_t *mfptr = meth; - if (__unlikely(mfptr->fptr == NULL)) + if (__unlikely(mfptr->fptr == NULL)) { mfptr = jl_compile_for_dispatch(mfptr); + if (!mfptr->fptr) + jl_generate_fptr(mfptr); + } if (mfptr->jlcall_api == 0) return mfptr->fptr(args[0], &args[1], nargs-1); else if (mfptr->jlcall_api == 1) diff --git a/test/ccall.jl b/test/ccall.jl index b4fd41f3e3c5a..fcba7186240e7 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -47,7 +47,7 @@ end ## Tests for passing and returning structs -let +let a, ci_ary, x a = 20 + 51im x = ccall((:ctest, libccalltest), Complex{Int}, (Complex{Int},), a) @@ -66,7 +66,7 @@ let Libc.free(convert(Ptr{Void},x)) end -let +let a, b, x a = 2.84 + 5.2im x = ccall((:cgtest, libccalltest), Complex128, (Complex128,), a) @@ -80,7 +80,7 @@ let @test a == 2.84 + 5.2im end -let +let a, b, x a = 3.34f0 + 53.2f0im x = ccall((:cftest, libccalltest), Complex64, (Complex64,), a) @@ -97,7 +97,7 @@ end ## Tests for native Julia data types -let +let a a = 2.84 + 5.2im @test_throws MethodError ccall((:cptest, libccalltest), Ptr{Complex{Int}}, (Ptr{Complex{Int}},), a) @@ -113,7 +113,7 @@ type Struct1 end copy(a::Struct1) = Struct1(a.x, a.y) -let +let a, b, a2, x a = Struct1(352.39422f23, 19.287577) b = Float32(123.456) @@ -127,7 +127,7 @@ let @test x.y ≈ a.y - 2*b end -let +let a, b, x, y a = Complex{Int32}(Int32(10),Int32(31)) b = Int32(42) @@ -140,7 +140,7 @@ let @test x == a + b*1 - b*2im end -let +let a, b, x, y, z a = Complex{Int64}(Int64(20),Int64(51)) b = Int64(42) @@ -162,7 +162,7 @@ type Struct4 z::Int32 end -let +let a, b, x a = Struct4(-512275808,882558299,-2133022131) b = Int32(42) @@ -180,7 +180,7 @@ type Struct5 a::Int32 end -let +let a, b, x a = Struct5(1771319039, 406394736, -1269509787, -745020976) b = Int32(42) @@ -198,7 +198,7 @@ type Struct6 z::Int64 end -let +let a, b, x a = Struct6(-654017936452753226, -5573248801240918230, -983717165097205098) b = Int64(42) @@ -214,7 +214,7 @@ type Struct7 y::Cchar end -let +let a, b, x a = Struct7(-384082741977533896, 'h') b = Int8(42) @@ -229,7 +229,7 @@ type Struct8 y::Cchar end -let +let a, b, x a = Struct8(-384082896, 'h') b = Int8(42) @@ -244,7 +244,7 @@ type Struct9 y::Int16 end -let +let a, b, x a = Struct9(-394092996, -3840) b = Int16(42) @@ -261,7 +261,7 @@ type Struct10 a::Cchar end -let +let a, b, x a = Struct10('0', '1', '2', '3') b = Int8(2) @@ -277,7 +277,7 @@ type Struct11 x::Complex64 end -let +let a, b, x a = Struct11(0.8877077f0 + 0.4591081f0im) b = Float32(42) @@ -291,7 +291,7 @@ type Struct12 y::Complex64 end -let +let a, b, x a = Struct12(0.8877077f5 + 0.4591081f2im, 0.0004842868f0 - 6982.3265f3im) b = Float32(42) @@ -305,7 +305,7 @@ type Struct13 x::Complex128 end -let +let a, b, x a = Struct13(42968.97560380495 - 803.0576845153616im) b = Float64(42) @@ -319,7 +319,7 @@ type Struct14 y::Float32 end -let +let a, b, x a = Struct14(0.024138331f0, 0.89759064f32) b = Float32(42) @@ -334,7 +334,7 @@ type Struct15 y::Float64 end -let +let a, b, x a = Struct15(4.180997967273657, -0.404218594294923) b = Float64(42) @@ -353,7 +353,7 @@ type Struct16 c::Float64 end -let +let a, b, x a = Struct16(0.1604656f0, 0.6297606f0, 0.83588994f0, 0.6460273620993535, 0.9472692581106656, 0.47328535437352093) b = Float32(42) @@ -368,7 +368,7 @@ let @test x.c ≈ a.c - b*6 end -let +let a, b, x a = Int128(0x7f00123456789abc)<<64 + typemax(UInt64) b = Int64(1) @@ -385,7 +385,7 @@ type Struct_Big end copy(a::Struct_Big) = Struct_Big(a.x, a.y, a.z) -let +let a, a2, x a = Struct_Big(424,-5,Int8('Z')) a2 = copy(a) @@ -443,12 +443,16 @@ function verify_huge(init, a, b) end # make sure b was modifed as expected a1, b1 = getfield(a, 1), getfield(b, 1) - if isa(a1, Tuple) - @test oftype(a1[1], a1[1] * 39) === b1[1] + while isa(a1, Tuple) @test a1[2:end] === b1[2:end] - else - @test oftype(a1, a1 * 39) === b1 + a1 = a1[1] + b1 = b1[1] end + if isa(a1, VecElement) + a1 = a1.value + b1 = b1.value + end + @test oftype(a1, a1 * 39) === b1 for i = 2:nfields(a) @test getfield(a, i) === getfield(b, i) end @@ -458,7 +462,8 @@ macro test_huge(i, b, init) ty = Symbol("Struct_huge", i, b) return quote let a = $ty($(esc(init))...), f - f(b) = ccall(($f, libccalltest), $ty, (Cchar, $ty, Cchar), '0' + $i, a, $b) + f(b) = ccall(($f, libccalltest), $ty, (Cchar, $ty, Cchar), '0' + $i, a, $b[1]) + #code_llvm(f, typeof((a,))) verify_huge($ty($(esc(init))...), a, f(a)) end end @@ -605,7 +610,10 @@ end # SIMD Registers typealias VecReg{N,T} NTuple{N,VecElement{T}} +typealias V2xF32 VecReg{2,Float32} typealias V4xF32 VecReg{4,Float32} +typealias V2xF64 VecReg{2,Float64} +typealias V2xI32 VecReg{2,Int32} typealias V4xI32 VecReg{4,Int32} immutable Struct_AA64_1 @@ -628,8 +636,64 @@ immutable Struct_AA64_4 v1::VecReg{8,Int16} end -if Sys.ARCH === :x86_64 +type Struct_huge1_ppc64 + m::Int64 + v::V4xF32 +end + +type Struct_huge2_ppc64 + v1::V4xF32 + v2::V2xI32 +end + +type Struct_huge3_ppc64 + v1::V4xF32 + f::NTuple{4,Float32} +end + +type Struct_huge4_ppc64 + v1::V2xF32 + v2::V2xF64 +end + +type Struct_huge5_ppc64 + v1::NTuple{9,V4xF32} +end + +type Struct_huge6_ppc64 + v1::NTuple{8,V4xF32} + v2::V4xF32 +end + +type Struct_huge7_ppc64 + v1::VecReg{3,Int32} + v2::VecReg{3,Int32} +end +type Struct_huge1_ppc64_hva + v1::NTuple{8,V4xF32} +end + +type Struct_huge2_ppc64_hva + v1::NTuple{2,NTuple{2,V4xF32}} +end + +type Struct_huge3_ppc64_hva + vf1::V4xF32 + vf2::Tuple{NTuple{2,V4xF32}} +end + +type Struct_huge4_ppc64_hva + v1::V4xI32 + v2::V4xF32 +end + +type Struct_huge5_ppc64_hva + v1::V4xI32 + v2::V2xF64 +end + +if Sys.ARCH === :x86_64 function test_sse(a1::V4xF32,a2::V4xF32,a3::V4xF32,a4::V4xF32) ccall((:test_m128, libccalltest), V4xF32, (V4xF32,V4xF32,V4xF32,V4xF32), a1, a2, a3, a4) end @@ -653,6 +717,7 @@ if Sys.ARCH === :x86_64 # cfunction round-trip @test rt_sse(a1,a2,a3,a4) == r end + elseif Sys.ARCH === :aarch64 for v1 in 1:99:1000, v2 in -100:-1999:-20000 @test ccall((:test_aa64_i128_1, libccalltest), Int128, @@ -712,4 +777,68 @@ elseif Sys.ARCH === :aarch64 VecElement(Float32(v1_22 - v2_22)))) @test res === expected end + +elseif Sys.ARCH === :powerpc64le +@test_huge 1 "_ppc64" (1, (2.0, 3.0, 4.0, 5.0),) +@test_huge 2 "_ppc64" ((1.0, 2.0, 3.0, 4.0), (11, 12)) +@test_huge 3 "_ppc64" ((1, 2, 3, 4), (11.0, 12.0, 13.0, 14.0)) +@test_huge 4 "_ppc64" ((1, 2), (11.0, 12.0)) +@test_huge 5 "_ppc64" ((((1.0, 2.0, 3.0, 4.0), + (5.0, 6.0, 7.0, 8.0), + (11.0, 12.0, 13.0, 14.0), + (15.0, 16.0, 17.0, 18.0), + (21.0, 22.0, 23.0, 24.0), + (25.0, 26.0, 27.0, 28.0), + (31.0, 32.0, 33.0, 34.0), + (35.0, 36.0, 37.0, 38.0), + (41.0, 42.0, 43.0, 44.0)),)) +@test_huge 6 "_ppc64" ((((1.0, 2.0, 3.0, 4.0), + (5.0, 6.0, 7.0, 8.0), + (11.0, 12.0, 13.0, 14.0), + (15.0, 16.0, 17.0, 18.0), + (21.0, 22.0, 23.0, 24.0), + (25.0, 26.0, 27.0, 28.0), + (31.0, 32.0, 33.0, 34.0), + (35.0, 36.0, 37.0, 38.0)), + (41.0, 42.0, 43.0, 44.0))) +@test_huge 1 "_ppc64_hva" ((((1.0, 2.0, 3.0, 4.0), + (5.0, 6.0, 7.0, 8.0), + (11.0, 12.0, 13.0, 14.0), + (15.0, 16.0, 17.0, 18.0), + (21.0, 22.0, 23.0, 24.0), + (25.0, 26.0, 27.0, 28.0), + (31.0, 32.0, 33.0, 34.0), + (35.0, 36.0, 37.0, 38.0)),)) +@test_huge 2 "_ppc64_hva" (((((1.0, 2.0, 3.0, 4.0), + (5.0, 6.0, 7.0, 8.0)), + ((11.0, 12.0, 13.0, 14.0), + (15.0, 16.0, 17.0, 18.0))),)) +@test_huge 3 "_ppc64_hva" (((1.0, 2.0, 3.0, 4.0), + (((11.0, 12.0, 13.0, 14.0), + (15.0, 16.0, 17.0, 18.0)),))) +@test_huge 4 "_ppc64_hva" (((1, 2, 3, 4), + (11.0, 12.0, 13.0, 14.0))) +@test_huge 5 "_ppc64_hva" (((1, 2, 3, 4), + (11.0, 12.0))) + +@test 18451 == ccall((:test_ppc64_vec1long, libccalltest), Int64, + (Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, Struct_huge1_ppc64), + 1, 2, 3, 4, 5, 6, 7, 8, 9, Struct_huge1_ppc64(18000, (100, 101, 102, 103))) + +@test 941 == ccall((:test_ppc64_vec1long_vec, libccalltest), Int64, + (Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, V4xF32), + 11, 12, 13, 14, 15, 16, 17, 18, 19, (200, 201, 202, 203)) + +@test V4xF32((614232, 614218, 614204, 614190)) == + ccall((:test_ppc64_vec2, libccalltest), V4xF32, + (Int64, V4xF32, V4xF32, V4xF32, V4xF32, + V4xF32, V4xF32, V4xF32, V4xF32, V4xF32, + V4xF32, V4xF32, V4xF32, V4xF32, V4xF32), + 600000, (4, 3, 2, 1), (5, 4, 3, 2), (6, 5, 4, 3), (7, 6, 5, 4), + (14, 13, 12, 11), (15, 14, 13, 12), (16, 15, 14, 13), (17, 16, 15, 14), (18, 17, 16, 15), + (1024, 1023, 1022, 1021), (1025, 1024, 1023, 1022), (1026, 1025, 1024, 1023), (1027, 1026, 1025, 1024), (10028, 10027, 10026, 10025)) + +else +warn("ccall: no VecReg tests run for this platform") + end From 5ec747b09207eb5dc07f768184bfcebb8cb00bfe Mon Sep 17 00:00:00 2001 From: 8ring Date: Fri, 1 Jul 2016 16:56:50 -0400 Subject: [PATCH 0183/1117] Corrected Chinese text for UDHR Corrected Chinese text in UniversalDeclarationOfHumanRightsstart. --- doc/manual/variables.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/variables.rst b/doc/manual/variables.rst index c2d374de36094..c4a251fc76a9f 100644 --- a/doc/manual/variables.rst +++ b/doc/manual/variables.rst @@ -45,8 +45,8 @@ the language will not treat variables differently based on their names). julia> customary_phrase = "Hello world!" "Hello world!" - julia> UniversalDeclarationOfHumanRightsStart = "人人生而自由，在尊严和权力上一律平等。" - "人人生而自由，在尊严和权力上一律平等。" + julia> UniversalDeclarationOfHumanRightsStart = "人人生而自由，在尊严和权利上一律平等。" + "人人生而自由，在尊严和权利上一律平等。" .. raw:: latex From ee7f42e4a613f465ee9ef08489a9994babe70da6 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 1 Jul 2016 19:24:39 -0400 Subject: [PATCH 0184/1117] Fix assertion failure on LLVM 3.8+ --- src/jitlayers.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 097565f955a8f..7b554811add50 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -544,24 +544,25 @@ static T *addComdat(T *G) // Add comdat information to make MSVC link.exe happy // it's valid to emit this for ld.exe too, // but makes it very slow to link for no benefit -#if defined(_COMPILER_MICROSOFT_) if (G->getParent() == shadow_output) { +#if defined(_COMPILER_MICROSOFT_) Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); // ELF only supports Comdat::Any jl_Comdat->setSelectionKind(Comdat::NoDuplicates); G->setComdat(jl_Comdat); - } #endif +#if defined(_CPU_X86_64_) + // Add unwind exception personalities to functions to handle async exceptions + assert(!juliapersonality_func || juliapersonality_func->getParent() == shadow_output); + if (Function *F = dyn_cast(G)) + F->setPersonalityFn(juliapersonality_func); +#endif + } // add __declspec(dllexport) to everything marked for export if (G->getLinkage() == GlobalValue::ExternalLinkage) G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); else G->setDLLStorageClass(GlobalValue::DefaultStorageClass); -#if defined(_CPU_X86_64_) - // Add unwind exception personalities to functions to handle async exceptions - if (Function *F = dyn_cast(G)) - F->setPersonalityFn(juliapersonality_func); -#endif } #endif return G; From 60a7202964d835ff089f85edbfce3febda411c03 Mon Sep 17 00:00:00 2001 From: Dmitri Iouchtchenko Date: Fri, 1 Jul 2016 19:49:02 -0400 Subject: [PATCH 0185/1117] Add missing backslashes in readdlm docstrings --- base/docs/helpdb/Base.jl | 8 ++++---- doc/stdlib/io-network.rst | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 418b4b728fdf6..477c66f370745 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -3643,14 +3643,14 @@ readdlm(source, delim::Char, eol::Char) """ readdlm(source, delim::Char, T::Type; options...) -The end of line delimiter is taken as `n`. +The end of line delimiter is taken as `\\n`. """ readdlm(source, delim::Char, T::Type) """ readdlm(source, delim::Char; options...) -The end of line delimiter is taken as `n`. If all data is numeric, the result will be a +The end of line delimiter is taken as `\\n`. If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. """ @@ -3660,7 +3660,7 @@ readdlm(source, delim::Char) readdlm(source, T::Type; options...) The columns are assumed to be separated by one or more whitespaces. The end of line -delimiter is taken as `n`. +delimiter is taken as `\\n`. """ readdlm(source, T::Type) @@ -3668,7 +3668,7 @@ readdlm(source, T::Type) readdlm(source; options...) The columns are assumed to be separated by one or more whitespaces. The end of line -delimiter is taken as `n`. If all data is numeric, the result will be a numeric array. If +delimiter is taken as `\\n`. If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. """ diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 3e2885e1635d1..c855dc0ed8df7 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -588,25 +588,25 @@ Text I/O .. Docstring generated from Julia source - The end of line delimiter is taken as ``n``\ . + The end of line delimiter is taken as ``\n``\ . .. function:: readdlm(source, delim::Char; options...) .. Docstring generated from Julia source - The end of line delimiter is taken as ``n``\ . If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. + The end of line delimiter is taken as ``\n``\ . If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. .. function:: readdlm(source, T::Type; options...) .. Docstring generated from Julia source - The columns are assumed to be separated by one or more whitespaces. The end of line delimiter is taken as ``n``\ . + The columns are assumed to be separated by one or more whitespaces. The end of line delimiter is taken as ``\n``\ . .. function:: readdlm(source; options...) .. Docstring generated from Julia source - The columns are assumed to be separated by one or more whitespaces. The end of line delimiter is taken as ``n``\ . If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. + The columns are assumed to be separated by one or more whitespaces. The end of line delimiter is taken as ``\n``\ . If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. .. function:: writedlm(f, A, delim='\\t') From d606bc6d127d28935a72a0508ae85da7a2a8c4c0 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Fri, 24 Jun 2016 16:47:31 -0400 Subject: [PATCH 0186/1117] Fix #16326: use `typeof` in code_* for all :call expressions Update docs/utils.jl to pass test (h/t MichaelHatherly). --- base/docs/utils.jl | 8 ++------ base/interactiveutil.jl | 3 ++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/base/docs/utils.jl b/base/docs/utils.jl index 02300396ae118..0224ffda1991f 100644 --- a/base/docs/utils.jl +++ b/base/docs/utils.jl @@ -150,12 +150,8 @@ repl(str::AbstractString) = :(apropos($str)) repl(other) = :(@doc $(esc(other))) function _repl(x) - docs = :(@doc $(esc(x))) - if isexpr(x, :call) - # Handles function call syntax where each argument is an atom (symbol, number, etc.) - t = Base.gen_call_with_extracted_types(doc, x) - (isexpr(t, :call, 3) && t.args[1] === doc) && (docs = t) - end + docs = (isexpr(x, :call) && !any(isexpr(x, :(::)) for x in x.args)) ? + Base.gen_call_with_extracted_types(doc, x) : :(@doc $(esc(x))) if isfield(x) quote if isa($(esc(x.args[1])), DataType) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 22821a8c4fd6f..ce4274d2551a1 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -278,7 +278,8 @@ typesof(args...) = Tuple{map(a->(isa(a,Type) ? Type{a} : typeof(a)), args)...} gen_call_with_extracted_types(fcn, ex0::Symbol) = Expr(:call, fcn, Meta.quot(ex0)) function gen_call_with_extracted_types(fcn, ex0) if isa(ex0, Expr) && - any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) + (any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) || + ex0.head == :call) # keyword args not used in dispatch, so just remove them args = filter(a->!(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) return Expr(:call, fcn, esc(args[1]), From 8e461534c31256a2c2de7d5e6eae5eca916b4ed9 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Thu, 30 Jun 2016 21:14:04 -0400 Subject: [PATCH 0187/1117] Fix #17219: conditionally lower ccall first arg Without this compile step, ccall uses the first slotted version of the variable regardless of whether it has been reassigned in the function body. Fixes #17219 Fixes aviks/JavaCall.jl#37 See https://groups.google.com/forum/#!msg/julia-users/DyTQmsQaQl4/KmxQHSqACQAJ --- src/julia-syntax.scm | 8 ++++++-- test/ccall.jl | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 2a081e9e5a7bb..a1574692dcbed 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3018,8 +3018,12 @@ f(x) = yt(x) ((call new) (let* ((ccall? (and (eq? (car e) 'call) (equal? (cadr e) '(core ccall)))) (args (if ccall? - ;; NOTE: first 3 arguments of ccall must be left in place - (append (list-head (cdr e) 4) + ;; NOTE: 2nd and 3rd arguments of ccall must be left in place + ;; the 1st should be compiled if an atom. + (append (list (cadr e)) + (cond (atom? (caddr e) (compile-args (list (caddr e)) break-labels)) + (else (caddr e))) + (list-head (cdddr e) 2) (compile-args (list-tail e 5) break-labels)) (compile-args (cdr e) break-labels))) (callex (cons (car e) args))) diff --git a/test/ccall.jl b/test/ccall.jl index fcba7186240e7..20e7245f880c5 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -575,6 +575,13 @@ foo13031(x,y,z) = z foo13031p = cfunction(foo13031, Cint, (Ref{Tuple{}},Ref{Tuple{}},Cint)) ccall(foo13031p, Cint, (Ref{Tuple{}},Ref{Tuple{}},Cint), (), (), 8) +# issue 17219 +function ccall_reassigned_ptr(ptr::Ptr{Void}) + ptr = Libdl.dlsym(Libdl.dlopen(libccalltest), "test_echo_p") + ccall(ptr, Any, (Any,), "foo") +end +@test ccall_reassigned_ptr(C_NULL) == "foo" + # @threadcall functionality threadcall_test_func(x) = @threadcall((:testUcharX, libccalltest), Int32, (UInt8,), x % UInt8) From 5aa8c5fc32a990e19d08a59f18d5bfb13438990a Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 1 Jul 2016 23:54:59 -0400 Subject: [PATCH 0188/1117] Enable ifunc on ARM/AArch64/Power/Power64 for GCC 4.9+ --- src/threading.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/threading.c b/src/threading.c index 0dd109c96cdad..67d6be607f3b5 100644 --- a/src/threading.c +++ b/src/threading.c @@ -146,7 +146,10 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) // is not guaranteed to be reliable, we still need to fallback to the wrapper // version as the symbol address if we didn't find the static version in `ifunc`. #if defined(__GLIBC__) && (defined(_CPU_X86_64_) || defined(_CPU_X86_) || \ - defined(_CPU_AARCH64_) || defined(_CPU_ARM_)) + ((defined(_CPU_AARCH64_) || defined(_CPU_ARM_) || \ + defined(_CPU_PPC64_) || defined(_CPU_PPC_)) && \ + (__GNUC__ >= 5 || __GNUC_MINOR__ >= 9))) +// Skip the `__GNUC__ >= 4` check since we require GCC 4.7+ // Only enable this on architectures that are tested. // For example, GCC doesn't seem to support the `ifunc` attribute on power yet. # if __GLIBC_PREREQ(2, 12) From a4cba2ff57546d9dc9f9ae26fbb4aeb28d7cefa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Sat, 2 Jul 2016 10:57:29 +0200 Subject: [PATCH 0189/1117] Speed up repl dot and method completion --- base/precompile.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base/precompile.jl b/base/precompile.jl index 54d5300fa770a..9c8519886a1de 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -477,3 +477,7 @@ precompile(Base.set_valid_processes, (Array{Int, 1}, )) sprint(Markdown.term, @doc mean) sprint(Docs.repl_search, "mean") sprint(Docs.repl_corrections, "meen") + +# Speed up repl completions +Base.REPLCompletions.completions("IOBuffer().",11) +Base.REPLCompletions.completions("max([1],",8) From ff13f870133bc82e388f15143a1e7c26b64eb295 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Fri, 1 Jul 2016 11:49:57 +0200 Subject: [PATCH 0190/1117] Fix type instability in decompose(::Float64) on 32-bit Need at least 64-bit to store the result of Float64 decomposition. (A very good use case for return type declarations.) --- base/hashing2.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/base/hashing2.jl b/base/hashing2.jl index 714cbb59d44fc..3d29fcef204c5 100644 --- a/base/hashing2.jl +++ b/base/hashing2.jl @@ -96,7 +96,7 @@ Special values: decompose(x::Integer) = x, 0, 1 decompose(x::Rational) = num(x), 0, den(x) -function decompose(x::Float16) +function decompose(x::Float16)::NTuple{3,Int} isnan(x) && return 0, 0, 0 isinf(x) && return ifelse(x < 0, -1, 1), 0, 0 n = reinterpret(UInt16, x) @@ -104,10 +104,10 @@ function decompose(x::Float16) e = (n & 0x7c00 >> 10) % Int s |= Int16(e != 0) << 10 d = ifelse(signbit(x), -1, 1) - Int(s), Int(e - 25 + (e == 0)), d + s, e - 25 + (e == 0), d end -function decompose(x::Float32) +function decompose(x::Float32)::NTuple{3,Int} isnan(x) && return 0, 0, 0 isinf(x) && return ifelse(x < 0, -1, 1), 0, 0 n = reinterpret(UInt32, x) @@ -115,10 +115,10 @@ function decompose(x::Float32) e = (n & 0x7f800000 >> 23) % Int s |= Int32(e != 0) << 23 d = ifelse(signbit(x), -1, 1) - Int(s), Int(e - 150 + (e == 0)), d + s, e - 150 + (e == 0), d end -function decompose(x::Float64) +function decompose(x::Float64)::Tuple{Int64, Int, Int} isnan(x) && return 0, 0, 0 isinf(x) && return ifelse(x < 0, -1, 1), 0, 0 n = reinterpret(UInt64, x) @@ -126,19 +126,19 @@ function decompose(x::Float64) e = (n & 0x7ff0000000000000 >> 52) % Int s |= Int64(e != 0) << 52 d = ifelse(signbit(x), -1, 1) - s, Int(e - 1075 + (e == 0)), d + s, e - 1075 + (e == 0), d end -function decompose(x::BigFloat) - isnan(x) && return big(0), 0, 0 - isinf(x) && return big(x.sign), 0, 0 - x == 0 && return big(0), 0, Int(x.sign) +function decompose(x::BigFloat)::Tuple{BigInt, Int, Int} + isnan(x) && return 0, 0, 0 + isinf(x) && return x.sign, 0, 0 + x == 0 && return 0, 0, x.sign s = BigInt() s.size = cld(x.prec, 8*sizeof(GMP.Limb)) # limbs b = s.size * sizeof(GMP.Limb) # bytes ccall((:__gmpz_realloc2, :libgmp), Void, (Ptr{BigInt}, Culong), &s, 8b) # bits ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Csize_t), s.d, x.d, b) # bytes - s, Int(x.exp - 8b), Int(x.sign) + s, x.exp - 8b, x.sign end ## streamlined hashing for smallish rational types ## From 8e05e888fbd10a658ebed496dd977d5cbd1d23b4 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Sat, 2 Jul 2016 12:16:07 +0200 Subject: [PATCH 0191/1117] Fix various corner cases with operators on Numbers These only affected particular type combinations, with either type instabilities which affected inference of return type or plain failures. Tests are included in the next commit. --- base/complex.jl | 2 +- base/gmp.jl | 1 + base/intfuncs.jl | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base/complex.jl b/base/complex.jl index 5aa2ce2f318a0..59dbd99132814 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -461,7 +461,7 @@ function ^{T<:AbstractFloat}(z::Complex{T}, p::Complex{T}) if p==2 #square zr, zi = reim(z) x = (zr-zi)*(zr+zi) - y = 2zr*zi + y = T(2)*zr*zi if isnan(x) if isinf(y) x = copysign(zero(T),zr) diff --git a/base/gmp.jl b/base/gmp.jl index 3c221fbb71527..3b63f121ec13f 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -428,6 +428,7 @@ end ^(x::BigInt , y::Bool ) = y ? x : one(x) ^(x::BigInt , y::Integer) = bigint_pow(x, y) ^(x::Integer, y::BigInt ) = bigint_pow(BigInt(x), y) +^(x::Bool , y::BigInt ) = Base.power_by_squaring(x, y) function powermod(x::BigInt, p::BigInt, m::BigInt) r = BigInt() diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 4c4f6b51f3e0d..f7b3eee752ce5 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -186,7 +186,7 @@ ndigits0z(x::Integer) = ndigits0z(unsigned(abs(x))) const ndigits_max_mul = Core.sizeof(Int) == 4 ? 69000000 : 290000000000000000 -function ndigits0znb(n::Int, b::Int) +function ndigits0znb(n::Signed, b::Int) d = 0 while n != 0 n = cld(n,b) From 46917dcc4af784be898b40d532371ad959d63ef0 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Sat, 2 Jul 2016 12:56:58 +0200 Subject: [PATCH 0192/1117] promote_op() improvements Use common promote_op() based on one() for all Number types: this fixes promote_op(+, ::Bool), which returned Int, and promote_op(==, ::Complex, ::Complex), which returned Complex{Bool}. Add fallback definitions returning Any so that broadcast() computes the appropriate return type, instead of using promote_type() which is often wrong. This fixes broadcast() when the return type is different from the input type. Add systematic tests for operators and their return types. --- base/bool.jl | 4 --- base/complex.jl | 11 -------- base/int.jl | 2 -- base/irrationals.jl | 7 +++++ base/number.jl | 12 ++++++++ base/pkg/resolve/fieldvalue.jl | 1 + base/promotion.jl | 4 +-- test/arrayops.jl | 7 ++--- test/broadcast.jl | 15 +++++++++- test/linalg/dense.jl | 3 ++ test/numbers.jl | 50 ++++++++++++++++++++++++++++++++++ 11 files changed, 90 insertions(+), 26 deletions(-) diff --git a/base/bool.jl b/base/bool.jl index 753d838abeae9..90917d1d736d0 100644 --- a/base/bool.jl +++ b/base/bool.jl @@ -71,7 +71,3 @@ fld(x::Bool, y::Bool) = div(x,y) cld(x::Bool, y::Bool) = div(x,y) rem(x::Bool, y::Bool) = y ? false : throw(DivideError()) mod(x::Bool, y::Bool) = rem(x,y) - -promote_op(op, ::Type{Bool}, ::Type{Bool}) = typeof(op(true, true)) -promote_op(::typeof(^), ::Type{Bool}, ::Type{Bool}) = Bool -promote_op{T<:Integer}(::typeof(^), ::Type{Bool}, ::Type{T}) = Bool diff --git a/base/complex.jl b/base/complex.jl index 59dbd99132814..07e004b3519cf 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -26,17 +26,6 @@ promote_rule{T<:Real,S<:Real}(::Type{Complex{T}}, ::Type{S}) = promote_rule{T<:Real,S<:Real}(::Type{Complex{T}}, ::Type{Complex{S}}) = Complex{promote_type(T,S)} -promote_op{T<:Real,S<:Real}(op, ::Type{Complex{T}}, ::Type{Complex{S}}) = - Complex{promote_op(op,T,S)} -promote_op{T<:Real,S<:Real}(op, ::Type{Complex{T}}, ::Type{S}) = - Complex{promote_op(op,T,S)} -promote_op{T<:Real,S<:Real}(op, ::Type{T}, ::Type{Complex{S}}) = - Complex{promote_op(op,T,S)} -promote_op{T<:Integer,S<:Integer}(::typeof(^), ::Type{T}, ::Type{Complex{S}}) = - Complex{Float64} -promote_op{T<:Integer,S<:Integer}(::typeof(.^), ::Type{T}, ::Type{Complex{S}}) = - Complex{Float64} - widen{T}(::Type{Complex{T}}) = Complex{widen(T)} real(z::Complex) = z.re diff --git a/base/int.jl b/base/int.jl index 5322eceddd82e..ca3242cc682f5 100644 --- a/base/int.jl +++ b/base/int.jl @@ -305,8 +305,6 @@ promote_rule{T<:BitSigned64}(::Type{UInt64}, ::Type{T}) = UInt64 promote_rule{T<:Union{UInt32, UInt64}}(::Type{T}, ::Type{Int128}) = Int128 promote_rule{T<:BitSigned}(::Type{UInt128}, ::Type{T}) = UInt128 -promote_op{R<:Integer,S<:Integer}(op, ::Type{R}, ::Type{S}) = typeof(op(one(R), one(S))) - ## traits ## typemin(::Type{Int8 }) = Int8(-128) diff --git a/base/irrationals.jl b/base/irrationals.jl index e4347f6b8dedd..27699a16a2112 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -10,6 +10,13 @@ promote_rule{s}(::Type{Irrational{s}}, ::Type{Float32}) = Float32 promote_rule{s,t}(::Type{Irrational{s}}, ::Type{Irrational{t}}) = Float64 promote_rule{s,T<:Number}(::Type{Irrational{s}}, ::Type{T}) = promote_type(Float64,T) +promote_op{S<:Irrational,T<:Irrational}(op::Any, ::Type{S}, ::Type{T}) = + promote_op(op, Float64, Float64) +promote_op{S<:Irrational,T<:Number}(op::Any, ::Type{S}, ::Type{T}) = + promote_op(op, Float64, T) +promote_op{S<:Irrational,T<:Number}(op::Any, ::Type{T}, ::Type{S}) = + promote_op(op, T, Float64) + convert(::Type{AbstractFloat}, x::Irrational) = Float64(x) convert(::Type{Float16}, x::Irrational) = Float16(Float32(x)) convert{T<:Real}(::Type{Complex{T}}, x::Irrational) = convert(Complex{T}, convert(T,x)) diff --git a/base/number.jl b/base/number.jl index 218c883e5f2d4..3f8cbfc37c610 100644 --- a/base/number.jl +++ b/base/number.jl @@ -63,4 +63,16 @@ zero{T<:Number}(::Type{T}) = convert(T,0) one(x::Number) = oftype(x,1) one{T<:Number}(::Type{T}) = convert(T,1) +promote_op{R,S<:Number}(::Type{R}, ::Type{S}) = (@_pure_meta; R) # to fix ambiguities +function promote_op{T<:Number}(op, ::Type{T}) + S = typeof(op(one(T))) + # preserve the most general (abstract) type when possible + return isleaftype(T) ? S : typejoin(S, T) +end +function promote_op{R<:Number,S<:Number}(op, ::Type{R}, ::Type{S}) + T = typeof(op(one(R), one(S))) + # preserve the most general (abstract) type when possible + return isleaftype(R) && isleaftype(S) ? T : typejoin(R, S, T) +end + factorial(x::Number) = gamma(x + 1) # fallback for x not Integer diff --git a/base/pkg/resolve/fieldvalue.jl b/base/pkg/resolve/fieldvalue.jl index fe175ce356b0b..a3666388d35a7 100644 --- a/base/pkg/resolve/fieldvalue.jl +++ b/base/pkg/resolve/fieldvalue.jl @@ -42,6 +42,7 @@ Base.typemin(::Type{FieldValue}) = (x=typemin(Int); y=typemin(VersionWeight); Fi Base.:-(a::FieldValue, b::FieldValue) = FieldValue(a.l0-b.l0, a.l1-b.l1, a.l2-b.l2, a.l3-b.l3, a.l4-b.l4) Base.:+(a::FieldValue, b::FieldValue) = FieldValue(a.l0+b.l0, a.l1+b.l1, a.l2+b.l2, a.l3+b.l3, a.l4+b.l4) +Base.promote_op(::Union{typeof(+), typeof(-)}, ::Type{FieldValue}, ::Type{FieldValue}) = FieldValue function Base.isless(a::FieldValue, b::FieldValue) a.l0 < b.l0 && return true diff --git a/base/promotion.jl b/base/promotion.jl index 7b6ebf0f406cb..778de13c004c9 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -222,10 +222,8 @@ minmax(x::Real, y::Real) = minmax(promote(x, y)...) # for the multiplication of two types, # promote_op{R<:MyType,S<:MyType}(::typeof(*), ::Type{R}, ::Type{S}) = MyType{multype(R,S)} promote_op(::Any) = (@_pure_meta; Bottom) -promote_op(::Any, T) = (@_pure_meta; T) +promote_op(::Any, ::Any, ::Any...) = (@_pure_meta; Any) promote_op{T}(::Type{T}, ::Any) = (@_pure_meta; T) -promote_op{R,S}(::Any, ::Type{R}, ::Type{S}) = (@_pure_meta; promote_type(R, S)) -promote_op(op, T, S, U, V...) = (@_pure_meta; promote_op(op, T, promote_op(op, S, U, V...))) ## catch-alls to prevent infinite recursion when definitions are missing ## diff --git a/test/arrayops.jl b/test/arrayops.jl index 171cbda41e1bf..66bb95f096563 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1408,7 +1408,7 @@ b = rand(6,7) # return type declarations (promote_op) module RetTypeDecl using Base.Test - import Base: +, *, .*, zero + import Base: +, *, .*, convert immutable MeterUnits{T,P} <: Number val::T @@ -1422,11 +1422,8 @@ module RetTypeDecl (*){T,pow}(x::Int, y::MeterUnits{T,pow}) = MeterUnits{typeof(x*one(T)),pow}(x*y.val) (*){T}(x::MeterUnits{T,1}, y::MeterUnits{T,1}) = MeterUnits{T,2}(x.val*y.val) (.*){T}(x::MeterUnits{T,1}, y::MeterUnits{T,1}) = MeterUnits{T,2}(x.val*y.val) - zero{T,pow}(x::MeterUnits{T,pow}) = MeterUnits{T,pow}(zero(T)) - - Base.promote_op{R,S}(::typeof(+), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),1} + convert{T,pow}(::Type{MeterUnits{T,pow}}, y::Real) = MeterUnits{T,pow}(convert(T,y)) Base.promote_op{R,S}(::typeof(*), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),2} - Base.promote_op{R,S}(::typeof(.*), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),2} @test @inferred(m+[m,m]) == [m+m,m+m] @test @inferred([m,m]+m) == [m+m,m+m] diff --git a/test/broadcast.jl b/test/broadcast.jl index ee6dc7f9ff0d4..0b43510974ae7 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -165,7 +165,7 @@ m = [1:2;]' @test @inferred([0,1.2].+reshape([0,-2],1,1,2)) == reshape([0 -2; 1.2 -0.8],2,1,2) rt = Base.return_types(.+, Tuple{Array{Float64, 3}, Array{Int, 1}}) @test length(rt) == 1 && rt[1] == Array{Float64, 3} -rt = Base.return_types(broadcast, Tuple{Function, Array{Float64, 3}, Array{Int, 1}}) +rt = Base.return_types(broadcast, Tuple{typeof(.+), Array{Float64, 3}, Array{Int, 3}}) @test length(rt) == 1 && rt[1] == Array{Float64, 3} rt = Base.return_types(broadcast!, Tuple{Function, Array{Float64, 3}, Array{Float64, 3}, Array{Int, 1}}) @test length(rt) == 1 && rt[1] == Array{Float64, 3} @@ -200,3 +200,16 @@ end # issue #4883 @test isa(broadcast(tuple, [1 2 3], ["a", "b", "c"]), Matrix{Tuple{Int,String}}) @test isa(broadcast((x,y)->(x==1?1.0:x,y), [1 2 3], ["a", "b", "c"]), Matrix{Tuple{Real,String}}) +let a = length.(["foo", "bar"]) + @test isa(a, Vector{Int}) + @test a == [3, 3] +end +let a = sin.([1, 2]) + @test isa(a, Vector{Float64}) + @test a ≈ [0.8414709848078965, 0.9092974268256817] +end + +# PR 16988 +@test Base.promote_op(+, Bool) === Int +@test isa(broadcast(+, true), Array{Int,0}) +@test Base.promote_op(Float64, Bool) === Float64 diff --git a/test/linalg/dense.jl b/test/linalg/dense.jl index e3593fb5b49cc..3f1c7a3c45995 100644 --- a/test/linalg/dense.jl +++ b/test/linalg/dense.jl @@ -361,6 +361,9 @@ let @test S*T == [z z; 0 0] end +# similar issue for Array{Real} +@test Real[1 2] * Real[1.5; 2.0] == [5.5] + # Matrix exponential for elty in (Float32, Float64, Complex64, Complex128) A1 = convert(Matrix{elty}, [4 2 0; 1 4 1; 1 1 4]) diff --git a/test/numbers.jl b/test/numbers.jl index 97c368601670b..64b3174fe484f 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2762,3 +2762,53 @@ testmi(typemax(UInt32)-UInt32(1000):typemax(UInt32), map(UInt32, 1:100)) # issue #16282 @test_throws MethodError 3 // 4.5im + +# PR #16995 +let types = (Base.BitInteger_types..., BigInt, Bool, + Rational{Int}, Rational{BigInt}, + Float16, Float32, Float64, BigFloat, + Complex{Int}, Complex{UInt}, Complex32, Complex64, Complex128) + for S in types + for op in (+, -) + T = @inferred Base.promote_op(op, S) + t = @inferred op(one(S)) + @test T === typeof(t) + end + end + + @test @inferred(Base.promote_op(!, Bool)) === Bool + + for R in types, S in types + for op in (+, -, *, /, ^) + T = @inferred Base.promote_op(op, R, S) + t = @inferred op(one(R), one(S)) + @test T === typeof(t) + end + end +end + +let types = (Base.BitInteger_types..., BigInt, Bool, + Rational{Int}, Rational{BigInt}, + Float16, Float32, Float64, BigFloat) + for S in types, T in types + for op in (<, >, <=, >=, (==)) + @test @inferred(Base.promote_op(op, S, T)) === Bool + end + end +end + +let types = (Base.BitInteger_types..., BigInt, Bool) + for S in types + T = @inferred Base.promote_op(~, S) + t = @inferred ~one(S) + @test T === typeof(t) + end + + for S in types, T in types + for op in (&, |, <<, >>, (>>>), %, ÷) + T = @inferred Base.promote_op(op, S, T) + t = @inferred op(one(S), one(T)) + @test T === typeof(t) + end + end +end From 20555f683548b085c8d45222756b068b12a124ee Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 1 Jul 2016 23:32:17 -0400 Subject: [PATCH 0193/1117] Fix handling of `Array` argument type in `ccall`. Fixes #17204 --- src/abi_aarch64.cpp | 5 +++-- src/abi_arm.cpp | 5 +++-- src/abi_win64.cpp | 4 +++- src/abi_x86.cpp | 4 +++- src/ccall.cpp | 3 ++- test/ccall.jl | 11 +++++++++++ 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/abi_aarch64.cpp b/src/abi_aarch64.cpp index 3015cc3fb18e3..5b87548a50f9f 100644 --- a/src/abi_aarch64.cpp +++ b/src/abi_aarch64.cpp @@ -356,7 +356,8 @@ static Type *classify_arg(jl_value_t *ty, bool *fpreg, bool *onstack, bool use_sret(AbiState*, jl_value_t *ty) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) + // Assume (jl_is_datatype(ty) && !jl_is_abstracttype(ty) && + // !jl_is_array_type(ty)) // Section 5.5 // If the type, T, of the result of a function is such that // @@ -375,7 +376,7 @@ bool use_sret(AbiState*, jl_value_t *ty) Type *preferred_llvm_type(jl_value_t *ty, bool) { - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty)) + if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_array_type(ty)) return NULL; jl_datatype_t *dt = (jl_datatype_t*)ty; if (Type *fptype = get_llvm_fp_or_vectype(dt)) diff --git a/src/abi_arm.cpp b/src/abi_arm.cpp index 65a4dd188559e..acadd5c901ef4 100644 --- a/src/abi_arm.cpp +++ b/src/abi_arm.cpp @@ -206,7 +206,8 @@ static void classify_return_arg(jl_value_t *ty, bool *reg, bool use_sret(AbiState *state, jl_value_t *ty) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) + // Assume (jl_is_datatype(ty) && !jl_is_abstracttype(ty) && + // !jl_is_array_type(ty)) bool reg = false; bool onstack = false; @@ -254,7 +255,7 @@ static void classify_arg(jl_value_t *ty, bool *reg, Type *preferred_llvm_type(jl_value_t *ty, bool isret) { - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty)) + if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_array_type(ty)) return NULL; jl_datatype_t *dt = (jl_datatype_t*)ty; diff --git a/src/abi_win64.cpp b/src/abi_win64.cpp index 9ecf8cd3e68e9..3e8c441e79351 100644 --- a/src/abi_win64.cpp +++ b/src/abi_win64.cpp @@ -46,7 +46,9 @@ const AbiState default_abi_state = {}; bool use_sret(AbiState *state, jl_value_t *ty) { - if(!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) + // Assume (jl_is_datatype(ty) && !jl_is_abstracttype(ty) && + // !jl_is_array_type(ty)) + if (jl_is_cpointer_type(ty)) return false; size_t size = jl_datatype_size(ty); if (size <= 8 || is_native_simd_type(ty)) diff --git a/src/abi_x86.cpp b/src/abi_x86.cpp index 4e345e20ab7cf..8b5d34304f8b3 100644 --- a/src/abi_x86.cpp +++ b/src/abi_x86.cpp @@ -56,7 +56,9 @@ inline bool is_complex128(jl_value_t *ty) bool use_sret(AbiState *state, jl_value_t *ty) { - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) + // Assume (jl_is_datatype(ty) && !jl_is_abstracttype(ty) && + // !jl_is_array_type(ty)) + if (jl_is_cpointer_type(ty)) return false; size_t size = jl_datatype_size(ty); if (size == 0) diff --git a/src/ccall.cpp b/src/ccall.cpp index d7db816a56046..322a23d0bdb8e 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -892,7 +892,8 @@ static std::string generate_func_sig( if (*prt == NULL) *prt = *lrt; - if (jl_is_datatype(rt) && !jl_is_abstracttype(rt) && use_sret(&abi, rt)) { + if (jl_is_datatype(rt) && !jl_is_abstracttype(rt) && + !jl_is_array_type(rt) && use_sret(&abi, rt)) { paramattrs.push_back(AttrBuilder()); paramattrs[0].clear(); #if !defined(_OS_WINDOWS_) || defined(LLVM35) // llvm used to use the old mingw ABI, skipping this marking works around that difference diff --git a/test/ccall.jl b/test/ccall.jl index 20e7245f880c5..0129954c87bb5 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -849,3 +849,14 @@ else warn("ccall: no VecReg tests run for this platform") end + +# Special calling convention for `Array` +function f17204(a) + b = similar(a) + for i in eachindex(a) + b[i] = a[i] + 10 + end + return b +end +@test ccall(cfunction(f17204, Vector{Any}, Tuple{Vector{Any}}), + Vector{Any}, (Vector{Any},), Any[1:10;]) == Any[11:20;] From bd361551455cea13a76c4e2c3d5234a695951c51 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Sat, 2 Jul 2016 15:02:50 +0200 Subject: [PATCH 0194/1117] Don't store source expressions in docstrings Currently each docstring stores the `Expr` that it documents. With Documenter.jl I've not found any use for looking at the source expression since we can get the useful information through reflection easier than traversing raw expressions. Instead we store the `Binding` and type signature in place of the source, which is much more useful when auto-generating docs. This was undocumented and I've not found anything relying on it within `METADATA` so should be fine to just remove. As a bonus removing the source expressions saves 800KB in each of the `sys.*` files. --- base/docs/Docs.jl | 5 ++--- test/docs.jl | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index c238f1fc8bd20..754c63607c310 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -229,6 +229,8 @@ function doc!(b::Binding, str::DocStr, sig::ANY = Union{}) push!(m.order, sig) end m.docs[sig] = str + str.data[:binding] = b + str.data[:typesig] = sig return b end @@ -436,7 +438,6 @@ Build a `Dict` expression containing metadata captured from the expression `expr Fields that may be included in the returned `Dict`: -- `:source`: Source code for the given `expr`. - `:path`: String representing the file where `expr` is defined. - `:linenumber`: Linenumber where `expr` is defined. - `:module`: Module where the docstring is defined. @@ -444,8 +445,6 @@ Fields that may be included in the returned `Dict`: """ function metadata(expr) args = [] - # Source code for the object being documented. - push!(args, :($(Pair)(:source, $(quot(expr))))) # Filename and linenumber of the docstring. push!(args, :($(Pair)(:path, $(Base).@__FILE__))) push!(args, :($(Pair)(:linenumber, $(unsafe_load(cglobal(:jl_lineno, Cint)))))) diff --git a/test/docs.jl b/test/docs.jl index a706eaa07bcb4..563bc9366f98f 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -181,6 +181,10 @@ let f = @var(DocsTest.f) md = meta(DocsTest)[f] @test docstrings_equal(md.docs[Tuple{Any}], doc"f-1") @test docstrings_equal(md.docs[Tuple{Any,Any}], doc"f-2") + @test md.docs[Tuple{Any}].data[:binding] === f + @test md.docs[Tuple{Any}].data[:typesig] === Tuple{Any} + @test md.docs[Tuple{Any,Any}].data[:binding] === f + @test md.docs[Tuple{Any,Any}].data[:typesig] === Tuple{Any,Any} end let s = @var(DocsTest.s) From 5a8301d0d87e645502226fd82a5795684f3c2a67 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 28 Jun 2016 04:20:46 -0400 Subject: [PATCH 0195/1117] Remove 'if false' dead code blocks, use test_broken for the ones in test/spawn.jl --- base/coreimg.jl | 9 --------- base/sysimg.jl | 10 ---------- test/spawn.jl | 22 +++++++++------------- 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/base/coreimg.jl b/base/coreimg.jl index 4dfce3786e285..c2322a78d8917 100644 --- a/base/coreimg.jl +++ b/base/coreimg.jl @@ -3,15 +3,6 @@ Main.Core.eval(Main.Core, :(baremodule Inference using Core.Intrinsics import Core: print, println, show, write, unsafe_write, STDOUT, STDERR -if false # show that the IO system is already (relatively) operational - print("HELLO") - println(" WORLD") - show("αβγ :)"); println() - println(STDERR, "TEST") - println(STDERR, STDERR) - println(STDERR, 'a') - println(STDERR, 'α') -end ccall(:jl_set_istopmod, Void, (Bool,), false) diff --git a/base/sysimg.jl b/base/sysimg.jl index 7e588b56d01a3..022c4a9dac16c 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -27,16 +27,6 @@ if false show(io::IO, x::ANY) = Core.show(io, x) print(io::IO, a::ANY...) = Core.print(io, a...) println(io::IO, x::ANY...) = Core.println(io, x...) - if false # show that the IO system now (relatively) operational - print("HELLO") - println(" WORLD") - show("αβγ :)"); println() - println(STDERR, "TEST") - println(STDERR, STDERR) - println(STDERR, 'a') - println(STDERR, 'α') - show(STDOUT, 'α') - end end ## Load essential files and libraries diff --git a/test/spawn.jl b/test/spawn.jl index b544a52c54b1c..84046640e727f 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -53,26 +53,22 @@ if valgrind_off @test_throws Base.UVError run(`foo_is_not_a_valid_command`) end -if false - prefixer(prefix, sleep) = `perl -nle '$|=1; print "'$prefix' ", $_; sleep '$sleep';'` - @test success(pipeline(`perl -le '$|=1; for(0..2){ print; sleep 1 }'`, +prefixer(prefix, sleep) = `perl -nle '$|=1; print "'$prefix' ", $_; sleep '$sleep';'` +@test success(pipeline(`perl -le '$|=1; for(0..2){ print; sleep 1 }'`, prefixer("A",2) & prefixer("B",2))) - @test success(pipeline(`perl -le '$|=1; for(0..2){ print; sleep 1 }'`, +@test success(pipeline(`perl -le '$|=1; for(0..2){ print; sleep 1 }'`, prefixer("X",3) & prefixer("Y",3) & prefixer("Z",3), prefixer("A",2) & prefixer("B",2))) -end @test success(`true`) @test !success(`false`) @test success(pipeline(`true`, `true`)) -if false - @test success(ignorestatus(`false`)) - @test success(pipeline(ignorestatus(`false`), `true`)) - @test !success(pipeline(ignorestatus(`false`), `false`)) - @test !success(ignorestatus(`false`) & `false`) - @test success(ignorestatus(pipeline(`false`, `false`))) - @test success(ignorestatus(`false` & `false`)) -end +@test_broken success(ignorestatus(`false`)) +@test_broken success(pipeline(ignorestatus(`false`), `true`)) +@test !success(pipeline(ignorestatus(`false`), `false`)) +@test !success(ignorestatus(`false`) & `false`) +@test_broken success(ignorestatus(pipeline(`false`, `false`))) +@test_broken success(ignorestatus(`false` & `false`)) # STDIN Redirection file = tempname() From 6cd3f56f6bc994240a75c6405768b382762e5e03 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 28 Jun 2016 16:44:43 -0400 Subject: [PATCH 0196/1117] Restrict tryparse_internal on Bool to Union{String,SubString} Add a test of the error throwing branch of tryparse_internal which has had a typo in it since a68f950d14806d4de7c214727fb7980ebd67192f --- base/parse.jl | 12 ++++++++---- test/parse.jl | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/base/parse.jl b/base/parse.jl index af17b09998f13..7b39f4b8e06c6 100644 --- a/base/parse.jl +++ b/base/parse.jl @@ -128,12 +128,16 @@ function tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, startpos::I return Nullable{T}(n) end -function tryparse_internal(::Type{Bool}, sbuff::AbstractString, startpos::Int, endpos::Int, base::Integer, raise::Bool) +function tryparse_internal(::Type{Bool}, sbuff::Union{String,SubString}, + startpos::Int, endpos::Int, base::Integer, raise::Bool) len = endpos-startpos+1 p = pointer(sbuff)+startpos-1 - (len == 4) && (0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), p, "true", 4)) && (return Nullable(true)) - (len == 5) && (0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), p, "false", 5)) && (return Nullable(false)) - raise && throw(ArgumentError("invalid Bool representation: $(repr(SubString(s,startpos,endpos)))")) + (len == 4) && (0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), + p, "true", 4)) && (return Nullable(true)) + (len == 5) && (0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), + p, "false", 5)) && (return Nullable(false)) + raise && throw(ArgumentError("invalid Bool representation: " * + repr(SubString(sbuff, startpos, endpos)))) Nullable{Bool}() end diff --git a/test/parse.jl b/test/parse.jl index 3fc6b8869215f..cd57a6b159c7c 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -537,3 +537,6 @@ end @test get(tryparse(Bool, "false")) === get(Nullable{Bool}(false)) @test_throws ArgumentError parse(Int, "2", 1) @test_throws ArgumentError parse(Int, "2", 63) + +# error throwing branch from #10560 +@test_throws ArgumentError Base.tryparse_internal(Bool, "foo", 1, 2, 10, true) From bfa2ee06899da96e7b32217846cac9daf7114a75 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Sat, 2 Jul 2016 12:27:27 -0400 Subject: [PATCH 0197/1117] Add test for #17096 --- test/reflection.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/reflection.jl b/test/reflection.jl index d09b836e65934..89cd17593f4f0 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -38,8 +38,16 @@ test_code_reflections(test_ast_reflection, code_lowered) test_code_reflections(test_ast_reflection, code_typed) test_code_reflections(test_bin_reflection, code_llvm) test_code_reflections(test_bin_reflection, code_native) + +# Issue #16326 +let OLDSTDOUT = STDOUT + redirect_stdout(open(tempname(),"w")) + @test try @code_native map(y->abs(y), rand(3)); true; catch false; end + redirect_stdout(OLDSTDOUT) end +end # module ReflectionTest + # code_warntype module WarnType using Base.Test From 9fc2652f42af52510a5ebd812ed31d30dd43c8b1 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 29 Jun 2016 11:18:27 -0400 Subject: [PATCH 0198/1117] Delete MIME macro now, it's been deprecated for 3 years ref af8c18c1f088e59b3bc0249fd4e4bfd27851606d --- base/deprecated.jl | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 8e59e53e1269e..b22c1a3cc6523 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -788,19 +788,6 @@ function symperm{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}) "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) end -# needs to be a macro so that we can use ::@mime(s) in type declarations -eval(Multimedia, quote -export @MIME -macro MIME(s) - Base.warn_once("@MIME(\"\") is deprecated, use MIME\"\" instead.") - if isa(s,AbstractString) - :(MIME{$(Expr(:quote, Symbol(s)))}) - else - :(MIME{Symbol($s)}) - end -end -end) - # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 From 98631d172b138eccb865738bca364c4274121b81 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 29 Jun 2016 11:50:17 -0400 Subject: [PATCH 0199/1117] Reword cholfact docs to remove incorrect reference to structure --- base/sparse/cholmod.jl | 54 ++++++++++++++++++++---------------------- doc/stdlib/linalg.rst | 12 +++++----- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/base/sparse/cholmod.jl b/base/sparse/cholmod.jl index 6ae290c3449ff..3d0b57e30a3ca 100644 --- a/base/sparse/cholmod.jl +++ b/base/sparse/cholmod.jl @@ -1259,11 +1259,10 @@ end """ cholfact!(F::Factor, A; shift = 0.0) -> CHOLMOD.Factor -Compute the Cholesky (``LL'``) factorization of `A`, reusing the symbolic factorization `F`. -`A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, or -`Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't -have the type tag, its structure and values must still be -symmetric/Hermitian. +Compute the Cholesky (``LL'``) factorization of `A`, reusing the symbolic +factorization `F`. `A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, +or `Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't +have the type tag, it must still be symmetric or Hermitian. ** Note ** @@ -1303,22 +1302,22 @@ end Compute the Cholesky factorization of a sparse positive definite matrix `A`. `A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, or `Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't -have the type tag, its structure and values must still be -symmetric/Hermitian. +have the type tag, it must still be symmetric or Hermitian. A fill-reducing permutation is used. `F = cholfact(A)` is most frequently used to solve systems of equations with `F\\b`, but also the methods `diag`, `det`, `logdet` are defined for `F`. You can also extract individual factors from `F`, using `F[:L]`. -However, since pivoting is on by default, -the factorization is internally represented as `A == P'*L*L'*P` with a permutation matrix `P`; +However, since pivoting is on by default, the factorization is internally +represented as `A == P'*L*L'*P` with a permutation matrix `P`; using just `L` without accounting for `P` will give incorrect answers. To include the effects of permutation, -it's typically preferable to extact "combined" factors like `PtL = F[:PtL]` (the equivalent of `P'*L`) -and `LtP = F[:UP]` (the equivalent of `L'*P`). +it's typically preferable to extact "combined" factors like `PtL = F[:PtL]` +(the equivalent of `P'*L`) and `LtP = F[:UP]` (the equivalent of `L'*P`). -Setting optional `shift` keyword argument computes the factorization of `A+shift*I` instead of `A`. -If the `perm` argument is nonempty, -it should be a permutation of `1:size(A,1)` giving the ordering to use (instead of CHOLMOD's default AMD ordering). +Setting optional `shift` keyword argument computes the factorization of +`A+shift*I` instead of `A`. If the `perm` argument is nonempty, +it should be a permutation of `1:size(A,1)` giving the ordering to use +(instead of CHOLMOD's default AMD ordering). ** Note ** @@ -1360,8 +1359,7 @@ end Compute the ``LDL'`` factorization of `A`, reusing the symbolic factorization `F`. `A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, or `Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't -have the type tag, its structure and values must still be -symmetric/Hermitian. +have the type tag, it must still be symmetric or Hermitian. ** Note ** @@ -1401,23 +1399,23 @@ end Compute the ``LDL'`` factorization of a sparse matrix `A`. `A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}`, or `Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't -have the type tag, its structure and values must still be -symmetric/Hermitian. -A fill-reducing permutation is used. -`F = ldltfact(A)` is most frequently used to solve systems of equations `A*x = b` with `F\\b`. -The returned factorization object `F` also supports the methods `diag`, +have the type tag, it must still be symmetric or Hermitian. +A fill-reducing permutation is used. `F = ldltfact(A)` is most frequently +used to solve systems of equations `A*x = b` with `F\\b`. The returned +factorization object `F` also supports the methods `diag`, `det`, and `logdet`. You can extract individual factors from `F` using `F[:L]`. -However, since pivoting is on by default, -the factorization is internally represented as `A == P'*L*D*L'*P` with a permutation matrix `P`; +However, since pivoting is on by default, the factorization is internally +represented as `A == P'*L*D*L'*P` with a permutation matrix `P`; using just `L` without accounting for `P` will give incorrect answers. -To include the effects of permutation, -it's typically preferable to extact "combined" factors like `PtL = F[:PtL]` (the equivalent of +To include the effects of permutation, it is typically preferable to extact +"combined" factors like `PtL = F[:PtL]` (the equivalent of `P'*L`) and `LtP = F[:UP]` (the equivalent of `L'*P`). The complete list of supported factors is `:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP`. -Setting optional `shift` keyword argument computes the factorization of `A+shift*I` instead of `A`. -If the `perm` argument is nonempty, -it should be a permutation of `1:size(A,1)` giving the ordering to use (instead of CHOLMOD's default AMD ordering). +Setting optional `shift` keyword argument computes the factorization of +`A+shift*I` instead of `A`. If the `perm` argument is nonempty, +it should be a permutation of `1:size(A,1)` giving the ordering to use +(instead of CHOLMOD's default AMD ordering). ** Note ** diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index f2be13a8b3930..627f85bc26c1a 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -322,7 +322,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the Cholesky factorization of a sparse positive definite matrix ``A``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . A fill-reducing permutation is used. ``F = cholfact(A)`` is most frequently used to solve systems of equations with ``F\b``\ , but also the methods ``diag``\ , ``det``\ , ``logdet`` are defined for ``F``\ . You can also extract individual factors from ``F``\ , using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). + Compute the Cholesky factorization of a sparse positive definite matrix ``A``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . Note that even if ``A`` doesn't have the type tag, it must still be symmetric or Hermitian. A fill-reducing permutation is used. ``F = cholfact(A)`` is most frequently used to solve systems of equations with ``F\b``\ , but also the methods ``diag``\ , ``det``\ , ``logdet`` are defined for ``F``\ . You can also extract individual factors from ``F``\ , using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). Setting optional ``shift`` keyword argument computes the factorization of ``A+shift*I`` instead of ``A``\ . If the ``perm`` argument is nonempty, it should be a permutation of ``1:size(A,1)`` giving the ordering to use (instead of CHOLMOD's default AMD ordering). @@ -336,7 +336,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the Cholesky (:math:`LL'`\ ) factorization of ``A``\ , reusing the symbolic factorization ``F``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . + Compute the Cholesky (:math:`LL'`\ ) factorization of ``A``\ , reusing the symbolic factorization ``F``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . Note that even if ``A`` doesn't have the type tag, it must still be symmetric or Hermitian. ** Note ** @@ -392,7 +392,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the :math:`LDL'` factorization of a sparse matrix ``A``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . Note that even if ``A`` doesn't have the type tag, its structure must still be symmetric/Hermitian. A fill-reducing permutation is used. ``F = ldltfact(A)`` is most frequently used to solve systems of equations ``A*x = b`` with ``F\b``\ . The returned factorization object ``F`` also supports the methods ``diag``\ , ``det``\ , and ``logdet``\ . You can extract individual factors from ``F`` using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*D*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it's typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). The complete list of supported factors is ``:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP``\ . + Compute the :math:`LDL'` factorization of a sparse matrix ``A``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . Note that even if ``A`` doesn't have the type tag, it must still be symmetric or Hermitian. A fill-reducing permutation is used. ``F = ldltfact(A)`` is most frequently used to solve systems of equations ``A*x = b`` with ``F\b``\ . The returned factorization object ``F`` also supports the methods ``diag``\ , ``det``\ , and ``logdet``\ . You can extract individual factors from ``F`` using ``F[:L]``\ . However, since pivoting is on by default, the factorization is internally represented as ``A == P'*L*D*L'*P`` with a permutation matrix ``P``\ ; using just ``L`` without accounting for ``P`` will give incorrect answers. To include the effects of permutation, it is typically preferable to extact "combined" factors like ``PtL = F[:PtL]`` (the equivalent of ``P'*L``\ ) and ``LtP = F[:UP]`` (the equivalent of ``L'*P``\ ). The complete list of supported factors is ``:L, :PtL, :D, :UP, :U, :LD, :DU, :PtLD, :DUP``\ . Setting optional ``shift`` keyword argument computes the factorization of ``A+shift*I`` instead of ``A``\ . If the ``perm`` argument is nonempty, it should be a permutation of ``1:size(A,1)`` giving the ordering to use (instead of CHOLMOD's default AMD ordering). @@ -406,7 +406,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the :math:`LDL'` factorization of ``A``\ , reusing the symbolic factorization ``F``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . + Compute the :math:`LDL'` factorization of ``A``\ , reusing the symbolic factorization ``F``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . Note that even if ``A`` doesn't have the type tag, it must still be symmetric or Hermitian. ** Note ** @@ -546,13 +546,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the LQ factorization of ``A``\ , using the input matrix as a workspace. See also :func:`lq\ . + Compute the LQ factorization of ``A``\ , using the input matrix as a workspace. See also :func:`lq`\ . .. function:: lqfact(A) -> LQ .. Docstring generated from Julia source - Compute the LQ factorization of ``A``\ . See also :func:`lq\ . + Compute the LQ factorization of ``A``\ . See also :func:`lq`\ . .. function:: lq(A; [thin=true]) -> L, Q From 838399fa70253c9c36e542b7bf1dadc184392ab9 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 1 Jul 2016 05:03:34 -0400 Subject: [PATCH 0200/1117] Remove references to mojibake, temporary fork name of utf8proc --- LICENSE.md | 2 +- test/unicode/utf8proc.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index fbd0efad2ef9c..132ce06028af9 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -41,7 +41,7 @@ own licenses: - [LIBUNWIND](http://git.savannah.gnu.org/gitweb/?p=libunwind.git;a=blob_plain;f=LICENSE;hb=master) [MIT] - [LIBUV](https://github.com/joyent/libuv/blob/master/LICENSE) [MIT] - [LLVM](http://llvm.org/releases/3.7.0/LICENSE.TXT) [BSD-3, effectively] -- [UTF8PROC](https://github.com/JuliaLang/libmojibake) [MIT] +- [UTF8PROC](https://github.com/JuliaLang/utf8proc) [MIT] Julia's standard library uses the following external libraries, which have diff --git a/test/unicode/utf8proc.jl b/test/unicode/utf8proc.jl index 1ecd4b35f5a80..0f1e3b0727e26 100644 --- a/test/unicode/utf8proc.jl +++ b/test/unicode/utf8proc.jl @@ -75,7 +75,7 @@ end @test normalize_string("\U1e9b\U0323", :NFKD) == "s\U0323\U0307" @test normalize_string("\U1e9b\U0323", :NFKC) == "\U1e69" -#issue #5939 uft8proc/libmojibake character predicates +#issue #5939 uft8proc character predicates let alower=['a', 'd', 'j', 'y', 'z'] ulower=['α', 'β', 'γ', 'δ', 'ф', 'я'] From 5df3334da8c8f5f61d656483d023d2b5f10fd04b Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 24 May 2016 16:53:04 -0700 Subject: [PATCH 0201/1117] editorial revisions to some comments --- base/docs/helpdb/Base.jl | 2 +- base/linalg/qr.jl | 18 +++++++++--------- base/sparse/spqr.jl | 20 ++++++++++---------- doc/stdlib/io-network.rst | 2 +- src/gc.c | 6 +++--- test/compile.jl | 2 +- test/replutil.jl | 2 +- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 477c66f370745..a821a03fbd5fc 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -3623,7 +3623,7 @@ the file is large, and is only read once and not written to. If `ignore_invalid_chars` is `true`, bytes in `source` with invalid character encoding will be ignored. Otherwise an error is thrown indicating the offending character position. -If `quotes` is `true`, column enclosed within double-quote (\") characters are allowed to +If `quotes` is `true`, columns enclosed within double-quote (\") characters are allowed to contain new lines and column delimiters. Double-quote characters within a quoted field must be escaped with another double-quote. Specifying `dims` as a tuple of the expected rows and columns (including header, if any) may speed up reading of large files. If `comments` is diff --git a/base/linalg/qr.jl b/base/linalg/qr.jl index 42f4f77fd5056..46899f0e7bc21 100644 --- a/base/linalg/qr.jl +++ b/base/linalg/qr.jl @@ -585,7 +585,7 @@ end # With a real lhs and complex rhs with the same precision, we can reinterpret the complex # rhs as a real rhs with twice the number of columns. -# convenience to compute the return size correctly for vector and matrices +# convenience methods to compute the return size correctly for vectors and matrices _ret_size(A::Factorization, b::AbstractVector) = (max(size(A, 2), length(b)),) _ret_size(A::Factorization, B::AbstractMatrix) = (max(size(A, 2), size(B, 1)), size(B, 2)) @@ -593,18 +593,18 @@ function (\){T<:BlasReal}(A::Union{QR{T},QRCompactWY{T},QRPivoted{T}}, BIn::VecO m, n = size(A) m == size(BIn, 1) || throw(DimensionMismatch("left hand side has $m rows, but right hand side has $(size(BIn,1)) rows")) -# | z | z | reinterpret | x | x | x | x | transpose | x | y | reshape | x | y | x | y | -# | z | z | -> | y | y | y | y | -> | x | y | -> | x | y | x | y | -# | x | y | -# | x | y | +# |z1|z3| reinterpret |x1|x2|x3|x4| transpose |x1|y1| reshape |x1|y1|x3|y3| +# |z2|z4| -> |y1|y2|y3|y4| -> |x2|y2| -> |x2|y2|x4|y4| +# |x3|y3| +# |x4|y4| B = reshape(transpose(reinterpret(T, BIn, (2, length(BIn)))), size(BIn, 1), 2*size(BIn, 2)) X = A_ldiv_B!(A, _append_zeros(B, T, n)) -# | z | z | reinterpret | x | x | x | x | transpose | x | y | reshape | x | y | x | y | -# | z | z | <- | y | y | y | y | <- | x | y | <- | x | y | x | y | -# | x | y | -# | x | y | +# |z1|z3| reinterpret |x1|x2|x3|x4| transpose |x1|y1| reshape |x1|y1|x3|y3| +# |z2|z4| <- |y1|y2|y3|y4| <- |x2|y2| <- |x2|y2|x4|y4| +# |x3|y3| +# |x4|y4| XX = reinterpret(Complex{T}, transpose(reshape(X, div(length(X), 2), 2)), _ret_size(A, BIn)) return _cut_B(XX, 1:n) end diff --git a/base/sparse/spqr.jl b/base/sparse/spqr.jl index 735267cede66f..636afffe4a096 100644 --- a/base/sparse/spqr.jl +++ b/base/sparse/spqr.jl @@ -142,25 +142,25 @@ qrfact(A::SparseMatrixCSC) = qrfact(A, Val{true}) # With a real lhs and complex rhs with the same precision, we can reinterpret # the complex rhs as a real rhs with twice the number of columns # -# This definition is similar to the definition in factorization.jl except the we here have to use -# \ instead of A_ldiv_B! because of limitations in SPQR +# This definition is similar to the definition in factorization.jl except that +# here we have to use \ instead of A_ldiv_B! because of limitations in SPQR ## Two helper methods _ret_size(F::Factorization, b::AbstractVector) = (size(F, 2),) _ret_size(F::Factorization, B::AbstractMatrix) = (size(F, 2), size(B, 2)) function (\)(F::Factorization{Float64}, B::VecOrMat{Complex{Float64}}) -# | z | z | reinterpret | x | x | x | x | transpose | x | y | reshape | x | y | x | y | -# | z | z | -> | y | y | y | y | -> | x | y | -> | x | y | x | y | -# | x | y | -# | x | y | +# |z1|z3| reinterpret |x1|x2|x3|x4| transpose |x1|y1| reshape |x1|y1|x3|y3| +# |z2|z4| -> |y1|y2|y3|y4| -> |x2|y2| -> |x2|y2|x4|y4| +# |x3|y3| +# |x4|y4| c2r = reshape(transpose(reinterpret(Float64, B, (2, length(B)))), size(B, 1), 2*size(B, 2)) x = F\c2r -# | z | z | reinterpret | x | x | x | x | transpose | x | y | reshape | x | y | x | y | -# | z | z | <- | y | y | y | y | <- | x | y | <- | x | y | x | y | -# | x | y | -# | x | y | +# |z1|z3| reinterpret |x1|x2|x3|x4| transpose |x1|y1| reshape |x1|y1|x3|y3| +# |z2|z4| <- |y1|y2|y3|y4| <- |x2|y2| <- |x2|y2|x4|y4| +# |x3|y3| +# |x4|y4| return reinterpret(Complex{Float64}, transpose(reshape(x, (length(x) >> 1), 2)), _ret_size(F, B)) end diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index c855dc0ed8df7..6a78e42592b52 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -576,7 +576,7 @@ Text I/O If ``ignore_invalid_chars`` is ``true``\ , bytes in ``source`` with invalid character encoding will be ignored. Otherwise an error is thrown indicating the offending character position. - If ``quotes`` is ``true``\ , column enclosed within double-quote (") characters are allowed to contain new lines and column delimiters. Double-quote characters within a quoted field must be escaped with another double-quote. Specifying ``dims`` as a tuple of the expected rows and columns (including header, if any) may speed up reading of large files. If ``comments`` is ``true``\ , lines beginning with ``comment_char`` and text following ``comment_char`` in any line are ignored. + If ``quotes`` is ``true``\ , columns enclosed within double-quote (") characters are allowed to contain new lines and column delimiters. Double-quote characters within a quoted field must be escaped with another double-quote. Specifying ``dims`` as a tuple of the expected rows and columns (including header, if any) may speed up reading of large files. If ``comments`` is ``true``\ , lines beginning with ``comment_char`` and text following ``comment_char`` in any line are ignored. .. function:: readdlm(source, delim::Char, eol::Char; options...) diff --git a/src/gc.c b/src/gc.c index e0ef7a3b5a0f7..a8657fad95388 100644 --- a/src/gc.c +++ b/src/gc.c @@ -7,8 +7,8 @@ extern "C" { #endif // Protect all access to `finalizer_list_marked` and `to_finalize`. -// For accessing `ptls->finalizers`, the lock is needed if a thread is -// is going to realloc the buffer (of it's own list) or accessing the +// For accessing `ptls->finalizers`, the lock is needed if a thread +// is going to realloc the buffer (of its own list) or accessing the // list of another thread static jl_mutex_t finalizers_lock; @@ -115,7 +115,7 @@ static void run_finalizer(jl_value_t *o, jl_value_t *ff) } // if `need_sync` is true, the `list` is the `finalizers` list of another -// thead and we need additional synchronizations +// thread and we need additional synchronizations static void finalize_object(arraylist_t *list, jl_value_t *o, arraylist_t *copied_list, int need_sync) { diff --git a/test/compile.jl b/test/compile.jl index 379d3e4028533..b0170a3015ccc 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -35,7 +35,7 @@ try # issue 16529 (adding a method to a type with no instances) (::Task)(::UInt8, ::UInt16, ::UInt32) = 2 - # issue 16471 (capturing references to an kwfunc) + # issue 16471 (capturing references to a kwfunc) Base.Test.@test_throws ErrorException Core.kwfunc(Base.nothing) Base.nothing(::UInt8, ::UInt16, ::UInt32; x = 52) = x const nothingkw = Core.kwfunc(Base.nothing) diff --git a/test/replutil.jl b/test/replutil.jl index a272c23a4dba1..3bd83c8f7f29a 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -390,7 +390,7 @@ withenv("JULIA_EDITOR" => nothing, "VISUAL" => nothing, "EDITOR" => nothing) do @test Base.editor() == ["/Applications/Sublime Text.app/Contents/SharedSupport/bin/subl", "-w"] end -# Issue #14684: `display` should prints associative types in full. +# Issue #14684: `display` should print associative types in full. let d = Dict(1 => 2, 3 => 45) buf = IOBuffer() td = TextDisplay(buf) From cdcb587ab25abf7662f4101e511fae06132ff35b Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 8 Jun 2016 01:21:00 -0700 Subject: [PATCH 0202/1117] Misc OCD whitespace adjustments Use 4 space indent in some doc code examples --- base/inference.jl | 2 +- base/multi.jl | 1 - base/random.jl | 1 - base/sharedarray.jl | 1 - doc/manual/variables-and-scoping.rst | 22 +++++++++++----------- src/ccalltest.c | 4 ++-- src/jitlayers.cpp | 2 -- src/jltypes.c | 2 +- test/fft.jl | 6 +----- test/file.jl | 1 - test/linalg/hessenberg.jl | 3 +-- test/math.jl | 2 -- test/parse.jl | 8 ++++---- 13 files changed, 21 insertions(+), 34 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 0f2c36e8d7e65..2f5b5d5010497 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -5,7 +5,7 @@ import Core: _apply, svec, apply_type, Builtin, IntrinsicFunction #### parameters limiting potentially-infinite types #### const MAX_TYPEUNION_LEN = 3 const MAX_TYPE_DEPTH = 7 -const MAX_TUPLETYPE_LEN = 15 +const MAX_TUPLETYPE_LEN = 15 const MAX_TUPLE_DEPTH = 4 const MAX_TUPLE_SPLAT = 16 diff --git a/base/multi.jl b/base/multi.jl index 548516c14f31a..2b52b033a8dca 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -345,7 +345,6 @@ function send_connection_hdr(w::Worker, cookie=true) # For a connection initiated from the remote side to us, we only send the version, # else when we initiate a connection we first send the cookie followed by our version. # The remote side validates the cookie. - if cookie write(w.w_stream, LPROC.cookie) end diff --git a/base/random.jl b/base/random.jl index b8e8c90d0a220..641721bfd1d31 100644 --- a/base/random.jl +++ b/base/random.jl @@ -1190,7 +1190,6 @@ randexp(dims::Int...) = randexp!(Array{Float64}(dims)) randexp(rng::AbstractRNG, dims::Dims) = randexp!(rng, Array{Float64}(dims)) randexp(rng::AbstractRNG, dims::Int...) = randexp!(rng, Array{Float64}(dims)) - ## random UUID generation immutable UUID diff --git a/base/sharedarray.jl b/base/sharedarray.jl index dca0bf88c0e0e..894c9eb7358b5 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -53,7 +53,6 @@ If an `init` function of the type `initfn(S::SharedArray)` is specified, it is c the participating workers. """ function SharedArray{T,N}(::Type{T}, dims::Dims{N}; init=false, pids=Int[]) - isbits(T) || throw(ArgumentError("type of SharedArray elements must be bits types, got $(T)")) pids, onlocalhost = shared_pids(pids) diff --git a/doc/manual/variables-and-scoping.rst b/doc/manual/variables-and-scoping.rst index 480e836bf9a82..7840eb03ae636 100644 --- a/doc/manual/variables-and-scoping.rst +++ b/doc/manual/variables-and-scoping.rst @@ -400,8 +400,8 @@ storage. Here is an example where the behavior of ``let`` is needed:: Fs = Array{Any}(2) i = 1 while i <= 2 - Fs[i] = ()->i - i += 1 + Fs[i] = ()->i + i += 1 end julia> Fs[1]() @@ -418,10 +418,10 @@ behave identically. We can use ``let`` to create a new binding for Fs = Array{Any}(2) i = 1 while i <= 2 - let i = i - Fs[i] = ()->i - end - i += 1 + let i = i + Fs[i] = ()->i + end + i += 1 end julia> Fs[1]() @@ -437,11 +437,11 @@ block without creating any new bindings: .. doctest:: julia> let - local x = 1 - let - local x = 2 - end - x + local x = 1 + let + local x = 2 + end + x end 1 diff --git a/src/ccalltest.c b/src/ccalltest.c index e9c46e5d789a3..e01aafa300fb2 100644 --- a/src/ccalltest.c +++ b/src/ccalltest.c @@ -535,7 +535,7 @@ JL_DLLEXPORT void *test_echo_p(void *p) { #include -JL_DLLEXPORT __m128i test_m128i(__m128i a, __m128i b, __m128i c, __m128i d ) +JL_DLLEXPORT __m128i test_m128i(__m128i a, __m128i b, __m128i c, __m128i d) { // 64-bit x86 has only level 2 SSE, which does not have a <4 x int32> multiplication, // so we use floating-point instead, and assume caller knows about the hack. @@ -544,7 +544,7 @@ JL_DLLEXPORT __m128i test_m128i(__m128i a, __m128i b, __m128i c, __m128i d ) _mm_cvtepi32_ps(_mm_sub_epi32(c,d))))); } -JL_DLLEXPORT __m128 test_m128(__m128 a, __m128 b, __m128 c, __m128 d ) +JL_DLLEXPORT __m128 test_m128(__m128 a, __m128 b, __m128 c, __m128 d) { return _mm_add_ps(a, _mm_mul_ps(b, _mm_sub_ps(c, d))); } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 7b554811add50..37d3061ef086a 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1079,7 +1079,6 @@ void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sys if (!err.empty()) jl_safe_printf("%s\n", err.c_str()); else { - #ifndef LLVM37 bc_FOS.reset(new formatted_raw_ostream(*bc_OS.get())); #endif @@ -1103,7 +1102,6 @@ void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sys if (!err.empty()) jl_safe_printf("%s\n", err.c_str()); else { - #ifndef LLVM37 obj_FOS.reset(new formatted_raw_ostream(*obj_OS.get())); #endif diff --git a/src/jltypes.c b/src/jltypes.c index 39c0478477e10..12fb96270138b 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2115,7 +2115,7 @@ void jl_precompute_memoized_dt(jl_datatype_t *dt) if (d > dt->depth) dt->depth = d; if (!dt->hastypevars) - dt->hastypevars = jl_has_typevars__(p, 0, NULL, 0); + dt->hastypevars = jl_has_typevars__(p, 0, NULL, 0); if (!dt->haswildcard) dt->haswildcard = jl_has_typevars__(p, 1, NULL, 0); if (dt->isleaftype) diff --git a/test/fft.jl b/test/fft.jl index b851def9aff9f..6aca96d874a02 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -121,7 +121,6 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), # The following capabilities are FFTW only. # They are not available in MKL, and hence do not test them. if Base.fftw_vendor() != :mkl - ifft3_fft3_m3d = fi(f(m3d)) fftd3_m3d = f(m3d,3) @@ -156,9 +155,8 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), @test pfftd3_m3d[i] ≈ true_fftd3_m3d[i] @test pifftd3_fftd3_m3d[i] ≈ m3d[i] @test pfft!d3_m3d[i] ≈ true_fftd3_m3d[i] - @test pifft!d3_fftd3_m3d[i] ≈ m3d[i] + @test pifft!d3_fftd3_m3d[i] ≈ m3d[i] end - end # if fftw_vendor() != :mkl # rfft/rfftn @@ -206,7 +204,6 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), end if Base.fftw_vendor() != :mkl - rfftn_m3d = rfft(m3d) rfftd3_m3d = rfft(m3d,3) @test size(rfftd3_m3d) == size(fftd3_m3d) @@ -225,7 +222,6 @@ for (f,fi,pf,pfi) in ((fft,ifft,plan_fft,plan_ifft), for i = 1:3, j = 1:3, k = 1:2 @test rfftn_m3d[i,j,k] ≈ fftn_m3d[i,j,k] end - end # !mkl end diff --git a/test/file.jl b/test/file.jl index 3c1e4381473e5..ef60ff00fd69c 100644 --- a/test/file.jl +++ b/test/file.jl @@ -1088,7 +1088,6 @@ test2_12992() test2_12992() # issue 13559 - if !is_windows() function test_13559() fn = tempname() diff --git a/test/linalg/hessenberg.jl b/test/linalg/hessenberg.jl index 80d7406f2a801..8ea3b290056bc 100644 --- a/test/linalg/hessenberg.jl +++ b/test/linalg/hessenberg.jl @@ -6,7 +6,6 @@ using Base.Test using Base.LinAlg: BlasComplex, BlasFloat, BlasReal, QRPivoted let n = 10 - srand(1234321) Areal = randn(n,n)/2 @@ -19,7 +18,7 @@ let n = 10 complex(Areal, Aimg) : Areal) - debug && println("\ntype of a: ", eltya, " type of b: ", eltyb, "\n") +debug && println("\ntype of a: ", eltya, " type of b: ", eltyb, "\n") if eltya != BigFloat H = hessfact(A) diff --git a/test/math.jl b/test/math.jl index 09d3cf814d52f..33902962b3999 100644 --- a/test/math.jl +++ b/test/math.jl @@ -47,7 +47,6 @@ for T in (Float16,Float32,Float64) (prevfloat(realmin(T)), nextfloat(one(T),-2)), (nextfloat(zero(T),3), T(0.75)), (nextfloat(zero(T)), T(0.5))] - n = Int(log2(a/b)) @test frexp(a) == (b,n) @test ldexp(b,n) == a @@ -539,7 +538,6 @@ end # digamma for elty in (Float32, Float64) - @test digamma(convert(elty, 9)) ≈ convert(elty, 2.140641477955609996536345) @test digamma(convert(elty, 2.5)) ≈ convert(elty, 0.7031566406452431872257) @test digamma(convert(elty, 0.1)) ≈ convert(elty, -10.42375494041107679516822) diff --git a/test/parse.jl b/test/parse.jl index cd57a6b159c7c..734a4d94c5176 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -516,13 +516,13 @@ end let err = try include_string("module A - function broken() + function broken() - x[1] = some_func( + x[1] = some_func( - end + end - end") + end") catch e e end From d03dc239b617e3e399a6e23b3772802322aaefce Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 9 Jun 2016 19:16:01 -0700 Subject: [PATCH 0203/1117] Use long form function definition of ==(::String, ::String) --- base/strings/basic.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 7308123929a07..167bef7bcbd44 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -128,9 +128,11 @@ isless(a::AbstractString, b::AbstractString) = cmp(a,b) < 0 cmp(a::String, b::String) = lexcmp(a.data, b.data) cmp(a::Symbol, b::Symbol) = Int(sign(ccall(:strcmp, Int32, (Cstring, Cstring), a, b))) -==(a::String, b::String) = - (len = length(a.data)) == length(b.data) && - ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), a.data, b.data, len) == 0 +function ==(a::String, b::String) + len = length(a.data) + return len == length(b.data) && + 0 == ccall(:memcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}, UInt), a.data, b.data, len) +end isless(a::Symbol, b::Symbol) = cmp(a,b) < 0 ## Generic validation functions ## From fabe67bf70030d509bda26d4557bdd683ced3908 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 2 Jul 2016 11:12:59 -0700 Subject: [PATCH 0204/1117] clean up temporary file when done --- test/reflection.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/reflection.jl b/test/reflection.jl index 89cd17593f4f0..3f34fd8c08bcd 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -40,8 +40,9 @@ test_code_reflections(test_bin_reflection, code_llvm) test_code_reflections(test_bin_reflection, code_native) # Issue #16326 -let OLDSTDOUT = STDOUT - redirect_stdout(open(tempname(),"w")) +mktemp() do f, io + OLDSTDOUT = STDOUT + redirect_stdout(io) @test try @code_native map(y->abs(y), rand(3)); true; catch false; end redirect_stdout(OLDSTDOUT) end From 1f1fcebc504e31d6a3d04fd7362dc98cf3de781a Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 2 Jul 2016 14:55:06 -0700 Subject: [PATCH 0205/1117] Reword docstrings for test_broken and test_skip Use :inert instead of :quote in test_broken and test_skip for consistency with the change made in #16082 --- base/test.jl | 118 ++++++++++++++++++++++---------------------- doc/stdlib/test.rst | 24 ++++----- 2 files changed, 70 insertions(+), 72 deletions(-) diff --git a/base/test.jl b/base/test.jl index 0dea0230b9f37..dfaa967cb67f8 100644 --- a/base/test.jl +++ b/base/test.jl @@ -26,7 +26,7 @@ export GenericString Result All tests produce a result object. This object may or may not be -'stored', depending on whether the test is part of a test set. +stored, depending on whether the test is part of a test set. """ abstract Result @@ -90,8 +90,8 @@ end The test condition couldn't be evaluated due to an exception, or it evaluated to something other than a `Bool`. -In the case of `@test_broken` it is used to alert for an -unexpected `Pass` `Result`. +In the case of `@test_broken` it is used to indicate that an +unexpected `Pass` `Result` occurred. """ type Error <: Result test_type::Symbol @@ -130,9 +130,8 @@ end """ Broken -The test condition is the expected (failed) result of a broken test. -This will result in a error if the test succeeds, forcing the tester -to update the test. +The test condition is the expected (failed) result of a broken test, +or was explicitly skipped with `@test_skip`. """ type Broken <: Result test_type::Symbol @@ -187,7 +186,7 @@ Returns a `Pass` `Result` if it does, a `Fail` `Result` if it is `false`, and an `Error` `Result` if it could not be evaluated. """ macro test(ex) - orig_ex = Expr(:inert,ex) + orig_ex = Expr(:inert, ex) result = get_test_result(ex) :(do_test($result, $orig_ex)) end @@ -195,13 +194,13 @@ end """ @test_broken ex -For use to indicate a test that should pass but currently consistently -fails. Tests that the expression `ex` evaluates to `false` or causes an -exception. Returns a `Broken` `Result` if it does, and an `Error` `Result` -if it is `true`. +Indicates a test that should pass but currently consistently fails. +Tests that the expression `ex` evaluates to `false` or causes an +exception. Returns a `Broken` `Result` if it does, or an `Error` `Result` +if the exception evaluates to `true`. """ macro test_broken(ex) - orig_ex = Expr(:quote,ex) + orig_ex = Expr(:inert, ex) result = get_test_result(ex) # code to call do_test with execution result and original expr :(do_broken_test($result, $orig_ex)) @@ -210,12 +209,12 @@ end """ @test_skip ex -For use to indicate a test that should pass but currently intermittently -fails. Does not evaluate the expression, which makes it useful for tests -of not-yet-implemented functionality. +Marks a test that should not be executed but should be included in test +summary reporting as `Broken`. This can be useful for tests that intermittently +fail, or tests of not-yet-implemented functionality. """ macro test_skip(ex) - orig_ex = Expr(:quote, ex) + orig_ex = Expr(:inert, ex) testres = :(Broken(:skipped, $orig_ex)) :(record(get_testset(), $testres)) end @@ -226,7 +225,7 @@ end # evaluate each term in the comparison individually so the results # can be displayed nicely. function get_test_result(ex) - orig_ex = Expr(:inert,ex) + orig_ex = Expr(:inert, ex) # Normalize comparison operator calls to :comparison expressions if isa(ex, Expr) && ex.head == :call && length(ex.args)==3 && (ex.args[1] === :(==) || Base.operator_precedence(ex.args[1]) == comparison_prec) @@ -256,7 +255,7 @@ end # An internal function, called by the code generated by the @test # macro to actually perform the evaluation and manage the result. function do_test(result::ExecutionResult, orig_expr) - # get_testset() returns the most recently added tests set + # get_testset() returns the most recently added test set # We then call record() with this test set and the test result if isa(result, Returned) # expr, in the case of a comparison, will contain the @@ -302,7 +301,7 @@ end Tests that the expression `ex` throws an exception of type `extype`. """ macro test_throws(extype, ex) - orig_ex = Expr(:inert,ex) + orig_ex = Expr(:inert, ex) result = quote try Returned($(esc(ex)), nothing) @@ -318,7 +317,7 @@ end # to evaluate and catch the thrown exception - if it exists function do_test_throws(result::ExecutionResult, orig_expr, extype) if isa(result, Threw) - # Check the right type of exception was thrown + # Check that the right type of exception was thrown if isa(result.exception, extype) testres = Pass(:test_throws, orig_expr, extype, result.exception) else @@ -391,9 +390,8 @@ end fallback_testset = FallbackTestSet() # Records nothing, and throws an error immediately whenever a Fail or -# Error occurs. Takes no action in the event of a Pass result -record(ts::FallbackTestSet, t::Pass) = t -record(ts::FallbackTestSet, t::Broken) = t +# Error occurs. Takes no action in the event of a Pass or Broken result +record(ts::FallbackTestSet, t::Union{Pass,Broken}) = t function record(ts::FallbackTestSet, t::Union{Fail,Error}) println(t) error("There was an error during testing") @@ -417,9 +415,8 @@ type DefaultTestSet <: AbstractTestSet end DefaultTestSet(desc) = DefaultTestSet(desc, [], false) -# For a passing result, simply store the result -record(ts::DefaultTestSet, t::Pass) = (push!(ts.results, t); t) -record(ts::DefaultTestSet, t::Broken) = (push!(ts.results, t); t) +# For a passing or broken result, simply store the result +record(ts::DefaultTestSet, t::Union{Pass,Broken}) = (push!(ts.results, t); t) # For the other result types, immediately print the error message # but do not terminate. Print a backtrace. @@ -453,42 +450,42 @@ function finish(ts::DefaultTestSet) # Calculate the overall number for each type so each of # the test result types are aligned passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = get_test_counts(ts) - total_pass = passes + c_passes - total_fail = fails + c_fails - total_error = errors + c_errors + total_pass = passes + c_passes + total_fail = fails + c_fails + total_error = errors + c_errors total_broken = broken + c_broken - dig_pass = total_pass > 0 ? ndigits(total_pass) : 0 - dig_fail = total_fail > 0 ? ndigits(total_fail) : 0 - dig_error = total_error > 0 ? ndigits(total_error) : 0 + dig_pass = total_pass > 0 ? ndigits(total_pass) : 0 + dig_fail = total_fail > 0 ? ndigits(total_fail) : 0 + dig_error = total_error > 0 ? ndigits(total_error) : 0 dig_broken = total_broken > 0 ? ndigits(total_broken) : 0 total = total_pass + total_fail + total_error + total_broken dig_total = total > 0 ? ndigits(total) : 0 # For each category, take max of digits and header width if there are # tests of that type - pass_width = dig_pass > 0 ? max(length("Pass"), dig_pass) : 0 - fail_width = dig_fail > 0 ? max(length("Fail"), dig_fail) : 0 - error_width = dig_error > 0 ? max(length("Error"), dig_error) : 0 + pass_width = dig_pass > 0 ? max(length("Pass"), dig_pass) : 0 + fail_width = dig_fail > 0 ? max(length("Fail"), dig_fail) : 0 + error_width = dig_error > 0 ? max(length("Error"), dig_error) : 0 broken_width = dig_broken > 0 ? max(length("Broken"), dig_broken) : 0 - total_width = dig_total > 0 ? max(length("Total"), dig_total) : 0 + total_width = dig_total > 0 ? max(length("Total"), dig_total) : 0 # Calculate the alignment of the test result counts by # recursively walking the tree of test sets align = max(get_alignment(ts, 0), length("Test Summary:")) # Print the outer test set header once - print_with_color(:white, rpad("Test Summary:",align," ")" | ") + print_with_color(:white, rpad("Test Summary:",align," "), " | ") if pass_width > 0 print_with_color(:green, lpad("Pass",pass_width," "), " ") end if fail_width > 0 - print_with_color(:red, lpad("Fail",fail_width," ")," ") + print_with_color(:red, lpad("Fail",fail_width," "), " ") end if error_width > 0 - print_with_color(:red, lpad("Error",error_width," ")," ") + print_with_color(:red, lpad("Error",error_width," "), " ") end if broken_width > 0 - print_with_color(:yellow, lpad("Broken",broken_width," ")," ") + print_with_color(:yellow, lpad("Broken",broken_width," "), " ") end if total_width > 0 - print_with_color(:blue, lpad("Total",total_width," ")) + print_with_color(:blue, lpad("Total",total_width, " ")) end println() # Recursively print a summary at every level @@ -526,9 +523,9 @@ function get_test_counts(ts::DefaultTestSet) passes, fails, errors, broken = 0, 0, 0, 0 c_passes, c_fails, c_errors, c_broken = 0, 0, 0, 0 for t in ts.results - isa(t, Pass) && (passes += 1) - isa(t, Fail) && (fails += 1) - isa(t, Error) && (errors += 1) + isa(t, Pass) && (passes += 1) + isa(t, Fail) && (fails += 1) + isa(t, Error) && (errors += 1) isa(t, Broken) && (broken += 1) if isa(t, DefaultTestSet) np, nf, ne, nb, ncp, ncf, nce , ncb = get_test_counts(t) @@ -545,9 +542,9 @@ end # Recursive function that prints out the results at each level of # the tree of test sets function print_counts(ts::DefaultTestSet, depth, align, - pass_width, fail_width, error_width, broken_width, total_width) + pass_width, fail_width, error_width, broken_width, total_width) # Count results by each type at this level, and recursively - # through and child test sets + # through any child test sets passes, fails, errors, broken, c_passes, c_fails, c_errors, c_broken = get_test_counts(ts) subtotal = passes + fails + errors + broken + c_passes + c_fails + c_errors + c_broken @@ -583,7 +580,7 @@ function print_counts(ts::DefaultTestSet, depth, align, if nb > 0 print_with_color(:yellow, lpad(string(nb), broken_width, " "), " ") elseif broken_width > 0 - # No errors at this level, but some at another level + # None broken at this level, but some at another level print(lpad(" ", broken_width), " ") end @@ -599,7 +596,7 @@ function print_counts(ts::DefaultTestSet, depth, align, for t in ts.results if isa(t, DefaultTestSet) print_counts(t, depth + 1, align, - pass_width, fail_width, error_width, broken_width, total_width) + pass_width, fail_width, error_width, broken_width, total_width) end end end @@ -658,7 +655,7 @@ function testset_beginend(args, tests) if desc === nothing desc = "test set" end - # if we're at the top level we'll default to DefaultTestSet. Otherwise + # If we're at the top level we'll default to DefaultTestSet. Otherwise # default to the type of the parent testset if testsettype === nothing testsettype = :(get_testset_depth() == 0 ? DefaultTestSet : typeof(get_testset())) @@ -666,7 +663,7 @@ function testset_beginend(args, tests) # Generate a block of code that initializes a new testset, adds # it to the task local storage, evaluates the test(s), before - # finally removing the testset and giving it a change to take + # finally removing the testset and giving it a chance to take # action (such as reporting the results) quote ts = $(testsettype)($desc; $options...) @@ -688,7 +685,7 @@ end Generate the code for a `@testset` with a `for` loop argument """ function testset_forloop(args, testloop) - # pull out the loop variables. We might need them for generating the + # Pull out the loop variables. We might need them for generating the # description and we'll definitely need them for generating the # comprehension expression at the end loopvars = Expr[] @@ -707,10 +704,10 @@ function testset_forloop(args, testloop) if desc === nothing # No description provided. Generate from the loop variable names v = loopvars[1].args[1] - desc = Expr(:string,"$v = ", esc(v)) # first variable + desc = Expr(:string, "$v = ", esc(v)) # first variable for l = loopvars[2:end] v = l.args[1] - push!(desc.args,", $v = ") + push!(desc.args, ", $v = ") push!(desc.args, esc(v)) end end @@ -728,7 +725,7 @@ function testset_forloop(args, testloop) try $(esc(tests)) catch err - # something in the test block threw an error. Count that as an + # Something in the test block threw an error. Count that as an # error in this test set record(ts, Error(:nontest_error, :(), err, catch_backtrace())) end @@ -859,7 +856,8 @@ test_approx_eq(va, vb, astr, bstr) = """ @test_approx_eq_eps(a, b, tol) -Test two floating point numbers `a` and `b` for equality taking in account a margin of tolerance given by `tol`. +Test two floating point numbers `a` and `b` for equality taking into account +a margin of tolerance given by `tol`. """ macro test_approx_eq_eps(a, b, c) :(test_approx_eq($(esc(a)), $(esc(b)), $(esc(c)), $(string(a)), $(string(b)))) @@ -868,7 +866,8 @@ end """ @test_approx_eq(a, b) -Test two floating point numbers `a` and `b` for equality taking in account small numerical errors. +Test two floating point numbers `a` and `b` for equality taking into account +small numerical errors. """ macro test_approx_eq(a, b) :(test_approx_eq($(esc(a)), $(esc(b)), $(string(a)), $(string(b)))) @@ -878,7 +877,7 @@ end @inferred f(x) Tests that the call expression `f(x)` returns a value of the same type -inferred by the compiler. It's useful to check for type stability. +inferred by the compiler. It is useful to check for type stability. `f(x)` can be any call expression. Returns the result of `f(x)` if the types match, @@ -933,7 +932,7 @@ end # Test approximate equality of vectors or columns of matrices modulo floating # point roundoff and phase (sign) differences. # -# This function is design to test for equality between vectors of floating point +# This function is designed to test for equality between vectors of floating point # numbers when the vectors are defined only up to a global phase or sign, such as # normalized eigenvectors or singular vectors. The global phase is usually # defined consistently, but may occasionally change due to small differences in @@ -949,7 +948,6 @@ end # nothing. function test_approx_eq_modphase{S<:Real,T<:Real}( a::StridedVecOrMat{S}, b::StridedVecOrMat{T}, err=nothing) - m, n = size(a) @test n==size(b, 2) && m==size(b, 1) err === nothing && (err=m^3*(eps(S)+eps(T))) @@ -1002,7 +1000,7 @@ end """ The `GenericString` can be used to test generic string APIs that program to -the `AbstractString` interface. In order to ensure that functions can work +the `AbstractString` interface, in order to ensure that functions can work with string types besides the standard `String` type. """ immutable GenericString <: AbstractString diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index ef5de78ce02f9..ace00c15bcd27 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -92,7 +92,7 @@ symbols, an ``Error`` object is returned and an exception is thrown:: in do_test at test.jl:191 If we expect that evaluating an expression *should* throw an exception, -then we can use :func:`@test_throws` to check this occurs:: +then we can use :func:`@test_throws` to check that this occurs:: julia> @test_throws MethodError foo(:cat) Test Passed @@ -140,7 +140,7 @@ We can put our tests for the ``foo(x)`` function in a test set:: Test Summary: | Pass Total Foo Tests | 3 3 -Test sets can all also be nested:: +Test sets can also be nested:: julia> @testset "Foo Tests" begin @testset "Animals" begin @@ -183,7 +183,7 @@ the details for the failed test sets will be shown:: Foo Tests | 3 1 4 Animals | 2 2 Arrays | 1 1 2 - ERROR: Some tests did not pass: 3 passed, 1 failed, 0 errored. + ERROR: Some tests did not pass: 3 passed, 1 failed, 0 errored, 0 broken. in finish at test.jl:362 @@ -236,19 +236,19 @@ writing new tests. .. Docstring generated from Julia source - Test two floating point numbers ``a`` and ``b`` for equality taking in account small numerical errors. + Test two floating point numbers ``a`` and ``b`` for equality taking into account small numerical errors. .. function:: @test_approx_eq_eps(a, b, tol) .. Docstring generated from Julia source - Test two floating point numbers ``a`` and ``b`` for equality taking in account a margin of tolerance given by ``tol``\ . + Test two floating point numbers ``a`` and ``b`` for equality taking into account a margin of tolerance given by ``tol``\ . .. function:: @inferred f(x) .. Docstring generated from Julia source - Tests that the call expression ``f(x)`` returns a value of the same type inferred by the compiler. It's useful to check for type stability. + Tests that the call expression ``f(x)`` returns a value of the same type inferred by the compiler. It is useful to check for type stability. ``f(x)`` can be any call expression. Returns the result of ``f(x)`` if the types match, and an ``Error`` ``Result`` if it finds different types. @@ -296,17 +296,17 @@ and alerts the user via an ``Error`` if the test succeeds. .. Docstring generated from Julia source - For use to indicate a test that should pass but currently consistently fails. Tests that the expression ``ex`` evaluates to ``false`` or causes an exception. Returns a ``Broken`` ``Result`` if it does, and an ``Error`` ``Result`` if it is ``true``\ . + Indicates a test that should pass but currently consistently fails. Tests that the expression ``ex`` evaluates to ``false`` or causes an exception. Returns a ``Broken`` ``Result`` if it does, or an ``Error`` ``Result`` if the exception evaluates to ``true``\ . -:func:`@test_skip` is also available for use to skip a test without -evaluation in the case of intermittent failures. This test will not run but -gives a `Broken` `Result`. +:func:`@test_skip` is also available to skip a test without evaluation, but +counting the skipped test in the test set reporting. The test will not run but +gives a ``Broken`` ``Result``. .. function:: @test_skip ex .. Docstring generated from Julia source - For use to indicate a test that should pass but currently intermittently fails. Does not evaluate the expression, which makes it useful for tests of not-yet-implemented functionality. + Marks a test that should not be executed but should be included in test summary reporting as ``Broken``\ . This can be useful for tests that intermittently fail, or tests of not-yet-implemented functionality. Creating Custom ``AbstractTestSet`` Types ----------------------------------------- @@ -347,7 +347,7 @@ they are executed, but any result accumulation is the responsibility of the ``Base.Test`` also makes sure that nested ``@testset`` invocations use the same ``AbstractTestSet`` subtype as their parent unless it is set explicitly. It does -not propagate any properties of the testset option inheritance behavior can be +not propagate any properties of the testset. Option inheritance behavior can be implemented by packages using the stack infrastructure that ``Base.Test`` provides. From 8b5b38195370fe48e0935c30495d2bb666b4d101 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 2 Jul 2016 17:09:00 -0700 Subject: [PATCH 0206/1117] Wrap long lines in signals-win.c and indent line continuation in .travis.yml minor comment revisions in array.c --- .travis.yml | 2 +- base/test.jl | 2 +- doc/stdlib/test.rst | 2 +- src/array.c | 5 ++--- src/signals-win.c | 19 ++++++++++++++----- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc87e6ca68b39..5749b77932f45 100644 --- a/.travis.yml +++ b/.travis.yml @@ -102,7 +102,7 @@ script: fi - cd .. && mv julia julia2 - /tmp/julia/bin/julia --precompiled=no -e 'true' && - /tmp/julia/bin/julia-debug --precompiled=no -e 'true' + /tmp/julia/bin/julia-debug --precompiled=no -e 'true' - /tmp/julia/bin/julia -e 'versioninfo()' - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN && diff --git a/base/test.jl b/base/test.jl index dfaa967cb67f8..b0382f3284f5c 100644 --- a/base/test.jl +++ b/base/test.jl @@ -197,7 +197,7 @@ end Indicates a test that should pass but currently consistently fails. Tests that the expression `ex` evaluates to `false` or causes an exception. Returns a `Broken` `Result` if it does, or an `Error` `Result` -if the exception evaluates to `true`. +if the expression evaluates to `true`. """ macro test_broken(ex) orig_ex = Expr(:inert, ex) diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index ace00c15bcd27..05ea608d300ac 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -296,7 +296,7 @@ and alerts the user via an ``Error`` if the test succeeds. .. Docstring generated from Julia source - Indicates a test that should pass but currently consistently fails. Tests that the expression ``ex`` evaluates to ``false`` or causes an exception. Returns a ``Broken`` ``Result`` if it does, or an ``Error`` ``Result`` if the exception evaluates to ``true``\ . + Indicates a test that should pass but currently consistently fails. Tests that the expression ``ex`` evaluates to ``false`` or causes an exception. Returns a ``Broken`` ``Result`` if it does, or an ``Error`` ``Result`` if the expression evaluates to ``true``\ . :func:`@test_skip` is also available to skip a test without evaluation, but counting the skipped test in the test set reporting. The test will not run but diff --git a/src/array.c b/src/array.c index 9580a3eb74d55..0f78e651b410c 100644 --- a/src/array.c +++ b/src/array.c @@ -956,16 +956,15 @@ static NOINLINE ssize_t jl_array_ptr_copy_backward(jl_value_t *owner, } // Unsafe, assume inbounds and that dest and src have the same eltype -// `doffs` and `soffs` are zero based. JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array_t *src, void **src_p, ssize_t n) { assert(dest->flags.ptrarray && src->flags.ptrarray); jl_value_t *owner = jl_array_owner(dest); - // Destination is old and doesn't refer any young object + // Destination is old and doesn't refer to any young object if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { jl_value_t *src_owner = jl_array_owner(src); - // Source is young or might refer young objects + // Source is young or might refer to young objects if (!(jl_astaggedvalue(src_owner)->bits.gc & GC_OLD)) { ssize_t done; if (dest_p < src_p || dest_p > src_p + n) { diff --git a/src/signals-win.c b/src/signals-win.c index 8569658c93664..37fcb1b4e1da5 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -95,7 +95,8 @@ void restore_signals(void) // turn on ctrl-c handler SetConsoleCtrlHandler(NULL, 0); // see if SetThreadStackGuarantee exists - pSetThreadStackGuarantee = (BOOL (*)(PULONG)) jl_dlsym_e(jl_kernel32_handle, "SetThreadStackGuarantee"); + pSetThreadStackGuarantee = (BOOL (*)(PULONG)) jl_dlsym_e(jl_kernel32_handle, + "SetThreadStackGuarantee"); } void jl_throw_in_ctx(jl_value_t *excpt, CONTEXT *ctxThread, int bt) @@ -186,10 +187,12 @@ static LONG WINAPI _exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo, switch (ExceptionInfo->ExceptionRecord->ExceptionCode) { case EXCEPTION_INT_DIVIDE_BY_ZERO: fpreset(); - jl_throw_in_ctx(jl_diverror_exception, ExceptionInfo->ContextRecord,in_ctx); + jl_throw_in_ctx(jl_diverror_exception, + ExceptionInfo->ContextRecord,in_ctx); return EXCEPTION_CONTINUE_EXECUTION; case EXCEPTION_STACK_OVERFLOW: - jl_throw_in_ctx(jl_stackovf_exception, ExceptionInfo->ContextRecord,in_ctx&&pSetThreadStackGuarantee); + jl_throw_in_ctx(jl_stackovf_exception, + ExceptionInfo->ContextRecord,in_ctx&&pSetThreadStackGuarantee); return EXCEPTION_CONTINUE_EXECUTION; case EXCEPTION_ACCESS_VIOLATION: if (jl_addr_is_safepoint(ExceptionInfo->ExceptionRecord->ExceptionInformation[1])) { @@ -210,7 +213,8 @@ static LONG WINAPI _exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo, return EXCEPTION_CONTINUE_EXECUTION; } if (ExceptionInfo->ExceptionRecord->ExceptionInformation[0] == 1) { // writing to read-only memory (e.g. mmap) - jl_throw_in_ctx(jl_readonlymemory_exception, ExceptionInfo->ContextRecord,in_ctx); + jl_throw_in_ctx(jl_readonlymemory_exception, + ExceptionInfo->ContextRecord,in_ctx); return EXCEPTION_CONTINUE_EXECUTION; } } @@ -278,7 +282,12 @@ static LONG WINAPI exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo) } #if defined(_CPU_X86_64_) -JL_DLLEXPORT EXCEPTION_DISPOSITION __julia_personality(PEXCEPTION_RECORD ExceptionRecord, void *EstablisherFrame, PCONTEXT ContextRecord, void *DispatcherContext) { +JL_DLLEXPORT EXCEPTION_DISPOSITION __julia_personality( + PEXCEPTION_RECORD ExceptionRecord, + void *EstablisherFrame, + PCONTEXT ContextRecord, + void *DispatcherContext) +{ EXCEPTION_POINTERS ExceptionInfo; ExceptionInfo.ExceptionRecord = ExceptionRecord; ExceptionInfo.ContextRecord = ContextRecord; From 194c7ecccf058da55e0d26939e94db138210ecd4 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 3 Jul 2016 07:34:32 -0500 Subject: [PATCH 0207/1117] Type{Tuple{}} should not be a subtype of Type{Tuple}. Fixes #17198 --- src/jltypes.c | 2 +- test/core.jl | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index 12fb96270138b..2df57706e2826 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2609,7 +2609,7 @@ static int jl_tuple_subtype_(jl_value_t **child, size_t clenr, if (cseq && !pseq) break; if (ci >= clenf) { - result = pi >= plenf || pseq; + result = pi >= plenf || (pseq && !invariant); break; } if (pi >= plenf && !pseq) diff --git a/test/core.jl b/test/core.jl index 8a9007e4d00bd..e4c32b60dd554 100644 --- a/test/core.jl +++ b/test/core.jl @@ -45,8 +45,8 @@ isnot(x,y) = !is(x,y) @test !(Type{Rational{Int}} <: Type{Rational}) @test Tuple{} <: Tuple{Vararg} @test Tuple{} <: NTuple{TypeVar(:N,true)} -@test Type{Tuple{}} <: Type{Tuple{Vararg}} -@test Type{Tuple{}} <: Type{NTuple{TypeVar(:N,true)}} +@test !(Type{Tuple{}} <: Type{Tuple{Vararg}}) +@test !(Type{Tuple{}} <: Type{NTuple{TypeVar(:N,true)}}) let T = TypeVar(:T,true) @testintersect(Array{Bottom},AbstractArray{T}, Bottom, isnot) @testintersect(Tuple{Type{Ptr{UInt8}},Ptr{Bottom}}, @@ -307,6 +307,9 @@ nttest1{n}(x::NTuple{n,Int}) = n @test Tuple{Vararg{Int32}} <: NTuple{TypeVar(:T),Int32} @test Tuple{Int32,Vararg{Int32}} <: NTuple{TypeVar(:T),Int32} +# #17198 +@test_throws MethodError convert(Tuple{Int}, (1.0, 2.0, 3.0)) + # type declarations abstract Sup_{A,B} From fab2c2ae6be3658b777427d1379b858be407635f Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 3 Jul 2016 08:50:07 -0500 Subject: [PATCH 0208/1117] Fix Cartesian.exprresolve for change in conditional parsing Ref. #15524 --- base/cartesian.jl | 6 +++--- test/cartesian.jl | 3 +++ test/choosetests.jl | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 test/cartesian.jl diff --git a/base/cartesian.jl b/base/cartesian.jl index 16d20243c574d..2d1065b955953 100644 --- a/base/cartesian.jl +++ b/base/cartesian.jl @@ -321,7 +321,7 @@ end function lreplace!(ex::Expr, r::LReplace) # Curly-brace notation, which acts like parentheses if ex.head == :curly && length(ex.args) == 2 && isa(ex.args[1], Symbol) && endswith(string(ex.args[1]), "_") - excurly = Base.Cartesian.exprresolve(lreplace!(ex.args[2], r)) + excurly = exprresolve(lreplace!(ex.args[2], r)) if isa(excurly, Number) return Symbol(ex.args[1],excurly) else @@ -369,8 +369,8 @@ exprresolve_arith(arg) = false, 0 exprresolve_conditional(b::Bool) = true, b function exprresolve_conditional(ex::Expr) - if ex.head == :comparison && isa(ex.args[1], Number) && isa(ex.args[3], Number) - return true, exprresolve_cond_dict[ex.args[2]](ex.args[1], ex.args[3]) + if ex.head == :call && ex.args[1] ∈ keys(exprresolve_cond_dict) && isa(ex.args[2], Number) && isa(ex.args[3], Number) + return true, exprresolve_cond_dict[ex.args[1]](ex.args[2], ex.args[3]) end false, false end diff --git a/test/cartesian.jl b/test/cartesian.jl new file mode 100644 index 0000000000000..907905aff13b3 --- /dev/null +++ b/test/cartesian.jl @@ -0,0 +1,3 @@ +@test Base.Cartesian.exprresolve(:(1 + 3)) == 4 +ex = Base.Cartesian.exprresolve(:(if 5 > 4; :x; else :y; end)) +@test ex.args[2] == QuoteNode(:x) diff --git a/test/choosetests.jl b/test/choosetests.jl index 6a9fb13463df7..9818d9acbeb12 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -33,7 +33,7 @@ function choosetests(choices = []) "markdown", "base64", "serialize", "misc", "threads", "enums", "cmdlineargs", "i18n", "workspace", "libdl", "int", "checked", "intset", "floatfuncs", "compile", "parallel", "inline", - "boundscheck", "error", "ambiguous", "offsetarray" + "boundscheck", "error", "ambiguous", "offsetarray", "cartesian" ] if Base.USE_GPL_LIBS From 8709655b068faf8d1115bb42bef3fb5f7fe8033a Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Sat, 2 Jul 2016 18:52:41 -0700 Subject: [PATCH 0209/1117] Some power machines advertise themselves as powerpc64 and others as ppc64 --- Make.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Make.inc b/Make.inc index ed81c6e1a04e6..7d790227c0187 100644 --- a/Make.inc +++ b/Make.inc @@ -636,8 +636,8 @@ else ISX86:=0 endif -# If we are running on powerpc64, set certain options automatically -ifneq (,$(findstring powerpc64,$(ARCH))) +# If we are running on powerpc64 or ppc64, set certain options automatically +ifneq (,$(filter $(ARCH), powerpc64 ppc64)) JCFLAGS += -fsigned-char override LLVM_VER:=3.8.0 From 83360014508e3df285f56e2331118edf540495e9 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 2 Jan 2016 17:13:28 -0500 Subject: [PATCH 0210/1117] Add GMP patch to fix abort in number test --- deps/gmp.mk | 1 + deps/patches/gmp-exception.patch | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 deps/patches/gmp-exception.patch diff --git a/deps/gmp.mk b/deps/gmp.mk index 87a2d3e984c8c..1d06f24c88ab2 100644 --- a/deps/gmp.mk +++ b/deps/gmp.mk @@ -18,6 +18,7 @@ $(SRCDIR)/srccache/gmp-$(GMP_VER)/configure: $(SRCDIR)/srccache/gmp-$(GMP_VER).t cd $(dir $<) && $(TAR) jxf $< touch -c $@ $(SRCDIR)/srccache/gmp-$(GMP_VER)/patched: $(SRCDIR)/srccache/gmp-$(GMP_VER)/configure + cd $(dir $@) && patch < $(SRCDIR)/patches/gmp-exception.patch echo 1 > $@ $(BUILDDIR)/gmp-$(GMP_VER)/config.status: $(SRCDIR)/srccache/gmp-$(GMP_VER)/configure $(SRCDIR)/srccache/gmp-$(GMP_VER)/patched mkdir -p $(dir $@) diff --git a/deps/patches/gmp-exception.patch b/deps/patches/gmp-exception.patch new file mode 100644 index 0000000000000..e4672be10244b --- /dev/null +++ b/deps/patches/gmp-exception.patch @@ -0,0 +1,35 @@ +diff -r 842c2ba359bf errno.c +--- a/errno.c Sun Jan 24 22:06:51 2016 +0100 ++++ b/errno.c Thu Jan 28 13:37:54 2016 -0500 +@@ -33,24 +33,24 @@ + see https://www.gnu.org/licenses/. */ + + #include ++ ++#include ++ + #include "gmp.h" + #include "gmp-impl.h" + + int gmp_errno = 0; + + +-/* The deliberate divide by zero triggers an exception on most systems. On +- those where it doesn't, for example power and powerpc, use abort instead. +- +- Enhancement: Perhaps raise(SIGFPE) (or the same with kill()) would be +- better than abort. Perhaps it'd be possible to get the BSD style +- FPE_INTDIV_TRAP parameter in there too. */ +- ++/* Use SIGFPE on systems which have it. Otherwise, deliberate divide ++ by zero, which triggers an exception on most systems. On those ++ where it doesn't, for example power and powerpc, use abort instead. */ + void + __gmp_exception (int error_bit) + { + gmp_errno |= error_bit; + __gmp_junk = 10 / __gmp_0; ++ raise (SIGFPE); + abort (); + } + From 75d43411e43f2746345541f5e86ae287207c40b1 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Sun, 3 Jul 2016 11:32:23 -0700 Subject: [PATCH 0211/1117] Tweak powerpc arch for correct detection. --- Make.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index 7d790227c0187..8d34d4be08880 100644 --- a/Make.inc +++ b/Make.inc @@ -637,7 +637,7 @@ ISX86:=0 endif # If we are running on powerpc64 or ppc64, set certain options automatically -ifneq (,$(filter $(ARCH), powerpc64 ppc64)) +ifneq (,$(filter $(ARCH), powerpc64 ppc64le)) JCFLAGS += -fsigned-char override LLVM_VER:=3.8.0 From a8bb5588e88f2d00740831ef845610c9fea06dbb Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 24 Jun 2016 15:58:45 -0400 Subject: [PATCH 0212/1117] Start removing implicit TLS lookup Add `typedef jl_ptls_t` to simplify code. --- src/builtins.c | 9 +-- src/codegen.cpp | 2 +- src/debuginfo.cpp | 8 +-- src/gc-debug.c | 10 +-- src/gc.c | 144 +++++++++++++++++++++++-------------------- src/gc.h | 4 +- src/gf.c | 5 +- src/init.c | 29 +++++---- src/jl_uv.c | 9 +-- src/jlapi.c | 13 ++-- src/julia.h | 4 +- src/julia_internal.h | 15 ++--- src/julia_threads.h | 53 ++++++++-------- src/signals-mach.c | 18 +++--- src/signals-unix.c | 30 +++++---- src/signals-win.c | 22 ++++--- src/stackwalk.c | 19 +++--- src/task.c | 46 ++++++++------ src/threading.c | 80 +++++++++++++----------- src/threading.h | 3 +- ui/repl.c | 2 +- 21 files changed, 280 insertions(+), 245 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index cd91a4fe58382..88df1d39f24dc 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -194,7 +194,7 @@ JL_CALLABLE(jl_f_throw) JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_task_t *current_task = ptls->current_task; // Must have no safepoint eh->prev = current_task->eh; @@ -1583,9 +1583,10 @@ JL_DLLEXPORT size_t jl_static_show_func_sig(JL_STREAM *s, jl_value_t *type) JL_DLLEXPORT void jl_(void *jl_value) { - jl_jmp_buf *old_buf = jl_safe_restore; + jl_ptls_t ptls = jl_get_ptls_states(); + jl_jmp_buf *old_buf = ptls->safe_restore; jl_jmp_buf buf; - jl_safe_restore = &buf; + ptls->safe_restore = &buf; if (!jl_setjmp(buf, 0)) { jl_static_show((JL_STREAM*)STDERR_FILENO, (jl_value_t*)jl_value); jl_printf((JL_STREAM*)STDERR_FILENO,"\n"); @@ -1593,7 +1594,7 @@ JL_DLLEXPORT void jl_(void *jl_value) else { jl_printf((JL_STREAM*)STDERR_FILENO, "\n!!! ERROR in jl_ -- ABORTING !!!\n"); } - jl_safe_restore = old_buf; + ptls->safe_restore = old_buf; } JL_DLLEXPORT void jl_breakpoint(jl_value_t *v) diff --git a/src/codegen.cpp b/src/codegen.cpp index 6277b576a43af..363daed26aee8 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1334,7 +1334,7 @@ static uint64_t compute_obj_symsize(const object::ObjectFile *obj, uint64_t offs extern "C" JL_DLLEXPORT const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); std::string code; llvm::raw_string_ostream stream(code); #ifndef LLVM37 diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 00e91725497b3..0b265cf9d98df 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -248,7 +248,7 @@ class JuliaJITEventListener: public JITEventListener virtual void NotifyFunctionEmitted(const Function &F, void *Code, size_t Size, const EmittedFunctionDetails &Details) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); // This function modify linfo->fptr in GC safe region. // This should be fine since the GC won't scan this field. int8_t gc_state = jl_gc_safe_enter(ptls); @@ -307,7 +307,7 @@ class JuliaJITEventListener: public JITEventListener virtual void NotifyObjectEmitted(const ObjectImage &obj) #endif { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); // This function modify linfo->fptr in GC safe region. // This should be fine since the GC won't scan this field. int8_t gc_state = jl_gc_safe_enter(ptls); @@ -1183,7 +1183,7 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int64_t *se extern "C" JL_DLLEXPORT jl_value_t *jl_get_dobj_data(uint64_t fptr) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); // Used by Gallium.jl const object::ObjectFile *object = NULL; DIContext *context; @@ -1205,7 +1205,7 @@ JL_DLLEXPORT jl_value_t *jl_get_dobj_data(uint64_t fptr) extern "C" JL_DLLEXPORT uint64_t jl_get_section_start(uint64_t fptr) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); // Used by Gallium.jl int8_t gc_state = jl_gc_safe_enter(ptls); std::map &objmap = jl_jit_events->getObjectMap(); diff --git a/src/gc-debug.c b/src/gc-debug.c index d9442090c3f16..2ecb5aadaaec0 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -139,7 +139,7 @@ static void clear_mark(int bits) for (int j = 0; j < 32; j++) { if ((line >> j) & 1) { jl_gc_pagemeta_t *pg = page_metadata(region->pages[pg_i*32 + j].data + GC_PAGE_OFFSET); - jl_tls_states_t *ptls2 = jl_all_tls_states[pg->thread_n]; + jl_ptls_t ptls2 = jl_all_tls_states[pg->thread_n]; jl_gc_pool_t *pool = &ptls2->heap.norm_pools[pg->pool_n]; pv = (jl_taggedvalue_t*)(pg->data + GC_PAGE_OFFSET); char *lim = (char*)pv + GC_PAGE_SZ - GC_PAGE_OFFSET - pool->osize; @@ -173,7 +173,7 @@ static void gc_verify_track(void) pre_mark(); gc_mark_object_list(&to_finalize, 0); for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; gc_mark_object_list(&ptls2->finalizers, 0); } gc_mark_object_list(&finalizer_list_marked, 0); @@ -219,7 +219,7 @@ void gc_verify(void) pre_mark(); gc_mark_object_list(&to_finalize, 0); for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; gc_mark_object_list(&ptls2->finalizers, 0); } gc_mark_object_list(&finalizer_list_marked, 0); @@ -660,7 +660,7 @@ void gc_time_mark_pause(int64_t t0, int64_t scanned_bytes, int64_t last_remset_len = 0; int64_t remset_nptr = 0; for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; last_remset_len += ptls2->heap.last_remset->len; remset_nptr = ptls2->heap.remset_nptr; } @@ -779,7 +779,7 @@ void gc_stats_all_pool(void) size_t nb=0, w, tw=0, no=0,tp=0, nold=0,noldbytes=0, np, nol; for (int i = 0; i < JL_GC_N_POOLS; i++) { for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; size_t b = pool_stats(&ptls2->heap.norm_pools[i], &w, &np, &nol); nb += b; no += (b / ptls2->heap.norm_pools[i].osize); diff --git a/src/gc.c b/src/gc.c index a8657fad95388..e57cc04914261 100644 --- a/src/gc.c +++ b/src/gc.c @@ -71,7 +71,7 @@ NOINLINE uintptr_t gc_get_stack_ptr(void) static void jl_gc_wait_for_the_world(void) { for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; // FIXME: The acquire load pairs with the release stores // in the signal handler of safepoint so we are sure that // all the stores on those threads are visible. However, @@ -205,9 +205,8 @@ static void run_finalizers(void) arraylist_free(&copied_list); } -JL_DLLEXPORT void jl_gc_enable_finalizers(int on) +JL_DLLEXPORT void jl_gc_enable_finalizers(jl_ptls_t ptls, int on) { - jl_tls_states_t *ptls = jl_get_ptls_states(); int old_val = ptls->finalizers_inhibited; int new_val = old_val + (on ? -1 : 1); ptls->finalizers_inhibited = new_val; @@ -237,14 +236,14 @@ static void schedule_all_finalizers(arraylist_t *flist) void jl_gc_run_all_finalizers(void) { for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; schedule_all_finalizers(&ptls2->finalizers); } schedule_all_finalizers(&finalizer_list_marked); run_finalizers(); } -static void gc_add_finalizer_(jl_tls_states_t *ptls, void *v, void *f) +static void gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) { int8_t gc_state = jl_gc_unsafe_enter(ptls); arraylist_t *a = &ptls->finalizers; @@ -262,13 +261,13 @@ static void gc_add_finalizer_(jl_tls_states_t *ptls, void *v, void *f) jl_gc_unsafe_leave(ptls, gc_state); } -STATIC_INLINE void gc_add_ptr_finalizer(jl_tls_states_t *ptls, +STATIC_INLINE void gc_add_ptr_finalizer(jl_ptls_t ptls, jl_value_t *v, void *f) { gc_add_finalizer_(ptls, (void*)(((uintptr_t)v) | 1), f); } -JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_tls_states_t *ptls, +JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_ptls_t ptls, jl_value_t *v, jl_function_t *f) { if (__unlikely(jl_typeis(f, jl_voidpointer_type))) { @@ -279,21 +278,14 @@ JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_tls_states_t *ptls, } } -JL_DLLEXPORT void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) -{ - jl_tls_states_t *ptls = jl_get_ptls_states(); - jl_gc_add_finalizer_th(ptls, v, f); -} - -JL_DLLEXPORT void jl_gc_add_ptr_finalizer(jl_tls_states_t *ptls, - jl_value_t *v, void *f) +JL_DLLEXPORT void jl_gc_add_ptr_finalizer(jl_ptls_t ptls, jl_value_t *v, void *f) { gc_add_ptr_finalizer(ptls, v, f); } JL_DLLEXPORT void jl_finalize(jl_value_t *o) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); JL_LOCK_NOGC(&finalizers_lock); // Copy the finalizers into a temporary list so that code in the finalizer // won't change the list as we loop through them. @@ -305,7 +297,7 @@ JL_DLLEXPORT void jl_finalize(jl_value_t *o) // No need to check the to_finalize list since the user is apparently // still holding a reference to the object for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; finalize_object(&ptls2->finalizers, o, &copied_list, ptls != ptls2); } finalize_object(&finalizer_list_marked, o, &copied_list, 0); @@ -430,6 +422,7 @@ STATIC_INLINE void gc_update_heap_size(int64_t sz_ub, int64_t sz_est) static inline int gc_setmark_big(jl_taggedvalue_t *o, int mark_mode) { + jl_ptls_t ptls = jl_get_ptls_states(); if (gc_verifying) { o->bits.gc = mark_mode; return 0; @@ -441,7 +434,7 @@ static inline int gc_setmark_big(jl_taggedvalue_t *o, int mark_mode) // Reset the object as if it was just allocated hdr->age = 0; gc_big_object_unlink(hdr); - gc_big_object_link(hdr, &jl_thread_heap.big_objects); + gc_big_object_link(hdr, &ptls->heap.big_objects); bits = GC_CLEAN; mark_mode = GC_MARKED; } @@ -550,7 +543,7 @@ static inline int maybe_collect(void) jl_gc_collect(0); return 1; } - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_gc_safepoint_(ptls); return 0; } @@ -559,17 +552,18 @@ static inline int maybe_collect(void) JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc_1w(); jl_set_typeof(wr, jl_weakref_type); wr->value = value; // NOTE: wb not needed here - arraylist_push(&jl_thread_heap.weak_refs, wr); + arraylist_push(&ptls->heap.weak_refs, wr); return wr; } static void sweep_weak_refs(void) { for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; size_t n = 0; size_t ndel = 0; size_t l = ptls2->heap.weak_refs.len; @@ -601,6 +595,7 @@ static void sweep_weak_refs(void) static NOINLINE jl_taggedvalue_t *alloc_big(size_t sz) { + jl_ptls_t ptls = jl_get_ptls_states(); maybe_collect(); size_t offs = offsetof(bigval_t, header); size_t allocsz = LLT_ALIGN(sz + offs, JL_CACHE_BYTE_ALIGNMENT); @@ -617,7 +612,7 @@ static NOINLINE jl_taggedvalue_t *alloc_big(size_t sz) v->sz = allocsz; v->header = 0; v->age = 0; - gc_big_object_link(v, &jl_thread_heap.big_objects); + gc_big_object_link(v, &ptls->heap.big_objects); return (jl_taggedvalue_t*)&v->header; } @@ -664,18 +659,19 @@ static bigval_t **sweep_big_list(int sweep_full, bigval_t **pv) static void sweep_big(int sweep_full) { + jl_ptls_t ptls = jl_get_ptls_states(); gc_time_big_start(); for (int i = 0;i < jl_n_threads;i++) sweep_big_list(sweep_full, &jl_all_tls_states[i]->heap.big_objects); if (sweep_full) { bigval_t **last_next = sweep_big_list(sweep_full, &big_objects_marked); // Move all survivors from big_objects_marked list to big_objects list. - if (jl_thread_heap.big_objects) - jl_thread_heap.big_objects->prev = last_next; - *last_next = jl_thread_heap.big_objects; - jl_thread_heap.big_objects = big_objects_marked; - if (jl_thread_heap.big_objects) - jl_thread_heap.big_objects->prev = &jl_thread_heap.big_objects; + if (ptls->heap.big_objects) + ptls->heap.big_objects->prev = last_next; + *last_next = ptls->heap.big_objects; + ptls->heap.big_objects = big_objects_marked; + if (ptls->heap.big_objects) + ptls->heap.big_objects->prev = &ptls->heap.big_objects; big_objects_marked = NULL; } gc_time_big_end(); @@ -686,17 +682,18 @@ static void sweep_big(int sweep_full) void jl_gc_track_malloced_array(jl_array_t *a) { // This is **NOT** a GC safe point. + jl_ptls_t ptls = jl_get_ptls_states(); mallocarray_t *ma; - if (jl_thread_heap.mafreelist == NULL) { + if (ptls->heap.mafreelist == NULL) { ma = (mallocarray_t*)malloc(sizeof(mallocarray_t)); } else { - ma = jl_thread_heap.mafreelist; - jl_thread_heap.mafreelist = ma->next; + ma = ptls->heap.mafreelist; + ptls->heap.mafreelist = ma->next; } ma->a = a; - ma->next = jl_thread_heap.mallocarrays; - jl_thread_heap.mallocarrays = ma; + ma->next = ptls->heap.mallocarrays; + ptls->heap.mallocarrays = ma; } void jl_gc_count_allocd(size_t sz) @@ -738,7 +735,7 @@ static void sweep_malloced_arrays(void) { gc_time_mallocd_array_start(); for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; mallocarray_t *ma = ptls2->heap.mallocarrays; mallocarray_t **pma = &ptls2->heap.mallocarrays; while (ma != NULL) { @@ -765,7 +762,7 @@ static void sweep_malloced_arrays(void) static inline jl_taggedvalue_t *reset_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_taggedvalue_t *fl) { pg->nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / p->osize; - jl_tls_states_t *ptls2 = jl_all_tls_states[pg->thread_n]; + jl_ptls_t ptls2 = jl_all_tls_states[pg->thread_n]; pg->pool_n = p - ptls2->heap.norm_pools; memset(pg->ages, 0, GC_PAGE_SZ / 8 / p->osize + 1); jl_taggedvalue_t *beg = (jl_taggedvalue_t*)(pg->data + GC_PAGE_OFFSET); @@ -780,6 +777,7 @@ static inline jl_taggedvalue_t *reset_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg static NOINLINE void add_page(jl_gc_pool_t *p) { + jl_ptls_t ptls = jl_get_ptls_states(); char *data = (char*)jl_gc_alloc_page(); if (data == NULL) jl_throw(jl_memory_exception); @@ -787,12 +785,12 @@ static NOINLINE void add_page(jl_gc_pool_t *p) pg->data = data; pg->osize = p->osize; pg->ages = (uint8_t*)malloc(GC_PAGE_SZ / 8 / p->osize + 1); - pg->thread_n = ti_tid; + pg->thread_n = ptls->tid; jl_taggedvalue_t *fl = reset_page(p, pg, p->newpages); p->newpages = fl; } -static inline jl_taggedvalue_t *__pool_alloc(jl_tls_states_t *ptls, +static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, int osize, int end_offset) { @@ -854,27 +852,27 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_tls_states_t *ptls, // use this variant when osize is statically known // and is definitely in sizeclasses // GC_POOL_END_OFS uses an integer division -static inline jl_taggedvalue_t *_pool_alloc(jl_tls_states_t *ptls, +static inline jl_taggedvalue_t *_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, int osize) { return __pool_alloc(ptls, p, osize, GC_POOL_END_OFS(osize)); } -static inline jl_taggedvalue_t *pool_alloc(jl_tls_states_t *ptls, +static inline jl_taggedvalue_t *pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p) { return __pool_alloc(ptls, p, p->osize, p->end_offset); } // Size includes the tag!! -JL_DLLEXPORT void *jl_gc_pool_alloc(jl_tls_states_t *ptls, jl_gc_pool_t *p, +JL_DLLEXPORT void *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, int osize, int end_offset) { return jl_valueof(__pool_alloc(ptls, p, osize, end_offset)); } // Size includes the tag!! -JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_tls_states_t *ptls, size_t allocsz) +JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz) { return jl_valueof(alloc_big(allocsz)); } @@ -939,7 +937,7 @@ int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset) int klass = szclass(allocsz); *osize = sizeclasses[klass]; *end_offset = GC_POOL_END_OFS(*osize); - return (int)(intptr_t)(&((jl_tls_states_t*)0)->heap.norm_pools[klass]); + return (int)(intptr_t)(&((jl_ptls_t)0)->heap.norm_pools[klass]); } // sweep phase @@ -1075,7 +1073,7 @@ static void sweep_pool_region(jl_taggedvalue_t ***pfl, int region_i, int sweep_f jl_gc_pagemeta_t *pg = ®ion->meta[pg_i*32 + j]; int p_n = pg->pool_n; int t_n = pg->thread_n; - jl_tls_states_t *ptls2 = jl_all_tls_states[t_n]; + jl_ptls_t ptls2 = jl_all_tls_states[t_n]; jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; int osize = pg->osize; pfl[t_n * JL_GC_N_POOLS + p_n] = sweep_page(p, pg, pfl[t_n * JL_GC_N_POOLS + p_n], sweep_full, osize); @@ -1106,7 +1104,7 @@ static void gc_sweep_pool(int sweep_full) // update metadata of pages that were pointed to by freelist or newpages from a pool // i.e. pages being the current allocation target for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; for (int i = 0; i < JL_GC_N_POOLS; i++) { jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; jl_taggedvalue_t *last = p->freelist; @@ -1137,7 +1135,7 @@ static void gc_sweep_pool(int sweep_full) // null out terminal pointers of free lists and cache back pg->nfree in the jl_gc_pool_t for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; for (int i = 0; i < JL_GC_N_POOLS; i++) { jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; *pfl[t_i * JL_GC_N_POOLS + i] = NULL; @@ -1175,7 +1173,7 @@ static void grow_mark_stack(void) static void reset_remset(void) { for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; arraylist_t *tmp = ptls2->heap.remset; ptls2->heap.remset = ptls2->heap.last_remset; ptls2->heap.last_remset = tmp; @@ -1186,6 +1184,7 @@ static void reset_remset(void) JL_DLLEXPORT void jl_gc_queue_root(jl_value_t *ptr) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_taggedvalue_t *o = jl_astaggedvalue(ptr); #ifndef JULIA_ENABLE_THREADING // Disable this assert since it can happen with multithreading (same @@ -1198,19 +1197,20 @@ JL_DLLEXPORT void jl_gc_queue_root(jl_value_t *ptr) // write GC_OLD to the GC bits outside GC. This could cause // duplicated objects in the remset but that shouldn't be a problem. o->bits.gc = GC_MARKED; - arraylist_push(jl_thread_heap.remset, ptr); - jl_thread_heap.remset_nptr++; // conservative + arraylist_push(ptls->heap.remset, ptr); + ptls->heap.remset_nptr++; // conservative } void gc_queue_binding(jl_binding_t *bnd) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_taggedvalue_t *buf = jl_astaggedvalue(bnd); #ifndef JULIA_ENABLE_THREADING // Will fail for multithreading. See `jl_gc_queue_root` assert(buf->bits.gc == GC_OLD_MARKED); #endif buf->bits.gc = GC_MARKED; - arraylist_push(&jl_thread_heap.rem_bindings, bnd); + arraylist_push(&ptls->heap.rem_bindings, bnd); } static int push_root(jl_value_t *v, int d, int); @@ -1308,7 +1308,7 @@ static void gc_mark_task_stack(jl_task_t *ta, int d) { int stkbuf = (ta->stkbuf != (void*)(intptr_t)-1 && ta->stkbuf != NULL); int16_t tid = ta->tid; - jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_ptls_t ptls2 = jl_all_tls_states[tid]; if (stkbuf) { #ifdef COPY_STACKS gc_setmark_buf(ta->stkbuf, jl_astaggedvalue(ta)->bits.gc, ta->bufsz); @@ -1377,6 +1377,7 @@ JL_DLLEXPORT void jl_gc_lookfor(jl_value_t *v) { lookforme = v; } // v is added to the remset static int push_root(jl_value_t *v, int d, int bits) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(v != NULL); jl_value_t *vt = jl_typeof(v); int refyoung = 0, nptr = 0; @@ -1530,9 +1531,9 @@ static int push_root(jl_value_t *v, int d, int bits) if (gc_verifying) return bits; if ((bits == GC_OLD_MARKED) && refyoung) { - jl_thread_heap.remset_nptr += nptr; + ptls->heap.remset_nptr += nptr; // v is an old object referencing young objects - arraylist_push(jl_thread_heap.remset, v); + arraylist_push(ptls->heap.remset, v); } return bits; @@ -1567,7 +1568,7 @@ void pre_mark(void) size_t i; for(i=0; i < jl_n_threads; i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; // current_module might not have a value when the thread is not // running. if (ptls2->current_module) @@ -1660,7 +1661,7 @@ static volatile uint64_t jl_gc_disable_counter = 0; JL_DLLEXPORT int jl_gc_enable(int on) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); int prev = !ptls->disable_gc; ptls->disable_gc = (on == 0); if (on && !prev) { @@ -1680,7 +1681,8 @@ JL_DLLEXPORT int jl_gc_enable(int on) } JL_DLLEXPORT int jl_gc_is_enabled(void) { - return !jl_get_ptls_states()->disable_gc; + jl_ptls_t ptls = jl_get_ptls_states(); + return !ptls->disable_gc; } JL_DLLEXPORT int64_t jl_gc_total_bytes(void) @@ -1718,7 +1720,7 @@ static void _jl_gc_collect(int full, char *stack_hi) // 1. mark every object in the remset reset_remset(); for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; // avoid counting remembered objects & bindings twice in perm_scanned_bytes for (int i = 0; i < ptls2->heap.last_remset->len; i++) { jl_value_t *item = (jl_value_t*)ptls2->heap.last_remset->items[i]; @@ -1738,7 +1740,7 @@ static void _jl_gc_collect(int full, char *stack_hi) // 2. mark every object in a remembered binding for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; int n_bnd_refyoung = 0; for (int i = 0; i < ptls2->heap.rem_bindings.len; i++) { jl_binding_t *ptr = (jl_binding_t*)ptls2->heap.rem_bindings.items[i]; @@ -1767,7 +1769,7 @@ static void _jl_gc_collect(int full, char *stack_hi) // `finalizer_list` by `post_mark` size_t orig_marked_len = finalizer_list_marked.len; for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; post_mark(&ptls2->finalizers); } if (prev_sweep_full) { @@ -1775,7 +1777,7 @@ static void _jl_gc_collect(int full, char *stack_hi) orig_marked_len = 0; } for (int i = 0;i < jl_n_threads;i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[i]; + jl_ptls_t ptls2 = jl_all_tls_states[i]; gc_mark_object_list(&ptls2->finalizers, 0); } gc_mark_object_list(&finalizer_list_marked, orig_marked_len); @@ -1850,7 +1852,7 @@ static void _jl_gc_collect(int full, char *stack_hi) // 6. if it is a quick sweep, put back the remembered objects in queued state // so that we don't trigger the barrier again on them. for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t_i]; + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; if (!sweep_full) { for (int i = 0; i < ptls2->heap.remset->len; i++) { jl_astaggedvalue(ptls2->heap.remset->items[i])->bits.gc = GC_MARKED; @@ -1887,7 +1889,7 @@ static void _jl_gc_collect(int full, char *stack_hi) JL_DLLEXPORT void jl_gc_collect(int full) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); if (jl_gc_disable_counter) { gc_num.deferred_alloc += (gc_num.allocd + gc_num.interval); gc_num.allocd = -(int64_t)gc_num.interval; @@ -1933,7 +1935,7 @@ JL_DLLEXPORT void jl_gc_collect(int full) void *allocb(size_t sz) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_taggedvalue_t *b = NULL; size_t allocsz = sz + sizeof(jl_taggedvalue_t); if (allocsz < sz) // overflow in adding offs, size was "negative" @@ -1951,7 +1953,7 @@ void *allocb(size_t sz) JL_DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); size_t allocsz = sz + sizeof(jl_taggedvalue_t); if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); @@ -1963,7 +1965,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) JL_DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); const int sz = sizeof(jl_taggedvalue_t); void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); return jl_valueof(tag); @@ -1971,7 +1973,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) JL_DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT); void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); @@ -1980,7 +1982,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) JL_DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 2, JL_SMALL_BYTE_ALIGNMENT); void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); @@ -1989,7 +1991,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 3, JL_SMALL_BYTE_ALIGNMENT); void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); @@ -1997,7 +1999,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) } // Per-thread initialization (when threading is fully implemented) -void jl_mk_thread_heap(jl_tls_states_t *ptls) +void jl_mk_thread_heap(jl_ptls_t ptls) { jl_thread_heap_t *heap = &ptls->heap; const int *szc = sizeclasses; @@ -2174,6 +2176,12 @@ JL_DLLEXPORT void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, return b; } +JL_DLLEXPORT void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + jl_gc_add_finalizer_th(ptls, v, f); +} + #ifdef __cplusplus } #endif diff --git a/src/gc.h b/src/gc.h index 2d6cbf6bf538c..0c6310aa5bda2 100644 --- a/src/gc.h +++ b/src/gc.h @@ -147,7 +147,7 @@ typedef struct { uint16_t osize; // size of each object in this page uint16_t fl_begin_offset; // offset of first free object in this page uint16_t fl_end_offset; // offset of last free object in this page - uint16_t thread_n; // index (into jl_thread_heap) of heap that owns this page + uint16_t thread_n; // thread id of the heap that owns this page char *data; uint8_t *ages; } jl_gc_pagemeta_t; @@ -279,8 +279,6 @@ void gc_mark_object_list(arraylist_t *list, size_t start); void visit_mark_stack(void); void gc_debug_init(void); -#define jl_thread_heap (jl_get_ptls_states()->heap) - // GC pages void jl_gc_init_page(void); diff --git a/src/gf.c b/src/gf.c index f592733ab2392..7f5262808603e 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1087,6 +1087,7 @@ void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletyp void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_value_t *fargs[3] = { (jl_value_t*)jl_methoderror_type, (jl_value_t*)f, @@ -1099,8 +1100,8 @@ void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args) jl_printf((JL_STREAM*)STDERR_FILENO, "A method error occurred before the base MethodError type was defined. Aborting...\n"); jl_static_show((JL_STREAM*)STDERR_FILENO,(jl_value_t*)f); jl_printf((JL_STREAM*)STDERR_FILENO,"\n"); jl_static_show((JL_STREAM*)STDERR_FILENO,args); jl_printf((JL_STREAM*)STDERR_FILENO,"\n"); - jl_bt_size = rec_backtrace(jl_bt_data, JL_MAX_BT_SIZE); - jl_critical_error(0, NULL, jl_bt_data, &jl_bt_size); + ptls->bt_size = rec_backtrace(ptls->bt_data, JL_MAX_BT_SIZE); + jl_critical_error(0, NULL, ptls->bt_data, &ptls->bt_size); abort(); } // not reached diff --git a/src/init.c b/src/init.c index bf0a45448c2fe..4f2d3532423ce 100644 --- a/src/init.c +++ b/src/init.c @@ -87,6 +87,7 @@ size_t jl_page_size; void jl_init_stack_limits(int ismaster) { + jl_ptls_t ptls = jl_get_ptls_states(); #ifdef _OS_WINDOWS_ (void)ismaster; # ifdef _COMPILER_MICROSOFT_ @@ -104,8 +105,8 @@ void jl_init_stack_limits(int ismaster) # endif # endif // https://en.wikipedia.org/wiki/Win32_Thread_Information_Block - jl_stack_hi = (char*)tib[1]; // Stack Base / Bottom of stack (high address) - jl_stack_lo = (char*)tib[2]; // Stack Limit / Ceiling of stack (low address) + ptls->stack_hi = (char*)tib[1]; // Stack Base / Bottom of stack (high address) + ptls->stack_lo = (char*)tib[2]; // Stack Limit / Ceiling of stack (low address) #else # ifdef JULIA_ENABLE_THREADING // Only use pthread_*_np functions to get stack address for non-master @@ -119,8 +120,8 @@ void jl_init_stack_limits(int ismaster) size_t stacksize; pthread_attr_getstack(&attr, &stackaddr, &stacksize); pthread_attr_destroy(&attr); - jl_stack_lo = (char*)stackaddr; - jl_stack_hi = (char*)stackaddr + stacksize; + ptls->stack_lo = (char*)stackaddr; + ptls->stack_hi = (char*)stackaddr + stacksize; return; # elif defined(_OS_DARWIN_) extern void *pthread_get_stackaddr_np(pthread_t thread); @@ -128,8 +129,8 @@ void jl_init_stack_limits(int ismaster) pthread_t thread = pthread_self(); void *stackaddr = pthread_get_stackaddr_np(thread); size_t stacksize = pthread_get_stacksize_np(thread); - jl_stack_lo = (char*)stackaddr; - jl_stack_hi = (char*)stackaddr + stacksize; + ptls->stack_lo = (char*)stackaddr; + ptls->stack_hi = (char*)stackaddr + stacksize; return; # elif defined(_OS_FREEBSD_) pthread_attr_t attr; @@ -139,8 +140,8 @@ void jl_init_stack_limits(int ismaster) size_t stacksize; pthread_attr_getstack(&attr, &stackaddr, &stacksize); pthread_attr_destroy(&attr); - jl_stack_lo = (char*)stackaddr; - jl_stack_hi = (char*)stackaddr + stacksize; + ptls->stack_lo = (char*)stackaddr; + ptls->stack_hi = (char*)stackaddr + stacksize; return; # else # warning "Getting stack size for thread is not supported." @@ -152,8 +153,8 @@ void jl_init_stack_limits(int ismaster) struct rlimit rl; getrlimit(RLIMIT_STACK, &rl); size_t stack_size = rl.rlim_cur; - jl_stack_hi = (char*)&stack_size; - jl_stack_lo = jl_stack_hi - stack_size; + ptls->stack_hi = (char*)&stack_size; + ptls->stack_lo = ptls->stack_hi - stack_size; #endif } @@ -525,7 +526,8 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) static void jl_set_io_wait(int v) { - jl_get_ptls_states()->io_wait = v; + jl_ptls_t ptls = jl_get_ptls_states(); + ptls->io_wait = v; } void _julia_init(JL_IMAGE_SEARCH rel) @@ -534,6 +536,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) // Make sure we finalize the tls callback before starting any threads. jl_get_ptls_states_getter(); #endif + jl_ptls_t ptls = jl_get_ptls_states(); jl_safepoint_init(); libsupport_init(); ios_set_io_wait_func = jl_set_io_wait; @@ -614,7 +617,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_init_frontend(); jl_init_types(); jl_init_tasks(); - jl_init_root_task(jl_stack_lo, jl_stack_hi-jl_stack_lo); + jl_init_root_task(ptls->stack_lo, ptls->stack_hi-ptls->stack_lo); init_stdio(); // libuv stdio cleanup depends on jl_init_tasks() because JL_TRY is used in jl_atexit_hook() @@ -793,7 +796,7 @@ void jl_get_builtin_hooks(void) { int t; for (t = 0; t < jl_n_threads; t++) { - jl_tls_states_t *ptls2 = jl_all_tls_states[t]; + jl_ptls_t ptls2 = jl_all_tls_states[t]; ptls2->root_task->tls = jl_nothing; ptls2->root_task->consumers = jl_nothing; ptls2->root_task->donenotify = jl_nothing; diff --git a/src/jl_uv.c b/src/jl_uv.c index e4beeafc1f178..3cdc6c61340c1 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -136,7 +136,7 @@ JL_DLLEXPORT void *jl_uv_write_handle(uv_write_t *req) { return req->handle; } JL_DLLEXPORT int jl_run_once(uv_loop_t *loop) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); if (loop) { loop->stop_flag = 0; jl_gc_safepoint_(ptls); @@ -147,7 +147,7 @@ JL_DLLEXPORT int jl_run_once(uv_loop_t *loop) JL_DLLEXPORT void jl_run_event_loop(uv_loop_t *loop) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); if (loop) { loop->stop_flag = 0; jl_gc_safepoint_(ptls); @@ -157,7 +157,7 @@ JL_DLLEXPORT void jl_run_event_loop(uv_loop_t *loop) JL_DLLEXPORT int jl_process_events(uv_loop_t *loop) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); if (loop) { loop->stop_flag = 0; jl_gc_safepoint_(ptls); @@ -351,7 +351,8 @@ JL_DLLEXPORT int jl_fs_chown(char *path, int uid, int gid) JL_DLLEXPORT int jl_fs_write(int handle, const char *data, size_t len, int64_t offset) { - if (jl_safe_restore) + jl_ptls_t ptls = jl_get_ptls_states(); + if (ptls->safe_restore) return write(handle, data, len); uv_fs_t req; uv_buf_t buf[1]; diff --git a/src/jlapi.c b/src/jlapi.c index 250726dbd70a7..945e1f32d6d2e 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -229,7 +229,8 @@ JL_DLLEXPORT void jl_sigatomic_begin(void) JL_DLLEXPORT void jl_sigatomic_end(void) { - if (jl_get_ptls_states()->defer_signal == 0) + jl_ptls_t ptls = jl_get_ptls_states(); + if (ptls->defer_signal == 0) jl_error("sigatomic_end called in non-sigatomic region"); JL_SIGATOMIC_END(); } @@ -326,31 +327,31 @@ JL_DLLEXPORT jl_value_t *(jl_typeof)(jl_value_t *v) JL_DLLEXPORT int8_t (jl_gc_unsafe_enter)(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); return jl_gc_unsafe_enter(ptls); } JL_DLLEXPORT void (jl_gc_unsafe_leave)(int8_t state) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_gc_unsafe_leave(ptls, state); } JL_DLLEXPORT int8_t (jl_gc_safe_enter)(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); return jl_gc_safe_enter(ptls); } JL_DLLEXPORT void (jl_gc_safe_leave)(int8_t state) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_gc_safe_leave(ptls, state); } JL_DLLEXPORT void (jl_gc_safepoint)(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_gc_safepoint_(ptls); } diff --git a/src/julia.h b/src/julia.h index 201212a78073d..d8f6ec87dd89e 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1508,7 +1508,7 @@ static inline void jl_lock_frame_pop(void) STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_task_t *current_task = ptls->current_task; // `eh` may not be `jl_current_task->eh`. See `jl_pop_handler` // This function should **NOT** have any safepoint before the ones at the @@ -1532,7 +1532,7 @@ STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh) jl_gc_safepoint_(ptls); } if (old_defer_signal && !eh->defer_signal) { - jl_sigint_safepoint(); + jl_sigint_safepoint(ptls); } } diff --git a/src/julia_internal.h b/src/julia_internal.h index 801997e027a42..ac386d0be9996 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -32,20 +32,17 @@ typedef void (*tracer_cb)(jl_value_t *tracee); void jl_call_tracer(tracer_cb callback, jl_value_t *tracee); extern size_t jl_page_size; -#define jl_stack_lo (jl_get_ptls_states()->stack_lo) -#define jl_stack_hi (jl_get_ptls_states()->stack_hi) extern jl_function_t *jl_typeinf_func; #if defined(JL_USE_INTEL_JITEVENTS) extern unsigned sig_stack_size; #endif -#define jl_safe_restore jl_get_ptls_states()->safe_restore JL_DLLEXPORT extern int jl_lineno; JL_DLLEXPORT extern const char *jl_filename; -JL_DLLEXPORT void *jl_gc_pool_alloc(jl_tls_states_t *ptls, jl_gc_pool_t *p, +JL_DLLEXPORT void *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, int osize, int end_offset); -JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_tls_states_t *ptls, size_t allocsz); +JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz); int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset); STATIC_INLINE jl_value_t *newobj(jl_value_t *type, size_t nfields) @@ -155,7 +152,7 @@ JL_CALLABLE(jl_f_intrinsic_call); extern jl_function_t *jl_unprotect_stack_func; void jl_install_default_signal_handlers(void); void restore_signals(void); -void jl_install_thread_signal_handler(void); +void jl_install_thread_signal_handler(jl_ptls_t ptls); jl_fptr_t jl_get_builtin_fptr(jl_value_t *b); @@ -250,7 +247,7 @@ void jl_init_restored_modules(jl_array_t *init_order); void jl_init_signal_async(void); void jl_init_debuginfo(void); void jl_init_runtime_ccall(void); -void jl_mk_thread_heap(jl_tls_states_t *ptls); +void jl_mk_thread_heap(jl_ptls_t ptls); void _julia_init(JL_IMAGE_SEARCH rel); @@ -309,7 +306,7 @@ void jl_wake_libuv(void); jl_get_ptls_states_func jl_get_ptls_states_getter(void); static inline void jl_set_gc_and_wait(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); // reading own gc state doesn't need atomic ops since no one else // should store to it. int8_t state = jl_gc_state(ptls); @@ -376,8 +373,6 @@ typedef unw_cursor_t bt_cursor_t; # define JL_UNW_HAS_FORMAT_IP 1 # endif #endif -#define jl_bt_data (jl_get_ptls_states()->bt_data) -#define jl_bt_size (jl_get_ptls_states()->bt_size) size_t rec_backtrace(uintptr_t *data, size_t maxsize); size_t rec_backtrace_ctx(uintptr_t *data, size_t maxsize, bt_context_t *ctx); #ifdef LIBOSXUNWIND diff --git a/src/julia_threads.h b/src/julia_threads.h index 89cd65ceca0c8..48b03f91d93ea 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -111,6 +111,7 @@ typedef struct _jl_tls_states_t { int finalizers_inhibited; arraylist_t finalizers; } jl_tls_states_t; +typedef jl_tls_states_t *jl_ptls_t; #ifdef __MIC__ # define jl_cpu_pause() _mm_delay_64(100) @@ -337,7 +338,7 @@ JL_DLLEXPORT void (jl_cpu_pause)(void); JL_DLLEXPORT void (jl_cpu_wake)(void); // Accessing the tls variables, gc safepoint and gc states -JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void); +JL_DLLEXPORT JL_CONST_FUNC jl_ptls_t (jl_get_ptls_states)(void); // This triggers a SegFault when we are in GC // Assign it to a variable to make sure the compiler emit the load // and to avoid Clang warning for -Wunused-volatile-lvalue @@ -347,32 +348,32 @@ JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void); jl_signal_fence(); \ (void)safepoint_load; \ } while (0) -#define jl_sigint_safepoint() do { \ - jl_signal_fence(); \ - size_t safepoint_load = jl_get_ptls_states()->safepoint[-1]; \ - jl_signal_fence(); \ - (void)safepoint_load; \ +#define jl_sigint_safepoint(ptls) do { \ + jl_signal_fence(); \ + size_t safepoint_load = ptls->safepoint[-1]; \ + jl_signal_fence(); \ + (void)safepoint_load; \ } while (0) #ifndef JULIA_ENABLE_THREADING extern JL_DLLEXPORT jl_tls_states_t jl_tls_states; #define jl_get_ptls_states() (&jl_tls_states) #define jl_gc_state(ptls) ((int8_t)0) -STATIC_INLINE int8_t jl_gc_state_set(jl_tls_states_t *ptls, - int8_t state, int8_t old_state) +STATIC_INLINE int8_t jl_gc_state_set(jl_ptls_t ptls, int8_t state, + int8_t old_state) { (void)ptls; (void)state; return old_state; } #else // ifndef JULIA_ENABLE_THREADING -typedef jl_tls_states_t *(*jl_get_ptls_states_func)(void); +typedef jl_ptls_t (*jl_get_ptls_states_func)(void); #if !defined(_OS_DARWIN_) && !defined(_OS_WINDOWS_) JL_DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f); #endif // Make sure jl_gc_state() is always a rvalue #define jl_gc_state(ptls) ((int8_t)ptls->gc_state) -STATIC_INLINE int8_t jl_gc_state_set(jl_tls_states_t *ptls, - int8_t state, int8_t old_state) +STATIC_INLINE int8_t jl_gc_state_set(jl_ptls_t ptls, int8_t state, + int8_t old_state) { ptls->gc_state = state; // A safe point is required if we transition from GC-safe region to @@ -382,7 +383,7 @@ STATIC_INLINE int8_t jl_gc_state_set(jl_tls_states_t *ptls, return old_state; } #endif // ifndef JULIA_ENABLE_THREADING -STATIC_INLINE int8_t jl_gc_state_save_and_set(jl_tls_states_t *ptls, +STATIC_INLINE int8_t jl_gc_state_save_and_set(jl_ptls_t ptls, int8_t state) { return jl_gc_state_set(ptls, state, jl_gc_state(ptls)); @@ -400,7 +401,7 @@ JL_DLLEXPORT void (jl_gc_safepoint)(void); #define JL_SIGATOMIC_END() do { \ jl_signal_fence(); \ if (--jl_get_ptls_states()->defer_signal == 0) { \ - jl_sigint_safepoint(); \ + jl_sigint_safepoint(jl_get_ptls_states()); \ } \ } while (0) @@ -410,7 +411,7 @@ typedef struct { uint32_t count; } jl_mutex_t; -JL_DLLEXPORT void jl_gc_enable_finalizers(int on); +JL_DLLEXPORT void jl_gc_enable_finalizers(jl_ptls_t ptls, int on); static inline void jl_lock_frame_push(jl_mutex_t *lock); static inline void jl_lock_frame_pop(void); @@ -436,7 +437,7 @@ static inline void jl_mutex_wait(jl_mutex_t *lock, int safepoint) return; } if (safepoint) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_gc_safepoint_(ptls); } jl_cpu_pause(); @@ -451,10 +452,11 @@ static inline void jl_mutex_lock_nogc(jl_mutex_t *lock) static inline void jl_mutex_lock(jl_mutex_t *lock) { + jl_ptls_t ptls = jl_get_ptls_states(); JL_SIGATOMIC_BEGIN(); jl_mutex_wait(lock, 1); jl_lock_frame_push(lock); - jl_gc_enable_finalizers(0); + jl_gc_enable_finalizers(ptls, 0); } static inline void jl_mutex_unlock_nogc(jl_mutex_t *lock) @@ -469,8 +471,9 @@ static inline void jl_mutex_unlock_nogc(jl_mutex_t *lock) static inline void jl_mutex_unlock(jl_mutex_t *lock) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_mutex_unlock_nogc(lock); - jl_gc_enable_finalizers(1); + jl_gc_enable_finalizers(ptls, 1); jl_lock_frame_pop(); JL_SIGATOMIC_END(); } @@ -486,15 +489,15 @@ static inline void jl_mutex_check_type(jl_mutex_t *m) { (void)m; } -#define JL_LOCK(m) do { \ - JL_SIGATOMIC_BEGIN(); \ - jl_gc_enable_finalizers(0); \ - jl_mutex_check_type(m); \ +#define JL_LOCK(m) do { \ + JL_SIGATOMIC_BEGIN(); \ + jl_gc_enable_finalizers(jl_get_ptls_states(), 0); \ + jl_mutex_check_type(m); \ } while (0) -#define JL_UNLOCK(m) do { \ - jl_gc_enable_finalizers(1); \ - jl_mutex_check_type(m); \ - JL_SIGATOMIC_END(); \ +#define JL_UNLOCK(m) do { \ + jl_gc_enable_finalizers(jl_get_ptls_states(), 1); \ + jl_mutex_check_type(m); \ + JL_SIGATOMIC_END(); \ } while (0) #define JL_LOCK_NOGC(m) jl_mutex_check_type(m) #define JL_UNLOCK_NOGC(m) jl_mutex_check_type(m) diff --git a/src/signals-mach.c b/src/signals-mach.c index 1f5a9950f4357..dd44d67f527fe 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -27,7 +27,7 @@ void jl_mach_gc_end(void) uintptr_t item = (uintptr_t)suspended_threads.items[i]; int16_t tid = (int16_t)item; int8_t gc_state = (int8_t)(item >> 8); - jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_ptls_t ptls2 = jl_all_tls_states[tid]; jl_atomic_store_release(&ptls2->gc_state, gc_state); thread_resume(pthread_mach_thread_np(ptls2->system_id)); } @@ -36,7 +36,7 @@ void jl_mach_gc_end(void) // Suspend the thread and return `1` if the GC is running. // Otherwise return `0` -static int jl_mach_gc_wait(jl_tls_states_t *ptls2, +static int jl_mach_gc_wait(jl_ptls_t ptls2, mach_port_t thread, int16_t tid) { jl_mutex_lock_nogc(&safepoint_lock); @@ -126,7 +126,7 @@ void jl_throw_in_thread(int tid, mach_port_t thread, jl_value_t *exception) x86_thread_state64_t state; kern_return_t ret = thread_get_state(thread, x86_THREAD_STATE64, (thread_state_t)&state, &count); HANDLE_MACH_ERROR("thread_get_state", ret); - jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_ptls_t ptls2 = jl_all_tls_states[tid]; ptls2->bt_size = rec_backtrace_ctx(ptls2->bt_data, JL_MAX_BT_SIZE, (bt_context_t*)&state); @@ -166,9 +166,9 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, #endif int16_t tid; #ifdef JULIA_ENABLE_THREADING - jl_tls_states_t *ptls2 = NULL; + jl_ptls_t ptls2 = NULL; for (tid = 0;tid < jl_n_threads;tid++) { - jl_tls_states_t *_ptls2 = jl_all_tls_states[tid]; + jl_ptls_t _ptls2 = jl_all_tls_states[tid]; if (pthread_mach_thread_np(_ptls2->system_id) == thread) { ptls2 = _ptls2; break; @@ -182,7 +182,7 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, return KERN_INVALID_ARGUMENT; } #else - jl_tls_states_t *ptls2 = &jl_tls_states; + jl_ptls_t ptls2 = &jl_tls_states; tid = 0; #endif kern_return_t ret = thread_get_state(thread, x86_EXCEPTION_STATE64, (thread_state_t)&exc_state, &exc_count); @@ -247,7 +247,7 @@ static void attach_exception_port(thread_port_t thread) static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) { - jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_ptls_t ptls2 = jl_all_tls_states[tid]; mach_port_t tid_port = pthread_mach_thread_np(ptls2->system_id); kern_return_t ret = thread_suspend(tid_port); @@ -267,7 +267,7 @@ static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) static void jl_thread_resume(int tid, int sig) { - jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_ptls_t ptls2 = jl_all_tls_states[tid]; mach_port_t thread = pthread_mach_thread_np(ptls2->system_id); kern_return_t ret = thread_resume(thread); HANDLE_MACH_ERROR("thread_resume", ret); @@ -277,7 +277,7 @@ static void jl_thread_resume(int tid, int sig) // or if SIGINT happens too often. static void jl_try_deliver_sigint(void) { - jl_tls_states_t *ptls2 = jl_all_tls_states[0]; + jl_ptls_t ptls2 = jl_all_tls_states[0]; mach_port_t thread = pthread_mach_thread_np(ptls2->system_id); kern_return_t ret = thread_suspend(thread); diff --git a/src/signals-unix.c b/src/signals-unix.c index fd66d6334f985..26715c12e930d 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -48,9 +48,10 @@ static bt_context_t *jl_to_bt_context(void *sigctx) static void JL_NORETURN jl_throw_in_ctx(jl_value_t *e, void *sigctx) { - if (!jl_safe_restore) - jl_bt_size = rec_backtrace_ctx(jl_bt_data, JL_MAX_BT_SIZE, - jl_to_bt_context(sigctx)); + jl_ptls_t ptls = jl_get_ptls_states(); + if (!ptls->safe_restore) + ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, + jl_to_bt_context(sigctx)); jl_exception_in_transit = e; // TODO throw the error by modifying sigctx for supported platforms // This will avoid running the atexit handler on the signal stack @@ -60,7 +61,7 @@ static void JL_NORETURN jl_throw_in_ctx(jl_value_t *e, void *sigctx) static pthread_t signals_thread; -static int is_addr_on_stack(jl_tls_states_t *ptls, void *addr) +static int is_addr_on_stack(jl_ptls_t ptls, void *addr) { #ifdef COPY_STACKS return ((char*)addr > (char*)ptls->stack_lo-3000000 && @@ -73,9 +74,11 @@ static int is_addr_on_stack(jl_tls_states_t *ptls, void *addr) void sigdie_handler(int sig, siginfo_t *info, void *context) { + jl_ptls_t ptls = jl_get_ptls_states(); sigset_t sset; uv_tty_reset_mode(); - jl_critical_error(sig, jl_to_bt_context(context), jl_bt_data, &jl_bt_size); + jl_critical_error(sig, jl_to_bt_context(context), + ptls->bt_data, &ptls->bt_size); sigfillset(&sset); sigprocmask(SIG_UNBLOCK, &sset, NULL); signal(sig, SIG_DFL); @@ -103,6 +106,7 @@ static void jl_unblock_signal(int sig) static void segv_handler(int sig, siginfo_t *info, void *context) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(sig == SIGSEGV || sig == SIGBUS); if (jl_addr_is_safepoint((uintptr_t)info->si_addr)) { @@ -110,7 +114,7 @@ static void segv_handler(int sig, siginfo_t *info, void *context) #ifdef JULIA_ENABLE_THREADING jl_set_gc_and_wait(); // Do not raise sigint on worker thread - if (ti_tid != 0) + if (ptls->tid != 0) return; #endif if (jl_get_ptls_states()->defer_signal) { @@ -122,7 +126,7 @@ static void segv_handler(int sig, siginfo_t *info, void *context) } return; } - if (jl_safe_restore || is_addr_on_stack(jl_get_ptls_states(), info->si_addr)) { // stack overflow, or restarting jl_ + if (ptls->safe_restore || is_addr_on_stack(jl_get_ptls_states(), info->si_addr)) { // stack overflow, or restarting jl_ jl_unblock_signal(sig); jl_throw_in_ctx(jl_stackovf_exception, context); } @@ -164,7 +168,7 @@ static pthread_cond_t signal_caught_cond; static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) { pthread_mutex_lock(&in_signal_lock); - jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_ptls_t ptls2 = jl_all_tls_states[tid]; jl_atomic_store_release(&ptls2->signal_request, 1); pthread_kill(ptls2->system_id, SIGUSR2); pthread_cond_wait(&signal_caught_cond, &in_signal_lock); // wait for thread to acknowledge @@ -175,7 +179,7 @@ static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) static void jl_thread_resume(int tid, int sig) { (void)sig; - jl_tls_states_t *ptls2 = jl_all_tls_states[tid]; + jl_ptls_t ptls2 = jl_all_tls_states[tid]; jl_atomic_store_release(&ptls2->signal_request, 1); pthread_cond_broadcast(&exit_signal_cond); pthread_cond_wait(&signal_caught_cond, &in_signal_lock); // wait for thread to acknowledge @@ -187,7 +191,7 @@ static void jl_thread_resume(int tid, int sig) // or if SIGINT happens too often. static void jl_try_deliver_sigint(void) { - jl_tls_states_t *ptls2 = jl_all_tls_states[0]; + jl_ptls_t ptls2 = jl_all_tls_states[0]; jl_safepoint_enable_sigint(); jl_wake_libuv(); jl_atomic_store_release(&ptls2->signal_request, 2); @@ -202,7 +206,7 @@ static void jl_try_deliver_sigint(void) // is reached void usr2_handler(int sig, siginfo_t *info, void *ctx) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); sig_atomic_t request = jl_atomic_exchange(&ptls->signal_request, 0); if (request == 1) { signal_context = jl_to_bt_context(ctx); @@ -319,7 +323,7 @@ static void *alloc_sigstack(size_t size) return (void*)((char*)stackbuff + pagesz); } -void jl_install_thread_signal_handler(void) +void jl_install_thread_signal_handler(jl_ptls_t ptls) { void *signal_stack = alloc_sigstack(sig_stack_size); stack_t ss; @@ -341,7 +345,7 @@ void jl_install_thread_signal_handler(void) } #endif - jl_get_ptls_states()->signal_stack = signal_stack; + ptls->signal_stack = signal_stack; } void jl_sigsetset(sigset_t *sset) diff --git a/src/signals-win.c b/src/signals-win.c index 37fcb1b4e1da5..a3df41188222f 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -43,7 +43,7 @@ static char *strsignal(int sig) static void jl_try_throw_sigint(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_safepoint_enable_sigint(); jl_wake_libuv(); int force = jl_check_force_sigint(); @@ -59,6 +59,7 @@ static void jl_try_throw_sigint(void) void __cdecl crt_sig_handler(int sig, int num) { + jl_ptls_t ptls = jl_get_ptls_states(); CONTEXT Context; switch (sig) { case SIGFPE: @@ -85,7 +86,7 @@ void __cdecl crt_sig_handler(int sig, int num) default: // SIGSEGV, (SSIGTERM, IGILL) memset(&Context, 0, sizeof(Context)); RtlCaptureContext(&Context); - jl_critical_error(sig, &Context, jl_bt_data, &jl_bt_size); + jl_critical_error(sig, &Context, ptls->bt_data, &ptls->bt_size); raise(sig); } } @@ -101,6 +102,7 @@ void restore_signals(void) void jl_throw_in_ctx(jl_value_t *excpt, CONTEXT *ctxThread, int bt) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(excpt != NULL); #if defined(_CPU_X86_64_) DWORD64 Rsp = (ctxThread->Rsp&(DWORD64)-16) - 8; @@ -109,7 +111,8 @@ void jl_throw_in_ctx(jl_value_t *excpt, CONTEXT *ctxThread, int bt) #else #error WIN16 not supported :P #endif - jl_bt_size = bt ? rec_backtrace_ctx(jl_bt_data, JL_MAX_BT_SIZE, ctxThread) : 0; + ptls->bt_size = bt ? rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, + ctxThread) : 0; jl_exception_in_transit = excpt; #if defined(_CPU_X86_64_) *(DWORD64*)Rsp = 0; @@ -127,7 +130,7 @@ HANDLE hMainThread = INVALID_HANDLE_VALUE; // Try to throw the exception in the master thread. static void jl_try_deliver_sigint(void) { - jl_tls_states_t *ptls2 = jl_all_tls_states[0]; + jl_ptls_t ptls2 = jl_all_tls_states[0]; jl_safepoint_enable_sigint(); jl_wake_libuv(); if ((DWORD)-1 == SuspendThread(hMainThread)) { @@ -183,6 +186,7 @@ static BOOL WINAPI sigint_handler(DWORD wsig) //This needs winapi types to guara static LONG WINAPI _exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo, int in_ctx) { + jl_ptls_t ptls = jl_get_ptls_states(); if (ExceptionInfo->ExceptionRecord->ExceptionFlags == 0) { switch (ExceptionInfo->ExceptionRecord->ExceptionCode) { case EXCEPTION_INT_DIVIDE_BY_ZERO: @@ -199,10 +203,10 @@ static LONG WINAPI _exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo, #ifdef JULIA_ENABLE_THREADING jl_set_gc_and_wait(); // Do not raise sigint on worker thread - if (ti_tid != 0) + if (ptls->tid != 0) return EXCEPTION_CONTINUE_EXECUTION; #endif - if (jl_get_ptls_states()->defer_signal) { + if (ptls->defer_signal) { jl_safepoint_defer_sigint(); } else if (jl_safepoint_consume_sigint()) { @@ -266,7 +270,8 @@ static LONG WINAPI _exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo, jl_safe_printf(" at 0x%Ix -- ", (size_t)ExceptionInfo->ExceptionRecord->ExceptionAddress); jl_gdblookup((uintptr_t)ExceptionInfo->ExceptionRecord->ExceptionAddress); - jl_critical_error(0, ExceptionInfo->ContextRecord, jl_bt_data, &jl_bt_size); + jl_critical_error(0, ExceptionInfo->ContextRecord, + ptls->bt_data, &ptls->bt_size); static int recursion = 0; if (recursion++) exit(1); @@ -409,8 +414,9 @@ void jl_install_default_signal_handlers(void) SetUnhandledExceptionFilter(exception_handler); } -void jl_install_thread_signal_handler(void) +void jl_install_thread_signal_handler(jl_ptls_t ptls) { + (void)ptls; // Ensure the stack overflow handler has enough space to collect the backtrace ULONG StackSizeInBytes = sig_stack_size; if (pSetThreadStackGuarantee) { diff --git a/src/stackwalk.c b/src/stackwalk.c index fcac8a1d9294b..d6bd218e2b741 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -26,6 +26,7 @@ static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp); size_t jl_unw_stepn(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, size_t maxsize) { + jl_ptls_t ptls = jl_get_ptls_states(); volatile size_t n = 0; uintptr_t nullsp; #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) @@ -33,10 +34,10 @@ size_t jl_unw_stepn(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, size_t ma jl_in_stackwalk = 1; #endif #if !defined(_OS_WINDOWS_) - jl_jmp_buf *old_buf = jl_safe_restore; + jl_jmp_buf *old_buf = ptls->safe_restore; jl_jmp_buf buf; if (!jl_setjmp(buf, 0)) { - jl_safe_restore = &buf; + ptls->safe_restore = &buf; #endif while (1) { if (n >= maxsize) { @@ -57,7 +58,7 @@ size_t jl_unw_stepn(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, size_t ma // reader happy. if (n > 0) n -= 1; } - jl_safe_restore = old_buf; + ptls->safe_restore = old_buf; #endif #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) jl_in_stackwalk = 0; @@ -121,6 +122,7 @@ JL_DLLEXPORT jl_value_t *jl_backtrace_from_here(int returnsp) JL_DLLEXPORT jl_value_t *jl_get_backtrace(void) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_svec_t *tp = NULL; jl_array_t *bt = NULL; JL_GC_PUSH2(&tp, &bt); @@ -128,8 +130,8 @@ JL_DLLEXPORT jl_value_t *jl_get_backtrace(void) tp = jl_svec2(jl_voidpointer_type, jl_box_long(1)); array_ptr_void_type = jl_apply_type((jl_value_t*)jl_array_type, tp); } - bt = jl_alloc_array_1d(array_ptr_void_type, jl_bt_size); - memcpy(bt->data, jl_bt_data, jl_bt_size * sizeof(void*)); + bt = jl_alloc_array_1d(array_ptr_void_type, ptls->bt_size); + memcpy(bt->data, ptls->bt_data, ptls->bt_size * sizeof(void*)); JL_GC_POP(); return (jl_value_t*)bt; } @@ -369,7 +371,7 @@ size_t rec_backtrace_ctx_dwarf(uintptr_t *data, size_t maxsize, JL_DLLEXPORT jl_value_t *jl_lookup_code_address(void *ip, int skipC) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); jl_frame_t *frames = NULL; int8_t gc_state = jl_gc_safe_enter(ptls); int n = jl_getFunctionInfo(&frames, (uintptr_t)ip, skipC, 0); @@ -432,9 +434,10 @@ JL_DLLEXPORT void jl_gdblookup(uintptr_t ip) JL_DLLEXPORT void jlbacktrace(void) { - size_t n = jl_bt_size; // jl_bt_size > 40 ? 40 : jl_bt_size; + jl_ptls_t ptls = jl_get_ptls_states(); + size_t n = ptls->bt_size; // ptls->bt_size > 40 ? 40 : ptls->bt_size; for (size_t i=0; i < n; i++) - jl_gdblookup(jl_bt_data[i] - 1); + jl_gdblookup(ptls->bt_data[i] - 1); } #ifdef __cplusplus diff --git a/src/task.c b/src/task.c index 4d8cc8617b87f..9312ff099b48e 100644 --- a/src/task.c +++ b/src/task.c @@ -97,6 +97,7 @@ static int mangle_pointers; static void _probe_arch(void) { + jl_ptls_t ptls = jl_get_ptls_states(); struct _probe_data p; memset(p.probe_env, 0, sizeof(jl_jmp_buf)); memset(p.probe_sameAR, 0, sizeof(jl_jmp_buf)); @@ -112,12 +113,12 @@ static void _probe_arch(void) #if defined(__linux__) && defined(__i386__) char **s = (char**)p.ref_probe; - mangle_pointers = !(s[4] > jl_stack_lo && - s[4] < jl_stack_hi); + mangle_pointers = !(s[4] > ptls->stack_lo && + s[4] < ptls->stack_hi); #elif defined(__linux__) && defined(__x86_64__) char **s = (char**)p.ref_probe; - mangle_pointers = !(s[6] > jl_stack_lo && - s[6] < jl_stack_hi); + mangle_pointers = !(s[6] > ptls->stack_lo && + s[6] < ptls->stack_hi); #else mangle_pointers = 0; #endif @@ -140,7 +141,7 @@ jl_datatype_t *jl_task_type; #define ASM_COPY_STACKS #endif -static void NOINLINE save_stack(jl_tls_states_t *ptls, jl_task_t *t) +static void NOINLINE save_stack(jl_ptls_t ptls, jl_task_t *t) { if (t->state == done_sym || t->state == failed_sym) return; @@ -163,7 +164,7 @@ static void NOINLINE save_stack(jl_tls_states_t *ptls, jl_task_t *t) jl_gc_wb_back(t); } -static void NOINLINE restore_stack(jl_tls_states_t *ptls, jl_task_t *t, +static void NOINLINE restore_stack(jl_ptls_t ptls, jl_task_t *t, jl_jmp_buf *where, char *p) { char *_x = (char*)ptls->stackbase - t->ssize; @@ -185,6 +186,7 @@ static jl_function_t *task_done_hook_func=NULL; static void JL_NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) { + jl_ptls_t ptls = jl_get_ptls_states(); if (t->exception != jl_nothing) t->state = failed_sym; else @@ -195,7 +197,7 @@ static void JL_NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) #ifdef COPY_STACKS t->stkbuf = (void*)(intptr_t)-1; #endif - if (ti_tid != 0) { + if (ptls->tid != 0) { // For now, only thread 0 runs the task scheduler. // The others return to the thread loop jl_switchto(jl_root_task, jl_nothing); @@ -225,13 +227,15 @@ static void throw_if_exception_set(jl_task_t *t) static void record_backtrace(void) { - jl_bt_size = rec_backtrace(jl_bt_data, JL_MAX_BT_SIZE); + jl_ptls_t ptls = jl_get_ptls_states(); + ptls->bt_size = rec_backtrace(ptls->bt_data, JL_MAX_BT_SIZE); } static void NOINLINE JL_NORETURN start_task(void) { + jl_ptls_t ptls = jl_get_ptls_states(); // this runs the first time we switch to a task - jl_task_t *t = jl_current_task; + jl_task_t *t = ptls->current_task; jl_value_t *res; t->started = 1; if (t->exception != NULL && t->exception != jl_nothing) { @@ -240,14 +244,14 @@ static void NOINLINE JL_NORETURN start_task(void) } else { JL_TRY { - if (jl_get_ptls_states()->defer_signal) { - jl_get_ptls_states()->defer_signal = 0; - jl_sigint_safepoint(); + if (ptls->defer_signal) { + ptls->defer_signal = 0; + jl_sigint_safepoint(ptls); } res = jl_apply(&t->start, 1); } JL_CATCH { - res = jl_exception_in_transit; + res = ptls->exception_in_transit; t->exception = res; jl_gc_wb(t, res); } @@ -260,7 +264,7 @@ static void NOINLINE JL_NORETURN start_task(void) #ifdef COPY_STACKS void NOINLINE jl_set_base_ctx(char *__stk) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); ptls->stackbase = (char*)(((uintptr_t)__stk + sizeof(*__stk))&-16); // also ensures stackbase is 16-byte aligned #ifndef ASM_COPY_STACKS if (jl_setjmp(ptls->base_ctx, 1)) { @@ -281,7 +285,7 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) #endif } -static void ctx_switch(jl_tls_states_t *ptls, jl_task_t *t, jl_jmp_buf *where) +static void ctx_switch(jl_ptls_t ptls, jl_task_t *t, jl_jmp_buf *where) { if (t == ptls->current_task) return; @@ -365,7 +369,7 @@ static void ctx_switch(jl_tls_states_t *ptls, jl_task_t *t, jl_jmp_buf *where) JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); if (t == ptls->current_task) { throw_if_exception_set(t); return arg; @@ -391,7 +395,7 @@ JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) sig_atomic_t other_defer_signal = ptls->defer_signal; ptls->defer_signal = defer_signal; if (other_defer_signal && !defer_signal) - jl_sigint_safepoint(); + jl_sigint_safepoint(ptls); return val; } @@ -495,7 +499,7 @@ static void init_task(jl_task_t *t, char *stack) // yield to exception handler void JL_NORETURN throw_internal(jl_value_t *e) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); ptls->io_wait = 0; if (ptls->safe_restore) jl_longjmp(*ptls->safe_restore, 1); @@ -518,8 +522,9 @@ void JL_NORETURN throw_internal(jl_value_t *e) // record backtrace and raise an error JL_DLLEXPORT void jl_throw(jl_value_t *e) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(e != NULL); - if (!jl_safe_restore) + if (!ptls->safe_restore) record_backtrace(); throw_internal(e); } @@ -649,6 +654,7 @@ void jl_init_tasks(void) // Initialize a root task using the given stack. void jl_init_root_task(void *stack, size_t ssize) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_current_task = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); jl_set_typeof(jl_current_task, jl_task_type); #ifdef COPY_STACKS @@ -672,7 +678,7 @@ void jl_init_root_task(void *stack, size_t ssize) jl_current_task->backtrace = jl_nothing; jl_current_task->eh = NULL; jl_current_task->gcstack = NULL; - jl_current_task->tid = ti_tid; + jl_current_task->tid = ptls->tid; #ifdef JULIA_ENABLE_THREADING arraylist_new(&jl_current_task->locks, 0); #endif diff --git a/src/threading.c b/src/threading.c index 67d6be607f3b5..d15737c773941 100644 --- a/src/threading.c +++ b/src/threading.c @@ -70,20 +70,20 @@ __attribute__((constructor)) void jl_mac_init_tls(void) pthread_key_create(&jl_tls_key, NULL); } -JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) +JL_DLLEXPORT JL_CONST_FUNC jl_ptls_t (jl_get_ptls_states)(void) { void *ptls = pthread_getspecific(jl_tls_key); if (__unlikely(!ptls)) { ptls = calloc(1, sizeof(jl_tls_states_t)); pthread_setspecific(jl_tls_key, ptls); } - return (jl_tls_states_t*)ptls; + return (jl_ptls_t)ptls; } // This is only used after the tls is already initialized on the thread -static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_fast(void) +static JL_CONST_FUNC jl_ptls_t jl_get_ptls_states_fast(void) { - return (jl_tls_states_t*)pthread_getspecific(jl_tls_key); + return (jl_ptls_t)pthread_getspecific(jl_tls_key); } jl_get_ptls_states_func jl_get_ptls_states_getter(void) @@ -121,9 +121,9 @@ BOOLEAN WINAPI DllMain(IN HINSTANCE hDllHandle, IN DWORD nReason, return 1; // success } -JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) +JL_DLLEXPORT JL_CONST_FUNC jl_ptls_t (jl_get_ptls_states)(void) { - return (jl_tls_states_t*)TlsGetValue(jl_tls_key); + return (jl_ptls_t)TlsGetValue(jl_tls_key); } jl_get_ptls_states_func jl_get_ptls_states_getter(void) @@ -163,18 +163,18 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) # endif #endif // fallback provided for embedding -static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_fallback(void) +static JL_CONST_FUNC jl_ptls_t jl_get_ptls_states_fallback(void) { static __thread jl_tls_states_t tls_states; return &tls_states; } #ifdef JL_TLS_USE_IFUNC JL_DLLEXPORT JL_CONST_FUNC __attribute__((weak)) -jl_tls_states_t *jl_get_ptls_states_static(void); +jl_ptls_t jl_get_ptls_states_static(void); #endif -static jl_tls_states_t *jl_get_ptls_states_init(void); +static jl_ptls_t jl_get_ptls_states_init(void); static jl_get_ptls_states_func jl_tls_states_cb = jl_get_ptls_states_init; -static jl_tls_states_t *jl_get_ptls_states_init(void) +static jl_ptls_t jl_get_ptls_states_init(void) { // This 2-step initialization is used to detect calling // `jl_set_ptls_states_getter` after the address of the TLS variables @@ -193,7 +193,7 @@ static jl_tls_states_t *jl_get_ptls_states_init(void) return cb(); } -static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_wrapper(void) +static JL_CONST_FUNC jl_ptls_t jl_get_ptls_states_wrapper(void) { return (*jl_tls_states_cb)(); } @@ -227,10 +227,10 @@ static jl_get_ptls_states_func jl_get_ptls_states_resolve(void) return jl_tls_states_cb; } -JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) +JL_DLLEXPORT JL_CONST_FUNC jl_ptls_t (jl_get_ptls_states)(void) __attribute__((ifunc ("jl_get_ptls_states_resolve"))); #else // JL_TLS_USE_IFUNC -JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) +JL_DLLEXPORT JL_CONST_FUNC jl_ptls_t (jl_get_ptls_states)(void) { return jl_get_ptls_states_wrapper(); } @@ -245,7 +245,7 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) # endif #else JL_DLLEXPORT jl_tls_states_t jl_tls_states; -JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) +JL_DLLEXPORT JL_CONST_FUNC jl_ptls_t (jl_get_ptls_states)(void) { return &jl_tls_states; } @@ -253,16 +253,20 @@ JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void) // thread ID JL_DLLEXPORT int jl_n_threads; // # threads we're actually using -jl_tls_states_t **jl_all_tls_states; +jl_ptls_t *jl_all_tls_states; // return calling thread's ID // Also update the suspended_threads list in signals-mach when changing the // type of the thread id. -JL_DLLEXPORT int16_t jl_threadid(void) { return ti_tid; } +JL_DLLEXPORT int16_t jl_threadid(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return ptls->tid; +} static void ti_initthread(int16_t tid) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); #ifndef _OS_WINDOWS_ ptls->system_id = pthread_self(); #endif @@ -288,7 +292,7 @@ static void ti_initthread(int16_t tid) } ptls->bt_data = (uintptr_t*)bt_data; jl_mk_thread_heap(ptls); - jl_install_thread_signal_handler(); + jl_install_thread_signal_handler(ptls); jl_all_tls_states[tid] = ptls; } @@ -343,7 +347,7 @@ static uv_barrier_t thread_init_done; // thread function: used by all except the main thread void ti_threadfun(void *arg) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); ti_threadarg_t *ta = (ti_threadarg_t *)arg; ti_threadgroup_t *tg; ti_threadwork_t *work; @@ -353,7 +357,7 @@ void ti_threadfun(void *arg) jl_init_stack_limits(0); // set up tasking - jl_init_root_task(jl_stack_lo, jl_stack_hi - jl_stack_lo); + jl_init_root_task(ptls->stack_lo, ptls->stack_hi - ptls->stack_lo); #ifdef COPY_STACKS jl_set_base_ctx((char*)&arg); #endif @@ -370,7 +374,7 @@ void ti_threadfun(void *arg) uv_barrier_wait(&thread_init_done); // initialize this thread in the thread group tg = ta->tg; - ti_threadgroup_initthread(tg, ti_tid); + ti_threadgroup_initthread(tg, ptls->tid); // free the thread argument here free(ta); @@ -381,11 +385,11 @@ void ti_threadfun(void *arg) uint64_t tstart = uv_hrtime(); #endif - ti_threadgroup_fork(tg, ti_tid, (void **)&work); + ti_threadgroup_fork(tg, ptls->tid, (void **)&work); #if PROFILE_JL_THREADING uint64_t tfork = uv_hrtime(); - fork_ns[ti_tid] += tfork - tstart; + fork_ns[ptls->tid] += tfork - tstart; #endif if (work) { @@ -412,14 +416,14 @@ void ti_threadfun(void *arg) #if PROFILE_JL_THREADING uint64_t tuser = uv_hrtime(); - user_ns[ti_tid] += tuser - tfork; + user_ns[ptls->tid] += tuser - tfork; #endif - ti_threadgroup_join(tg, ti_tid); + ti_threadgroup_join(tg, ptls->tid); #if PROFILE_JL_THREADING uint64_t tjoin = uv_hrtime(); - join_ns[ti_tid] += tjoin - tuser; + join_ns[ptls->tid] += tjoin - tuser; #endif // TODO: @@ -501,7 +505,7 @@ static int check_tls_cb(struct dl_phdr_info *info, size_t size, void *_data) static void jl_check_tls(void) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); check_tls_cb_t data = {0}; dl_iterate_phdr(check_tls_cb, &data); if (data.total_size == 0) @@ -544,7 +548,7 @@ void jl_init_threading(void) if (jl_n_threads <= 0) jl_n_threads = 1; - jl_all_tls_states = (jl_tls_states_t**)malloc(jl_n_threads * sizeof(void*)); + jl_all_tls_states = (jl_ptls_t*)malloc(jl_n_threads * sizeof(void*)); #if PROFILE_JL_THREADING // set up space for profiling information @@ -560,6 +564,7 @@ void jl_init_threading(void) void jl_start_threads(void) { + jl_ptls_t ptls = jl_get_ptls_states(); char *cp, mask[UV_CPU_SETSIZE]; int i, exclusive; uv_thread_t uvtid; @@ -603,7 +608,7 @@ void jl_start_threads(void) ti_threadgroup_create(1, jl_n_threads, 1, &tgworld); for (i = 0; i < jl_n_threads; ++i) ti_threadgroup_addthread(tgworld, i, NULL); - ti_threadgroup_initthread(tgworld, ti_tid); + ti_threadgroup_initthread(tgworld, ptls->tid); // give the threads the world thread group; they will block waiting for fork for (i = 0; i < jl_n_threads - 1; ++i) { @@ -620,11 +625,12 @@ void jl_start_threads(void) // TODO: is this needed? where/when/how to call it? void jl_shutdown_threading(void) { + jl_ptls_t ptls = jl_get_ptls_states(); // stop the spinning threads by sending them a command ti_threadwork_t *work = &threadwork; work->command = TI_THREADWORK_DONE; - ti_threadgroup_fork(tgworld, ti_tid, (void **)&work); + ti_threadgroup_fork(tgworld, ptls->tid, (void **)&work); sleep(1); @@ -646,7 +652,7 @@ JL_DLLEXPORT void *jl_threadgroup(void) { return (void *)tgworld; } // and run it in all threads JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) { - jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_ptls_t ptls = jl_get_ptls_states(); // GC safe #if PROFILE_JL_THREADING uint64_t tstart = uv_hrtime(); @@ -674,11 +680,11 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) // fork the world thread group ti_threadwork_t *tw = &threadwork; - ti_threadgroup_fork(tgworld, ti_tid, (void **)&tw); + ti_threadgroup_fork(tgworld, ptls->tid, (void **)&tw); #if PROFILE_JL_THREADING uint64_t tfork = uv_hrtime(); - fork_ns[ti_tid] += (tfork - tcompile); + fork_ns[ptls->tid] += (tfork - tcompile); #endif // this thread must do work too (TODO: reduction?) @@ -686,17 +692,17 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) #if PROFILE_JL_THREADING uint64_t trun = uv_hrtime(); - user_ns[ti_tid] += (trun - tfork); + user_ns[ptls->tid] += (trun - tfork); #endif jl_gc_state_set(ptls, JL_GC_STATE_SAFE, 0); // wait for completion (TODO: nowait?) - ti_threadgroup_join(tgworld, ti_tid); + ti_threadgroup_join(tgworld, ptls->tid); jl_gc_state_set(ptls, 0, JL_GC_STATE_SAFE); #if PROFILE_JL_THREADING uint64_t tjoin = uv_hrtime(); - join_ns[ti_tid] += (tjoin - trun); + join_ns[ptls->tid] += (tjoin - trun); #endif JL_GC_POP(); @@ -769,7 +775,7 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) void jl_init_threading(void) { - static jl_tls_states_t *_jl_all_tls_states; + static jl_ptls_t _jl_all_tls_states; jl_all_tls_states = &_jl_all_tls_states; jl_n_threads = 1; diff --git a/src/threading.h b/src/threading.h index 7be1146312b55..3d70d86b511b8 100644 --- a/src/threading.h +++ b/src/threading.h @@ -14,8 +14,7 @@ extern "C" { #define PROFILE_JL_THREADING 1 // thread ID -#define ti_tid (jl_get_ptls_states()->tid) -extern jl_tls_states_t **jl_all_tls_states; +extern jl_ptls_t *jl_all_tls_states; extern JL_DLLEXPORT int jl_n_threads; // # threads we're actually using // thread state diff --git a/ui/repl.c b/ui/repl.c index 185c9a8222b0d..e781aeae23070 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -35,7 +35,7 @@ extern "C" { #endif #if defined(JULIA_ENABLE_THREADING) && !defined(_OS_DARWIN_) && !defined(_OS_WINDOWS_) -JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_static(void) +JL_DLLEXPORT JL_CONST_FUNC jl_ptls_t jl_get_ptls_states_static(void) { static __attribute__((tls_model("local-exec"))) __thread jl_tls_states_t tls_states; return &tls_states; From 3e1aaac1b1133009810a3bce5cbf6e6c126f690c Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 24 Jun 2016 16:27:07 -0400 Subject: [PATCH 0213/1117] Remove most of the tls getter call in the GC --- base/base.jl | 3 +- src/gc-debug.c | 26 ++--- src/gc.c | 255 ++++++++++++++++++++++--------------------- src/gc.h | 10 +- src/julia_internal.h | 5 +- 5 files changed, 156 insertions(+), 143 deletions(-) diff --git a/base/base.jl b/base/base.jl index 61bd5006e67e3..910db8d8a6fec 100644 --- a/base/base.jl +++ b/base/base.jl @@ -82,7 +82,8 @@ function finalizer{T}(o::T, f::Ptr{Void}) Core.getptls(), o, f) end -finalize(o::ANY) = ccall(:jl_finalize, Void, (Any,), o) +finalize(o::ANY) = ccall(:jl_finalize_th, Void, (Ptr{Void}, Any,), + Core.getptls(), o) gc(full::Bool=true) = ccall(:jl_gc_collect, Void, (Cint,), full) gc_enable(on::Bool) = ccall(:jl_gc_enable, Cint, (Cint,), on)!=0 diff --git a/src/gc-debug.c b/src/gc-debug.c index 2ecb5aadaaec0..44400d17f451a 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -164,20 +164,20 @@ static void restore(void) } } -static void gc_verify_track(void) +static void gc_verify_track(jl_ptls_t ptls) { do { arraylist_push(&lostval_parents_done, lostval); jl_printf(JL_STDERR, "Now looking for %p =======\n", lostval); clear_mark(GC_CLEAN); - pre_mark(); - gc_mark_object_list(&to_finalize, 0); + pre_mark(ptls); + gc_mark_object_list(ptls, &to_finalize, 0); for (int i = 0;i < jl_n_threads;i++) { jl_ptls_t ptls2 = jl_all_tls_states[i]; - gc_mark_object_list(&ptls2->finalizers, 0); + gc_mark_object_list(ptls, &ptls2->finalizers, 0); } - gc_mark_object_list(&finalizer_list_marked, 0); - visit_mark_stack(); + gc_mark_object_list(ptls, &finalizer_list_marked, 0); + visit_mark_stack(ptls); if (lostval_parents.len == 0) { jl_printf(JL_STDERR, "Could not find the missing link. We missed a toplevel root. This is odd.\n"); break; @@ -209,21 +209,21 @@ static void gc_verify_track(void) } while(lostval != NULL); } -void gc_verify(void) +void gc_verify(jl_ptls_t ptls) { lostval = NULL; lostval_parents.len = 0; lostval_parents_done.len = 0; clear_mark(GC_CLEAN); gc_verifying = 1; - pre_mark(); - gc_mark_object_list(&to_finalize, 0); + pre_mark(ptls); + gc_mark_object_list(ptls, &to_finalize, 0); for (int i = 0;i < jl_n_threads;i++) { jl_ptls_t ptls2 = jl_all_tls_states[i]; - gc_mark_object_list(&ptls2->finalizers, 0); + gc_mark_object_list(ptls, &ptls2->finalizers, 0); } - gc_mark_object_list(&finalizer_list_marked, 0); - visit_mark_stack(); + gc_mark_object_list(ptls, &finalizer_list_marked, 0); + visit_mark_stack(ptls); int clean_len = bits_save[GC_CLEAN].len; for(int i = 0; i < clean_len + bits_save[GC_OLD].len; i++) { jl_taggedvalue_t *v = (jl_taggedvalue_t*)bits_save[i >= clean_len ? GC_OLD : GC_CLEAN].items[i >= clean_len ? i - clean_len : i]; @@ -243,7 +243,7 @@ void gc_verify(void) return; } restore(); - gc_verify_track(); + gc_verify_track(ptls); gc_debug_print_status(); gc_debug_critical_error(); abort(); diff --git a/src/gc.c b/src/gc.c index e57cc04914261..82d37a55ca565 100644 --- a/src/gc.c +++ b/src/gc.c @@ -283,9 +283,8 @@ JL_DLLEXPORT void jl_gc_add_ptr_finalizer(jl_ptls_t ptls, jl_value_t *v, void *f gc_add_ptr_finalizer(ptls, v, f); } -JL_DLLEXPORT void jl_finalize(jl_value_t *o) +JL_DLLEXPORT void jl_finalize_th(jl_ptls_t ptls, jl_value_t *o) { - jl_ptls_t ptls = jl_get_ptls_states(); JL_LOCK_NOGC(&finalizers_lock); // Copy the finalizers into a temporary list so that code in the finalizer // won't change the list as we loop through them. @@ -420,9 +419,9 @@ STATIC_INLINE void gc_update_heap_size(int64_t sz_ub, int64_t sz_est) last_full_live_est = sz_est; } -static inline int gc_setmark_big(jl_taggedvalue_t *o, int mark_mode) +static inline int gc_setmark_big(jl_ptls_t ptls, jl_taggedvalue_t *o, + int mark_mode) { - jl_ptls_t ptls = jl_get_ptls_states(); if (gc_verifying) { o->bits.gc = mark_mode; return 0; @@ -460,11 +459,11 @@ static inline int gc_setmark_big(jl_taggedvalue_t *o, int mark_mode) return mark_mode; } -static inline int gc_setmark_pool_(jl_taggedvalue_t *o, int mark_mode, - region_t *r) +static inline int gc_setmark_pool_(jl_ptls_t ptls, jl_taggedvalue_t *o, + int mark_mode, region_t *r) { #ifdef MEMDEBUG - return gc_setmark_big(o, mark_mode); + return gc_setmark_big(ptls, o, mark_mode); #endif assert(r != NULL); if (gc_verifying) { @@ -504,21 +503,22 @@ static inline int gc_setmark_pool_(jl_taggedvalue_t *o, int mark_mode, return mark_mode; } -static inline int gc_setmark_pool(jl_taggedvalue_t *o, int mark_mode) +static inline int gc_setmark_pool(jl_ptls_t ptls, jl_taggedvalue_t *o, + int mark_mode) { - return gc_setmark_pool_(o, mark_mode, find_region(o)); + return gc_setmark_pool_(ptls, o, mark_mode, find_region(o)); } -static inline int gc_setmark(jl_value_t *v, int sz) +static inline int gc_setmark(jl_ptls_t ptls, jl_value_t *v, int sz) { jl_taggedvalue_t *o = jl_astaggedvalue(v); if (sz <= GC_MAX_SZCLASS) - return gc_setmark_pool(o, GC_MARKED); + return gc_setmark_pool(ptls, o, GC_MARKED); else - return gc_setmark_big(o, GC_MARKED); + return gc_setmark_big(ptls, o, GC_MARKED); } -inline void gc_setmark_buf(void *o, int mark_mode, size_t minsz) +inline void gc_setmark_buf(jl_ptls_t ptls, void *o, int mark_mode, size_t minsz) { jl_taggedvalue_t *buf = jl_astaggedvalue(o); // If the object is larger than the max pool size it can't be a pool object. @@ -528,22 +528,21 @@ inline void gc_setmark_buf(void *o, int mark_mode, size_t minsz) if (minsz <= GC_MAX_SZCLASS) { region_t *r = find_region(buf); if (r) { - gc_setmark_pool_(buf, mark_mode, r); + gc_setmark_pool_(ptls, buf, mark_mode, r); return; } } - gc_setmark_big(buf, mark_mode); + gc_setmark_big(ptls, buf, mark_mode); } #define should_collect() (__unlikely(gc_num.allocd>0)) -static inline int maybe_collect(void) +static inline int maybe_collect(jl_ptls_t ptls) { if (should_collect() || gc_debug_check_other()) { jl_gc_collect(0); return 1; } - jl_ptls_t ptls = jl_get_ptls_states(); jl_gc_safepoint_(ptls); return 0; } @@ -593,10 +592,9 @@ static void sweep_weak_refs(void) // big value list -static NOINLINE jl_taggedvalue_t *alloc_big(size_t sz) +static NOINLINE jl_taggedvalue_t *alloc_big(jl_ptls_t ptls, size_t sz) { - jl_ptls_t ptls = jl_get_ptls_states(); - maybe_collect(); + maybe_collect(ptls); size_t offs = offsetof(bigval_t, header); size_t allocsz = LLT_ALIGN(sz + offs, JL_CACHE_BYTE_ALIGNMENT); if (allocsz < sz) // overflow in adding offs, size was "negative" @@ -657,9 +655,8 @@ static bigval_t **sweep_big_list(int sweep_full, bigval_t **pv) return pv; } -static void sweep_big(int sweep_full) +static void sweep_big(jl_ptls_t ptls, int sweep_full) { - jl_ptls_t ptls = jl_get_ptls_states(); gc_time_big_start(); for (int i = 0;i < jl_n_threads;i++) sweep_big_list(sweep_full, &jl_all_tls_states[i]->heap.big_objects); @@ -777,6 +774,8 @@ static inline jl_taggedvalue_t *reset_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg static NOINLINE void add_page(jl_gc_pool_t *p) { + // Do not pass in `ptls` as argument. This slows down the fast path + // in pool_alloc significantly jl_ptls_t ptls = jl_get_ptls_states(); char *data = (char*)jl_gc_alloc_page(); if (data == NULL) @@ -790,12 +789,11 @@ static NOINLINE void add_page(jl_gc_pool_t *p) p->newpages = fl; } -static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, - jl_gc_pool_t *p, int osize, - int end_offset) +static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, + int osize, int end_offset) { #ifdef MEMDEBUG - return alloc_big(osize); + return alloc_big(ptls, osize); #endif jl_taggedvalue_t *v, *end; // FIXME - need JL_ATOMIC_FETCH_AND_ADD here @@ -874,7 +872,7 @@ JL_DLLEXPORT void *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, // Size includes the tag!! JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz) { - return jl_valueof(alloc_big(allocsz)); + return jl_valueof(alloc_big(ptls, allocsz)); } // pools are 16376 bytes large (GC_POOL_SZ - GC_PAGE_OFFSET) @@ -1088,10 +1086,10 @@ static void sweep_pool_region(jl_taggedvalue_t ***pfl, int region_i, int sweep_f region->lb = lb; } -static void gc_sweep_other(int sweep_full) +static void gc_sweep_other(jl_ptls_t ptls, int sweep_full) { sweep_malloced_arrays(); - sweep_big(sweep_full); + sweep_big(ptls, sweep_full); } static void gc_sweep_pool(int sweep_full) @@ -1213,12 +1211,12 @@ void gc_queue_binding(jl_binding_t *bnd) arraylist_push(&ptls->heap.rem_bindings, bnd); } -static int push_root(jl_value_t *v, int d, int); +static int push_root(jl_ptls_t ptls, jl_value_t *v, int d, int); #ifdef JL_DEBUG_BUILD static void *volatile gc_findval; // for usage from gdb, for finding the gc-root for a value #endif // Returns whether the object is young -static inline int gc_push_root(void *v, int d) // v isa jl_value_t* +static inline int gc_push_root(jl_ptls_t ptls, void *v, int d) // v isa jl_value_t* { #ifdef JL_DEBUG_BUILD if (v == gc_findval) @@ -1229,19 +1227,20 @@ static inline int gc_push_root(void *v, int d) // v isa jl_value_t* verify_val(v); int bits = o->bits.gc; if (!gc_marked(bits)) - return !gc_old(push_root((jl_value_t*)v, d, bits)); + return !gc_old(push_root(ptls, (jl_value_t*)v, d, bits)); return !gc_old(bits); } void jl_gc_setmark(jl_value_t *v) // TODO rename this as it is misleading now { + jl_ptls_t ptls = jl_get_ptls_states(); jl_taggedvalue_t *o = jl_astaggedvalue(v); if (!gc_marked(o->bits.gc)) { - gc_setmark_pool(o, GC_MARKED); + gc_setmark_pool(ptls, o, GC_MARKED); } } -NOINLINE static int gc_mark_module(jl_module_t *m, int d) +NOINLINE static int gc_mark_module(jl_ptls_t ptls, jl_module_t *m, int d) { size_t i; int refyoung = 0; @@ -1249,7 +1248,7 @@ NOINLINE static int gc_mark_module(jl_module_t *m, int d) for(i=1; i < m->bindings.size; i+=2) { if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; - gc_setmark_buf(b, jl_astaggedvalue(m)->bits.gc, + gc_setmark_buf(ptls, b, jl_astaggedvalue(m)->bits.gc, sizeof(jl_binding_t)); void *vb = jl_astaggedvalue(b); verify_parent1("module", m, &vb, "binding_buff"); @@ -1257,10 +1256,10 @@ NOINLINE static int gc_mark_module(jl_module_t *m, int d) if (b->value != NULL) { verify_parent2("module", m, &b->value, "binding(%s)", jl_symbol_name(b->name)); - refyoung |= gc_push_root(b->value, d); + refyoung |= gc_push_root(ptls, b->value, d); } if (b->globalref != NULL) - refyoung |= gc_push_root(b->globalref, d); + refyoung |= gc_push_root(ptls, b->globalref, d); } } // this is only necessary because bindings for "using" modules @@ -1268,17 +1267,18 @@ NOINLINE static int gc_mark_module(jl_module_t *m, int d) // after "using" it but before accessing it, this array might // contain the only reference. for(i=0; i < m->usings.len; i++) { - refyoung |= gc_push_root(m->usings.items[i], d); + refyoung |= gc_push_root(ptls, m->usings.items[i], d); } if (m->parent) { - refyoung |= gc_push_root(m->parent, d); + refyoung |= gc_push_root(ptls, m->parent, d); } return refyoung; } -static void gc_mark_stack(jl_value_t *ta, jl_gcframe_t *s, intptr_t offset, int d) +static void gc_mark_stack(jl_ptls_t ptls, jl_value_t *ta, jl_gcframe_t *s, + intptr_t offset, int d) { while (s != NULL) { s = (jl_gcframe_t*)((char*)s + offset); @@ -1288,7 +1288,7 @@ static void gc_mark_stack(jl_value_t *ta, jl_gcframe_t *s, intptr_t offset, int for(size_t i=0; i < nr; i++) { jl_value_t **ptr = (jl_value_t**)((char*)rts[i] + offset); if (*ptr != NULL) { - gc_push_root(*ptr, d); + gc_push_root(ptls, *ptr, d); } } } @@ -1296,7 +1296,7 @@ static void gc_mark_stack(jl_value_t *ta, jl_gcframe_t *s, intptr_t offset, int for(size_t i=0; i < nr; i++) { if (rts[i] != NULL) { verify_parent2("task", ta, &rts[i], "stack(%d)", (int)i); - gc_push_root(rts[i], d); + gc_push_root(ptls, rts[i], d); } } } @@ -1304,24 +1304,25 @@ static void gc_mark_stack(jl_value_t *ta, jl_gcframe_t *s, intptr_t offset, int } } -static void gc_mark_task_stack(jl_task_t *ta, int d) +static void gc_mark_task_stack(jl_ptls_t ptls, jl_task_t *ta, int d) { int stkbuf = (ta->stkbuf != (void*)(intptr_t)-1 && ta->stkbuf != NULL); int16_t tid = ta->tid; jl_ptls_t ptls2 = jl_all_tls_states[tid]; if (stkbuf) { #ifdef COPY_STACKS - gc_setmark_buf(ta->stkbuf, jl_astaggedvalue(ta)->bits.gc, ta->bufsz); + gc_setmark_buf(ptls, ta->stkbuf, jl_astaggedvalue(ta)->bits.gc, + ta->bufsz); #else // stkbuf isn't owned by julia for the root task if (ta != ptls2->root_task) { - gc_setmark_buf(ta->stkbuf, jl_astaggedvalue(ta)->bits.gc, + gc_setmark_buf(ptls, ta->stkbuf, jl_astaggedvalue(ta)->bits.gc, ta->ssize); } #endif } if (ta == ptls2->current_task) { - gc_mark_stack((jl_value_t*)ta, ptls2->pgcstack, 0, d); + gc_mark_stack(ptls, (jl_value_t*)ta, ptls2->pgcstack, 0, d); } else if (stkbuf) { intptr_t offset; @@ -1330,24 +1331,24 @@ static void gc_mark_task_stack(jl_task_t *ta, int d) #else offset = 0; #endif - gc_mark_stack((jl_value_t*)ta, ta->gcstack, offset, d); + gc_mark_stack(ptls, (jl_value_t*)ta, ta->gcstack, offset, d); } } -NOINLINE static void gc_mark_task(jl_task_t *ta, int d) +NOINLINE static void gc_mark_task(jl_ptls_t ptls, jl_task_t *ta, int d) { - if (ta->parent) gc_push_root(ta->parent, d); - gc_push_root(ta->tls, d); - gc_push_root(ta->consumers, d); - gc_push_root(ta->donenotify, d); - gc_push_root(ta->exception, d); - if (ta->backtrace) gc_push_root(ta->backtrace, d); - if (ta->start) gc_push_root(ta->start, d); - if (ta->result) gc_push_root(ta->result, d); - gc_mark_task_stack(ta, d); + if (ta->parent) gc_push_root(ptls, ta->parent, d); + gc_push_root(ptls, ta->tls, d); + gc_push_root(ptls, ta->consumers, d); + gc_push_root(ptls, ta->donenotify, d); + gc_push_root(ptls, ta->exception, d); + if (ta->backtrace) gc_push_root(ptls, ta->backtrace, d); + if (ta->start) gc_push_root(ptls, ta->start, d); + if (ta->result) gc_push_root(ptls, ta->result, d); + gc_mark_task_stack(ptls, ta, d); } -void gc_mark_object_list(arraylist_t *list, size_t start) +void gc_mark_object_list(jl_ptls_t ptls, arraylist_t *list, size_t start) { void **items = list->items; size_t len = list->len; @@ -1360,7 +1361,7 @@ void gc_mark_object_list(arraylist_t *list, size_t start) i++; assert(i < len); } - gc_push_root(v, 0); + gc_push_root(ptls, v, 0); } } @@ -1375,20 +1376,19 @@ JL_DLLEXPORT void jl_gc_lookfor(jl_value_t *v) { lookforme = v; } // it does so assuming the gc bits of v are "bits" and returns the new bits of v // if v becomes GC_OLD_MARKED and some of its children are GC_MARKED (young), // v is added to the remset -static int push_root(jl_value_t *v, int d, int bits) +static int push_root(jl_ptls_t ptls, jl_value_t *v, int d, int bits) { - jl_ptls_t ptls = jl_get_ptls_states(); assert(v != NULL); jl_value_t *vt = jl_typeof(v); int refyoung = 0, nptr = 0; if (vt == (jl_value_t*)jl_weakref_type) { - bits = gc_setmark(v, sizeof(jl_weakref_t)); + bits = gc_setmark(ptls, v, sizeof(jl_weakref_t)); goto ret; } if ((jl_is_datatype(vt) && ((jl_datatype_t*)vt)->pointerfree)) { int sz = jl_datatype_size(vt); - bits = gc_setmark(v, sz); + bits = gc_setmark(ptls, v, sz); goto ret; } d++; @@ -1396,7 +1396,7 @@ static int push_root(jl_value_t *v, int d, int bits) // some values have special representations if (vt == (jl_value_t*)jl_simplevector_type) { size_t l = jl_svec_len(v); - bits = gc_setmark(v, l * sizeof(void*) + sizeof(jl_svec_t)); + bits = gc_setmark(ptls, v, l * sizeof(void*) + sizeof(jl_svec_t)); if (d >= MAX_MARK_DEPTH) goto queue_the_root; jl_value_t **data = jl_svec_data(v); @@ -1405,7 +1405,7 @@ static int push_root(jl_value_t *v, int d, int bits) jl_value_t *elt = data[i]; if (elt != NULL) { verify_parent2("svec", v, &data[i], "elem(%d)", (int)i); - refyoung |= gc_push_root(elt, d); + refyoung |= gc_push_root(ptls, elt, d); } } } @@ -1413,8 +1413,8 @@ static int push_root(jl_value_t *v, int d, int bits) jl_array_t *a = (jl_array_t*)v; jl_taggedvalue_t *o = jl_astaggedvalue(v); int todo = !gc_marked(bits); - bits = (a->flags.pooled ? gc_setmark_pool(o, GC_MARKED) : - gc_setmark_big(o, GC_MARKED)); + bits = (a->flags.pooled ? gc_setmark_pool(ptls, o, GC_MARKED) : + gc_setmark_big(ptls, o, GC_MARKED)); if (a->flags.how == 2 && todo) { objprofile_count(jl_malloc_tag, o->bits.gc == GC_OLD_MARKED, array_nbytes(a)); @@ -1429,15 +1429,15 @@ static int push_root(jl_value_t *v, int d, int bits) goto queue_the_root; if (a->flags.how == 3) { jl_value_t *owner = jl_array_data_owner(a); - refyoung |= gc_push_root(owner, d); + refyoung |= gc_push_root(ptls, owner, d); goto ret; } else if (a->flags.how == 1) { void *val_buf = jl_astaggedvalue((char*)a->data - a->offset*a->elsize); verify_parent1("array", v, &val_buf, "buffer ('loc' addr is meaningless)"); (void)val_buf; - gc_setmark_buf((char*)a->data - a->offset*a->elsize, o->bits.gc, - array_nbytes(a)); + gc_setmark_buf(ptls, (char*)a->data - a->offset*a->elsize, + o->bits.gc, array_nbytes(a)); } if (a->flags.ptrarray && a->data!=NULL) { size_t l = jl_array_len(a); @@ -1453,7 +1453,7 @@ static int push_root(jl_value_t *v, int d, int bits) jl_value_t *elt = ((jl_value_t**)data)[i]; if (elt != NULL) { verify_parent2("array", v, &((jl_value_t**)data)[i], "elem(%d)", (int)i); - refyoung |= gc_push_root(elt, d); + refyoung |= gc_push_root(ptls, elt, d); } // try to split large array marking (incremental mark TODO) // if (should_timeout() && l > 1000) goto queue_the_root; @@ -1463,17 +1463,17 @@ static int push_root(jl_value_t *v, int d, int bits) } else if (vt == (jl_value_t*)jl_module_type) { // should increase nptr here - bits = gc_setmark(v, sizeof(jl_module_t)); + bits = gc_setmark(ptls, v, sizeof(jl_module_t)); if (d >= MAX_MARK_DEPTH) goto queue_the_root; - refyoung |= gc_mark_module((jl_module_t*)v, d); + refyoung |= gc_mark_module(ptls, (jl_module_t*)v, d); } else if (vt == (jl_value_t*)jl_task_type) { // ditto nptr - bits = gc_setmark(v, sizeof(jl_task_t)); + bits = gc_setmark(ptls, v, sizeof(jl_task_t)); if (d >= MAX_MARK_DEPTH) goto queue_the_root; - gc_mark_task((jl_task_t*)v, d); + gc_mark_task(ptls, (jl_task_t*)v, d); // tasks should always be remarked since we do not trigger the write barrier // for stores to stack slots refyoung = 1; @@ -1494,7 +1494,7 @@ static int push_root(jl_value_t *v, int d, int bits) else { dtsz = jl_datatype_size(dt); } - bits = gc_setmark(v, dtsz); + bits = gc_setmark(ptls, v, dtsz); if (d >= MAX_MARK_DEPTH) goto queue_the_root; @@ -1512,12 +1512,12 @@ static int push_root(jl_value_t *v, int d, int bits) if (fld) { verify_parent2("object", v, slot, "field(%d)", i); //children[ci++] = fld; - refyoung |= gc_push_root(fld, d); + refyoung |= gc_push_root(ptls, fld, d); } } } //while(ci) - // refyoung |= gc_push_root(children[--ci], d); + // refyoung |= gc_push_root(ptls, children[--ci], d); } else { jl_printf(JL_STDOUT, "GC error (probable corruption) :\n"); @@ -1544,12 +1544,12 @@ static int push_root(jl_value_t *v, int d, int bits) return bits; } -void visit_mark_stack(void) +void visit_mark_stack(jl_ptls_t ptls) { while (mark_sp > 0 && !should_timeout()) { jl_value_t *v = mark_stack[--mark_sp]; assert(jl_astaggedvalue(v)->bits.gc); - push_root(v, 0, jl_astaggedvalue(v)->bits.gc); + push_root(ptls, v, 0, jl_astaggedvalue(v)->bits.gc); } assert(!mark_sp); } @@ -1560,11 +1560,11 @@ extern jl_array_t *jl_module_init_order; extern jl_typemap_entry_t *call_cache[N_CALL_CACHE]; // mark the initial root set -void pre_mark(void) +void pre_mark(jl_ptls_t ptls) { // modules - gc_push_root(jl_main_module, 0); - gc_push_root(jl_internal_main_module, 0); + gc_push_root(ptls, jl_main_module, 0); + gc_push_root(ptls, jl_internal_main_module, 0); size_t i; for(i=0; i < jl_n_threads; i++) { @@ -1572,35 +1572,35 @@ void pre_mark(void) // current_module might not have a value when the thread is not // running. if (ptls2->current_module) - gc_push_root(ptls2->current_module, 0); - gc_push_root(ptls2->current_task, 0); - gc_push_root(ptls2->root_task, 0); - gc_push_root(ptls2->exception_in_transit, 0); - gc_push_root(ptls2->task_arg_in_transit, 0); + gc_push_root(ptls, ptls2->current_module, 0); + gc_push_root(ptls, ptls2->current_task, 0); + gc_push_root(ptls, ptls2->root_task, 0); + gc_push_root(ptls, ptls2->exception_in_transit, 0); + gc_push_root(ptls, ptls2->task_arg_in_transit, 0); } // invisible builtin values if (jl_an_empty_vec_any != NULL) - gc_push_root(jl_an_empty_vec_any, 0); + gc_push_root(ptls, jl_an_empty_vec_any, 0); if (jl_module_init_order != NULL) - gc_push_root(jl_module_init_order, 0); - gc_push_root(jl_cfunction_list.unknown, 0); - gc_push_root(jl_anytuple_type_type, 0); - gc_push_root(jl_ANY_flag, 0); + gc_push_root(ptls, jl_module_init_order, 0); + gc_push_root(ptls, jl_cfunction_list.unknown, 0); + gc_push_root(ptls, jl_anytuple_type_type, 0); + gc_push_root(ptls, jl_ANY_flag, 0); for (i = 0; i < N_CALL_CACHE; i++) if (call_cache[i]) - gc_push_root(call_cache[i], 0); + gc_push_root(ptls, call_cache[i], 0); jl_mark_box_caches(); - //gc_push_root(jl_unprotect_stack_func, 0); - gc_push_root(jl_typetype_type, 0); + //gc_push_root(ptls, jl_unprotect_stack_func, 0); + gc_push_root(ptls, jl_typetype_type, 0); // constants - gc_push_root(jl_emptysvec, 0); - gc_push_root(jl_emptytuple, 0); - gc_push_root(jl_typeof(jl_emptytuple), 0); - gc_push_root(jl_true, 0); - gc_push_root(jl_false, 0); + gc_push_root(ptls, jl_emptysvec, 0); + gc_push_root(ptls, jl_emptytuple, 0); + gc_push_root(ptls, jl_typeof(jl_emptytuple), 0); + gc_push_root(ptls, jl_true, 0); + gc_push_root(ptls, jl_false, 0); } // find unmarked objects that need to be finalized from the finalizer list "list". @@ -1711,7 +1711,7 @@ void jl_gc_sync_total_bytes(void) {last_gc_total_bytes = jl_gc_total_bytes();} #define MIN_SCAN_BYTES 1024*1024 // Only one thread should be running in this function -static void _jl_gc_collect(int full, char *stack_hi) +static void _jl_gc_collect(jl_ptls_t ptls, int full, char *stack_hi) { uint64_t t0 = jl_hrtime(); int64_t last_perm_scanned_bytes = perm_scanned_bytes; @@ -1734,7 +1734,7 @@ static void _jl_gc_collect(int full, char *stack_hi) for (int i = 0; i < ptls2->heap.last_remset->len; i++) { jl_value_t *item = (jl_value_t*)ptls2->heap.last_remset->items[i]; - push_root(item, 0, GC_OLD_MARKED); + push_root(ptls, item, 0, GC_OLD_MARKED); } } @@ -1747,7 +1747,7 @@ static void _jl_gc_collect(int full, char *stack_hi) // A null pointer can happen here when the binding is cleaned up // as an exception is thrown after it was already queued (#10221) if (!ptr->value) continue; - if (gc_push_root(ptr->value, 0)) { + if (gc_push_root(ptls, ptr->value, 0)) { ptls2->heap.rem_bindings.items[n_bnd_refyoung] = ptr; n_bnd_refyoung++; } @@ -1756,8 +1756,8 @@ static void _jl_gc_collect(int full, char *stack_hi) } // 3. walk roots - pre_mark(); - visit_mark_stack(); + pre_mark(ptls); + visit_mark_stack(ptls); gc_num.since_sweep += gc_num.allocd + (int64_t)gc_num.interval; gc_settime_premark_end(); gc_time_mark_pause(t0, scanned_bytes, perm_scanned_bytes); @@ -1778,19 +1778,19 @@ static void _jl_gc_collect(int full, char *stack_hi) } for (int i = 0;i < jl_n_threads;i++) { jl_ptls_t ptls2 = jl_all_tls_states[i]; - gc_mark_object_list(&ptls2->finalizers, 0); + gc_mark_object_list(ptls, &ptls2->finalizers, 0); } - gc_mark_object_list(&finalizer_list_marked, orig_marked_len); + gc_mark_object_list(ptls, &finalizer_list_marked, orig_marked_len); // "Flush" the mark stack before flipping the reset_age bit // so that the objects are not incorrectly resetted. - visit_mark_stack(); + visit_mark_stack(ptls); mark_reset_age = 1; // Reset the age and old bit for any unmarked objects referenced by the // `to_finalize` list. These objects are only reachable from this list // and should not be referenced by any old objects so this won't break // the GC invariant. - gc_mark_object_list(&to_finalize, 0); - visit_mark_stack(); + gc_mark_object_list(ptls, &to_finalize, 0); + visit_mark_stack(ptls); mark_reset_age = 0; gc_settime_postmark_end(); @@ -1798,7 +1798,7 @@ static void _jl_gc_collect(int full, char *stack_hi) int64_t live_sz_est = scanned_bytes + perm_scanned_bytes; int64_t estimate_freed = live_sz_ub - live_sz_est; - gc_verify(); + gc_verify(ptls); gc_stats_all_pool(); gc_stats_big_obj(); @@ -1845,7 +1845,7 @@ static void _jl_gc_collect(int full, char *stack_hi) scanned_bytes = 0; // 5. start sweeping sweep_weak_refs(); - gc_sweep_other(sweep_full); + gc_sweep_other(ptls, sweep_full); gc_scrub(stack_hi); gc_sweep_pool(sweep_full); // sweeping is over @@ -1883,7 +1883,7 @@ static void _jl_gc_collect(int full, char *stack_hi) gc_num.freed = 0; if (recollect) { - _jl_gc_collect(0, stack_hi); + _jl_gc_collect(ptls, 0, stack_hi); } } @@ -1912,7 +1912,7 @@ JL_DLLEXPORT void jl_gc_collect(int full) if (!jl_gc_disable_counter) { JL_LOCK_NOGC(&finalizers_lock); - _jl_gc_collect(full, stack_hi); + _jl_gc_collect(ptls, full, stack_hi); JL_UNLOCK_NOGC(&finalizers_lock); } @@ -1941,7 +1941,7 @@ void *allocb(size_t sz) if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); if (allocsz > GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { - b = alloc_big(allocsz); + b = alloc_big(ptls, allocsz); b->header = jl_buff_tag; } else { @@ -1960,7 +1960,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) if (allocsz <= GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { return jl_valueof(pool_alloc(ptls, &ptls->heap.norm_pools[szclass(allocsz)])); } - return jl_valueof(alloc_big(allocsz)); + return jl_valueof(alloc_big(ptls, allocsz)); } JL_DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) @@ -2047,8 +2047,9 @@ void jl_gc_init(void) JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz) { + jl_ptls_t ptls = jl_get_ptls_states(); sz += JL_SMALL_BYTE_ALIGNMENT; - maybe_collect(); + maybe_collect(ptls); gc_num.allocd += sz; gc_num.malloc++; void *b = malloc(sz); @@ -2059,8 +2060,9 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz) JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz) { + jl_ptls_t ptls = jl_get_ptls_states(); nm += JL_SMALL_BYTE_ALIGNMENT; - maybe_collect(); + maybe_collect(ptls); gc_num.allocd += nm*sz; gc_num.malloc++; void *b = calloc(nm, sz); @@ -2078,9 +2080,10 @@ JL_DLLEXPORT void jl_gc_counted_free(void *p, size_t sz) JL_DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size_t sz) { + jl_ptls_t ptls = jl_get_ptls_states(); old += JL_SMALL_BYTE_ALIGNMENT; sz += JL_SMALL_BYTE_ALIGNMENT; - maybe_collect(); + maybe_collect(ptls); if (sz < old) gc_num.freed += (old - sz); else @@ -2134,7 +2137,8 @@ JL_DLLEXPORT void *jl_realloc(void *p, size_t sz) JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) { - maybe_collect(); + jl_ptls_t ptls = jl_get_ptls_states(); + maybe_collect(ptls); size_t allocsz = LLT_ALIGN(sz, JL_CACHE_BYTE_ALIGNMENT); if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); @@ -2149,7 +2153,8 @@ JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) JL_DLLEXPORT void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, int isaligned, jl_value_t *owner) { - maybe_collect(); + jl_ptls_t ptls = jl_get_ptls_states(); + maybe_collect(ptls); size_t allocsz = LLT_ALIGN(sz, JL_CACHE_BYTE_ALIGNMENT); if (allocsz < sz) // overflow in adding offs, size was "negative" @@ -2182,6 +2187,12 @@ JL_DLLEXPORT void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) jl_gc_add_finalizer_th(ptls, v, f); } +JL_DLLEXPORT void jl_finalize(jl_value_t *o) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + jl_finalize_th(ptls, o); +} + #ifdef __cplusplus } #endif diff --git a/src/gc.h b/src/gc.h index 0c6310aa5bda2..51b99842372ec 100644 --- a/src/gc.h +++ b/src/gc.h @@ -274,9 +274,9 @@ STATIC_INLINE void gc_big_object_link(bigval_t *hdr, bigval_t **list) *list = hdr; } -void pre_mark(void); -void gc_mark_object_list(arraylist_t *list, size_t start); -void visit_mark_stack(void); +void pre_mark(jl_ptls_t ptls); +void gc_mark_object_list(jl_ptls_t ptls, arraylist_t *list, size_t start); +void visit_mark_stack(jl_ptls_t ptls); void gc_debug_init(void); // GC pages @@ -349,7 +349,7 @@ STATIC_INLINE void gc_time_count_mallocd_array(int bits) #ifdef GC_VERIFY extern jl_value_t *lostval; -void gc_verify(void); +void gc_verify(jl_ptls_t ptls); void add_lostval_parent(jl_value_t *parent); #define verify_val(v) do { \ if (lostval == (jl_value_t*)(v) && (v) != 0) { \ @@ -380,7 +380,7 @@ void add_lostval_parent(jl_value_t *parent); #define verify_parent2(ty,obj,slot,arg1,arg2) verify_parent(ty,obj,slot,arg1,arg2) extern int gc_verifying; #else -#define gc_verify() +#define gc_verify(ptls) #define verify_val(v) #define verify_parent1(ty,obj,slot,arg1) #define verify_parent2(ty,obj,slot,arg1,arg2) diff --git a/src/julia_internal.h b/src/julia_internal.h index ac386d0be9996..f2db5c44192a8 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -109,7 +109,7 @@ void jl_gc_run_all_finalizers(void); void *allocb(size_t sz); void gc_queue_binding(jl_binding_t *bnd); -void gc_setmark_buf(void *buf, int, size_t); +void gc_setmark_buf(jl_ptls_t ptls, void *buf, int, size_t); STATIC_INLINE void jl_gc_wb_binding(jl_binding_t *bnd, void *val) // val isa jl_value_t* { @@ -122,7 +122,8 @@ STATIC_INLINE void jl_gc_wb_buf(void *parent, void *bufptr, size_t minsz) // par { // if parent is marked and buf is not if (__unlikely(jl_astaggedvalue(parent)->bits.gc & 1)) { - gc_setmark_buf(bufptr, 3, minsz); + jl_ptls_t ptls = jl_get_ptls_states(); + gc_setmark_buf(ptls, bufptr, 3, minsz); } } From d5a493b2aaf1a4d28e9af75c2b9a4093ec33151d Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 24 Jun 2016 19:31:55 -0400 Subject: [PATCH 0214/1117] Remove most implicit TLS getter calls --- src/alloc.c | 9 ++++--- src/ast.c | 36 ++++++++++++++++---------- src/builtins.c | 21 ++++++++------- src/ccall.cpp | 8 +++--- src/codegen.cpp | 10 +++++--- src/dump.c | 16 ++++++++---- src/gc.c | 27 ++++++++++---------- src/gf.c | 3 ++- src/init.c | 19 +++++++------- src/interpreter.c | 25 ++++++++++-------- src/jlapi.c | 8 +++--- src/jltypes.c | 11 ++++---- src/julia.h | 26 ++++++++++--------- src/julia_internal.h | 2 +- src/module.c | 17 +++++++----- src/signals-unix.c | 2 +- src/signals-win.c | 2 +- src/task.c | 61 +++++++++++++++++++++++--------------------- src/threading.c | 11 ++++---- src/toplevel.c | 53 +++++++++++++++++++++----------------- ui/repl.c | 10 +++++--- 21 files changed, 215 insertions(+), 162 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 79d64a8386dec..a7861bcb655f5 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -576,6 +576,7 @@ JL_DLLEXPORT void jl_method_init_properties(jl_method_t *m) JL_DLLEXPORT jl_method_t *jl_new_method_uninit(void) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_method_t *m = (jl_method_t*)newobj((jl_value_t*)jl_method_type, NWORDS(sizeof(jl_method_t))); @@ -584,7 +585,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(void) m->tvars = NULL; m->ambig = NULL; m->roots = NULL; - m->module = jl_current_module; + m->module = ptls->current_module; m->lambda_template = NULL; m->name = NULL; m->file = empty_sym; @@ -841,7 +842,8 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu JL_DLLEXPORT jl_typename_t *jl_new_typename(jl_sym_t *name) { - return jl_new_typename_in(name, jl_current_module); + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_new_typename_in(name, ptls->current_module); } jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_datatype_t *super, @@ -986,6 +988,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super int abstract, int mutabl, int ninitialized) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_datatype_t *t=NULL; jl_typename_t *tn=NULL; JL_GC_PUSH2(&t, &tn); @@ -1032,7 +1035,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super else { tn = jl_new_typename((jl_sym_t*)name); if (!abstract) { - tn->mt = jl_new_method_table(name, jl_current_module); + tn->mt = jl_new_method_table(name, ptls->current_module); jl_gc_wb(tn, tn->mt); } } diff --git a/src/ast.c b/src/ast.c index 3ce3fec653321..40e3a55264f6f 100644 --- a/src/ast.c +++ b/src/ast.c @@ -105,35 +105,39 @@ static value_t julia_to_scm(fl_context_t *fl_ctx, jl_value_t *v); value_t fl_defined_julia_global(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) { + jl_ptls_t ptls = jl_get_ptls_states(); // tells whether a var is defined in and *by* the current module argcount(fl_ctx, "defined-julia-global", nargs, 1); (void)tosymbol(fl_ctx, args[0], "defined-julia-global"); - if (jl_current_module == NULL) + if (ptls->current_module == NULL) return fl_ctx->F; jl_sym_t *var = jl_symbol(symbol_name(fl_ctx, args[0])); jl_binding_t *b = - (jl_binding_t*)ptrhash_get(&jl_current_module->bindings, var); - return (b != HT_NOTFOUND && b->owner==jl_current_module) ? fl_ctx->T : fl_ctx->F; + (jl_binding_t*)ptrhash_get(&ptls->current_module->bindings, var); + return (b != HT_NOTFOUND && b->owner==ptls->current_module) ? fl_ctx->T : fl_ctx->F; } value_t fl_current_julia_module(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) { + jl_ptls_t ptls = jl_get_ptls_states(); value_t opaque = cvalue(fl_ctx, jl_ast_ctx(fl_ctx)->jvtype, sizeof(void*)); - *(jl_value_t**)cv_data((cvalue_t*)ptr(opaque)) = (jl_value_t*)jl_current_module; + *(jl_value_t**)cv_data((cvalue_t*)ptr(opaque)) = (jl_value_t*)ptls->current_module; return opaque; } value_t fl_current_module_counter(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) { + jl_ptls_t ptls = jl_get_ptls_states(); static uint32_t fallback_counter = 0; - if (jl_current_module == NULL) + if (ptls->current_module == NULL) return fixnum(++fallback_counter); else - return fixnum(jl_module_next_counter(jl_current_module)); + return fixnum(jl_module_next_counter(ptls->current_module)); } value_t fl_invoke_julia_macro(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) { + jl_ptls_t ptls = jl_get_ptls_states(); if (nargs < 1) argcount(fl_ctx, "invoke-julia-macro", nargs, 1); jl_lambda_info_t *mfunc = NULL; @@ -158,7 +162,7 @@ value_t fl_invoke_julia_macro(fl_context_t *fl_ctx, value_t *args, uint32_t narg JL_CATCH { JL_GC_POP(); value_t opaque = cvalue(fl_ctx, jl_ast_ctx(fl_ctx)->jvtype, sizeof(void*)); - *(jl_value_t**)cv_data((cvalue_t*)ptr(opaque)) = jl_exception_in_transit; + *(jl_value_t**)cv_data((cvalue_t*)ptr(opaque)) = ptls->exception_in_transit; return fl_list2(fl_ctx, jl_ast_ctx(fl_ctx)->error_sym, opaque); } // protect result from GC, otherwise it could be freed during future @@ -172,7 +176,7 @@ value_t fl_invoke_julia_macro(fl_context_t *fl_ctx, value_t *args, uint32_t narg fl_gc_handle(fl_ctx, &scm); value_t scmresult; jl_module_t *defmod = mfunc->def->module; - if (defmod == NULL || defmod == jl_current_module) { + if (defmod == NULL || defmod == ptls->current_module) { scmresult = fl_cons(fl_ctx, scm, fl_ctx->F); } else { @@ -232,6 +236,7 @@ static jl_ast_context_list_t *jl_ast_ctx_freed = NULL; static jl_ast_context_t *jl_ast_ctx_enter(void) { + jl_ptls_t ptls = jl_get_ptls_states(); JL_SIGATOMIC_BEGIN(); JL_LOCK_NOGC(&flisp_lock); jl_ast_context_list_t *node; @@ -239,7 +244,7 @@ static jl_ast_context_t *jl_ast_ctx_enter(void) // First check if the current task is using one of the contexts for (node = jl_ast_ctx_using;node;(node = node->next)) { ctx = jl_ast_context_list_item(node); - if (ctx->task == jl_current_task) { + if (ctx->task == ptls->current_task) { ctx->ref++; JL_UNLOCK_NOGC(&flisp_lock); return ctx; @@ -251,7 +256,7 @@ static jl_ast_context_t *jl_ast_ctx_enter(void) jl_ast_context_list_insert(&jl_ast_ctx_using, node); ctx = jl_ast_context_list_item(node); ctx->ref = 1; - ctx->task = jl_current_task; + ctx->task = ptls->current_task; ctx->roots = NULL; JL_UNLOCK_NOGC(&flisp_lock); return ctx; @@ -260,7 +265,7 @@ static jl_ast_context_t *jl_ast_ctx_enter(void) ctx = (jl_ast_context_t*)calloc(1, sizeof(jl_ast_context_t)); // ctx->roots is NULL already due to calloc. ctx->ref = 1; - ctx->task = jl_current_task; + ctx->task = ptls->current_task; node = &ctx->list; jl_ast_context_list_insert(&jl_ast_ctx_using, node); JL_UNLOCK_NOGC(&flisp_lock); @@ -283,10 +288,11 @@ static void jl_ast_ctx_leave(jl_ast_context_t *ctx) void jl_init_frontend(void) { + jl_ptls_t ptls = jl_get_ptls_states(); if (jl_ast_ctx_using || jl_ast_ctx_freed) return; jl_ast_main_ctx.ref = 1; - jl_ast_main_ctx.task = jl_current_task; + jl_ast_main_ctx.task = ptls->current_task; jl_ast_context_list_insert(&jl_ast_ctx_using, &jl_ast_main_ctx.list); jl_init_ast_ctx(&jl_ast_main_ctx); // To match the one in jl_ast_ctx_leave @@ -344,6 +350,7 @@ extern int64_t conv_to_int64(void *data, numerictype_t tag); static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, int eo) { + jl_ptls_t ptls = jl_get_ptls_states(); if (fl_isnumber(fl_ctx, e)) { int64_t i64; if (isfixnum(e)) { @@ -444,7 +451,7 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, int eo) if (sym == top_sym) { scmv = scm_to_julia_(fl_ctx,car_(e),0); assert(jl_is_symbol(scmv)); - temp = jl_module_globalref(jl_base_relative_to(jl_current_module), (jl_sym_t*)scmv); + temp = jl_module_globalref(jl_base_relative_to(ptls->current_module), (jl_sym_t*)scmv); JL_GC_POP(); return temp; } @@ -644,6 +651,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len, jl_value_t *jl_parse_eval_all(const char *fname, const char *content, size_t contentlen) { + jl_ptls_t ptls = jl_get_ptls_states(); if (in_pure_callback) jl_error("cannot use include inside a generated function"); jl_ast_context_t *ctx = jl_ast_ctx_enter(); @@ -717,7 +725,7 @@ jl_value_t *jl_parse_eval_all(const char *fname, jl_rethrow(); else jl_rethrow_other(jl_new_struct(jl_loaderror_type, form, result, - jl_exception_in_transit)); + ptls->exception_in_transit)); } JL_GC_POP(); return result; diff --git a/src/builtins.c b/src/builtins.c index 88df1d39f24dc..86b3ed761d69f 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -210,9 +210,10 @@ JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh) JL_DLLEXPORT void jl_pop_handler(int n) { + jl_ptls_t ptls = jl_get_ptls_states(); if (__unlikely(n <= 0)) return; - jl_handler_t *eh = jl_current_task->eh; + jl_handler_t *eh = ptls->current_task->eh; while (--n > 0) eh = eh->prev; jl_eh_restore_state(eh); @@ -540,6 +541,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex) jl_value_t *jl_toplevel_eval_in_warn(jl_module_t *m, jl_value_t *ex, int delay_warn) { + jl_ptls_t ptls = jl_get_ptls_states(); static int jl_warn_on_eval = 0; int last_delay_warn = jl_warn_on_eval; if (m == NULL) @@ -548,8 +550,8 @@ jl_value_t *jl_toplevel_eval_in_warn(jl_module_t *m, jl_value_t *ex, int delay_w return jl_eval_global_var(m, (jl_sym_t*)ex); jl_value_t *v=NULL; int last_lineno = jl_lineno; - jl_module_t *last_m = jl_current_module; - jl_module_t *task_last_m = jl_current_task->current_module; + jl_module_t *last_m = ptls->current_module; + jl_module_t *task_last_m = ptls->current_task->current_module; if (!delay_warn && jl_options.incremental && jl_generating_output()) { if (m != last_m) { jl_printf(JL_STDERR, "WARNING: eval from module %s to %s: \n", @@ -567,27 +569,28 @@ jl_value_t *jl_toplevel_eval_in_warn(jl_module_t *m, jl_value_t *ex, int delay_w jl_error("eval cannot be used in a generated function"); JL_TRY { jl_warn_on_eval = delay_warn && (jl_warn_on_eval || m != last_m); // compute whether a warning was suppressed - jl_current_task->current_module = jl_current_module = m; + ptls->current_task->current_module = ptls->current_module = m; v = jl_toplevel_eval(ex); } JL_CATCH { jl_warn_on_eval = last_delay_warn; jl_lineno = last_lineno; - jl_current_module = last_m; - jl_current_task->current_module = task_last_m; + ptls->current_module = last_m; + ptls->current_task->current_module = task_last_m; jl_rethrow(); } jl_warn_on_eval = last_delay_warn; jl_lineno = last_lineno; - jl_current_module = last_m; - jl_current_task->current_module = task_last_m; + ptls->current_module = last_m; + ptls->current_task->current_module = task_last_m; assert(v); return v; } JL_CALLABLE(jl_f_isdefined) { - jl_module_t *m = jl_current_module; + jl_ptls_t ptls = jl_get_ptls_states(); + jl_module_t *m = ptls->current_module; jl_sym_t *s=NULL; JL_NARGSV(isdefined, 1); if (jl_is_array(args[0])) { diff --git a/src/ccall.cpp b/src/ccall.cpp index 322a23d0bdb8e..fe001801cfece 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1010,6 +1010,7 @@ static std::string generate_func_sig( // ccall(pointer, rettype, (argtypes...), args...) static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) { + jl_ptls_t ptls = jl_get_ptls_states(); JL_NARGSV(ccall, 3); jl_value_t *rt=NULL, *at=NULL; JL_GC_PUSH2(&rt, &at); @@ -1069,9 +1070,10 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } } if (rt == NULL) { - if (jl_exception_in_transit && jl_typeis(jl_exception_in_transit, - jl_undefvarerror_type) - && jl_is_symbol(args[2])) { + if (ptls->exception_in_transit && + jl_typeis(ptls->exception_in_transit, + jl_undefvarerror_type) && + jl_is_symbol(args[2])) { std::string msg = "ccall return type undefined: " + std::string(jl_symbol_name((jl_sym_t*)args[2])); emit_error(msg.c_str(), ctx); diff --git a/src/codegen.cpp b/src/codegen.cpp index 363daed26aee8..99f87aca3ad74 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -690,8 +690,9 @@ static void emit_write_barrier(jl_codectx_t*, Value*, Value*); static void jl_rethrow_with_add(const char *fmt, ...) { - if (jl_typeis(jl_exception_in_transit, jl_errorexception_type)) { - char *str = jl_string_data(jl_fieldref(jl_exception_in_transit,0)); + jl_ptls_t ptls = jl_get_ptls_states(); + if (jl_typeis(ptls->exception_in_transit, jl_errorexception_type)) { + char *str = jl_string_data(jl_fieldref(ptls->exception_in_transit,0)); char buf[1024]; va_list args; va_start(args, fmt); @@ -3295,7 +3296,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) Value *val = emit_jlcall(jlnew_func, typ, &args[1], nargs-1, ctx); return mark_julia_type(val, true, ty, ctx); } - else if (head == exc_sym) { // *jl_exception_in_transit + else if (head == exc_sym) { // *ptls->exception_in_transit return mark_julia_type(builder.CreateLoad(emit_exc_in_transit(ctx), /*isvolatile*/true), true, jl_any_type, ctx); @@ -4009,6 +4010,7 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, Function *f, bool sre // Compile to LLVM IR, using a specialized signature if applicable. static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(declarations && "Capturing declarations is always required"); // step 1. unpack AST and allocate codegen context for this function @@ -4025,7 +4027,7 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func ctx.arrayvars = &arrayvars; ctx.labels = &labels; ctx.handlers = &handlers; - ctx.module = lam->def ? lam->def->module : jl_current_module; + ctx.module = lam->def ? lam->def->module : ptls->current_module; ctx.linfo = lam; ctx.name = jl_symbol_name(lam->def ? lam->def->name : anonymous_sym); ctx.funcName = ctx.name; diff --git a/src/dump.c b/src/dump.c index 486f44e260a2f..4eb9295189a4f 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1796,7 +1796,9 @@ static void jl_finalize_serializer(ios_t *f) { } void jl_typemap_rehash(union jl_typemap_t ml, int8_t offs); -static void jl_reinit_item(ios_t *f, jl_value_t *v, int how, arraylist_t *tracee_list) { +static void jl_reinit_item(ios_t *f, jl_value_t *v, int how, arraylist_t *tracee_list) +{ + jl_ptls_t ptls = jl_get_ptls_states(); JL_TRY { switch (how) { case 1: { // rehash ObjectIdDict @@ -1846,11 +1848,13 @@ static void jl_reinit_item(ios_t *f, jl_value_t *v, int how, arraylist_t *tracee jl_printf(JL_STDERR, "WARNING: error while reinitializing value "); jl_static_show(JL_STDERR, v); jl_printf(JL_STDERR, ":\n"); - jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_static_show(JL_STDERR, ptls->exception_in_transit); jl_printf(JL_STDERR, "\n"); } } -static jl_array_t *jl_finalize_deserializer(ios_t *f, arraylist_t *tracee_list) { + +static jl_array_t *jl_finalize_deserializer(ios_t *f, arraylist_t *tracee_list) +{ jl_array_t *init_order = NULL; if (mode != MODE_MODULE) init_order = (jl_array_t*)jl_deserialize_value(f, NULL); @@ -1984,6 +1988,7 @@ JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname) static void jl_restore_system_image_from_stream(ios_t *f) { + jl_ptls_t ptls = jl_get_ptls_states(); JL_LOCK(&dump_lock); // Might GC int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; @@ -2011,7 +2016,7 @@ static void jl_restore_system_image_from_stream(ios_t *f) jl_symbol("Core")); jl_base_module = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol("Base")); - jl_current_module = jl_base_module; // run start_image in Base + ptls->current_module = jl_base_module; // run start_image in Base // ensure everything in deser_tag is reassociated with its GlobalValue for (i = 2; i < 255; i++) { @@ -2392,6 +2397,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname) void jl_init_serializer(void) { + jl_ptls_t ptls = jl_get_ptls_states(); htable_new(&ser_tag, 0); htable_new(&common_symbol_tag, 0); htable_new(&fptr_to_id, sizeof(id_to_fptrs)/sizeof(*id_to_fptrs)); @@ -2474,7 +2480,7 @@ void jl_init_serializer(void) jl_gotonode_type->name, jl_quotenode_type->name, jl_globalref_type->name, - jl_root_task, + ptls->root_task, NULL }; diff --git a/src/gc.c b/src/gc.c index 82d37a55ca565..330b7e37ba603 100644 --- a/src/gc.c +++ b/src/gc.c @@ -102,6 +102,7 @@ static void schedule_finalization(void *o, void *f) static void run_finalizer(jl_value_t *o, jl_value_t *ff) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(!jl_typeis(ff, jl_voidpointer_type)); jl_value_t *args[2] = {ff,o}; JL_TRY { @@ -109,7 +110,7 @@ static void run_finalizer(jl_value_t *o, jl_value_t *ff) } JL_CATCH { jl_printf(JL_STDERR, "error in running finalizer: "); - jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_static_show(JL_STDERR, ptls->exception_in_transit); jl_printf(JL_STDERR, "\n"); } } @@ -161,22 +162,22 @@ static void finalize_object(arraylist_t *list, jl_value_t *o, // The first two entries are assumed to be empty and the rest are assumed to // be pointers to `jl_value_t` objects -static void jl_gc_push_arraylist(arraylist_t *list) +static void jl_gc_push_arraylist(jl_ptls_t ptls, arraylist_t *list) { void **items = list->items; items[0] = (void*)(((uintptr_t)list->len - 2) << 1); - items[1] = jl_pgcstack; - jl_pgcstack = (jl_gcframe_t*)items; + items[1] = ptls->pgcstack; + ptls->pgcstack = (jl_gcframe_t*)items; } // Same assumption as `jl_gc_push_arraylist`. Requires the finalizers lock // to be hold for the current thread and will release the lock when the // function returns. -static void jl_gc_run_finalizers_in_list(arraylist_t *list) +static void jl_gc_run_finalizers_in_list(jl_ptls_t ptls, arraylist_t *list) { size_t len = list->len; jl_value_t **items = (jl_value_t**)list->items; - jl_gc_push_arraylist(list); + jl_gc_push_arraylist(ptls, list); JL_UNLOCK_NOGC(&finalizers_lock); for (size_t i = 2;i < len;i += 2) { run_finalizer(items[i], items[i + 1]); @@ -184,7 +185,7 @@ static void jl_gc_run_finalizers_in_list(arraylist_t *list) JL_GC_POP(); } -static void run_finalizers(void) +static void run_finalizers(jl_ptls_t ptls) { JL_LOCK_NOGC(&finalizers_lock); if (to_finalize.len == 0) { @@ -201,7 +202,7 @@ static void run_finalizers(void) arraylist_push(&copied_list, copied_list.items[0]); arraylist_push(&copied_list, copied_list.items[1]); // This releases the finalizers lock. - jl_gc_run_finalizers_in_list(&copied_list); + jl_gc_run_finalizers_in_list(ptls, &copied_list); arraylist_free(&copied_list); } @@ -212,7 +213,7 @@ JL_DLLEXPORT void jl_gc_enable_finalizers(jl_ptls_t ptls, int on) ptls->finalizers_inhibited = new_val; if (!new_val && old_val && !ptls->in_finalizer) { ptls->in_finalizer = 1; - run_finalizers(); + run_finalizers(ptls); ptls->in_finalizer = 0; } } @@ -233,14 +234,14 @@ static void schedule_all_finalizers(arraylist_t *flist) flist->len = 0; } -void jl_gc_run_all_finalizers(void) +void jl_gc_run_all_finalizers(jl_ptls_t ptls) { for (int i = 0;i < jl_n_threads;i++) { jl_ptls_t ptls2 = jl_all_tls_states[i]; schedule_all_finalizers(&ptls2->finalizers); } schedule_all_finalizers(&finalizer_list_marked); - run_finalizers(); + run_finalizers(ptls); } static void gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) @@ -302,7 +303,7 @@ JL_DLLEXPORT void jl_finalize_th(jl_ptls_t ptls, jl_value_t *o) finalize_object(&finalizer_list_marked, o, &copied_list, 0); if (copied_list.len > 2) { // This releases the finalizers lock. - jl_gc_run_finalizers_in_list(&copied_list); + jl_gc_run_finalizers_in_list(ptls, &copied_list); } else { JL_UNLOCK_NOGC(&finalizers_lock); @@ -1926,7 +1927,7 @@ JL_DLLEXPORT void jl_gc_collect(int full) if (!ptls->finalizers_inhibited) { int8_t was_in_finalizer = ptls->in_finalizer; ptls->in_finalizer = 1; - run_finalizers(); + run_finalizers(ptls); ptls->in_finalizer = was_in_finalizer; } } diff --git a/src/gf.c b/src/gf.c index 7f5262808603e..b7bfdff383001 100644 --- a/src/gf.c +++ b/src/gf.c @@ -82,6 +82,7 @@ JL_DLLEXPORT void jl_register_linfo_tracer(void (*callback)(jl_lambda_info_t *tr void jl_call_tracer(tracer_cb callback, jl_value_t *tracee) { + jl_ptls_t ptls = jl_get_ptls_states(); int last_in = in_pure_callback; JL_TRY { in_pure_callback = 1; @@ -91,7 +92,7 @@ void jl_call_tracer(tracer_cb callback, jl_value_t *tracee) JL_CATCH { in_pure_callback = last_in; jl_printf(JL_STDERR, "WARNING: tracer callback function threw an error:\n"); - jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_static_show(JL_STDERR, ptls->exception_in_transit); jl_printf(JL_STDERR, "\n"); jlbacktrace(); } diff --git a/src/init.c b/src/init.c index 4f2d3532423ce..dbef9f61be2fd 100644 --- a/src/init.c +++ b/src/init.c @@ -216,6 +216,7 @@ static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdow JL_DLLEXPORT void jl_atexit_hook(int exitcode) { + jl_ptls_t ptls = jl_get_ptls_states(); if (exitcode == 0) julia_save(); jl_print_gc_stats(JL_STDERR); if (jl_options.code_coverage) @@ -230,12 +231,12 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) } JL_CATCH { jl_printf(JL_STDERR, "\natexit hook threw an error: "); - jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_static_show(JL_STDERR, ptls->exception_in_transit); } } } - jl_gc_run_all_finalizers(); + jl_gc_run_all_finalizers(ptls); uv_loop_t *loop = jl_global_event_loop(); @@ -299,7 +300,7 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) //error handling -- continue cleanup, as much as possible uv_unref(item->h); jl_printf(JL_STDERR, "error during exit cleanup: close: "); - jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_static_show(JL_STDERR, ptls->exception_in_transit); item = next_shutdown_queue_item(item); } } @@ -638,7 +639,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_core_module = jl_new_module(jl_symbol("Core")); jl_type_type->name->mt->module = jl_core_module; jl_top_module = jl_core_module; - jl_current_module = jl_core_module; + ptls->current_module = jl_core_module; jl_init_intrinsic_functions(); jl_init_primitives(); jl_get_builtins(); @@ -646,9 +647,9 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_new_main_module(); jl_internal_main_module = jl_main_module; - jl_current_module = jl_core_module; + ptls->current_module = jl_core_module; for (int t = 0;t < jl_n_threads;t++) { - jl_all_tls_states[t]->root_task->current_module = jl_current_module; + jl_all_tls_states[t]->root_task->current_module = ptls->current_module; } jl_load("boot.jl"); @@ -663,7 +664,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) } JL_CATCH { jl_printf(JL_STDERR, "error during init:\n"); - jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_static_show(JL_STDERR, ptls->exception_in_transit); jl_printf(JL_STDERR, "\n"); jl_exit(1); } @@ -690,9 +691,9 @@ void _julia_init(JL_IMAGE_SEARCH rel) if (jl_base_module != NULL) { jl_add_standard_imports(jl_main_module); } - jl_current_module = jl_main_module; + ptls->current_module = jl_main_module; for (int t = 0;t < jl_n_threads;t++) { - jl_all_tls_states[t]->root_task->current_module = jl_current_module; + jl_all_tls_states[t]->root_task->current_module = ptls->current_module; } // This needs to be after jl_start_threads diff --git a/src/interpreter.c b/src/interpreter.c index 2aa62dfd765b1..c8e844d8d07bc 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -34,23 +34,24 @@ jl_value_t *jl_interpret_toplevel_expr(jl_value_t *e) JL_DLLEXPORT jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e, jl_lambda_info_t *lam) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_value_t *v=NULL; - jl_module_t *last_m = jl_current_module; - jl_module_t *task_last_m = jl_current_task->current_module; + jl_module_t *last_m = ptls->current_module; + jl_module_t *task_last_m = ptls->current_task->current_module; interpreter_state s; s.lam = lam; s.locals = NULL; s.sparam_vals = NULL; JL_TRY { - jl_current_task->current_module = jl_current_module = m; + ptls->current_task->current_module = ptls->current_module = m; v = eval(e, &s); } JL_CATCH { - jl_current_module = last_m; - jl_current_task->current_module = task_last_m; + ptls->current_module = last_m; + ptls->current_task->current_module = task_last_m; jl_rethrow(); } - jl_current_module = last_m; - jl_current_task->current_module = task_last_m; + ptls->current_module = last_m; + ptls->current_task->current_module = task_last_m; assert(v); return v; } @@ -146,6 +147,7 @@ static int jl_linfo_nssavalues(jl_lambda_info_t *li) static jl_value_t *eval(jl_value_t *e, interpreter_state *s) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_lambda_info_t *lam = s==NULL ? NULL : s->lam; if (jl_is_ssavalue(e)) { ssize_t id = ((jl_ssavalue_t*)e)->id; @@ -172,7 +174,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) } if (jl_is_quotenode(e)) return jl_fieldref(e,0); - jl_module_t *modu = (lam == NULL || lam->def == NULL) ? jl_current_module : lam->def->module; + jl_module_t *modu = (lam == NULL || lam->def == NULL) ? ptls->current_module : lam->def->module; if (jl_is_symbol(e)) { // bare symbols appear in toplevel exprs not wrapped in `thunk` jl_value_t *v = jl_get_global(modu, (jl_sym_t*)e); if (v == NULL) @@ -225,7 +227,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) return (jl_value_t*)jl_any_type; } else if (ex->head == exc_sym) { - return jl_exception_in_transit; + return ptls->exception_in_transit; } else if (ex->head == method_sym) { jl_sym_t *fname = (jl_sym_t*)args[0]; @@ -441,6 +443,7 @@ jl_value_t *jl_toplevel_eval_body(jl_array_t *stmts) static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, int start, int toplevel) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_handler_t __eh; size_t i=start, ns = jl_array_len(stmts); @@ -482,7 +485,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, int start, sym = (jl_value_t*)jl_globalref_name(sym); } else { - m = (s==NULL || s->lam==NULL || s->lam->def==NULL) ? jl_current_module : s->lam->def->module; + m = (s==NULL || s->lam==NULL || s->lam->def==NULL) ? ptls->current_module : s->lam->def->module; } assert(jl_is_symbol(sym)); JL_GC_PUSH1(&rhs); @@ -513,7 +516,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, int start, } else { #ifdef _OS_WINDOWS_ - if (jl_exception_in_transit == jl_stackovf_exception) + if (ptls->exception_in_transit == jl_stackovf_exception) _resetstkoflw(); #endif i = jl_unbox_long(jl_exprarg(stmt,0))-1; diff --git a/src/jlapi.c b/src/jlapi.c index 945e1f32d6d2e..0cc3d8fed4ea1 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -69,13 +69,15 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str) JL_DLLEXPORT jl_value_t *jl_exception_occurred(void) { - return jl_exception_in_transit == jl_nothing ? NULL : - jl_exception_in_transit; + jl_ptls_t ptls = jl_get_ptls_states(); + return ptls->exception_in_transit == jl_nothing ? NULL : + ptls->exception_in_transit; } JL_DLLEXPORT void jl_exception_clear(void) { - jl_exception_in_transit = jl_nothing; + jl_ptls_t ptls = jl_get_ptls_states(); + ptls->exception_in_transit = jl_nothing; } // get the name of a type as a string diff --git a/src/jltypes.c b/src/jltypes.c index 2df57706e2826..058d891fe79a9 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3456,6 +3456,7 @@ extern void jl_init_int32_int64_cache(void); void jl_init_types(void) { + jl_ptls_t ptls = jl_get_ptls_states(); arraylist_new(&partial_inst, 0); // create base objects jl_datatype_type = jl_new_uninitialized_datatype(11, 1); @@ -3473,7 +3474,7 @@ void jl_init_types(void) jl_any_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Any"), NULL, jl_emptysvec); jl_any_type->super = jl_any_type; jl_type_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Type"), jl_any_type, jl_emptysvec); - jl_type_type_mt = jl_new_method_table(jl_type_type->name->name, jl_current_module); + jl_type_type_mt = jl_new_method_table(jl_type_type->name->name, ptls->current_module); jl_type_type->name->mt = jl_type_type_mt; // initialize them. lots of cycles. @@ -3510,7 +3511,7 @@ void jl_init_types(void) jl_typename_type->name = jl_new_typename(jl_symbol("TypeName")); jl_typename_type->name->primary = (jl_value_t*)jl_typename_type; - jl_typename_type->name->mt = jl_new_method_table(jl_typename_type->name->name, jl_current_module); + jl_typename_type->name->mt = jl_new_method_table(jl_typename_type->name->name, ptls->current_module); jl_typename_type->super = jl_any_type; jl_typename_type->parameters = jl_emptysvec; jl_typename_type->name->names = jl_svec(8, jl_symbol("name"), jl_symbol("module"), @@ -3531,7 +3532,7 @@ void jl_init_types(void) jl_methtable_type->name = jl_new_typename(jl_symbol("MethodTable")); jl_methtable_type->name->primary = (jl_value_t*)jl_methtable_type; - jl_methtable_type->name->mt = jl_new_method_table(jl_methtable_type->name->name, jl_current_module); + jl_methtable_type->name->mt = jl_new_method_table(jl_methtable_type->name->name, ptls->current_module); jl_methtable_type->super = jl_any_type; jl_methtable_type->parameters = jl_emptysvec; jl_methtable_type->name->names = jl_svec(6, jl_symbol("name"), jl_symbol("defs"), @@ -3550,7 +3551,7 @@ void jl_init_types(void) jl_sym_type->name = jl_new_typename(jl_symbol("Symbol")); jl_sym_type->name->primary = (jl_value_t*)jl_sym_type; - jl_sym_type->name->mt = jl_new_method_table(jl_sym_type->name->name, jl_current_module); + jl_sym_type->name->mt = jl_new_method_table(jl_sym_type->name->name, ptls->current_module); jl_sym_type->super = jl_any_type; jl_sym_type->parameters = jl_emptysvec; jl_sym_type->name->names = jl_emptysvec; @@ -3567,7 +3568,7 @@ void jl_init_types(void) jl_simplevector_type->name = jl_new_typename(jl_symbol("SimpleVector")); jl_simplevector_type->name->primary = (jl_value_t*)jl_simplevector_type; - jl_simplevector_type->name->mt = jl_new_method_table(jl_simplevector_type->name->name, jl_current_module); + jl_simplevector_type->name->mt = jl_new_method_table(jl_simplevector_type->name->name, ptls->current_module); jl_simplevector_type->super = jl_any_type; jl_simplevector_type->parameters = jl_emptysvec; jl_simplevector_type->name->names = jl_svec(1, jl_symbol("length")); diff --git a/src/julia.h b/src/julia.h index d8f6ec87dd89e..7c2431c73b1e4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1189,7 +1189,6 @@ extern JL_DLLEXPORT jl_module_t *jl_internal_main_module; extern JL_DLLEXPORT jl_module_t *jl_core_module; extern JL_DLLEXPORT jl_module_t *jl_base_module; extern JL_DLLEXPORT jl_module_t *jl_top_module; -#define jl_current_module (jl_get_ptls_states()->current_module) JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name); // get binding for reading JL_DLLEXPORT jl_binding_t *jl_get_binding(jl_module_t *m, jl_sym_t *var); @@ -1463,11 +1462,6 @@ typedef struct _jl_task_t { #endif } jl_task_t; -#define jl_current_task (jl_get_ptls_states()->current_task) -#define jl_root_task (jl_get_ptls_states()->root_task) -#define jl_exception_in_transit (jl_get_ptls_states()->exception_in_transit) -#define jl_task_arg_in_transit (jl_get_ptls_states()->task_arg_in_transit) - JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize); JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg); JL_DLLEXPORT void JL_NORETURN jl_throw(jl_value_t *e); @@ -1477,10 +1471,11 @@ JL_DLLEXPORT void JL_NORETURN jl_rethrow_other(jl_value_t *e); #ifdef JULIA_ENABLE_THREADING static inline void jl_lock_frame_push(jl_mutex_t *lock) { + jl_ptls_t ptls = jl_get_ptls_states(); // For early bootstrap - if (__unlikely(!jl_current_task)) + if (__unlikely(!ptls->current_task)) return; - arraylist_t *locks = &jl_current_task->locks; + arraylist_t *locks = &ptls->current_task->locks; size_t len = locks->len; if (__unlikely(len >= locks->max)) { arraylist_grow(locks, 1); @@ -1492,8 +1487,9 @@ static inline void jl_lock_frame_push(jl_mutex_t *lock) } static inline void jl_lock_frame_pop(void) { - if (__likely(jl_current_task)) { - jl_current_task->locks.len--; + jl_ptls_t ptls = jl_get_ptls_states(); + if (__likely(ptls->current_task)) { + ptls->current_task->locks.len--; } } #else @@ -1510,7 +1506,7 @@ STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh) { jl_ptls_t ptls = jl_get_ptls_states(); jl_task_t *current_task = ptls->current_task; - // `eh` may not be `jl_current_task->eh`. See `jl_pop_handler` + // `eh` may not be `ptls->current_task->eh`. See `jl_pop_handler` // This function should **NOT** have any safepoint before the ones at the // end. sig_atomic_t old_defer_signal = ptls->defer_signal; @@ -1576,7 +1572,7 @@ void jl_longjmp(jmp_buf _Buf,int _Value); #define JL_CATCH \ else \ for (i__ca=1, jl_eh_restore_state(&__eh); i__ca; i__ca=0) \ - if (((jl_exception_in_transit==jl_stackovf_exception) && _resetstkoflw()) || 1) + if (((jl_get_ptls_states()->exception_in_transit==jl_stackovf_exception) && _resetstkoflw()) || 1) #else #define JL_CATCH \ else \ @@ -1753,6 +1749,12 @@ typedef struct { float value; } jl_nullable_float32_t; +#define jl_current_module (jl_get_ptls_states()->current_module) +#define jl_current_task (jl_get_ptls_states()->current_task) +#define jl_root_task (jl_get_ptls_states()->root_task) +#define jl_exception_in_transit (jl_get_ptls_states()->exception_in_transit) +#define jl_task_arg_in_transit (jl_get_ptls_states()->task_arg_in_transit) + #ifdef __cplusplus } #endif diff --git a/src/julia_internal.h b/src/julia_internal.h index f2db5c44192a8..4237c136901e0 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -105,7 +105,7 @@ void jl_gc_setmark(jl_value_t *v); void jl_gc_sync_total_bytes(void); void jl_gc_track_malloced_array(jl_array_t *a); void jl_gc_count_allocd(size_t sz); -void jl_gc_run_all_finalizers(void); +void jl_gc_run_all_finalizers(jl_ptls_t ptls); void *allocb(size_t sz); void gc_queue_binding(jl_binding_t *bnd); diff --git a/src/module.c b/src/module.c index 77a7144ddb32c..cdeb130cc8802 100644 --- a/src/module.c +++ b/src/module.c @@ -57,9 +57,10 @@ JL_DLLEXPORT jl_value_t *jl_f_new_module(jl_sym_t *name, uint8_t std_imports) JL_DLLEXPORT void jl_set_istopmod(uint8_t isprimary) { - jl_current_module->istopmod = 1; + jl_ptls_t ptls = jl_get_ptls_states(); + ptls->current_module->istopmod = 1; if (isprimary) - jl_top_module = jl_current_module; + jl_top_module = ptls->current_module; } JL_DLLEXPORT uint8_t jl_istopmod(jl_module_t *mod) @@ -449,7 +450,8 @@ JL_DLLEXPORT void jl_set_const(jl_module_t *m, jl_sym_t *var, jl_value_t *val) JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) { - if (m == NULL) m = jl_current_module; + jl_ptls_t ptls = jl_get_ptls_states(); + if (m == NULL) m = ptls->current_module; jl_binding_t *b = jl_get_binding(m, var); return b && b->constp; } @@ -529,13 +531,15 @@ JL_DLLEXPORT void jl_declare_constant(jl_binding_t *b) JL_DLLEXPORT jl_value_t *jl_get_current_module(void) { - return (jl_value_t*)jl_current_module; + jl_ptls_t ptls = jl_get_ptls_states(); + return (jl_value_t*)ptls->current_module; } JL_DLLEXPORT void jl_set_current_module(jl_value_t *m) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_typeis(m, jl_module_type)); - jl_current_module = (jl_module_t*)m; + ptls->current_module = (jl_module_t*)m; } JL_DLLEXPORT jl_value_t *jl_module_usings(jl_module_t *m) @@ -587,6 +591,7 @@ jl_function_t *jl_module_get_initializer(jl_module_t *m) JL_DLLEXPORT void jl_module_run_initializer(jl_module_t *m) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_function_t *f = jl_module_get_initializer(m); if (f == NULL) return; @@ -599,7 +604,7 @@ JL_DLLEXPORT void jl_module_run_initializer(jl_module_t *m) } else { jl_rethrow_other(jl_new_struct(jl_initerror_type, m->name, - jl_exception_in_transit)); + ptls->exception_in_transit)); } } } diff --git a/src/signals-unix.c b/src/signals-unix.c index 26715c12e930d..838b162cb5e45 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -52,7 +52,7 @@ static void JL_NORETURN jl_throw_in_ctx(jl_value_t *e, void *sigctx) if (!ptls->safe_restore) ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, jl_to_bt_context(sigctx)); - jl_exception_in_transit = e; + ptls->exception_in_transit = e; // TODO throw the error by modifying sigctx for supported platforms // This will avoid running the atexit handler on the signal stack // if no excepiton handler is registered. diff --git a/src/signals-win.c b/src/signals-win.c index a3df41188222f..a9b77c11a1ad2 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -113,7 +113,7 @@ void jl_throw_in_ctx(jl_value_t *excpt, CONTEXT *ctxThread, int bt) #endif ptls->bt_size = bt ? rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, ctxThread) : 0; - jl_exception_in_transit = excpt; + ptls->exception_in_transit = excpt; #if defined(_CPU_X86_64_) *(DWORD64*)Rsp = 0; ctxThread->Rsp = Rsp; diff --git a/src/task.c b/src/task.c index 9312ff099b48e..5da2b1a7e35b4 100644 --- a/src/task.c +++ b/src/task.c @@ -200,7 +200,7 @@ static void JL_NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) if (ptls->tid != 0) { // For now, only thread 0 runs the task scheduler. // The others return to the thread loop - jl_switchto(jl_root_task, jl_nothing); + jl_switchto(ptls->root_task, jl_nothing); gc_debug_critical_error(); abort(); } @@ -531,7 +531,8 @@ JL_DLLEXPORT void jl_throw(jl_value_t *e) JL_DLLEXPORT void jl_rethrow(void) { - throw_internal(jl_exception_in_transit); + jl_ptls_t ptls = jl_get_ptls_states(); + throw_internal(ptls->exception_in_transit); } JL_DLLEXPORT void jl_rethrow_other(jl_value_t *e) @@ -541,6 +542,7 @@ JL_DLLEXPORT void jl_rethrow_other(jl_value_t *e) JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) { + jl_ptls_t ptls = jl_get_ptls_states(); size_t pagesz = jl_page_size; jl_task_t *t = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); jl_set_typeof(t, jl_task_type); @@ -551,7 +553,7 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) ssize = LLT_ALIGN(ssize, pagesz); t->ssize = ssize; t->current_module = NULL; - t->parent = jl_current_task; + t->parent = ptls->current_task; t->tls = jl_nothing; t->consumers = jl_nothing; t->state = runnable_sym; @@ -607,7 +609,8 @@ JL_CALLABLE(jl_unprotect_stack) JL_DLLEXPORT jl_value_t *jl_get_current_task(void) { - return (jl_value_t*)jl_current_task; + jl_ptls_t ptls = jl_get_ptls_states(); + return (jl_value_t*)ptls->current_task; } jl_function_t *jl_unprotect_stack_func; @@ -655,38 +658,38 @@ void jl_init_tasks(void) void jl_init_root_task(void *stack, size_t ssize) { jl_ptls_t ptls = jl_get_ptls_states(); - jl_current_task = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); - jl_set_typeof(jl_current_task, jl_task_type); + ptls->current_task = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); + jl_set_typeof(ptls->current_task, jl_task_type); #ifdef COPY_STACKS - jl_current_task->ssize = 0; // size of saved piece - jl_current_task->bufsz = 0; - jl_current_task->stkbuf = NULL; + ptls->current_task->ssize = 0; // size of saved piece + ptls->current_task->bufsz = 0; + ptls->current_task->stkbuf = NULL; #else - jl_current_task->ssize = ssize; - jl_current_task->stkbuf = stack; + ptls->current_task->ssize = ssize; + ptls->current_task->stkbuf = stack; #endif - jl_current_task->started = 1; - jl_current_task->parent = jl_current_task; - jl_current_task->current_module = jl_current_module; - jl_current_task->tls = jl_nothing; - jl_current_task->consumers = jl_nothing; - jl_current_task->state = runnable_sym; - jl_current_task->start = NULL; - jl_current_task->result = jl_nothing; - jl_current_task->donenotify = jl_nothing; - jl_current_task->exception = jl_nothing; - jl_current_task->backtrace = jl_nothing; - jl_current_task->eh = NULL; - jl_current_task->gcstack = NULL; - jl_current_task->tid = ptls->tid; + ptls->current_task->started = 1; + ptls->current_task->parent = ptls->current_task; + ptls->current_task->current_module = ptls->current_module; + ptls->current_task->tls = jl_nothing; + ptls->current_task->consumers = jl_nothing; + ptls->current_task->state = runnable_sym; + ptls->current_task->start = NULL; + ptls->current_task->result = jl_nothing; + ptls->current_task->donenotify = jl_nothing; + ptls->current_task->exception = jl_nothing; + ptls->current_task->backtrace = jl_nothing; + ptls->current_task->eh = NULL; + ptls->current_task->gcstack = NULL; + ptls->current_task->tid = ptls->tid; #ifdef JULIA_ENABLE_THREADING - arraylist_new(&jl_current_task->locks, 0); + arraylist_new(&ptls->current_task->locks, 0); #endif - jl_root_task = jl_current_task; + ptls->root_task = ptls->current_task; - jl_exception_in_transit = (jl_value_t*)jl_nothing; - jl_task_arg_in_transit = (jl_value_t*)jl_nothing; + ptls->exception_in_transit = (jl_value_t*)jl_nothing; + ptls->task_arg_in_transit = (jl_value_t*)jl_nothing; } JL_DLLEXPORT int jl_is_task_started(jl_task_t *t) diff --git a/src/threading.c b/src/threading.c index d15737c773941..598bfc390266d 100644 --- a/src/threading.c +++ b/src/threading.c @@ -313,11 +313,12 @@ static void ti_init_master_thread(void) // all threads call this function to run user code static jl_value_t *ti_run_fun(jl_svec_t *args) { + jl_ptls_t ptls = jl_get_ptls_states(); JL_TRY { jl_apply(jl_svec_data(args), jl_svec_len(args)); } JL_CATCH { - return jl_exception_in_transit; + return ptls->exception_in_transit; } return jl_nothing; } @@ -404,11 +405,11 @@ void ti_threadfun(void *arg) // enter GC unsafe region when starting the work. int8_t gc_state = jl_gc_unsafe_enter(ptls); // This is probably always NULL for now - jl_module_t *last_m = jl_current_module; + jl_module_t *last_m = ptls->current_module; JL_GC_PUSH1(&last_m); - jl_current_module = work->current_module; + ptls->current_module = work->current_module; ti_run_fun(work->args); - jl_current_module = last_m; + ptls->current_module = last_m; JL_GC_POP(); jl_gc_unsafe_leave(ptls, gc_state); } @@ -671,7 +672,7 @@ JL_DLLEXPORT jl_value_t *jl_threading_run(jl_svec_t *args) threadwork.fun = NULL; threadwork.args = args; threadwork.ret = jl_nothing; - threadwork.current_module = jl_current_module; + threadwork.current_module = ptls->current_module; #if PROFILE_JL_THREADING uint64_t tcompile = uv_hrtime(); diff --git a/src/toplevel.c b/src/toplevel.c index 415e9c142753f..cbf7aa381dfd5 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -41,11 +41,13 @@ JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m) JL_DLLEXPORT jl_module_t *jl_new_main_module(void) { + jl_ptls_t ptls = jl_get_ptls_states(); if (jl_generating_output() && jl_options.incremental) jl_error("cannot call workspace() in incremental compile mode"); // switch to a new top-level module - if (jl_current_module != jl_main_module && jl_current_module != NULL && jl_main_module != NULL) + if (ptls->current_module != jl_main_module && + ptls->current_module != NULL && jl_main_module != NULL) jl_error("Main can only be replaced from the top level"); jl_module_t *old_main = jl_main_module; @@ -54,14 +56,14 @@ JL_DLLEXPORT jl_module_t *jl_new_main_module(void) jl_main_module->parent = jl_main_module; if (old_main) // don't block continued loading of incremental caches jl_main_module->uuid = old_main->uuid; - jl_current_module = jl_main_module; + ptls->current_module = jl_main_module; jl_core_module->parent = jl_main_module; jl_set_const(jl_main_module, jl_symbol("Core"), (jl_value_t*)jl_core_module); jl_set_global(jl_core_module, jl_symbol("Main"), (jl_value_t*)jl_main_module); - jl_current_task->current_module = jl_main_module; + ptls->current_task->current_module = jl_main_module; return old_main; } @@ -91,6 +93,7 @@ static void jl_module_load_time_initialize(jl_module_t *m) extern void jl_get_system_hooks(void); jl_value_t *jl_eval_module_expr(jl_expr_t *ex) { + jl_ptls_t ptls = jl_get_ptls_states(); static arraylist_t module_stack; static int initialized=0; static jl_module_t *outermost = NULL; @@ -99,7 +102,7 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) initialized = 1; } assert(ex->head == module_sym); - jl_module_t *last_module = jl_current_module; + jl_module_t *last_module = ptls->current_module; if (jl_array_len(ex->args) != 3 || !jl_is_expr(jl_exprarg(ex,2))) { jl_error("syntax: malformed module expression"); } @@ -108,7 +111,7 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) if (!jl_is_symbol(name)) { jl_type_error("module", (jl_value_t*)jl_sym_type, (jl_value_t*)name); } - jl_module_t *parent_module = jl_current_module; + jl_module_t *parent_module = ptls->current_module; jl_binding_t *b = jl_get_binding_wr(parent_module, name); jl_declare_constant(b); if (b->value != NULL) { @@ -145,8 +148,8 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) jl_value_t *defaultdefs = NULL, *form = NULL; JL_GC_PUSH3(&last_module, &defaultdefs, &form); - jl_module_t *task_last_m = jl_current_task->current_module; - jl_current_task->current_module = jl_current_module = newm; + jl_module_t *task_last_m = ptls->current_task->current_module; + ptls->current_task->current_module = ptls->current_module = newm; jl_module_t *prev_outermost = outermost; size_t stackidx = module_stack.len; if (outermost == NULL) @@ -168,15 +171,15 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) } } JL_CATCH { - jl_current_module = last_module; - jl_current_task->current_module = task_last_m; + ptls->current_module = last_module; + ptls->current_task->current_module = task_last_m; outermost = prev_outermost; module_stack.len = stackidx; jl_rethrow(); } JL_GC_POP(); - jl_current_module = last_module; - jl_current_task->current_module = task_last_m; + ptls->current_module = last_module; + ptls->current_task->current_module = task_last_m; outermost = prev_outermost; #if 0 @@ -202,7 +205,7 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) arraylist_push(&module_stack, newm); - if (outermost == NULL || jl_current_module == jl_main_module) { + if (outermost == NULL || ptls->current_module == jl_main_module) { JL_TRY { size_t i, l=module_stack.len; for(i = stackidx; i < l; i++) { @@ -317,6 +320,7 @@ static jl_value_t *require_func=NULL; static jl_module_t *eval_import_path_(jl_array_t *args, int retrying) { + jl_ptls_t ptls = jl_get_ptls_states(); // in .A.B.C, first find a binding for A in the chain of module scopes // following parent links. then evaluate the rest of the path from there. // in A.B, look for A in Main first. @@ -329,7 +333,7 @@ static jl_module_t *eval_import_path_(jl_array_t *args, int retrying) m = jl_main_module; } else { - m = jl_current_module; + m = ptls->current_module; while (1) { var = (jl_sym_t*)jl_array_ptr_ref(args,i); if (!jl_is_symbol(var)) jl_type_error("import or using", (jl_value_t*)jl_sym_type, (jl_value_t*)var); @@ -378,7 +382,7 @@ static jl_module_t *eval_import_path_(jl_array_t *args, int retrying) } if (retrying && require_func) { jl_printf(JL_STDERR, "WARNING: requiring \"%s\" in module \"%s\" did not define a corresponding module.\n", jl_symbol_name(var), - jl_symbol_name(jl_current_module->name)); + jl_symbol_name(ptls->current_module->name)); return NULL; } else { @@ -417,6 +421,7 @@ int jl_is_toplevel_only_expr(jl_value_t *e) jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast, int expanded) { + jl_ptls_t ptls = jl_get_ptls_states(); //jl_show(ex); //jl_printf(JL_STDOUT, "\n"); if (!jl_is_expr(e)) { @@ -444,7 +449,7 @@ jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast, int expanded) m = (jl_module_t*)jl_eval_global_var(m, name); if (!jl_is_module(m)) jl_errorf("invalid %s statement: name exists but does not refer to a module", jl_symbol_name(ex->head)); - jl_module_importall(jl_current_module, m); + jl_module_importall(ptls->current_module, m); return jl_nothing; } else if (ex->head == using_sym) { @@ -455,10 +460,10 @@ jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast, int expanded) jl_error("syntax: malformed \"using\" statement"); jl_module_t *u = (jl_module_t*)jl_eval_global_var(m, name); if (jl_is_module(u)) { - jl_module_using(jl_current_module, u); + jl_module_using(ptls->current_module, u); } else { - jl_module_use(jl_current_module, m, name); + jl_module_use(ptls->current_module, m, name); } return jl_nothing; } @@ -468,7 +473,7 @@ jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast, int expanded) jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, jl_array_len(ex->args)-1); if (!jl_is_symbol(name)) jl_error("syntax: malformed \"import\" statement"); - jl_module_import(jl_current_module, m, name); + jl_module_import(ptls->current_module, m, name); return jl_nothing; } else if (ex->head == export_sym) { @@ -476,7 +481,7 @@ jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast, int expanded) jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, i); if (!jl_is_symbol(name)) jl_error("syntax: malformed \"export\" statement"); - jl_module_export(jl_current_module, name); + jl_module_export(ptls->current_module, name); } return jl_nothing; } @@ -511,10 +516,10 @@ jl_value_t *jl_toplevel_eval_flex(jl_value_t *e, int fast, int expanded) thk = (jl_lambda_info_t*)jl_exprarg(ex,0); assert(jl_is_lambda_info(thk)); assert(jl_typeis(thk->code, jl_array_any_type)); - ewc = jl_eval_with_compiler_p(thk, (jl_array_t*)thk->code, fast, jl_current_module); + ewc = jl_eval_with_compiler_p(thk, (jl_array_t*)thk->code, fast, ptls->current_module); } else { - if (head && jl_eval_expr_with_compiler_p((jl_value_t*)ex, fast, jl_current_module)) { + if (head && jl_eval_expr_with_compiler_p((jl_value_t*)ex, fast, ptls->current_module)) { thk = jl_wrap_expr((jl_value_t*)ex); ewc = 1; } @@ -553,7 +558,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_value_t *v) JL_DLLEXPORT jl_value_t *jl_load(const char *fname) { - if (jl_current_module->istopmod) { + jl_ptls_t ptls = jl_get_ptls_states(); + if (ptls->current_module->istopmod) { jl_printf(JL_STDOUT, "%s\r\n", fname); #ifdef _OS_WINDOWS_ uv_run(uv_default_loop(), (uv_run_mode)1); @@ -632,6 +638,7 @@ void jl_check_static_parameter_conflicts(jl_method_t *m, jl_svec_t *t) JL_DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, jl_value_t **bp, jl_value_t *bp_owner, jl_binding_t *bnd) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_value_t *gf=NULL; assert(name && bp); @@ -645,7 +652,7 @@ JL_DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, jl_value_t **bp if (bnd) bnd->constp = 1; if (*bp == NULL) { - jl_module_t *module = (bnd ? bnd->owner : jl_current_module); + jl_module_t *module = (bnd ? bnd->owner : ptls->current_module); gf = (jl_value_t*)jl_new_generic_function(name, module); *bp = gf; if (bp_owner) jl_gc_wb(bp_owner, gf); diff --git a/ui/repl.c b/ui/repl.c index e781aeae23070..eb3d3184e42f0 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -470,12 +470,13 @@ void parse_opts(int *argcp, char ***argvp) static int exec_program(char *program) { + jl_ptls_t ptls = jl_get_ptls_states(); int err = 0; - again: ; + again: ; JL_TRY { if (err) { jl_value_t *errs = jl_stderr_obj(); - jl_value_t *e = jl_exception_in_transit; + jl_value_t *e = ptls->exception_in_transit; if (errs != NULL) { jl_show(errs, e); } @@ -528,6 +529,7 @@ static void print_profile(void) static NOINLINE int true_main(int argc, char *argv[]) { + jl_ptls_t ptls = jl_get_ptls_states(); if (jl_core_module != NULL) { jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); if (args == NULL) { @@ -573,7 +575,7 @@ static NOINLINE int true_main(int argc, char *argv[]) jl_value_t *val = (jl_value_t*)jl_eval_string(line); if (jl_exception_occurred()) { jl_printf(JL_STDERR, "error during run:\n"); - jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_static_show(JL_STDERR, ptls->exception_in_transit); jl_exception_clear(); } else if (val) { @@ -590,7 +592,7 @@ static NOINLINE int true_main(int argc, char *argv[]) line = NULL; } jl_printf(JL_STDERR, "\nparser error:\n"); - jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_static_show(JL_STDERR, ptls->exception_in_transit); jl_printf(JL_STDERR, "\n"); jlbacktrace(); } From 4a021a805eb4493cfcf153adbf3ce9253303a991 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 24 Jun 2016 22:49:48 -0400 Subject: [PATCH 0215/1117] Optimize GC allocation * Pass TLS pointer explicitly to GC allocation functions * Clean up GC allocation functions to take advantage of compiler constant propagation. * Do not initialize tag twice * Remove `jl_gc_pool_t::nfree` --- base/boot.jl | 3 +- src/alloc.c | 130 +++++++++-------- src/array.c | 34 ++--- src/ccall.cpp | 2 +- src/cgutils.cpp | 51 ++++--- src/codegen.cpp | 5 +- src/dump.c | 20 +-- src/gc.c | 292 +++++++++++++-------------------------- src/gc.h | 6 +- src/interpreter.c | 2 +- src/intrinsics.cpp | 11 +- src/jltypes.c | 24 ++-- src/julia_internal.h | 133 ++++++++++++++---- src/julia_threads.h | 1 - src/module.c | 8 +- src/runtime_intrinsics.c | 36 +++-- src/simplevector.c | 14 +- src/task.c | 12 +- src/typemap.c | 12 +- 19 files changed, 408 insertions(+), 388 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index d2cf29d60ebba..ba95bdf70a32a 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -248,7 +248,8 @@ end type WeakRef value WeakRef() = WeakRef(nothing) - WeakRef(v::ANY) = ccall(:jl_gc_new_weakref, Ref{WeakRef}, (Any,), v) + WeakRef(v::ANY) = ccall(:jl_gc_new_weakref_th, Ref{WeakRef}, + (Ptr{Void}, Any), getptls(), v) end TypeVar(n::Symbol) = diff --git a/src/alloc.c b/src/alloc.c index a7861bcb655f5..21a002ef48ecb 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -113,6 +113,7 @@ typedef struct { // Note that this function updates len static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_datatype(dt)); jl_datatype_t *bt = (jl_datatype_t*)dt; size_t nb = jl_datatype_size(bt); @@ -127,7 +128,7 @@ static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data); if (bt == jl_float64_type) return jl_box_float64(*(double*)data); - jl_value_t *v = (jl_value_t*)newobj((jl_value_t*)bt, NWORDS(nb)); + jl_value_t *v = jl_gc_alloc(ptls, nb, bt); switch (nb) { case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break; case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break; @@ -224,11 +225,12 @@ JL_DLLEXPORT int jl_field_isdefined(jl_value_t *v, size_t i) JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...) { + jl_ptls_t ptls = jl_get_ptls_states(); if (type->instance != NULL) return type->instance; va_list args; size_t nf = jl_datatype_nfields(type); va_start(args, type); - jl_value_t *jv = newstruct(type); + jl_value_t *jv = jl_gc_alloc(ptls, type->size, type); for(size_t i=0; i < nf; i++) { jl_set_nth_field(jv, i, va_arg(args, jl_value_t*)); } @@ -239,9 +241,10 @@ JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...) JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uint32_t na) { + jl_ptls_t ptls = jl_get_ptls_states(); if (type->instance != NULL) return type->instance; size_t nf = jl_datatype_nfields(type); - jl_value_t *jv = newstruct(type); + jl_value_t *jv = jl_gc_alloc(ptls, type->size, type); for(size_t i=0; i < na; i++) { jl_set_nth_field(jv, i, args[i]); } @@ -255,10 +258,12 @@ JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) { + jl_ptls_t ptls = jl_get_ptls_states(); if (type->instance != NULL) return type->instance; - jl_value_t *jv = newstruct(type); - if (type->size > 0) - memset(jl_data_ptr(jv), 0, type->size); + size_t size = type->size; + jl_value_t *jv = jl_gc_alloc(ptls, size, type); + if (size > 0) + memset(jl_data_ptr(jv), 0, size); return jv; } @@ -385,9 +390,10 @@ static void jl_lambda_info_set_ast(jl_lambda_info_t *li, jl_expr_t *ast) JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info_uninit(void) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_lambda_info_t *li = - (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type, - NWORDS(sizeof(jl_lambda_info_t))); + (jl_lambda_info_t*)jl_gc_alloc(ptls, sizeof(jl_lambda_info_t), + jl_lambda_info_type); li->code = NULL; li->slotnames = li->slotflags = NULL; li->slottypes = li->ssavaluetypes = NULL; @@ -578,8 +584,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(void) { jl_ptls_t ptls = jl_get_ptls_states(); jl_method_t *m = - (jl_method_t*)newobj((jl_value_t*)jl_method_type, - NWORDS(sizeof(jl_method_t))); + (jl_method_t*)jl_gc_alloc(ptls, sizeof(jl_method_t), jl_method_type); m->specializations.unknown = jl_nothing; m->sig = NULL; m->tvars = NULL; @@ -809,8 +814,10 @@ jl_sym_t *jl_demangle_typename(jl_sym_t *s) JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module) { - jl_methtable_t *mt = (jl_methtable_t*)jl_gc_allocobj(sizeof(jl_methtable_t)); - jl_set_typeof(mt, jl_methtable_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_methtable_t *mt = + (jl_methtable_t*)jl_gc_alloc(ptls, sizeof(jl_methtable_t), + jl_methtable_type); mt->name = jl_demangle_typename(name); mt->module = module; mt->defs.unknown = jl_nothing; @@ -825,7 +832,10 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *module) { - jl_typename_t *tn=(jl_typename_t*)newobj((jl_value_t*)jl_typename_type, NWORDS(sizeof(jl_typename_t))); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_typename_t *tn = + (jl_typename_t*)jl_gc_alloc(ptls, sizeof(jl_typename_t), + jl_typename_type); tn->name = name; tn->module = module; tn->primary = NULL; @@ -856,12 +866,14 @@ jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_datatype_t *super, JL_DLLEXPORT jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields, int8_t fielddesc_type) { + jl_ptls_t ptls = jl_get_ptls_states(); // fielddesc_type is specified manually for builtin types // and is (will be) calculated automatically for user defined types. uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); - jl_datatype_t *t = (jl_datatype_t*) - newobj((jl_value_t*)jl_datatype_type, - NWORDS(sizeof(jl_datatype_t) + nfields * fielddesc_size)); + jl_datatype_t *t = + (jl_datatype_t*)jl_gc_alloc(ptls, (sizeof(jl_datatype_t) + + nfields * fielddesc_size), + jl_datatype_type); // fielddesc_type should only be assigned here. It can cause data // corruption otherwise. t->fielddesc_type = fielddesc_type; @@ -1080,6 +1092,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_bitstype(jl_value_t *name, jl_datatype_t *sup JL_DLLEXPORT jl_value_t *jl_new_type_constructor(jl_svec_t *p, jl_value_t *body) { + jl_ptls_t ptls = jl_get_ptls_states(); #ifndef NDEBUG size_t i, np = jl_svec_len(p); for (i = 0; i < np; i++) { @@ -1087,7 +1100,9 @@ JL_DLLEXPORT jl_value_t *jl_new_type_constructor(jl_svec_t *p, jl_value_t *body) assert(jl_is_typevar(tv) && !tv->bound); } #endif - jl_typector_t *tc = (jl_typector_t*)newobj((jl_value_t*)jl_typector_type, NWORDS(sizeof(jl_typector_t))); + jl_typector_t *tc = + (jl_typector_t*)jl_gc_alloc(ptls, sizeof(jl_typector_t), + jl_typector_type); tc->parameters = p; tc->body = body; return (jl_value_t*)tc; @@ -1099,10 +1114,10 @@ JL_DLLEXPORT jl_value_t *jl_new_type_constructor(jl_svec_t *p, jl_value_t *body) #define BOXN_FUNC(nb,nw) \ JL_DLLEXPORT jl_value_t *jl_box##nb(jl_datatype_t *t, int##nb##_t x) \ { \ + jl_ptls_t ptls = jl_get_ptls_states(); \ assert(jl_isbits(t)); \ assert(jl_datatype_size(t) == sizeof(x)); \ - jl_value_t *v = (jl_value_t*)jl_gc_alloc_##nw##w(); \ - jl_set_typeof(v, t); \ + jl_value_t *v = jl_gc_alloc(ptls, nw * sizeof(void*), t); \ *(int##nb##_t*)jl_data_ptr(v) = x; \ return v; \ } @@ -1135,13 +1150,14 @@ UNBOX_FUNC(float32, float) UNBOX_FUNC(float64, double) UNBOX_FUNC(voidpointer, void*) -#define BOX_FUNC(typ,c_type,pfx,nw) \ - JL_DLLEXPORT jl_value_t *pfx##_##typ(c_type x) \ - { \ - jl_value_t *v = (jl_value_t*)jl_gc_alloc_##nw##w(); \ - jl_set_typeof(v, jl_##typ##_type); \ - *(c_type*)jl_data_ptr(v) = x; \ - return v; \ +#define BOX_FUNC(typ,c_type,pfx,nw) \ + JL_DLLEXPORT jl_value_t *pfx##_##typ(c_type x) \ + { \ + jl_ptls_t ptls = jl_get_ptls_states(); \ + jl_value_t *v = jl_gc_alloc(ptls, nw * sizeof(void*), \ + jl_##typ##_type); \ + *(c_type*)jl_data_ptr(v) = x; \ + return v; \ } BOX_FUNC(float32, float, jl_box, 1) BOX_FUNC(voidpointer, void*, jl_box, 1) @@ -1153,26 +1169,28 @@ BOX_FUNC(float64, double, jl_box, 2) #define NBOX_C 1024 -#define SIBOX_FUNC(typ,c_type,nw) \ - static jl_value_t *boxed_##typ##_cache[NBOX_C]; \ - JL_DLLEXPORT jl_value_t *jl_box_##typ(c_type x) \ - { \ - c_type idx = x+NBOX_C/2; \ - if ((u##c_type)idx < (u##c_type)NBOX_C) \ - return boxed_##typ##_cache[idx]; \ - jl_value_t *v = (jl_value_t*)jl_gc_alloc_##nw##w(); \ - jl_set_typeof(v, jl_##typ##_type); \ - *(c_type*)jl_data_ptr(v) = x; \ - return v; \ +#define SIBOX_FUNC(typ,c_type,nw)\ + static jl_value_t *boxed_##typ##_cache[NBOX_C]; \ + JL_DLLEXPORT jl_value_t *jl_box_##typ(c_type x) \ + { \ + jl_ptls_t ptls = jl_get_ptls_states(); \ + c_type idx = x+NBOX_C/2; \ + if ((u##c_type)idx < (u##c_type)NBOX_C) \ + return boxed_##typ##_cache[idx]; \ + jl_value_t *v = jl_gc_alloc(ptls, nw * sizeof(void*), \ + jl_##typ##_type); \ + *(c_type*)jl_data_ptr(v) = x; \ + return v; \ } #define UIBOX_FUNC(typ,c_type,nw) \ static jl_value_t *boxed_##typ##_cache[NBOX_C]; \ JL_DLLEXPORT jl_value_t *jl_box_##typ(c_type x) \ { \ + jl_ptls_t ptls = jl_get_ptls_states(); \ if (x < NBOX_C) \ return boxed_##typ##_cache[x]; \ - jl_value_t *v = (jl_value_t*)jl_gc_alloc_##nw##w(); \ - jl_set_typeof(v, jl_##typ##_type); \ + jl_value_t *v = jl_gc_alloc(ptls, nw * sizeof(void*), \ + jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } @@ -1236,23 +1254,23 @@ void jl_init_box_caches(void) } } -void jl_mark_box_caches(void) +void jl_mark_box_caches(jl_ptls_t ptls) { int64_t i; for(i=0; i < 256; i++) { - jl_gc_setmark(boxed_int8_cache[i]); - jl_gc_setmark(boxed_uint8_cache[i]); + jl_gc_setmark(ptls, boxed_int8_cache[i]); + jl_gc_setmark(ptls, boxed_uint8_cache[i]); } for(i=0; i < NBOX_C; i++) { - jl_gc_setmark(boxed_int16_cache[i]); - jl_gc_setmark(boxed_int32_cache[i]); - jl_gc_setmark(boxed_int64_cache[i]); - jl_gc_setmark(boxed_uint16_cache[i]); - jl_gc_setmark(boxed_uint32_cache[i]); - jl_gc_setmark(boxed_char_cache[i]); - jl_gc_setmark(boxed_uint64_cache[i]); - jl_gc_setmark(boxed_ssavalue_cache[i]); - jl_gc_setmark(boxed_slotnumber_cache[i]); + jl_gc_setmark(ptls, boxed_int16_cache[i]); + jl_gc_setmark(ptls, boxed_int32_cache[i]); + jl_gc_setmark(ptls, boxed_int64_cache[i]); + jl_gc_setmark(ptls, boxed_uint16_cache[i]); + jl_gc_setmark(ptls, boxed_uint32_cache[i]); + jl_gc_setmark(ptls, boxed_char_cache[i]); + jl_gc_setmark(ptls, boxed_uint64_cache[i]); + jl_gc_setmark(ptls, boxed_ssavalue_cache[i]); + jl_gc_setmark(ptls, boxed_slotnumber_cache[i]); } } @@ -1267,10 +1285,11 @@ JL_DLLEXPORT jl_value_t *jl_box_bool(int8_t x) jl_expr_t *jl_exprn(jl_sym_t *head, size_t n) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *ar = n==0 ? (jl_array_t*)jl_an_empty_vec_any : jl_alloc_vec_any(n); JL_GC_PUSH1(&ar); - jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc_3w(); assert(NWORDS(sizeof(jl_expr_t))==3); - jl_set_typeof(ex, jl_expr_type); + jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc(ptls, sizeof(jl_expr_t), + jl_expr_type); ex->head = head; ex->args = ar; ex->etype = (jl_value_t*)jl_any_type; @@ -1280,14 +1299,15 @@ jl_expr_t *jl_exprn(jl_sym_t *head, size_t n) JL_CALLABLE(jl_f__expr) { + jl_ptls_t ptls = jl_get_ptls_states(); JL_NARGSV(Expr, 1); JL_TYPECHK(Expr, symbol, args[0]); jl_array_t *ar = jl_alloc_vec_any(nargs-1); JL_GC_PUSH1(&ar); for(size_t i=0; i < nargs-1; i++) jl_array_ptr_set(ar, i, args[i+1]); - jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc_3w(); assert(NWORDS(sizeof(jl_expr_t))==3); - jl_set_typeof(ex, jl_expr_type); + jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc(ptls, sizeof(jl_expr_t), + jl_expr_type); ex->head = (jl_sym_t*)args[0]; ex->args = ar; ex->etype = (jl_value_t*)jl_any_type; diff --git a/src/array.c b/src/array.c index 0f78e651b410c..2992eefaf135f 100644 --- a/src/array.c +++ b/src/array.c @@ -54,6 +54,7 @@ size_t jl_arr_xtralloc_limit = 0; static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, int isunboxed, int elsz) { + jl_ptls_t ptls = jl_get_ptls_states(); size_t i, tot, nel=1; void *data; jl_array_t *a; @@ -90,9 +91,8 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, size_t doffs = tsz; tsz += tot; tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align whole object - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.how = 0; data = (char*)a + doffs; if (tot > 0 && !isunboxed) { @@ -104,11 +104,10 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, data = jl_gc_managed_malloc(tot); // Allocate the Array **after** allocating the data // to make sure the array is still young - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.how = 2; - jl_gc_track_malloced_array(a); + jl_gc_track_malloced_array(ptls, a); if (!isunboxed) memset(data, 0, tot); } @@ -173,6 +172,7 @@ static inline int is_ntuple_long(jl_value_t *v) JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_value_t *_dims) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *a; size_t ndims = jl_nfields(_dims); assert(is_ntuple_long(_dims)); @@ -180,9 +180,8 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT); - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->flags.ndims = ndims; a->offset = 0; @@ -239,6 +238,7 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { + jl_ptls_t ptls = jl_get_ptls_states(); size_t elsz; jl_array_t *a; jl_value_t *el_type = jl_tparam0(atype); @@ -251,9 +251,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN @@ -266,7 +265,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers if (own_buffer) { a->flags.how = 2; - jl_gc_track_malloced_array(a); + jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { @@ -282,6 +281,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *_dims, int own_buffer) { + jl_ptls_t ptls = jl_get_ptls_states(); size_t elsz, nel = 1; jl_array_t *a; size_t ndims = jl_nfields(_dims); @@ -306,9 +306,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN @@ -322,7 +321,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, a->flags.isaligned = 0; if (own_buffer) { a->flags.how = 2; - jl_gc_track_malloced_array(a); + jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { @@ -369,10 +368,10 @@ JL_DLLEXPORT jl_array_t *jl_pchar_to_array(const char *str, size_t len) JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a) { + jl_ptls_t ptls = jl_get_ptls_states(); if (!jl_typeis(a, jl_array_uint8_type)) jl_type_error("jl_array_to_string", (jl_value_t*)jl_array_uint8_type, (jl_value_t*)a); - jl_value_t *s = (jl_value_t*)jl_gc_alloc_1w(); - jl_set_typeof(s, jl_string_type); + jl_value_t *s = jl_gc_alloc(ptls, sizeof(void*), jl_string_type); jl_set_nth_field(s, 0, (jl_value_t*)a); return s; } @@ -565,6 +564,7 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i) // the **beginning** of the new buffer. static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(!a->flags.isshared || a->flags.how == 3); size_t elsz = a->elsize; size_t nbytes = newlen * elsz; @@ -593,12 +593,12 @@ static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen) #endif ) { a->data = jl_gc_managed_malloc(nbytes); - jl_gc_track_malloced_array(a); + jl_gc_track_malloced_array(ptls, a); a->flags.how = 2; a->flags.isaligned = 1; } else { - a->data = allocb(nbytes); + a->data = jl_gc_alloc_buf(ptls, nbytes); a->flags.how = 1; jl_gc_wb_buf(a, a->data, nbytes); } diff --git a/src/ccall.cpp b/src/ccall.cpp index fe001801cfece..05c2e6e08bd3f 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -854,7 +854,7 @@ static jl_cgval_t mark_or_box_ccall_result(Value *result, bool isboxed, jl_value int nb = sizeof(void*); // TODO: can this be tighter than tbaa_value? return mark_julia_type( - init_bits_value(emit_allocobj(ctx, nb), runtime_bt, result, tbaa_value), + init_bits_value(emit_allocobj(ctx, nb, runtime_bt), result, tbaa_value), true, (jl_value_t*)jl_pointer_type, ctx); } return mark_julia_type(result, isboxed, rt, ctx); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index ebbc9ef049998..5ab7d102f4cf1 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1333,28 +1333,25 @@ static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, size_ // --- boxing --- -static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size); -static void init_tag(Value *v, Value *jt) +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt); +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, + const jl_cgval_t &v); +static Value *init_bits_value(Value *newv, Value *v, MDNode *tbaa) { - tbaa_decorate(tbaa_tag, builder.CreateStore(jt, emit_typeptr_addr(v))); -} -static Value *init_bits_value(Value *newv, Value *jt, Value *v, MDNode *tbaa) -{ - init_tag(newv, jt); + // newv should already be tagged tbaa_decorate(tbaa, builder.CreateAlignedStore(v, builder.CreateBitCast(newv, PointerType::get(v->getType(),0)), sizeof(void*))); // min alignment in julia's gc is pointer-aligned return newv; } static Value *as_value(Type *t, const jl_cgval_t&); static Value *init_bits_cgval(Value *newv, const jl_cgval_t& v, MDNode *tbaa, jl_codectx_t *ctx) { - Value *jt = literal_pointer_val(v.typ); + // newv should already be tagged if (v.ispointer()) { - init_tag(newv, jt); builder.CreateMemCpy(newv, data_pointer(v, ctx, T_pint8), jl_datatype_size(v.typ), sizeof(void*)); return newv; } else { - return init_bits_value(newv, jt, v.V, tbaa); + return init_bits_value(newv, v.V, tbaa); } } @@ -1521,7 +1518,8 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) return literal_pointer_val(jb->instance); } else { - box = init_bits_cgval(emit_allocobj(ctx, jl_datatype_size(jt)), vinfo, jb->mutabl ? tbaa_mutab : tbaa_immut, ctx); + box = init_bits_cgval(emit_allocobj(ctx, jl_datatype_size(jt), vinfo), + vinfo, jb->mutabl ? tbaa_mutab : tbaa_immut, ctx); } if (gcrooted) { @@ -1555,23 +1553,33 @@ static void emit_cpointercheck(const jl_cgval_t &x, const std::string &msg, jl_c } // allocation for known size object -static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size) +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt) { int osize; int end_offset; int offset = jl_gc_classify_pools(static_size, &osize, &end_offset); Value *ptls_ptr = builder.CreateBitCast(ctx->ptlsStates, T_pint8); + Value *v; if (offset < 0) { Value *args[] = {ptls_ptr, ConstantInt::get(T_size, static_size + sizeof(void*))}; - return builder.CreateCall(prepare_call(jlalloc_big_func), - ArrayRef(args, 2)); + v = builder.CreateCall(prepare_call(jlalloc_big_func), + ArrayRef(args, 2)); + } + else { + Value *pool_ptr = builder.CreateConstGEP1_32(ptls_ptr, offset); + Value *args[] = {ptls_ptr, pool_ptr, ConstantInt::get(T_int32, osize), + ConstantInt::get(T_int32, end_offset)}; + v = builder.CreateCall(prepare_call(jlalloc_pool_func), + ArrayRef(args, 4)); } - Value *pool_ptr = builder.CreateConstGEP1_32(ptls_ptr, offset); - Value *args[] = {ptls_ptr, pool_ptr, ConstantInt::get(T_int32, osize), - ConstantInt::get(T_int32, end_offset)}; - return builder.CreateCall(prepare_call(jlalloc_pool_func), - ArrayRef(args, 4)); + tbaa_decorate(tbaa_tag, builder.CreateStore(jt, emit_typeptr_addr(v))); + return v; +} +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, + const jl_cgval_t &v) +{ + return emit_allocobj(ctx, static_size, literal_pointer_val(v.typ)); } // if ptr is NULL this emits a write barrier _back_ @@ -1720,10 +1728,9 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg f1 = boxed(fval_info, ctx); j++; } - Value *strct = emit_allocobj(ctx, sty->size); + Value *strct = emit_allocobj(ctx, sty->size, + literal_pointer_val((jl_value_t*)ty)); jl_cgval_t strctinfo = mark_julia_type(strct, true, ty, ctx); - tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)ty), - emit_typeptr_addr(strct))); if (f1) { jl_cgval_t f1info = mark_julia_type(f1, true, jl_any_type, ctx); if (!jl_subtype(expr_type(args[1],ctx), jl_field_type(sty,0), 0)) diff --git a/src/codegen.cpp b/src/codegen.cpp index 99f87aca3ad74..07c203fcf4032 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3709,9 +3709,8 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t (void)julia_type_to_llvm(jargty, &isboxed); if (isboxed) { // passed an unboxed T, but want something boxed - Value *mem = emit_allocobj(&ctx, jl_datatype_size(jargty)); - tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)jargty), - emit_typeptr_addr(mem))); + Value *mem = emit_allocobj(&ctx, jl_datatype_size(jargty), + literal_pointer_val((jl_value_t*)jargty)); tbaa_decorate(jl_is_mutable(jargty) ? tbaa_mutab : tbaa_immut, builder.CreateAlignedStore(val, builder.CreateBitCast(mem, val->getType()->getPointerTo()), diff --git a/src/dump.c b/src/dump.c index 4eb9295189a4f..926df27e559bf 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1311,6 +1311,7 @@ static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc) static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t **loc) { + jl_ptls_t ptls = jl_get_ptls_states(); int usetable = (mode != MODE_AST); size_t i; @@ -1422,7 +1423,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return (jl_value_t*)e; } else if (vtag == (jl_value_t*)jl_tvar_type) { - jl_tvar_t *tv = (jl_tvar_t*)newobj((jl_value_t*)jl_tvar_type, NWORDS(sizeof(jl_tvar_t))); + jl_tvar_t *tv = (jl_tvar_t*)jl_gc_alloc(ptls, sizeof(jl_tvar_t), + jl_tvar_type); if (usetable) arraylist_push(&backref_list, tv); tv->name = (jl_sym_t*)jl_deserialize_value(s, NULL); @@ -1436,8 +1438,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } else if (vtag == (jl_value_t*)jl_method_type) { jl_method_t *m = - (jl_method_t*)newobj((jl_value_t*)jl_method_type, - NWORDS(sizeof(jl_method_t))); + (jl_method_t*)jl_gc_alloc(ptls, sizeof(jl_method_t), + jl_method_type); if (usetable) arraylist_push(&backref_list, m); m->specializations.unknown = jl_deserialize_value(s, (jl_value_t**)&m->specializations); @@ -1473,8 +1475,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } else if (vtag == (jl_value_t*)jl_lambda_info_type) { jl_lambda_info_t *li = - (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type, - NWORDS(sizeof(jl_lambda_info_t))); + (jl_lambda_info_t*)jl_gc_alloc(ptls, sizeof(jl_lambda_info_t), + jl_lambda_info_type); if (usetable) arraylist_push(&backref_list, li); li->code = jl_deserialize_value(s, &li->code); jl_gc_wb(li, li->code); @@ -1606,7 +1608,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)SmallDataType_tag) { int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s) : read_int32(s)); - jl_value_t *v = jl_gc_allocobj(sz); + jl_value_t *v = jl_gc_alloc(ptls, sz, NULL); int pos = backref_list.len; if (usetable) arraylist_push(&backref_list, v); @@ -1666,7 +1668,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t backref_list.items[pos] = dt->instance; return dt->instance; } - jl_value_t *v = (jl_value_t*)jl_gc_alloc_0w(); + jl_value_t *v = (jl_value_t*)jl_gc_alloc(ptls, 0, NULL); if (usetable) { uintptr_t pos = backref_list.len; arraylist_push(&backref_list, (void*)v); @@ -2289,14 +2291,14 @@ static void jl_recache_types(void) } assert(dt); if (t != dt) { - jl_set_typeof(dt, (jl_value_t*)(intptr_t)0x10); // invalidate the old value to help catch errors + jl_set_typeof(dt, (void*)(intptr_t)0x10); // invalidate the old value to help catch errors if ((jl_value_t*)dt == o) { if (loc) *loc = (jl_value_t*)t; if (offs > 0) backref_list.items[offs] = t; } } if (t->instance != v) { - jl_set_typeof(v, (jl_value_t*)(intptr_t)0x20); // invalidate the old value to help catch errors + jl_set_typeof(v, (void*)(intptr_t)0x20); // invalidate the old value to help catch errors if (v == o) { *loc = t->instance; if (offs > 0) backref_list.items[offs] = t->instance; diff --git a/src/gc.c b/src/gc.c index 330b7e37ba603..d36dc9b03206a 100644 --- a/src/gc.c +++ b/src/gc.c @@ -100,9 +100,8 @@ static void schedule_finalization(void *o, void *f) arraylist_push(&to_finalize, f); } -static void run_finalizer(jl_value_t *o, jl_value_t *ff) +static void run_finalizer(jl_ptls_t ptls, jl_value_t *o, jl_value_t *ff) { - jl_ptls_t ptls = jl_get_ptls_states(); assert(!jl_typeis(ff, jl_voidpointer_type)); jl_value_t *args[2] = {ff,o}; JL_TRY { @@ -179,9 +178,8 @@ static void jl_gc_run_finalizers_in_list(jl_ptls_t ptls, arraylist_t *list) jl_value_t **items = (jl_value_t**)list->items; jl_gc_push_arraylist(ptls, list); JL_UNLOCK_NOGC(&finalizers_lock); - for (size_t i = 2;i < len;i += 2) { - run_finalizer(items[i], items[i + 1]); - } + for (size_t i = 2;i < len;i += 2) + run_finalizer(ptls, items[i], items[i + 1]); JL_GC_POP(); } @@ -262,14 +260,13 @@ static void gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) jl_gc_unsafe_leave(ptls, gc_state); } -STATIC_INLINE void gc_add_ptr_finalizer(jl_ptls_t ptls, - jl_value_t *v, void *f) +STATIC_INLINE void gc_add_ptr_finalizer(jl_ptls_t ptls, jl_value_t *v, void *f) { gc_add_finalizer_(ptls, (void*)(((uintptr_t)v) | 1), f); } -JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_ptls_t ptls, - jl_value_t *v, jl_function_t *f) +JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_ptls_t ptls, jl_value_t *v, + jl_function_t *f) { if (__unlikely(jl_typeis(f, jl_voidpointer_type))) { gc_add_ptr_finalizer(ptls, v, jl_unbox_voidpointer(f)); @@ -311,8 +308,6 @@ JL_DLLEXPORT void jl_finalize_th(jl_ptls_t ptls, jl_value_t *o) arraylist_free(&copied_list); } -#define GC_POOL_END_OFS(osize) ((((GC_PAGE_SZ - GC_PAGE_OFFSET)/(osize)) - 1)*(osize) + GC_PAGE_OFFSET) - // GC knobs and self-measurement variables static int64_t last_gc_total_bytes = 0; @@ -550,11 +545,11 @@ static inline int maybe_collect(jl_ptls_t ptls) // weak references -JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) +JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref_th(jl_ptls_t ptls, + jl_value_t *value) { - jl_ptls_t ptls = jl_get_ptls_states(); - jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc_1w(); - jl_set_typeof(wr, jl_weakref_type); + jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc(ptls, sizeof(void*), + jl_weakref_type); wr->value = value; // NOTE: wb not needed here arraylist_push(&ptls->heap.weak_refs, wr); return wr; @@ -593,7 +588,8 @@ static void sweep_weak_refs(void) // big value list -static NOINLINE jl_taggedvalue_t *alloc_big(jl_ptls_t ptls, size_t sz) +// Size includes the tag and the tag is not cleared!! +JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) { maybe_collect(ptls); size_t offs = offsetof(bigval_t, header); @@ -609,10 +605,9 @@ static NOINLINE jl_taggedvalue_t *alloc_big(jl_ptls_t ptls, size_t sz) memset(v, 0xee, allocsz); #endif v->sz = allocsz; - v->header = 0; v->age = 0; gc_big_object_link(v, &ptls->heap.big_objects); - return (jl_taggedvalue_t*)&v->header; + return jl_valueof(&v->header); } // Sweep list rooted at *pv, removing and freeing any unmarked objects. @@ -677,10 +672,9 @@ static void sweep_big(jl_ptls_t ptls, int sweep_full) // tracking Arrays with malloc'd storage -void jl_gc_track_malloced_array(jl_array_t *a) +void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) { // This is **NOT** a GC safe point. - jl_ptls_t ptls = jl_get_ptls_states(); mallocarray_t *ma; if (ptls->heap.mafreelist == NULL) { ma = (mallocarray_t*)malloc(sizeof(mallocarray_t)); @@ -773,7 +767,7 @@ static inline jl_taggedvalue_t *reset_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg return beg; } -static NOINLINE void add_page(jl_gc_pool_t *p) +static NOINLINE jl_taggedvalue_t *add_page(jl_gc_pool_t *p) { // Do not pass in `ptls` as argument. This slows down the fast path // in pool_alloc significantly @@ -788,15 +782,17 @@ static NOINLINE void add_page(jl_gc_pool_t *p) pg->thread_n = ptls->tid; jl_taggedvalue_t *fl = reset_page(p, pg, p->newpages); p->newpages = fl; + return fl; } -static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, - int osize, int end_offset) +// Size includes the tag and the tag is not cleared!! +JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, + int osize, int end_offset) { + assert(ptls->gc_state == 0); #ifdef MEMDEBUG - return alloc_big(ptls, osize); + return jl_gc_big_alloc(ptls, osize); #endif - jl_taggedvalue_t *v, *end; // FIXME - need JL_ATOMIC_FETCH_AND_ADD here if (__unlikely((gc_num.allocd += osize) >= 0) || gc_debug_check_pool()) { //gc_num.allocd -= osize; @@ -808,11 +804,9 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, } gc_num.poolalloc++; // first try to use the freelist - v = p->freelist; + jl_taggedvalue_t *v = p->freelist; if (v) { jl_taggedvalue_t *next = v->next; - v->header = 0; - p->nfree--; p->freelist = next; if (__unlikely(gc_page_data(v) != gc_page_data(next))) { // we only update pg's fields when the freelist changes page @@ -821,18 +815,14 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, assert(pg->osize == p->osize); pg->nfree = 0; pg->has_young = 1; - if (next) - p->nfree = page_metadata(next)->nfree; } - return v; + return jl_valueof(v); } // if the freelist is empty we reuse empty but not freed pages v = p->newpages; - if (__unlikely(!v)) { - add_page(p); - v = p->newpages; - } - end = (jl_taggedvalue_t*)&(gc_page_data(v)[end_offset]); + if (__unlikely(!v)) + v = add_page(p); + jl_taggedvalue_t *end = (jl_taggedvalue_t*)&(gc_page_data(v)[end_offset]); if (__likely(v != end)) { p->newpages = (jl_taggedvalue_t*)((char*)v + osize); } @@ -844,88 +834,7 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, pg->has_young = 1; p->newpages = v->next; } - v->header = 0; - return v; -} - -// use this variant when osize is statically known -// and is definitely in sizeclasses -// GC_POOL_END_OFS uses an integer division -static inline jl_taggedvalue_t *_pool_alloc(jl_ptls_t ptls, - jl_gc_pool_t *p, int osize) -{ - return __pool_alloc(ptls, p, osize, GC_POOL_END_OFS(osize)); -} - -static inline jl_taggedvalue_t *pool_alloc(jl_ptls_t ptls, - jl_gc_pool_t *p) -{ - return __pool_alloc(ptls, p, p->osize, p->end_offset); -} - -// Size includes the tag!! -JL_DLLEXPORT void *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, - int osize, int end_offset) -{ - return jl_valueof(__pool_alloc(ptls, p, osize, end_offset)); -} - -// Size includes the tag!! -JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz) -{ - return jl_valueof(alloc_big(ptls, allocsz)); -} - -// pools are 16376 bytes large (GC_POOL_SZ - GC_PAGE_OFFSET) -static const int sizeclasses[JL_GC_N_POOLS] = { -#ifdef _P64 - 8, -#else - 4, 8, 12, -#endif - - // 16 pools at 16-byte spacing - 16, 32, 48, 64, 80, 96, 112, 128, - 144, 160, 176, 192, 208, 224, 240, 256, - - // the following tables are computed for maximum packing efficiency via the formula: - // sz=(div(2^14-8,rng)÷16)*16; hcat(sz, (2^14-8)÷sz, 2^14-(2^14-8)÷sz.*sz)' - - // rng = 60:-4:32 (8 pools) - 272, 288, 304, 336, 368, 400, 448, 496, -// 60, 56, 53, 48, 44, 40, 36, 33, /pool -// 64, 256, 272, 256, 192, 384, 256, 16, bytes lost - - // rng = 30:-2:16 (8 pools) - 544, 576, 624, 672, 736, 816, 896, 1008, -// 30, 28, 26, 24, 22, 20, 18, 16, /pool -// 64, 256, 160, 256, 192, 64, 256, 256, bytes lost - - // rng = 15:-1:8 (8 pools) - 1088, 1168, 1248, 1360, 1488, 1632, 1808, 2032 -// 15, 14, 13, 12, 11, 10, 9, 8, /pool -// 64, 32, 160, 64, 16, 64, 112, 128, bytes lost -}; - -static inline int szclass(size_t sz) -{ -#ifdef _P64 - if (sz <= 8) - return 0; - const int N = 0; -#else - if (sz <= 12) - return (sz + 3) / 4 - 1; - const int N = 2; -#endif - if (sz <= 256) - return (sz + 15) / 16 + N; - if (sz <= 496) - return 16 - 16376 / 4 / LLT_ALIGN(sz, 16 * 4) + 16 + N; - if (sz <= 1008) - return 16 - 16376 / 2 / LLT_ALIGN(sz, 16 * 2) + 24 + N; - assert(sz <= GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t) && sizeclasses[JL_GC_N_POOLS-1] == GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)); - return 16 - 16376 / 1 / LLT_ALIGN(sz, 16 * 1) + 32 + N; + return jl_valueof(v); } int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset) @@ -933,8 +842,8 @@ int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset) if (sz > GC_MAX_SZCLASS) return -1; size_t allocsz = sz + sizeof(jl_taggedvalue_t); - int klass = szclass(allocsz); - *osize = sizeclasses[klass]; + int klass = jl_gc_szclass(allocsz); + *osize = jl_gc_sizeclasses[klass]; *end_offset = GC_POOL_END_OFS(*osize); return (int)(intptr_t)(&((jl_ptls_t)0)->heap.norm_pools[klass]); } @@ -1093,6 +1002,22 @@ static void gc_sweep_other(jl_ptls_t ptls, int sweep_full) sweep_big(ptls, sweep_full); } +static void gc_pool_sync_nfree(jl_gc_pagemeta_t *pg, jl_taggedvalue_t *last) +{ + assert(pg->fl_begin_offset != -1); + char *cur_pg = gc_page_data(last); + // Fast path for page that has no allocation + jl_taggedvalue_t *fl_beg = (jl_taggedvalue_t*)(cur_pg + pg->fl_begin_offset); + if (last == fl_beg) + return; + int nfree = 0; + do { + nfree++; + last = last->next; + } while (gc_page_data(last) == cur_pg); + pg->nfree = nfree; +} + static void gc_sweep_pool(int sweep_full) { gc_time_pool_start(); @@ -1109,8 +1034,8 @@ static void gc_sweep_pool(int sweep_full) jl_taggedvalue_t *last = p->freelist; if (last) { jl_gc_pagemeta_t *pg = page_metadata(last); + gc_pool_sync_nfree(pg, last); pg->has_young = 1; - pg->nfree = p->nfree; } p->freelist = NULL; pfl[t_i * JL_GC_N_POOLS + i] = &p->freelist; @@ -1132,15 +1057,10 @@ static void gc_sweep_pool(int sweep_full) } - // null out terminal pointers of free lists and cache back pg->nfree in the jl_gc_pool_t + // null out terminal pointers of free lists for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_ptls_t ptls2 = jl_all_tls_states[t_i]; for (int i = 0; i < JL_GC_N_POOLS; i++) { - jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; *pfl[t_i * JL_GC_N_POOLS + i] = NULL; - if (p->freelist) { - p->nfree = page_metadata(p->freelist)->nfree; - } } } @@ -1232,9 +1152,9 @@ static inline int gc_push_root(jl_ptls_t ptls, void *v, int d) // v isa jl_value return !gc_old(bits); } -void jl_gc_setmark(jl_value_t *v) // TODO rename this as it is misleading now +// TODO rename this as it is misleading now +void jl_gc_setmark(jl_ptls_t ptls, jl_value_t *v) { - jl_ptls_t ptls = jl_get_ptls_states(); jl_taggedvalue_t *o = jl_astaggedvalue(v); if (!gc_marked(o->bits.gc)) { gc_setmark_pool(ptls, o, GC_MARKED); @@ -1555,8 +1475,6 @@ void visit_mark_stack(jl_ptls_t ptls) assert(!mark_sp); } -void jl_mark_box_caches(void); - extern jl_array_t *jl_module_init_order; extern jl_typemap_entry_t *call_cache[N_CALL_CACHE]; @@ -1592,7 +1510,7 @@ void pre_mark(jl_ptls_t ptls) if (call_cache[i]) gc_push_root(ptls, call_cache[i], 0); - jl_mark_box_caches(); + jl_mark_box_caches(ptls); //gc_push_root(ptls, jl_unprotect_stack_func, 0); gc_push_root(ptls, jl_typetype_type, 0); @@ -1934,84 +1852,24 @@ JL_DLLEXPORT void jl_gc_collect(int full) // allocator entry points -void *allocb(size_t sz) +JL_DLLEXPORT jl_value_t *(jl_gc_alloc)(jl_ptls_t ptls, size_t sz, void *ty) { - jl_ptls_t ptls = jl_get_ptls_states(); - jl_taggedvalue_t *b = NULL; - size_t allocsz = sz + sizeof(jl_taggedvalue_t); - if (allocsz < sz) // overflow in adding offs, size was "negative" - jl_throw(jl_memory_exception); - if (allocsz > GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { - b = alloc_big(ptls, allocsz); - b->header = jl_buff_tag; - } - else { - b = pool_alloc(ptls, &ptls->heap.norm_pools[szclass(allocsz)]); - b->header = jl_buff_tag; - } - return jl_valueof(b); -} - -JL_DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - size_t allocsz = sz + sizeof(jl_taggedvalue_t); - if (allocsz < sz) // overflow in adding offs, size was "negative" - jl_throw(jl_memory_exception); - if (allocsz <= GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { - return jl_valueof(pool_alloc(ptls, &ptls->heap.norm_pools[szclass(allocsz)])); - } - return jl_valueof(alloc_big(ptls, allocsz)); -} - -JL_DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - const int sz = sizeof(jl_taggedvalue_t); - void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); - return jl_valueof(tag); -} - -JL_DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*), - JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); - return jl_valueof(tag); -} - -JL_DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 2, - JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); - return jl_valueof(tag); -} - -JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 3, - JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); - return jl_valueof(tag); + return jl_gc_alloc_(ptls, sz, ty); } // Per-thread initialization (when threading is fully implemented) void jl_mk_thread_heap(jl_ptls_t ptls) { jl_thread_heap_t *heap = &ptls->heap; - const int *szc = sizeclasses; jl_gc_pool_t *p = heap->norm_pools; for(int i=0; i < JL_GC_N_POOLS; i++) { - assert((szc[i] < 16 && szc[i] % sizeof(void*) == 0) || - (szc[i] % 16 == 0)); - p[i].osize = szc[i]; + assert((jl_gc_sizeclasses[i] < 16 && + jl_gc_sizeclasses[i] % sizeof(void*) == 0) || + (jl_gc_sizeclasses[i] % 16 == 0)); + p[i].osize = jl_gc_sizeclasses[i]; p[i].freelist = NULL; p[i].newpages = NULL; - p[i].end_offset = GC_POOL_END_OFS(szc[i]); + p[i].end_offset = GC_POOL_END_OFS(jl_gc_sizeclasses[i]); } arraylist_new(&heap->weak_refs, 0); heap->mallocarrays = NULL; @@ -2194,6 +2052,42 @@ JL_DLLEXPORT void jl_finalize(jl_value_t *o) jl_finalize_th(ptls, o); } +JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_new_weakref_th(ptls, value); +} + +JL_DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, sz, NULL); +} + +JL_DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, 0, NULL); +} + +JL_DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, sizeof(void*), NULL); +} + +JL_DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, sizeof(void*) * 2, NULL); +} + +JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, sizeof(void*) * 3, NULL); +} + #ifdef __cplusplus } #endif diff --git a/src/gc.h b/src/gc.h index 51b99842372ec..d59c7db7069a4 100644 --- a/src/gc.h +++ b/src/gc.h @@ -32,15 +32,10 @@ extern "C" { // manipulating mark bits -#define GC_PAGE_LG2 14 // log2(size of a page) -#define GC_PAGE_SZ (1 << GC_PAGE_LG2) // 16k -#define GC_PAGE_OFFSET (JL_SMALL_BYTE_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_SMALL_BYTE_ALIGNMENT)) - // 8G * 32768 = 2^48 // It's really unlikely that we'll actually allocate that much though... #define REGION_COUNT 32768 -#define jl_buff_tag ((uintptr_t)0x4eade800) #define jl_malloc_tag ((void*)0xdeadaa01) #define jl_singleton_tag ((void*)0xdeadaa02) @@ -278,6 +273,7 @@ void pre_mark(jl_ptls_t ptls); void gc_mark_object_list(jl_ptls_t ptls, arraylist_t *list, size_t start); void visit_mark_stack(jl_ptls_t ptls); void gc_debug_init(void); +void jl_mark_box_caches(jl_ptls_t ptls); // GC pages diff --git a/src/interpreter.c b/src/interpreter.c index c8e844d8d07bc..32900f7981f05 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -400,7 +400,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) } jl_compute_field_offsets(dt); if (para == (jl_value_t*)jl_emptysvec && jl_is_datatype_make_singleton(dt)) { - dt->instance = newstruct(dt); + dt->instance = jl_gc_alloc(ptls, 0, dt); jl_gc_wb(dt, dt->instance); } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 626774ff02be3..ffc6c4326f097 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -507,7 +507,8 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx return mark_julia_type(vx, false, bt, ctx); else return mark_julia_type( - init_bits_value(emit_allocobj(ctx, nb), boxed(bt_value, ctx), vx, tbaa_immut), + init_bits_value(emit_allocobj(ctx, nb, boxed(bt_value, ctx)), + vx, tbaa_immut), true, bt, ctx); } @@ -548,8 +549,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c Value *runtime_bt = boxed(bt_value, ctx); // XXX: emit type validity check on runtime_bt (bitstype of size nb) - Value *newobj = emit_allocobj(ctx, nb); - tbaa_decorate(tbaa_tag, builder.CreateStore(runtime_bt, emit_typeptr_addr(newobj))); + Value *newobj = emit_allocobj(ctx, nb, runtime_bt); if (!v.ispointer()) { tbaa_decorate(tbaa_value, builder.CreateAlignedStore(emit_unbox(llvmt, v, v.typ), builder.CreatePointerCast(newobj, llvmt->getPointerTo()), alignment)); } @@ -772,9 +772,8 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct } assert(jl_is_datatype(ety)); uint64_t size = jl_datatype_size(ety); - Value *strct = emit_allocobj(ctx, size); - tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)ety), - emit_typeptr_addr(strct))); + Value *strct = emit_allocobj(ctx, size, + literal_pointer_val((jl_value_t*)ety)); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); thePtr = builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1); diff --git a/src/jltypes.c b/src/jltypes.c index 058d891fe79a9..54285e7356d85 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -311,6 +311,7 @@ static jl_svec_t *jl_compute_type_union(jl_value_t **types, size_t ntypes) static jl_value_t *jl_type_union_v(jl_value_t **ts, size_t n) { + jl_ptls_t ptls = jl_get_ptls_states(); if (n == 0) return (jl_value_t*)jl_bottom_type; size_t i; for(i=0; i < n; i++) { @@ -323,7 +324,9 @@ static jl_value_t *jl_type_union_v(jl_value_t **ts, size_t n) if (jl_svec_len(types) == 0) return (jl_value_t*)jl_bottom_type; if (jl_svec_len(types) == 1) return jl_svecref(types, 0); JL_GC_PUSH1(&types); - jl_uniontype_t *tu = (jl_uniontype_t*)newobj((jl_value_t*)jl_uniontype_type,NWORDS(sizeof(jl_uniontype_t))); + jl_uniontype_t *tu = + (jl_uniontype_t*)jl_gc_alloc(ptls, sizeof(jl_uniontype_t), + jl_uniontype_type); tu->types = types; jl_gc_wb(tu, types); JL_GC_POP(); @@ -2129,6 +2132,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i int cacheable, jl_typestack_t *stack, jl_value_t **env, size_t n) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_typestack_t top; jl_typename_t *tn = dt->name; jl_value_t *tc = tn->primary; @@ -2245,7 +2249,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i ndt->pointerfree = dt->pointerfree; ndt->types = jl_emptysvec; if (jl_is_datatype_make_singleton(ndt)) { - ndt->instance = newstruct(ndt); + ndt->instance = jl_gc_alloc(ptls, 0, ndt); jl_gc_wb(ndt, ndt->instance); } } @@ -2266,7 +2270,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i if (cacheable) { jl_compute_field_offsets(ndt); if (jl_is_datatype_make_singleton(ndt)) { - ndt->instance = newstruct(ndt); + ndt->instance = jl_gc_alloc(ptls, 0, ndt); jl_gc_wb(ndt, ndt->instance); } } @@ -2511,6 +2515,7 @@ jl_datatype_t *jl_wrap_Type(jl_value_t *t) void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! { + jl_ptls_t ptls = jl_get_ptls_states(); inside_typedef = 0; assert(jl_is_datatype(t)); jl_typestack_t top; @@ -2550,7 +2555,7 @@ void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! if (ndt->uid) { // cacheable jl_compute_field_offsets(ndt); if (jl_is_datatype_make_singleton(ndt)) { - ndt->instance = newstruct(ndt); + ndt->instance = jl_gc_alloc(ptls, 0, ndt); jl_gc_wb(ndt, ndt->instance); } } @@ -3432,7 +3437,9 @@ jl_value_t *jl_type_match_morespecific(jl_value_t *a, jl_value_t *b) JL_DLLEXPORT jl_tvar_t *jl_new_typevar_(jl_sym_t *name, jl_value_t *lb, jl_value_t *ub, jl_value_t *b) { - jl_tvar_t *tv = (jl_tvar_t*)newobj((jl_value_t*)jl_tvar_type, 4); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_tvar_t *tv = (jl_tvar_t*)jl_gc_alloc(ptls, sizeof(jl_tvar_t), + jl_tvar_type); tv->name = name; tv->lb = lb; tv->ub = ub; @@ -3466,9 +3473,10 @@ void jl_init_types(void) jl_symbol_type = jl_sym_type; jl_simplevector_type = jl_new_uninitialized_datatype(1, 1); jl_methtable_type = jl_new_uninitialized_datatype(6, 1); - jl_nothing = jl_gc_alloc_0w(); + jl_nothing = jl_gc_alloc(ptls, 0, NULL); - jl_emptysvec = (jl_svec_t*)newobj((jl_value_t*)jl_simplevector_type, 1); + jl_emptysvec = (jl_svec_t*)jl_gc_alloc(ptls, sizeof(void*), + jl_simplevector_type); jl_svec_set_len_unsafe(jl_emptysvec, 0); jl_any_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Any"), NULL, jl_emptysvec); @@ -3628,7 +3636,7 @@ void jl_init_types(void) jl_tupletype_t *empty_tuple_type = jl_apply_tuple_type(jl_emptysvec); empty_tuple_type->uid = jl_assign_type_uid(); - jl_emptytuple = newstruct(empty_tuple_type); + jl_emptytuple = jl_gc_alloc(ptls, 0, empty_tuple_type); empty_tuple_type->instance = jl_emptytuple; // non-primitive definitions follow diff --git a/src/julia_internal.h b/src/julia_internal.h index 4237c136901e0..fbe97c63223e8 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -40,35 +40,116 @@ extern unsigned sig_stack_size; JL_DLLEXPORT extern int jl_lineno; JL_DLLEXPORT extern const char *jl_filename; -JL_DLLEXPORT void *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, - int osize, int end_offset); +JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, + int osize, int end_offset); JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz); int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset); -STATIC_INLINE jl_value_t *newobj(jl_value_t *type, size_t nfields) +// pools are 16376 bytes large (GC_POOL_SZ - GC_PAGE_OFFSET) +static const int jl_gc_sizeclasses[JL_GC_N_POOLS] = { +#ifdef _P64 + 8, +#else + 4, 8, 12, +#endif + + // 16 pools at 16-byte spacing + 16, 32, 48, 64, 80, 96, 112, 128, + 144, 160, 176, 192, 208, 224, 240, 256, + + // the following tables are computed for maximum packing efficiency via the formula: + // sz=(div(2^14-8,rng)÷16)*16; hcat(sz, (2^14-8)÷sz, 2^14-(2^14-8)÷sz.*sz)' + + // rng = 60:-4:32 (8 pools) + 272, 288, 304, 336, 368, 400, 448, 496, +// 60, 56, 53, 48, 44, 40, 36, 33, /pool +// 64, 256, 272, 256, 192, 384, 256, 16, bytes lost + + // rng = 30:-2:16 (8 pools) + 544, 576, 624, 672, 736, 816, 896, 1008, +// 30, 28, 26, 24, 22, 20, 18, 16, /pool +// 64, 256, 160, 256, 192, 64, 256, 256, bytes lost + + // rng = 15:-1:8 (8 pools) + 1088, 1168, 1248, 1360, 1488, 1632, 1808, 2032 +// 15, 14, 13, 12, 11, 10, 9, 8, /pool +// 64, 32, 160, 64, 16, 64, 112, 128, bytes lost +}; + +STATIC_INLINE int JL_CONST_FUNC jl_gc_szclass(size_t sz) +{ +#ifdef _P64 + if (sz <= 8) + return 0; + const int N = 0; +#else + if (sz <= 12) + return (sz + 3) / 4 - 1; + const int N = 2; +#endif + if (sz <= 256) + return (sz + 15) / 16 + N; + if (sz <= 496) + return 16 - 16376 / 4 / LLT_ALIGN(sz, 16 * 4) + 16 + N; + if (sz <= 1008) + return 16 - 16376 / 2 / LLT_ALIGN(sz, 16 * 2) + 24 + N; + return 16 - 16376 / 1 / LLT_ALIGN(sz, 16 * 1) + 32 + N; +} + +#ifdef __GNUC__ +# define jl_is_constexpr(e) __builtin_constant_p(e) +#else +# define jl_is_constexpr(e) (0) +#endif +#define JL_SMALL_BYTE_ALIGNMENT 16 +#define JL_CACHE_BYTE_ALIGNMENT 64 +#define GC_POOL_END_OFS(osize) ((((GC_PAGE_SZ - GC_PAGE_OFFSET)/(osize)) - 1)*(osize) + GC_PAGE_OFFSET) +#define GC_PAGE_LG2 14 // log2(size of a page) +#define GC_PAGE_SZ (1 << GC_PAGE_LG2) // 16k +#define GC_PAGE_OFFSET (JL_SMALL_BYTE_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_SMALL_BYTE_ALIGNMENT)) +#define GC_MAX_SZCLASS (2032-sizeof(void*)) + +STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) { - jl_value_t *jv = NULL; - switch (nfields) { - case 0: - jv = (jl_value_t*)jl_gc_alloc_0w(); break; - case 1: - jv = (jl_value_t*)jl_gc_alloc_1w(); break; - case 2: - jv = (jl_value_t*)jl_gc_alloc_2w(); break; - case 3: - jv = (jl_value_t*)jl_gc_alloc_3w(); break; - default: - jv = (jl_value_t*)jl_gc_allocobj(nfields * sizeof(void*)); + const size_t allocsz = sz + sizeof(jl_taggedvalue_t); + if (allocsz < sz) // overflow in adding offs, size was "negative" + jl_throw(jl_memory_exception); + jl_value_t *v; + if (allocsz <= GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { + int pool_id = jl_gc_szclass(allocsz); + jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id]; + int osize; + int endoff; + if (jl_is_constexpr(allocsz)) { + osize = jl_gc_sizeclasses[pool_id]; + endoff = GC_POOL_END_OFS(osize); + } + else { + osize = p->osize; + endoff = p->end_offset; + } + v = jl_gc_pool_alloc(ptls, p, osize, endoff); } - jl_set_typeof(jv, type); - return jv; + else { + v = jl_gc_big_alloc(ptls, allocsz); + } + jl_set_typeof(v, ty); + return v; } +JL_DLLEXPORT jl_value_t *jl_gc_alloc(jl_ptls_t ptls, size_t sz, void *ty); +// On GCC, only inline when sz is constant +#ifdef __GNUC__ +# define jl_gc_alloc(ptls, sz, ty) \ + (__builtin_constant_p(sz) ? jl_gc_alloc_(ptls, sz, ty) : \ + (jl_gc_alloc)(ptls, sz, ty)) +#else +# define jl_gc_alloc(ptls, sz) jl_gc_alloc_(ptls, sz, ty) +#endif -STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) +#define jl_buff_tag ((uintptr_t)0x4eade800) +STATIC_INLINE void *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz) { - jl_value_t *jv = (jl_value_t*)jl_gc_allocobj(type->size); - jl_set_typeof(jv, type); - return jv; + return jl_gc_alloc(ptls, sz, (void*)jl_buff_tag); } jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force); @@ -100,13 +181,11 @@ jl_tupletype_t *jl_argtype_with_function(jl_function_t *f, jl_tupletype_t *types JL_DLLEXPORT jl_value_t *jl_apply_2va(jl_value_t *f, jl_value_t **args, uint32_t nargs); -#define GC_MAX_SZCLASS (2032-sizeof(void*)) -void jl_gc_setmark(jl_value_t *v); +void jl_gc_setmark(jl_ptls_t ptls, jl_value_t *v); void jl_gc_sync_total_bytes(void); -void jl_gc_track_malloced_array(jl_array_t *a); +void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a); void jl_gc_count_allocd(size_t sz); void jl_gc_run_all_finalizers(jl_ptls_t ptls); -void *allocb(size_t sz); void gc_queue_binding(jl_binding_t *bnd); void gc_setmark_buf(jl_ptls_t ptls, void *buf, int, size_t); @@ -586,10 +665,6 @@ STATIC_INLINE void jl_free_aligned(void *p) } #endif -#define JL_SMALL_BYTE_ALIGNMENT 16 -#define JL_CACHE_BYTE_ALIGNMENT 64 - - // -- typemap.c -- // STATIC_INLINE int is_kind(jl_value_t *v) diff --git a/src/julia_threads.h b/src/julia_threads.h index 48b03f91d93ea..b4d07e797d65d 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -33,7 +33,6 @@ typedef struct { jl_taggedvalue_t *newpages; // root of list of chunks of free objects uint16_t end_offset; // stored to avoid computing it at each allocation uint16_t osize; // size of objects in this pool - uint16_t nfree; // number of free objects in page pointed into by free_list } jl_gc_pool_t; typedef struct { diff --git a/src/module.c b/src/module.c index cdeb130cc8802..77856cf2796ad 100644 --- a/src/module.c +++ b/src/module.c @@ -18,8 +18,9 @@ jl_module_t *jl_top_module=NULL; JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name) { - jl_module_t *m = (jl_module_t*)jl_gc_allocobj(sizeof(jl_module_t)); - jl_set_typeof(m, jl_module_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_module_t *m = (jl_module_t*)jl_gc_alloc(ptls, sizeof(jl_module_t), + jl_module_type); JL_GC_PUSH1(&m); assert(jl_is_symbol(name)); m->name = name; @@ -70,8 +71,9 @@ JL_DLLEXPORT uint8_t jl_istopmod(jl_module_t *mod) static jl_binding_t *new_binding(jl_sym_t *name) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_symbol(name)); - jl_binding_t *b = (jl_binding_t*)allocb(sizeof(jl_binding_t)); + jl_binding_t *b = (jl_binding_t*)jl_gc_alloc_buf(ptls, sizeof(jl_binding_t)); b->name = name; b->value = NULL; b->owner = NULL; diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 2375a340fa5a6..7d5b2bc2e923c 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -11,6 +11,7 @@ #include "julia.h" #include "julia_internal.h" #include "APInt-C.h" + const unsigned int host_char_bit = 8; // run time version of box/unbox intrinsic @@ -306,7 +307,8 @@ jl_value_t *jl_iintrinsic_1(jl_value_t *ty, jl_value_t *a, const char *name, static inline jl_value_t *jl_intrinsiclambda_ty1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, const void *voidlist) { - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)ty)->size, ty); intrinsic_1_t op = select_intrinsic_1(osize2, (const intrinsic_1_t*)voidlist); op(osize * host_char_bit, pa, jl_data_ptr(newv)); return newv; @@ -314,7 +316,8 @@ static inline jl_value_t *jl_intrinsiclambda_ty1(jl_value_t *ty, void *pa, unsig static inline jl_value_t *jl_intrinsiclambda_u1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, const void *voidlist) { - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)ty)->size, ty); intrinsic_u1_t op = select_intrinsic_u1(osize2, (const intrinsic_u1_t*)voidlist); unsigned cnt = op(osize * host_char_bit, pa); // TODO: the following memset/memcpy assumes little-endian @@ -342,6 +345,7 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *ty, jl_value_t *a) \ static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const char *name, intrinsic_cvt_t op, intrinsic_cvt_check_t check_op) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_value_t *aty = jl_typeof(a); if (!jl_is_bitstype(aty)) jl_errorf("%s: value is not a bitstype", name); @@ -352,7 +356,7 @@ static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const unsigned osize = jl_datatype_size(ty); if (check_op && check_op(isize, osize, pa)) jl_throw(jl_inexact_exception); - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)ty)->size, ty); op(aty == (jl_value_t*)jl_bool_type ? 1 : isize * host_char_bit, pa, osize * host_char_bit, jl_data_ptr(newv)); if (ty == (jl_value_t*)jl_bool_type) @@ -381,14 +385,15 @@ typedef void (fintrinsic_op1)(unsigned, void*, void*); static inline jl_value_t *jl_fintrinsic_1(jl_value_t *ty, jl_value_t *a, const char *name, fintrinsic_op1 *floatop, fintrinsic_op1 *doubleop) { + jl_ptls_t ptls = jl_get_ptls_states(); if (!jl_is_bitstype(jl_typeof(a))) jl_errorf("%s: value is not a bitstype", name); if (!jl_is_bitstype(ty)) jl_errorf("%s: type is not a bitstype", name); - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + unsigned sz2 = jl_datatype_size(ty); + jl_value_t *newv = jl_gc_alloc(ptls, sz2, ty); void *pa = jl_data_ptr(a), *pr = jl_data_ptr(newv); unsigned sz = jl_datatype_size(jl_typeof(a)); - unsigned sz2 = jl_datatype_size(ty); switch (sz) { /* choose the right size c-type operation based on the input */ case 4: @@ -517,7 +522,8 @@ jl_value_t *jl_iintrinsic_2(jl_value_t *a, jl_value_t *b, const char *name, static inline jl_value_t *jl_intrinsiclambda_2(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, const void *voidlist) { - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)ty)->size, ty); intrinsic_2_t op = select_intrinsic_2(sz2, (const intrinsic_2_t*)voidlist); op(sz * host_char_bit, pa, pb, jl_data_ptr(newv)); if (ty == (jl_value_t*)jl_bool_type) @@ -534,7 +540,8 @@ static inline jl_value_t *jl_intrinsiclambda_cmp(jl_value_t *ty, void *pa, void static inline jl_value_t *jl_intrinsiclambda_checked(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, const void *voidlist) { - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_value_t *newv = jl_gc_alloc(ptls, ((jl_datatype_t*)ty)->size, ty); intrinsic_checked_t op = select_intrinsic_checked(sz2, (const intrinsic_checked_t*)voidlist); int ovflw = op(sz * host_char_bit, pa, pb, jl_data_ptr(newv)); if (ovflw) @@ -551,14 +558,15 @@ static inline jl_value_t *jl_intrinsiclambda_checked(jl_value_t *ty, void *pa, v bi_intrinsic_ctype(OP, name, 64, double) \ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ { \ + jl_ptls_t ptls = jl_get_ptls_states();\ jl_value_t *ty = jl_typeof(a); \ if (jl_typeof(b) != ty) \ jl_error(#name ": types of a and b must match"); \ if (!jl_is_bitstype(ty)) \ jl_error(#name ": values are not bitstypes"); \ - jl_value_t *newv = newstruct((jl_datatype_t*)ty); \ - void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pr = jl_data_ptr(newv); \ int sz = jl_datatype_size(ty); \ + jl_value_t *newv = jl_gc_alloc(ptls, sz, ty); \ + void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pr = jl_data_ptr(newv); \ switch (sz) { \ /* choose the right size c-type operation */ \ case 4: \ @@ -605,14 +613,15 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ ter_intrinsic_ctype(OP, name, 64, double) \ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b, jl_value_t *c) \ { \ + jl_ptls_t ptls = jl_get_ptls_states();\ jl_value_t *ty = jl_typeof(a); \ if (jl_typeof(b) != ty || jl_typeof(c) != ty) \ jl_error(#name ": types of a, b, and c must match"); \ if (!jl_is_bitstype(ty)) \ jl_error(#name ": values are not bitstypes"); \ - jl_value_t *newv = newstruct((jl_datatype_t*)ty); \ + int sz = jl_datatype_size(ty); \ + jl_value_t *newv = jl_gc_alloc(ptls, sz, ty); \ void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pc = jl_data_ptr(c), *pr = jl_data_ptr(newv); \ - int sz = jl_datatype_size(ty); \ switch (sz) { \ /* choose the right size c-type operation */ \ case 4: \ @@ -892,14 +901,15 @@ un_fintrinsic(sqrt_float,sqrt_llvm) JL_DLLEXPORT jl_value_t *jl_powi_llvm(jl_value_t *a, jl_value_t *b) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_value_t *ty = jl_typeof(a); if (!jl_is_bitstype(ty)) jl_error("powi_llvm: a is not a bitstype"); if (!jl_is_bitstype(jl_typeof(b)) || jl_datatype_size(jl_typeof(b)) != 4) jl_error("powi_llvm: b is not a 32-bit bitstype"); - jl_value_t *newv = newstruct((jl_datatype_t*)ty); - void *pa = jl_data_ptr(a), *pr = jl_data_ptr(newv); int sz = jl_datatype_size(ty); + jl_value_t *newv = jl_gc_alloc(ptls, sz, ty); + void *pa = jl_data_ptr(a), *pr = jl_data_ptr(newv); switch (sz) { /* choose the right size c-type operation */ case 4: diff --git a/src/simplevector.c b/src/simplevector.c index 16006831a2af5..9322692059840 100644 --- a/src/simplevector.c +++ b/src/simplevector.c @@ -21,8 +21,9 @@ JL_DLLEXPORT jl_svec_t *jl_svec(size_t n, ...) JL_DLLEXPORT jl_svec_t *jl_svec1(void *a) { - jl_svec_t *v = (jl_svec_t*)jl_gc_alloc_2w(); - jl_set_typeof(v, jl_simplevector_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_svec_t *v = (jl_svec_t*)jl_gc_alloc(ptls, sizeof(void*) * 2, + jl_simplevector_type); jl_svec_set_len_unsafe(v, 1); jl_svecset(v, 0, a); return v; @@ -30,8 +31,9 @@ JL_DLLEXPORT jl_svec_t *jl_svec1(void *a) JL_DLLEXPORT jl_svec_t *jl_svec2(void *a, void *b) { - jl_svec_t *v = (jl_svec_t*)jl_gc_alloc_3w(); - jl_set_typeof(v, jl_simplevector_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_svec_t *v = (jl_svec_t*)jl_gc_alloc(ptls, sizeof(void*) * 3, + jl_simplevector_type); jl_svec_set_len_unsafe(v, 2); jl_svecset(v, 0, a); jl_svecset(v, 1, b); @@ -40,8 +42,10 @@ JL_DLLEXPORT jl_svec_t *jl_svec2(void *a, void *b) JL_DLLEXPORT jl_svec_t *jl_alloc_svec_uninit(size_t n) { + jl_ptls_t ptls = jl_get_ptls_states(); if (n == 0) return jl_emptysvec; - jl_svec_t *jv = (jl_svec_t*)newobj((jl_value_t*)jl_simplevector_type, n+1); + jl_svec_t *jv = (jl_svec_t*)jl_gc_alloc(ptls, (n + 1) * sizeof(void*), + jl_simplevector_type); jl_svec_set_len_unsafe(jv, n); return jv; } diff --git a/src/task.c b/src/task.c index 5da2b1a7e35b4..bcadd9497f777 100644 --- a/src/task.c +++ b/src/task.c @@ -149,7 +149,7 @@ static void NOINLINE save_stack(jl_ptls_t ptls, jl_task_t *t) size_t nb = (char*)ptls->stackbase - frame_addr; char *buf; if (t->stkbuf == NULL || t->bufsz < nb) { - buf = (char*)allocb(nb); + buf = (char*)jl_gc_alloc_buf(ptls, nb); t->stkbuf = buf; t->bufsz = nb; } @@ -544,8 +544,8 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) { jl_ptls_t ptls = jl_get_ptls_states(); size_t pagesz = jl_page_size; - jl_task_t *t = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); - jl_set_typeof(t, jl_task_type); + jl_task_t *t = (jl_task_t*)jl_gc_alloc(ptls, sizeof(jl_task_t), + jl_task_type); #ifndef COPY_STACKS if (ssize == 0) // unspecified -- pick some default size ssize = 1*1024*1024; // 1M (for now) @@ -575,7 +575,7 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) JL_GC_PUSH1(&t); size_t stkbuf_sz = ssize + pagesz + (pagesz - 1); - char *stk = allocb(stkbuf_sz); + char *stk = (char*)jl_gc_alloc_buf(ptls, stkbuf_sz); t->stkbuf = stk; jl_gc_wb_buf(t, t->stkbuf, stkbuf_sz); stk = (char*)LLT_ALIGN((uintptr_t)stk, pagesz); @@ -658,8 +658,8 @@ void jl_init_tasks(void) void jl_init_root_task(void *stack, size_t ssize) { jl_ptls_t ptls = jl_get_ptls_states(); - ptls->current_task = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); - jl_set_typeof(ptls->current_task, jl_task_type); + ptls->current_task = (jl_task_t*)jl_gc_alloc(ptls, sizeof(jl_task_t), + jl_task_type); #ifdef COPY_STACKS ptls->current_task->ssize = 0; // size of saved piece ptls->current_task->bufsz = 0; diff --git a/src/typemap.c b/src/typemap.c index 6e0764265219e..10fbd30db9a0f 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -826,8 +826,10 @@ static void jl_typemap_list_insert_sorted(jl_typemap_entry_t **pml, jl_value_t * static jl_typemap_level_t *jl_new_typemap_level(void) { - jl_typemap_level_t *cache = (jl_typemap_level_t*)jl_gc_allocobj(sizeof(jl_typemap_level_t)); - jl_set_typeof(cache, jl_typemap_level_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_typemap_level_t *cache = + (jl_typemap_level_t*)jl_gc_alloc(ptls, sizeof(jl_typemap_level_t), + jl_typemap_level_type); cache->key = NULL; cache->linear = (jl_typemap_entry_t*)jl_nothing; cache->any.unknown = jl_nothing; @@ -942,6 +944,7 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par const struct jl_typemap_info *tparams, jl_value_t **overwritten) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_tuple_type(type)); if (!simpletype) { simpletype = (jl_tupletype_t*)jl_nothing; @@ -973,8 +976,9 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par if (overwritten != NULL) *overwritten = NULL; - jl_typemap_entry_t *newrec = (jl_typemap_entry_t*)jl_gc_allocobj(sizeof(jl_typemap_entry_t)); - jl_set_typeof(newrec, jl_typemap_entry_type); + jl_typemap_entry_t *newrec = + (jl_typemap_entry_t*)jl_gc_alloc(ptls, sizeof(jl_typemap_entry_t), + jl_typemap_entry_type); newrec->sig = type; newrec->simplesig = simpletype; newrec->tvars = tvars; From 1dbfb3f3a0831e6c5f399c34b86faca4790dc40d Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Sun, 3 Jul 2016 22:55:57 +0200 Subject: [PATCH 0216/1117] More efficient conversion in ^(::Complex, ::Complex) (#17259) This can avoid a costly allocation of e.g. BigInt(2). --- base/complex.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/complex.jl b/base/complex.jl index 07e004b3519cf..afd39316cad6f 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -450,7 +450,7 @@ function ^{T<:AbstractFloat}(z::Complex{T}, p::Complex{T}) if p==2 #square zr, zi = reim(z) x = (zr-zi)*(zr+zi) - y = T(2)*zr*zi + y = T(2*zr*zi) if isnan(x) if isinf(y) x = copysign(zero(T),zr) From 1f34483699585adee9049ed6c731d339360739ad Mon Sep 17 00:00:00 2001 From: Siva Prasad Varma Date: Sun, 3 Jul 2016 15:59:51 -0700 Subject: [PATCH 0217/1117] Mention require's case-sensitivity in documenation (#17249) Mention require's case-sensitivity in documentation --- base/docs/helpdb/Base.jl | 4 +++- doc/stdlib/base.rst | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index a821a03fbd5fc..1f89d95b136e6 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1489,7 +1489,9 @@ This function is typically used to load library code, and is implicitly called b load packages. When searching for files, `require` first looks for package code under `Pkg.dir()`, then tries -paths in the global array `LOAD_PATH`. +paths in the global array `LOAD_PATH`. `require` is case-sensitive on all +platforms including those with case-insensitive filesystems like macOS and +Windows. """ require diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 5b6f24913c90d..1650062c0e4d3 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -126,7 +126,7 @@ Getting Around Loads a source files, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``\ ). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. - When searching for files, ``require`` first looks for package code under ``Pkg.dir()``\ , then tries paths in the global array ``LOAD_PATH``\ . + When searching for files, ``require`` first looks for package code under ``Pkg.dir()``\ , then tries paths in the global array ``LOAD_PATH``\ .``require`` is case-sensitive on all platforms including those with case-insensitive filesystems like macOS and Windows. .. function:: Base.compilecache(module::String) @@ -1427,4 +1427,3 @@ Internals .. Docstring generated from Julia source Compile the given function ``f`` for the argument tuple (of types) ``args``\ , but do not execute it. - From 8c56448081d99a869c76e97011e8454c400c75b0 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sun, 3 Jul 2016 20:31:48 -0400 Subject: [PATCH 0218/1117] Enable ifunc on ARM/AArch64/PPC/PPC64 only on GCC 5.0+ Closes #17264 --- src/threading.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/threading.c b/src/threading.c index 598bfc390266d..1c422009584ad 100644 --- a/src/threading.c +++ b/src/threading.c @@ -148,8 +148,7 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) #if defined(__GLIBC__) && (defined(_CPU_X86_64_) || defined(_CPU_X86_) || \ ((defined(_CPU_AARCH64_) || defined(_CPU_ARM_) || \ defined(_CPU_PPC64_) || defined(_CPU_PPC_)) && \ - (__GNUC__ >= 5 || __GNUC_MINOR__ >= 9))) -// Skip the `__GNUC__ >= 4` check since we require GCC 4.7+ + __GNUC__ >= 5)) // Only enable this on architectures that are tested. // For example, GCC doesn't seem to support the `ifunc` attribute on power yet. # if __GLIBC_PREREQ(2, 12) From d04f29535979d64695ff469660442426894662eb Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sun, 3 Jul 2016 19:26:13 -0700 Subject: [PATCH 0219/1117] Rerun genstdlib after second commit from #17249 --- doc/stdlib/base.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 1650062c0e4d3..daa5b1ec2ce8d 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -126,7 +126,7 @@ Getting Around Loads a source files, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``\ ). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. - When searching for files, ``require`` first looks for package code under ``Pkg.dir()``\ , then tries paths in the global array ``LOAD_PATH``\ .``require`` is case-sensitive on all platforms including those with case-insensitive filesystems like macOS and Windows. + When searching for files, ``require`` first looks for package code under ``Pkg.dir()``\ , then tries paths in the global array ``LOAD_PATH``\ . ``require`` is case-sensitive on all platforms including those with case-insensitive filesystems like macOS and Windows. .. function:: Base.compilecache(module::String) @@ -1427,3 +1427,4 @@ Internals .. Docstring generated from Julia source Compile the given function ``f`` for the argument tuple (of types) ``args``\ , but do not execute it. + From c2547f5ab1f0d516bbc21b05a181fce26aa25602 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 1 Jul 2016 04:04:48 -0400 Subject: [PATCH 0220/1117] move hidden fields of leaftype datatypes into a separate struct this shaves a sizeable amount off the system image :) also moves some of the common abi logic into the caller (fixing a minor bug too with sret handling and argument counting in needPassByRef) --- base/inference.jl | 2 +- base/reflection.jl | 26 +++++++- base/serialize.jl | 6 +- src/abi_aarch64.cpp | 34 ++++------ src/abi_arm.cpp | 33 +++------ src/abi_ppc64le.cpp | 20 ++---- src/abi_win32.cpp | 17 ++--- src/abi_win64.cpp | 24 +++---- src/abi_x86.cpp | 40 +++++------ src/abi_x86_64.cpp | 53 ++++++++------- src/abi_x86_vec.h | 14 ++-- src/alloc.c | 140 ++++++++++++++++++++++++++------------- src/array.c | 5 +- src/builtins.c | 8 +-- src/ccall.cpp | 38 +++++++---- src/cgutils.cpp | 46 ++++++++----- src/codegen.cpp | 2 +- src/dump.c | 64 +++++++++--------- src/gc.c | 13 +--- src/gf.c | 2 +- src/intrinsics.cpp | 8 +-- src/jltypes.c | 88 +++++++++++++----------- src/julia.h | 117 +++++++++++++++----------------- src/julia_internal.h | 1 + src/runtime_intrinsics.c | 4 +- src/sys.c | 6 +- src/typemap.c | 20 +++--- 27 files changed, 435 insertions(+), 396 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 2f5b5d5010497..446b033b4a075 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2387,7 +2387,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end if length(atypes)==3 && is(f,unbox) at3 = widenconst(atypes[3]) - if isa(at3,DataType) && !at3.mutable && at3.pointerfree + if isa(at3,DataType) && !at3.mutable && at3.layout != C_NULL && datatype_pointerfree(at3) # remove redundant unbox return (e.args[3],()) end diff --git a/base/reflection.jl b/base/reflection.jl index c62df75011d47..9e1e27a517d96 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -82,14 +82,36 @@ isconst(m::Module, s::Symbol) = # return an integer such that object_id(x)==object_id(y) if is(x,y) object_id(x::ANY) = ccall(:jl_object_id, UInt, (Any,), x) +immutable DataTypeLayout + nfields::UInt32 + alignment::UInt32 + # alignment : 28; + # haspadding : 1; + # pointerfree : 1; + # fielddesc_type : 2; +end + + # type predicates +datatype_alignment(dt::DataType) = dt.layout == C_NULL ? throw(UndefRefError()) : + Int(unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).alignment & 0x0FFFFFFF) + +datatype_haspadding(dt::DataType) = dt.layout == C_NULL ? throw(UndefRefError()) : + (unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).alignment >> 28) & 1 == 1 + +datatype_pointerfree(dt::DataType) = dt.layout == C_NULL ? throw(UndefRefError()) : + (unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).alignment >> 29) & 1 == 1 + +datatype_fielddesc_type(dt::DataType) = dt.layout == C_NULL ? throw(UndefRefError()) : + (unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).alignment >> 30) & 3 + isimmutable(x::ANY) = (@_pure_meta; (isa(x,Tuple) || !typeof(x).mutable)) isstructtype(t::DataType) = (@_pure_meta; nfields(t) != 0 || (t.size==0 && !t.abstract)) isstructtype(x) = (@_pure_meta; false) -isbits(t::DataType) = (@_pure_meta; !t.mutable & t.pointerfree & isleaftype(t)) +isbits(t::DataType) = (@_pure_meta; !t.mutable & (t.layout != C_NULL) && datatype_pointerfree(t)) isbits(t::Type) = (@_pure_meta; false) isbits(x) = (@_pure_meta; isbits(typeof(x))) -isleaftype(t::ANY) = (@_pure_meta; ccall(:jl_is_leaf_type, Int32, (Any,), t) != 0) +isleaftype(t::ANY) = (@_pure_meta; isa(t, DataType) && t.isleaftype) typeintersect(a::ANY,b::ANY) = (@_pure_meta; ccall(:jl_type_intersection, Any, (Any,Any), a, b)) typeseq(a::ANY,b::ANY) = (@_pure_meta; a<:b && b<:a) diff --git a/base/serialize.jl b/base/serialize.jl index 3f4be28406318..adb0048310034 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -2,7 +2,7 @@ module Serializer -import Base: GMP, Bottom, unsafe_convert, uncompressed_ast +import Base: GMP, Bottom, unsafe_convert, uncompressed_ast, datatype_pointerfree import Core: svec using Base: ViewIndex, index_lengths @@ -90,7 +90,7 @@ end # cycle handling function serialize_cycle(s::AbstractSerializer, x) - if !isimmutable(x) && !typeof(x).pointerfree + if !isimmutable(x) && !datatype_pointerfree(typeof(x)) offs = get(s.table, x, -1) if offs != -1 writetag(s.io, BACKREF_TAG) @@ -530,7 +530,7 @@ function deserialize(s::AbstractSerializer) end function deserialize_cycle(s::AbstractSerializer, x::ANY) - if !isimmutable(x) && !typeof(x).pointerfree + if !isimmutable(x) && !datatype_pointerfree(typeof(x)) s.table[s.counter] = x s.counter += 1 end diff --git a/src/abi_aarch64.cpp b/src/abi_aarch64.cpp index 5b87548a50f9f..61e1cb5d688b3 100644 --- a/src/abi_aarch64.cpp +++ b/src/abi_aarch64.cpp @@ -20,7 +20,9 @@ static Type *get_llvm_vectype(jl_datatype_t *dt) { // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt) // `!dt->mutabl && dt->pointerfree && !dt->haspadding && dt->nfields > 0` - size_t nfields = dt->nfields; + if (dt->layout == NULL) + return nullptr; + size_t nfields = dt->layout->nfields; assert(nfields > 0); if (nfields < 2) return nullptr; @@ -48,7 +50,7 @@ static Type *get_llvm_vectype(jl_datatype_t *dt) // `ft0` should be a `VecElement` type and the true element type // should be a `bitstype` if (ft0->name != jl_vecelement_typename || - ((jl_datatype_t*)jl_field_type(ft0, 0))->nfields) + ((jl_datatype_t*)jl_field_type(ft0, 0))->layout->nfields) return nullptr; for (size_t i = 1; i < nfields; i++) { if (jl_field_type(dt, i) != (jl_value_t*)ft0) { @@ -87,9 +89,9 @@ static Type *get_llvm_fptype(jl_datatype_t *dt) static Type *get_llvm_fp_or_vectype(jl_datatype_t *dt) { // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt) - if (dt->mutabl || !dt->pointerfree || dt->haspadding) + if (dt->mutabl || !dt->layout->pointerfree || dt->layout->haspadding) return nullptr; - return dt->nfields ? get_llvm_vectype(dt) : get_llvm_fptype(dt); + return dt->layout->nfields ? get_llvm_vectype(dt) : get_llvm_fptype(dt); } struct ElementType { @@ -179,7 +181,7 @@ static Type *isHFAorHVA(jl_datatype_t *dt, size_t &nele) // uniquely addressable members. // Maximum HFA and HVA size is 64 bytes (4 x fp128 or 16bytes vector) size_t dsz = dt->size; - if (dsz > 64 || !dt->pointerfree || dt->haspadding) + if (dsz > 64 || !dt->layout || !dt->layout->pointerfree || dt->layout->haspadding) return NULL; nele = 0; ElementType eltype; @@ -188,10 +190,8 @@ static Type *isHFAorHVA(jl_datatype_t *dt, size_t &nele) return NULL; } -void needPassByRef(AbiState*, jl_value_t *ty, bool *byRef, bool*) +void needPassByRef(AbiState*, jl_datatype_t *dt, bool *byRef, bool*) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) - jl_datatype_t *dt = (jl_datatype_t*)ty; // B.2 // If the argument type is an HFA or an HVA, then the argument is used // unmodified. @@ -227,12 +227,9 @@ bool need_private_copy(jl_value_t*, bool) // If the argument has to be passed on stack, we need to use sret. // // All the out parameters should be default to `false`. -static Type *classify_arg(jl_value_t *ty, bool *fpreg, bool *onstack, +static Type *classify_arg(jl_datatype_t *dt, bool *fpreg, bool *onstack, size_t *rewrite_len) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) - jl_datatype_t *dt = (jl_datatype_t*)ty; - // Based on section 5.4 C of the Procedure Call Standard // C.1 // If the argument is a Half-, Single-, Double- or Quad- precision @@ -354,10 +351,8 @@ static Type *classify_arg(jl_value_t *ty, bool *fpreg, bool *onstack, // } -bool use_sret(AbiState*, jl_value_t *ty) +bool use_sret(AbiState*, jl_datatype_t *dt) { - // Assume (jl_is_datatype(ty) && !jl_is_abstracttype(ty) && - // !jl_is_array_type(ty)) // Section 5.5 // If the type, T, of the result of a function is such that // @@ -370,21 +365,18 @@ bool use_sret(AbiState*, jl_value_t *ty) bool fpreg = false; bool onstack = false; size_t rewrite_len = 0; - classify_arg(ty, &fpreg, &onstack, &rewrite_len); + classify_arg(dt, &fpreg, &onstack, &rewrite_len); return onstack; } -Type *preferred_llvm_type(jl_value_t *ty, bool) +Type *preferred_llvm_type(jl_datatype_t *dt, bool) { - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_array_type(ty)) - return NULL; - jl_datatype_t *dt = (jl_datatype_t*)ty; if (Type *fptype = get_llvm_fp_or_vectype(dt)) return fptype; bool fpreg = false; bool onstack = false; size_t rewrite_len = 0; - if (Type *rewrite_ty = classify_arg(ty, &fpreg, &onstack, &rewrite_len)) + if (Type *rewrite_ty = classify_arg(dt, &fpreg, &onstack, &rewrite_len)) return ArrayType::get(rewrite_ty, rewrite_len); return NULL; } diff --git a/src/abi_arm.cpp b/src/abi_arm.cpp index acadd5c901ef4..a79d1de1a6ed3 100644 --- a/src/abi_arm.cpp +++ b/src/abi_arm.cpp @@ -24,7 +24,7 @@ namespace { typedef bool AbiState; AbiState default_abi_state = 0; -void needPassByRef(AbiState *state,jl_value_t *ty, bool *byRef, bool *inReg) +void needPassByRef(AbiState *state, jl_datatype_t *dt, bool *byRef, bool *inReg) { return; } @@ -87,7 +87,7 @@ static size_t isLegalHA(jl_datatype_t *dt, Type *&base) if (jl_is_structtype(dt)) { // Fast path checks before descending the type hierarchy // (4 x 128b vector == 64B max size) - if (dt->size > 64 || !dt->pointerfree || dt->haspadding) + if (dt->size > 64 || !dt->layout->pointerfree || dt->layout->haspadding) return 0; base = NULL; @@ -148,12 +148,9 @@ static void classify_cprc(jl_datatype_t *dt, bool *vfp) } } -static void classify_return_arg(jl_value_t *ty, bool *reg, +static void classify_return_arg(jl_datatype_t *dt, bool *reg, bool *onstack, bool *need_rewrite) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) - jl_datatype_t *dt = (jl_datatype_t*)ty; - // Based on section 5.4 of the Procedure Call Standard // VFP standard variant: see 6.1.2.2 @@ -204,15 +201,12 @@ static void classify_return_arg(jl_value_t *ty, bool *reg, *onstack = true; } -bool use_sret(AbiState *state, jl_value_t *ty) +bool use_sret(AbiState *state, jl_datatype_t *dt) { - // Assume (jl_is_datatype(ty) && !jl_is_abstracttype(ty) && - // !jl_is_array_type(ty)) - bool reg = false; bool onstack = false; bool need_rewrite = false; - classify_return_arg(ty, ®, &onstack, &need_rewrite); + classify_return_arg(dt, ®, &onstack, &need_rewrite); return onstack; } @@ -228,12 +222,9 @@ bool use_sret(AbiState *state, jl_value_t *ty) // If the argument has to be passed on stack, we need to use sret. // // All the out parameters should be default to `false`. -static void classify_arg(jl_value_t *ty, bool *reg, +static void classify_arg(jl_datatype_t *dt, bool *reg, bool *onstack, bool *need_rewrite) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) - jl_datatype_t *dt = (jl_datatype_t*)ty; - // Based on section 5.5 of the Procedure Call Standard // C.1.cp @@ -253,12 +244,8 @@ static void classify_arg(jl_value_t *ty, bool *reg, *need_rewrite = true; } -Type *preferred_llvm_type(jl_value_t *ty, bool isret) +Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) { - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_array_type(ty)) - return NULL; - jl_datatype_t *dt = (jl_datatype_t*)ty; - if (Type *fptype = get_llvm_fptype(dt)) return fptype; @@ -266,9 +253,9 @@ Type *preferred_llvm_type(jl_value_t *ty, bool isret) bool onstack = false; bool need_rewrite = false; if (isret) - classify_return_arg(ty, ®, &onstack, &need_rewrite); + classify_return_arg(dt, ®, &onstack, &need_rewrite); else - classify_arg(ty, ®, &onstack, &need_rewrite); + classify_arg(dt, ®, &onstack, &need_rewrite); if (!need_rewrite) return NULL; @@ -288,7 +275,7 @@ Type *preferred_llvm_type(jl_value_t *ty, bool isret) // For a Composite Type, the alignment of the copy will have 4-byte // alignment if its natural alignment is <= 4 and 8-byte alignment if // its natural alignment is >= 8 - size_t align = dt->alignment; + size_t align = dt->layout->alignment; if (align < 4) align = 4; if (align > 8) diff --git a/src/abi_ppc64le.cpp b/src/abi_ppc64le.cpp index 6fea221f06f85..40cd72b06b109 100644 --- a/src/abi_ppc64le.cpp +++ b/src/abi_ppc64le.cpp @@ -45,7 +45,7 @@ AbiState default_abi_state = 0; // count the homogeneous floating agregate size (saturating at max count of 8) static unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) { - size_t i, l = ty->nfields; + size_t i, l = ty->layout->nfields; // handle homogeneous float aggregates if (l == 0) { if (ty != jl_float64_type && ty != jl_float32_type) @@ -84,7 +84,7 @@ static unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) int n = 0; for (i = 0; i < l; i++) { jl_datatype_t *fld = (jl_datatype_t*)jl_field_type(ty, i); - if (!jl_is_datatype(fld)) + if (!jl_is_datatype(fld) || ((jl_datatype_t*)fld)->layout == NULL) return 9; n += isHFA((jl_datatype_t*)fld, ty0, hva); if (n > 8) @@ -93,10 +93,8 @@ static unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) return n; } -bool use_sret(AbiState *state, jl_value_t *ty) +bool use_sret(AbiState *state, jl_datatype_t *dt) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) - jl_datatype_t *dt = (jl_datatype_t*)ty; jl_datatype_t *ty0 = NULL; bool hva = false; if (dt->size > 16 && isHFA(dt, &ty0, &hva) > 8) @@ -104,23 +102,17 @@ bool use_sret(AbiState *state, jl_value_t *ty) return false; } -void needPassByRef(AbiState *state, jl_value_t *ty, bool *byRef, bool *inReg) +void needPassByRef(AbiState *state, jl_datatype_t *dt, bool *byRef, bool *inReg) { - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) - return; - jl_datatype_t *dt = (jl_datatype_t*)ty; jl_datatype_t *ty0 = NULL; bool hva = false; if (dt->size > 64 && isHFA(dt, &ty0, &hva) > 8) *byRef = true; } -Type *preferred_llvm_type(jl_value_t *ty, bool isret) +Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) { // Arguments are either scalar or passed by value - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) - return NULL; - jl_datatype_t *dt = (jl_datatype_t*)ty; size_t size = dt->size; // don't need to change bitstypes if (!dt->nfields) @@ -150,7 +142,7 @@ Type *preferred_llvm_type(jl_value_t *ty, bool isret) // rewrite integer-sized (non-HFA) struct to an array // the bitsize of the integer gives the desired alignment if (size > 8) { - if (dt->alignment <= 8) { + if (dt->layout->alignment <= 8) { return ArrayType::get(T_int64, (size + 7) / 8); } else { diff --git a/src/abi_win32.cpp b/src/abi_win32.cpp index c9cfd10f2b545..ad15596632784 100644 --- a/src/abi_win32.cpp +++ b/src/abi_win32.cpp @@ -40,10 +40,8 @@ typedef bool AbiState; AbiState default_abi_state = 0; -bool use_sret(AbiState *state, jl_value_t *ty) +bool use_sret(AbiState *state, jl_datatype_t *dt) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) - jl_datatype_t *dt = (jl_datatype_t*)ty; // Use sret if the size of the argument is not one of 1, 2, 4, 8 bytes // This covers the special case of Complex64 size_t size = dt->size; @@ -52,22 +50,17 @@ bool use_sret(AbiState *state, jl_value_t *ty) return true; } -void needPassByRef(AbiState *state, jl_value_t *ty, bool *byRef, bool *inReg) +void needPassByRef(AbiState *state, jl_datatype_t *dt, bool *byRef, bool *inReg) { - // Assume jl_is_datatype(ty) && !jl_is_abstracttype(ty) - jl_datatype_t *dt = (jl_datatype_t*)ty; // Use pass by reference for all structs - *byRef = dt->nfields > 0; + *byRef = dt->layout->nfields > 0; } -Type *preferred_llvm_type(jl_value_t *ty, bool isret) +Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) { // Arguments are either scalar or passed by value - if (!isret || !jl_is_datatype(ty) || jl_is_abstracttype(ty)) - return NULL; - jl_datatype_t *dt = (jl_datatype_t*)ty; // rewrite integer sized (non-sret) struct to the corresponding integer - if (!dt->nfields) + if (!dt->layout->nfields) return NULL; return Type::getIntNTy(jl_LLVMContext, dt->size * 8); } diff --git a/src/abi_win64.cpp b/src/abi_win64.cpp index 3e8c441e79351..af64698c3dbdd 100644 --- a/src/abi_win64.cpp +++ b/src/abi_win64.cpp @@ -44,33 +44,25 @@ struct AbiState { const AbiState default_abi_state = {}; -bool use_sret(AbiState *state, jl_value_t *ty) +bool use_sret(AbiState *state, jl_datatype_t *dt) { - // Assume (jl_is_datatype(ty) && !jl_is_abstracttype(ty) && - // !jl_is_array_type(ty)) - if (jl_is_cpointer_type(ty)) - return false; - size_t size = jl_datatype_size(ty); - if (size <= 8 || is_native_simd_type(ty)) + size_t size = jl_datatype_size(dt); + if (size <= 8 || is_native_simd_type(dt)) return false; return true; } -void needPassByRef(AbiState *state, jl_value_t *ty, bool *byRef, bool *inReg) +void needPassByRef(AbiState *state, jl_datatype_t *dt, bool *byRef, bool *inReg) { - if(!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) - return; - size_t size = jl_datatype_size(ty); + size_t size = jl_datatype_size(dt); if (size > 8) *byRef = true; } -Type *preferred_llvm_type(jl_value_t *ty, bool isret) +Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) { - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) - return NULL; - size_t size = jl_datatype_size(ty); - if (size > 0 && size <= 8 && !jl_is_bitstype(ty)) + size_t size = jl_datatype_size(dt); + if (size > 0 && size <= 8 && !jl_is_bitstype(dt)) return Type::getIntNTy(jl_LLVMContext, size*8); return NULL; } diff --git a/src/abi_x86.cpp b/src/abi_x86.cpp index 8b5d34304f8b3..8d0665bedc0b4 100644 --- a/src/abi_x86.cpp +++ b/src/abi_x86.cpp @@ -40,52 +40,44 @@ typedef bool AbiState; AbiState default_abi_state = 0; -inline bool is_complex64(jl_value_t *ty) +inline bool is_complex64(jl_datatype_t *dt) { - return jl_complex_type != NULL && jl_is_datatype(ty) && - ((jl_datatype_t*)ty)->name == jl_complex_type->name && - jl_tparam0(ty) == (jl_value_t*)jl_float32_type; + return jl_complex_type != NULL && jl_is_datatype(dt) && + ((jl_datatype_t*)dt)->name == jl_complex_type->name && + jl_tparam0(dt) == (jl_value_t*)jl_float32_type; } -inline bool is_complex128(jl_value_t *ty) +inline bool is_complex128(jl_datatype_t *dt) { - return jl_complex_type != NULL && jl_is_datatype(ty) && - ((jl_datatype_t*)ty)->name == jl_complex_type->name && - jl_tparam0(ty) == (jl_value_t*)jl_float64_type; + return jl_complex_type != NULL && jl_is_datatype(dt) && + ((jl_datatype_t*)dt)->name == jl_complex_type->name && + jl_tparam0(dt) == (jl_value_t*)jl_float64_type; } -bool use_sret(AbiState *state, jl_value_t *ty) +bool use_sret(AbiState *state, jl_datatype_t *dt) { - // Assume (jl_is_datatype(ty) && !jl_is_abstracttype(ty) && - // !jl_is_array_type(ty)) - if (jl_is_cpointer_type(ty)) - return false; - size_t size = jl_datatype_size(ty); + size_t size = jl_datatype_size(dt); if (size == 0) return false; - if (is_complex64(ty) || (jl_is_bitstype(ty) && size <= 8)) + if (is_complex64(dt) || (jl_is_bitstype(dt) && size <= 8)) return false; return true; } -void needPassByRef(AbiState *state, jl_value_t *ty, bool *byRef, bool *inReg) +void needPassByRef(AbiState *state, jl_datatype_t *dt, bool *byRef, bool *inReg) { - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) - return; - size_t size = jl_datatype_size(ty); - if (is_complex64(ty) || is_complex128(ty) || (jl_is_bitstype(ty) && size <= 8)) + size_t size = jl_datatype_size(dt); + if (is_complex64(dt) || is_complex128(dt) || (jl_is_bitstype(dt) && size <= 8)) return; *byRef = true; } -Type *preferred_llvm_type(jl_value_t *ty, bool isret) +Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) { if (!isret) return NULL; - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) - return NULL; // special case Complex{Float32} as a return type - if (is_complex64(ty)) + if (is_complex64(dt)) return T_int64; return NULL; } diff --git a/src/abi_x86_64.cpp b/src/abi_x86_64.cpp index 714839a7b4652..e3550ecb3b6cf 100644 --- a/src/abi_x86_64.cpp +++ b/src/abi_x86_64.cpp @@ -107,25 +107,25 @@ struct Classification { // make sure other half knows about it too: accum.addField(offset+16, ComplexX87); } */ -void classifyType(Classification& accum, jl_value_t *ty, uint64_t offset) +void classifyType(Classification& accum, jl_datatype_t *dt, uint64_t offset) { // Floating point types - if (ty == (jl_value_t*)jl_float64_type || ty == (jl_value_t*)jl_float32_type) { + if (dt == jl_float64_type || dt == jl_float32_type) { accum.addField(offset, Sse); } // Misc types - else if (!jl_is_datatype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty) || jl_is_abstracttype(ty)) { + else if (jl_is_cpointer_type((jl_value_t*)dt)) { accum.addField(offset, Integer); // passed as a pointer } // Ghost - else if (jl_datatype_size(ty) == 0) { + else if (jl_datatype_size(dt) == 0) { } // BitsTypes and not float, write as Integers - else if (jl_is_bitstype(ty)) { - if (jl_datatype_size(ty) <= 8) { - accum.addField(offset,Integer); + else if (jl_is_bitstype(dt)) { + if (jl_datatype_size(dt) <= 8) { + accum.addField(offset, Integer); } - else if (jl_datatype_size(ty) <= 16) { + else if (jl_datatype_size(dt) <= 16) { // Int128 or other 128bit wide INTEGER types accum.addField(offset, Integer); accum.addField(offset+8, Integer); @@ -135,15 +135,17 @@ void classifyType(Classification& accum, jl_value_t *ty, uint64_t offset) } } // struct types that map to SIMD registers - else if (is_native_simd_type(ty)) { + else if (is_native_simd_type(dt)) { accum.addField(offset, Sse); } // Other struct types - else if (jl_datatype_size(ty) <= 16) { + else if (jl_datatype_size(dt) <= 16) { size_t i; - for (i = 0; i < jl_datatype_nfields(ty); ++i) { - classifyType(accum, jl_field_type((jl_datatype_t*)ty,i), - offset + jl_field_offset((jl_datatype_t*)ty,i)); + for (i = 0; i < jl_datatype_nfields(dt); ++i) { + jl_value_t *ty = jl_field_type(dt, i); + if (!jl_is_datatype(ty) || ((jl_datatype_t*)ty)->layout == NULL || jl_is_array_type(ty)) + ty = (jl_value_t*)jl_voidpointer_type; + classifyType(accum, (jl_datatype_t*)ty, offset + jl_field_offset(dt, i)); } } else { @@ -151,16 +153,16 @@ void classifyType(Classification& accum, jl_value_t *ty, uint64_t offset) } } -Classification classify(jl_value_t *ty) +Classification classify(jl_datatype_t *dt) { Classification cl; - classifyType(cl, ty, 0); + classifyType(cl, dt, 0); return cl; } -bool use_sret(AbiState *state,jl_value_t *ty) +bool use_sret(AbiState *state, jl_datatype_t *dt) { - int sret = classify(ty).isMemory; + int sret = classify(dt).isMemory; if (sret) { assert(state->int_regs>0 && "No int regs available when determining sret-ness?"); state->int_regs--; @@ -168,9 +170,9 @@ bool use_sret(AbiState *state,jl_value_t *ty) return sret; } -void needPassByRef(AbiState *state, jl_value_t *ty, bool *byRef, bool *inReg) +void needPassByRef(AbiState *state, jl_datatype_t *dt, bool *byRef, bool *inReg) { - Classification cl = classify(ty); + Classification cl = classify(dt); if (cl.isMemory) { *byRef = true; return; @@ -189,7 +191,7 @@ void needPassByRef(AbiState *state, jl_value_t *ty, bool *byRef, bool *inReg) state->int_regs -= wanted.int_regs; state->sse_regs -= wanted.sse_regs; } - else if (jl_is_structtype(ty)) { + else if (jl_is_structtype(dt)) { // spill to memory even though we would ordinarily pass // it in registers *byRef = true; @@ -198,21 +200,18 @@ void needPassByRef(AbiState *state, jl_value_t *ty, bool *byRef, bool *inReg) // Called on behalf of ccall to determine preferred LLVM representation // for an argument or return value. -Type *preferred_llvm_type(jl_value_t *ty, bool isret) +Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) { (void) isret; // no need to rewrite these types (they are returned as pointers anyways) - if (!jl_is_datatype(ty) || jl_is_abstracttype(ty) || jl_is_cpointer_type(ty) || jl_is_array_type(ty)) - return NULL; - - if (is_native_simd_type(ty)) + if (is_native_simd_type(dt)) return NULL; - int size = jl_datatype_size(ty); + int size = jl_datatype_size(dt); if (size > 16 || size == 0) return NULL; - Classification cl = classify(ty); + Classification cl = classify(dt); if (cl.isMemory) return NULL; diff --git a/src/abi_x86_vec.h b/src/abi_x86_vec.h index f02a8ab21c41c..ac4c76f91e10f 100644 --- a/src/abi_x86_vec.h +++ b/src/abi_x86_vec.h @@ -4,22 +4,22 @@ #define ABI_X86_VEC_H // Determine if object of bitstype ty maps to a __m128, __m256, or __m512 type in C. -static bool is_native_simd_type(jl_value_t *ty) { - size_t size = jl_datatype_size(ty); - if (size!=16 && size!=32 && size!=64) +static bool is_native_simd_type(jl_datatype_t *dt) { + size_t size = jl_datatype_size(dt); + if (size != 16 && size != 32 && size != 64) // Wrong size for xmm, ymm, or zmm register. return false; - uint32_t n = jl_datatype_nfields(ty); + uint32_t n = jl_datatype_nfields(dt); if (n<2) // Not mapped to SIMD register. return false; - jl_value_t *ft0 = jl_field_type(ty, 0); + jl_value_t *ft0 = jl_field_type(dt, 0); for (uint32_t i = 1; i < n; ++i) - if (jl_field_type(ty, i)!=ft0) + if (jl_field_type(dt, i) != ft0) // Not homogeneous return false; // Type is homogeneous. Check if it maps to LLVM vector. - return jl_special_vector_alignment(n,ft0) != 0; + return jl_special_vector_alignment(n, ft0) != 0; } #endif diff --git a/src/alloc.c b/src/alloc.c index 21a002ef48ecb..ca68721489705 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -119,7 +119,7 @@ static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) size_t nb = jl_datatype_size(bt); if (nb == 0) return jl_new_struct_uninit(bt); - *len = LLT_ALIGN(*len, bt->alignment); + *len = LLT_ALIGN(*len, bt->layout->alignment); data = (char*)data + (*len); *len += nb; if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data); @@ -860,33 +860,87 @@ jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_datatype_t *super, jl_svec_t *parameters) { jl_datatype_t *dt = jl_new_datatype((jl_sym_t*)name, super, parameters, jl_emptysvec, jl_emptysvec, 1, 0, 0); - dt->pointerfree = 0; return dt; } -JL_DLLEXPORT jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields, int8_t fielddesc_type) +jl_datatype_t *jl_new_uninitialized_datatype() { jl_ptls_t ptls = jl_get_ptls_states(); - // fielddesc_type is specified manually for builtin types - // and is (will be) calculated automatically for user defined types. - uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); - jl_datatype_t *t = - (jl_datatype_t*)jl_gc_alloc(ptls, (sizeof(jl_datatype_t) + - nfields * fielddesc_size), - jl_datatype_type); - // fielddesc_type should only be assigned here. It can cause data - // corruption otherwise. - t->fielddesc_type = fielddesc_type; - t->nfields = nfields; - t->haspadding = 0; - t->pointerfree = 0; + jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ptls, sizeof(jl_datatype_t), jl_datatype_type); t->depth = 0; t->hastypevars = 0; t->haswildcard = 0; t->isleaftype = 1; + t->layout = NULL; return t; } +static struct _jl_datatype_layout_t *jl_get_layout( + uint32_t nfields, + uint32_t alignment, + int haspadding, + jl_fielddesc32_t desc[]) +{ + // compute the smallest fielddesc type that can hold the layout description + int fielddesc_type = 0; + if (nfields > 0) { + uint32_t max_size = 0; + uint32_t max_offset = desc[nfields - 1].offset; + for (size_t i = 0; i < nfields; i++) { + if (desc[i].size > max_size) + max_size = desc[i].size; + } + jl_fielddesc8_t maxdesc8 = { 0, max_size, max_offset }; + jl_fielddesc16_t maxdesc16 = { 0, max_size, max_offset }; + jl_fielddesc32_t maxdesc32 = { 0, max_size, max_offset }; + if (maxdesc8.size != max_size || maxdesc8.offset != max_offset) { + fielddesc_type = 1; + if (maxdesc16.size != max_size || maxdesc16.offset != max_offset) { + fielddesc_type = 2; + if (maxdesc32.size != max_size || maxdesc32.offset != max_offset) { + assert(0); // should have been verified by caller + } + } + } + } + + // allocate a new descriptor + uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); + struct _jl_datatype_layout_t *flddesc = (struct _jl_datatype_layout_t*)malloc( + sizeof(struct _jl_datatype_layout_t) + nfields * fielddesc_size); + flddesc->nfields = nfields; + flddesc->alignment = alignment; + flddesc->haspadding = haspadding; + flddesc->fielddesc_type = fielddesc_type; + + // fill out the fields of the new descriptor + jl_fielddesc8_t* desc8 = (jl_fielddesc8_t*)jl_dt_layout_fields(flddesc); + jl_fielddesc16_t* desc16 = (jl_fielddesc16_t*)jl_dt_layout_fields(flddesc); + jl_fielddesc32_t* desc32 = (jl_fielddesc32_t*)jl_dt_layout_fields(flddesc); + int ptrfree = 1; + for (size_t i = 0; i < nfields; i++) { + if (fielddesc_type == 0) { + desc8[i].offset = desc[i].offset; + desc8[i].size = desc[i].size; + desc8[i].isptr = desc[i].isptr; + } + else if (fielddesc_type == 1) { + desc16[i].offset = desc[i].offset; + desc16[i].size = desc[i].size; + desc16[i].isptr = desc[i].isptr; + } + else { + desc32[i].offset = desc[i].offset; + desc32[i].size = desc[i].size; + desc32[i].isptr = desc[i].isptr; + } + if (desc[i].isptr) + ptrfree = 0; + } + flddesc->pointerfree = ptrfree; + return flddesc; +} + // Determine if homogeneous tuple with fields of type t will have // a special alignment beyond normal Julia rules. // Return special alignment if one exists, 0 if normal alignment rules hold. @@ -931,65 +985,63 @@ unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) { void jl_compute_field_offsets(jl_datatype_t *st) { size_t sz = 0, alignm = 1; - int ptrfree = 1; int homogeneous = 1; jl_value_t *lastty = NULL; - assert(0 <= st->fielddesc_type && st->fielddesc_type <= 2); - - uint64_t max_offset = (((uint64_t)1) << - (1 << (3 + st->fielddesc_type))) - 1; + uint64_t max_offset = (((uint64_t)1) << 32) - 1; uint64_t max_size = max_offset >> 1; - for(size_t i=0; i < jl_datatype_nfields(st); i++) { + uint32_t nfields = jl_svec_len(st->types); + jl_fielddesc32_t desc[nfields]; + int haspadding = 0; + + for (size_t i = 0; i < nfields; i++) { jl_value_t *ty = jl_field_type(st, i); size_t fsz, al; - if (jl_isbits(ty) && jl_is_leaf_type(ty)) { + if (jl_isbits(ty) && jl_is_leaf_type(ty) && ((jl_datatype_t*)ty)->layout) { fsz = jl_datatype_size(ty); // Should never happen if (__unlikely(fsz > max_size)) jl_throw(jl_overflow_exception); - al = ((jl_datatype_t*)ty)->alignment; - jl_field_setisptr(st, i, 0); - if (((jl_datatype_t*)ty)->haspadding) - st->haspadding = 1; + al = ((jl_datatype_t*)ty)->layout->alignment; + desc[i].isptr = 0; + if (((jl_datatype_t*)ty)->layout->haspadding) + haspadding = 1; } else { fsz = sizeof(void*); if (fsz > MAX_ALIGN) fsz = MAX_ALIGN; al = fsz; - jl_field_setisptr(st, i, 1); - ptrfree = 0; + desc[i].isptr = 1; } if (al != 0) { size_t alsz = LLT_ALIGN(sz, al); if (sz & (al - 1)) - st->haspadding = 1; + haspadding = 1; sz = alsz; if (al > alignm) alignm = al; } homogeneous &= lastty==NULL || lastty==ty; lastty = ty; - jl_field_setoffset(st, i, sz); - jl_field_setsize(st, i, fsz); + desc[i].offset = sz; + desc[i].size = fsz; if (__unlikely(max_offset - sz < fsz)) jl_throw(jl_overflow_exception); sz += fsz; } if (homogeneous && lastty!=NULL && jl_is_tuple_type(st)) { // Some tuples become LLVM vectors with stronger alignment than what was calculated above. - unsigned al = jl_special_vector_alignment(jl_datatype_nfields(st), lastty); + unsigned al = jl_special_vector_alignment(nfields, lastty); assert(al % alignm == 0); if (al) alignm = al; } - st->alignment = alignm; st->size = LLT_ALIGN(sz, alignm); if (st->size > sz) - st->haspadding = 1; - st->pointerfree = ptrfree && !st->abstract; + haspadding = 1; + st->layout = jl_get_layout(nfields, alignm, haspadding, desc); } extern int jl_boot_file_loaded; @@ -1018,7 +1070,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super t = jl_uint8_type; } if (t == NULL) - t = jl_new_uninitialized_datatype(jl_svec_len(fnames), 2); // TODO + t = jl_new_uninitialized_datatype(); else tn = t->name; // init before possibly calling jl_new_typename @@ -1030,14 +1082,11 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super if (ftypes != NULL) jl_gc_wb(t, t->types); t->abstract = abstract; t->mutabl = mutabl; - t->pointerfree = 0; t->ninitialized = ninitialized; t->instance = NULL; t->struct_decl = NULL; t->ditype = NULL; t->size = 0; - t->alignment = 1; - t->haspadding = 0; if (tn == NULL) { t->name = NULL; @@ -1068,7 +1117,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super } else { t->uid = jl_assign_type_uid(); - if (t->types != NULL) + if (t->types != NULL && t->isleaftype) jl_compute_field_offsets(t); } JL_GC_POP(); @@ -1080,11 +1129,10 @@ JL_DLLEXPORT jl_datatype_t *jl_new_bitstype(jl_value_t *name, jl_datatype_t *sup { jl_datatype_t *bt = jl_new_datatype((jl_sym_t*)name, super, parameters, jl_emptysvec, jl_emptysvec, 0, 0, 0); - bt->size = nbits/8; - bt->alignment = bt->size; - if (bt->alignment > MAX_ALIGN) - bt->alignment = MAX_ALIGN; - bt->pointerfree = 1; + uint32_t nbytes = (nbits + 7) / 8; + uint32_t alignm = nbytes > MAX_ALIGN ? MAX_ALIGN : nbytes; + bt->size = nbytes; + bt->layout = jl_get_layout(0, alignm, 0, NULL); return bt; } diff --git a/src/array.c b/src/array.c index 2992eefaf135f..fecf3b4e222d7 100644 --- a/src/array.c +++ b/src/array.c @@ -22,9 +22,10 @@ extern "C" { // array constructors --------------------------------------------------------- -static inline int store_unboxed(jl_value_t *el_type) +static inline int store_unboxed(jl_value_t *el_type) // jl_isbits { - return jl_is_leaf_type(el_type) && jl_is_immutable(el_type) && jl_is_pointerfree((jl_datatype_t*)el_type); + return jl_is_leaf_type(el_type) && jl_is_immutable(el_type) && + ((jl_datatype_t*)el_type)->layout && ((jl_datatype_t*)el_type)->layout->pointerfree; } int jl_array_store_unboxed(jl_value_t *el_type) diff --git a/src/builtins.c b/src/builtins.c index 86b3ed761d69f..b9b4b30493929 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -282,7 +282,7 @@ static int NOINLINE compare_fields(jl_value_t *a, jl_value_t *b, jl_datatype_t * } else { jl_datatype_t *ft = (jl_datatype_t*)jl_field_type(dt, f); - if (!ft->haspadding) { + if (!ft->layout->haspadding) { eq = bits_equal(ao, bo, jl_field_size(dt, f)); } else { @@ -739,7 +739,7 @@ JL_CALLABLE(jl_f_fieldtype) int field_index; if (jl_is_long(args[1])) { field_index = jl_unbox_long(args[1]) - 1; - if (field_index < 0 || field_index >= jl_datatype_nfields(st)) + if (field_index < 0 || field_index >= jl_field_count(st)) jl_bounds_error(args[0], args[1]); } else { @@ -755,7 +755,7 @@ JL_CALLABLE(jl_f_nfields) jl_value_t *x = args[0]; if (!jl_is_datatype(x)) x = jl_typeof(x); - return jl_box_long(jl_datatype_nfields(x)); + return jl_box_long(jl_field_count(x)); } // conversion ----------------------------------------------------------------- @@ -1084,7 +1084,7 @@ static uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v) else { jl_datatype_t *fieldtype = (jl_datatype_t*)jl_field_type(dt, f); assert(jl_is_datatype(fieldtype) && !fieldtype->abstract && !fieldtype->mutabl); - if (fieldtype->haspadding) + if (fieldtype->layout->haspadding) u = jl_object_id_((jl_value_t*)fieldtype, (jl_value_t*)vo); else u = bits_hash(vo, jl_field_size(dt, f)); diff --git a/src/ccall.cpp b/src/ccall.cpp index 05c2e6e08bd3f..984faf61dd717 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -313,7 +313,7 @@ static Value *julia_to_native(Type *to, bool toboxed, jl_value_t *jlto, const jl else { prepare_call(builder.CreateMemCpy(slot, data_pointer(jvinfo, ctx, slot->getType()), (uint64_t)jl_datatype_size(ety), - (uint64_t)((jl_datatype_t*)ety)->alignment)->getCalledValue()); + (uint64_t)((jl_datatype_t*)ety)->layout->alignment)->getCalledValue()); mark_gc_use(jvinfo); } return slot; @@ -888,20 +888,24 @@ static std::string generate_func_sig( *prt = *lrt = T_void; } else { - *prt = sret ? NULL : preferred_llvm_type(rt, true); - if (*prt == NULL) - *prt = *lrt; - - if (jl_is_datatype(rt) && !jl_is_abstracttype(rt) && - !jl_is_array_type(rt) && use_sret(&abi, rt)) { + if (!jl_is_datatype(rt) || ((jl_datatype_t*)rt)->layout == NULL || jl_is_cpointer_type(rt) || jl_is_array_type(rt)) { + *prt = *lrt; // passed as pointer + } + else if (use_sret(&abi, (jl_datatype_t*)rt)) { paramattrs.push_back(AttrBuilder()); paramattrs[0].clear(); #if !defined(_OS_WINDOWS_) || defined(LLVM35) // llvm used to use the old mingw ABI, skipping this marking works around that difference paramattrs[0].addAttribute(Attribute::StructRet); #endif paramattrs[0].addAttribute(Attribute::NoAlias); - fargt_sig.push_back(PointerType::get(*prt, 0)); + fargt_sig.push_back(PointerType::get(*lrt, 0)); sret = 1; + *prt = *lrt; + } + else { + *prt = preferred_llvm_type((jl_datatype_t*)rt, true); + if (*prt == NULL) + *prt = *lrt; } } @@ -954,14 +958,22 @@ static std::string generate_func_sig( // Whether or not to pass this in registers bool inReg = false; - if (jl_is_datatype(tti) && !jl_is_abstracttype(tti)) { - needPassByRef(&abi, tti, &byRef, &inReg); - } + Type *pat; + if (!jl_is_datatype(tti) || ((jl_datatype_t*)tti)->layout == NULL || jl_is_array_type(tti)) + tti = (jl_value_t*)jl_voidpointer_type; // passed as pointer - Type *pat = byRef ? PointerType::get(t, 0) : preferred_llvm_type(tti, false); - if (pat == NULL) { + needPassByRef(&abi, (jl_datatype_t*)tti, &byRef, &inReg); + if (jl_is_cpointer_type(tti)) { pat = t; } + else if (byRef) { + pat = PointerType::get(t, 0); + } + else { + pat = preferred_llvm_type((jl_datatype_t*)tti, false); + if (pat == NULL) + pat = t; + } byRefList.push_back(byRef); inRegList.push_back(inReg); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 5ab7d102f4cf1..7eef89a7e1e51 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -115,11 +115,19 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed if (jl_is_bitstype(jt)) { uint64_t SizeInBits = 8*jdt->size; #ifdef LLVM37 - llvm::DIType *t = dbuilder->createBasicType(jl_symbol_name(jdt->name->name),SizeInBits,8*jdt->alignment,llvm::dwarf::DW_ATE_unsigned); + llvm::DIType *t = dbuilder->createBasicType( + jl_symbol_name(jdt->name->name), + SizeInBits, + 8 * jdt->layout->alignment, + llvm::dwarf::DW_ATE_unsigned); jdt->ditype = t; return t; #else - DIType t = dbuilder->createBasicType(jl_symbol_name(jdt->name->name),SizeInBits,8*jdt->alignment,llvm::dwarf::DW_ATE_unsigned); + DIType t = dbuilder->createBasicType( + jl_symbol_name(jdt->name->name), + SizeInBits, + 8 * jdt->layout->alignment, + llvm::dwarf::DW_ATE_unsigned); MDNode *M = t; jdt->ditype = M; return t; @@ -138,8 +146,8 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed jl_symbol_name(jdt->name->name), // Name NULL, // File 0, // LineNumber - 8*jdt->size, // SizeInBits - 8*jdt->alignment, // AlignmentInBits + 8 * jdt->size, // SizeInBits + 8 * jdt->layout->alignment, // AlignmentInBits 0, // Flags NULL, // DerivedFrom DINodeArray(), // Elements @@ -415,8 +423,8 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, bool *isboxed) *jl_ExecutionEngine->getDataLayout(); #endif unsigned llvm_alignment = DL.getABITypeAlignment((Type*)jst->struct_decl); - unsigned julia_alignment = jst->alignment; - assert(llvm_alignment==julia_alignment); + unsigned julia_alignment = jst->layout->alignment; + assert(llvm_alignment == julia_alignment); #endif } return (Type*)jst->struct_decl; @@ -511,6 +519,7 @@ static Value *emit_typeof(Value *tt) T_pjlvalue); return tt; } + static jl_cgval_t emit_typeof(const jl_cgval_t &p, jl_codectx_t *ctx) { // given p, compute its type @@ -523,6 +532,7 @@ static jl_cgval_t emit_typeof(const jl_cgval_t &p, jl_codectx_t *ctx) aty = (jl_value_t*)jl_typeof(jl_tparam0(aty)); return mark_julia_const(aty); } + static Value *emit_typeof_boxed(const jl_cgval_t &p, jl_codectx_t *ctx) { return boxed(emit_typeof(p, ctx), ctx); @@ -540,12 +550,13 @@ static Value *emit_datatype_types(Value *dt) static Value *emit_datatype_nfields(Value *dt) { - Value *nf = tbaa_decorate(tbaa_const, builder. - CreateLoad(builder. - CreateBitCast(builder. - CreateGEP(builder.CreateBitCast(dt, T_pint8), - ConstantInt::get(T_size, offsetof(jl_datatype_t, nfields))), - T_pint32))); + Value *nf = tbaa_decorate(tbaa_const, builder.CreateLoad( + tbaa_decorate(tbaa_const, builder.CreateLoad( + builder.CreateBitCast( + builder.CreateGEP( + builder.CreateBitCast(dt, T_pint8), + ConstantInt::get(T_size, offsetof(jl_datatype_t, types))), + T_pint32->getPointerTo()))))); #ifdef _P64 nf = builder.CreateSExt(nf, T_int64); #endif @@ -827,7 +838,7 @@ static Value *emit_bounds_check(const jl_cgval_t &ainfo, jl_value_t *ty, Value * // Parameter ptr should be the pointer argument for the LoadInst or StoreInst. // It is currently unused, but might be used in the future for a more precise answer. static unsigned julia_alignment(Value* /*ptr*/, jl_value_t *jltype, unsigned alignment) { - if (!alignment && ((jl_datatype_t*)jltype)->alignment > MAX_ALIGN) { + if (!alignment && ((jl_datatype_t*)jltype)->layout->alignment > MAX_ALIGN) { // Type's natural alignment exceeds strictest alignment promised in heap, so return the heap alignment. return MAX_ALIGN; } @@ -862,7 +873,7 @@ static jl_cgval_t typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, // elt = data; //} //else { - Instruction *load = builder.CreateAlignedLoad(data, julia_alignment(data, jltype, alignment), false); + Instruction *load = builder.CreateAlignedLoad(data, isboxed ? alignment : julia_alignment(data, jltype, alignment), false); if (tbaa) { elt = tbaa_decorate(tbaa, load); } @@ -881,12 +892,13 @@ static void typed_store(Value *ptr, Value *idx_0based, const jl_cgval_t &rhs, Value *parent, // for the write barrier, NULL if no barrier needed unsigned alignment = 0, bool root_box = true) // if the value to store needs a box, should we root it ? { - Type *elty = julia_type_to_llvm(jltype); + bool isboxed; + Type *elty = julia_type_to_llvm(jltype, &isboxed); assert(elty != NULL); if (type_is_ghost(elty)) return; Value *r; - if (jl_isbits(jltype) && ((jl_datatype_t*)jltype)->size > 0) { + if (!isboxed) { r = emit_unbox(elty, rhs, jltype); } else { @@ -898,7 +910,7 @@ static void typed_store(Value *ptr, Value *idx_0based, const jl_cgval_t &rhs, data = builder.CreateBitCast(ptr, PointerType::get(elty, 0)); else data = ptr; - Instruction *store = builder.CreateAlignedStore(r, builder.CreateGEP(data, idx_0based), julia_alignment(r, jltype, alignment)); + Instruction *store = builder.CreateAlignedStore(r, builder.CreateGEP(data, idx_0based), isboxed ? alignment : julia_alignment(r, jltype, alignment)); if (tbaa) tbaa_decorate(tbaa, store); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 07c203fcf4032..3d964a2781bfe 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1993,7 +1993,7 @@ static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, if (at->isAggregateType()) { // Struct or Array assert(arg1.ispointer() && arg2.ispointer()); size_t sz = jl_datatype_size(arg1.typ); - if (sz > 512 && !((jl_datatype_t*)arg1.typ)->haspadding) { + if (sz > 512 && !((jl_datatype_t*)arg1.typ)->layout->haspadding) { #ifdef LLVM37 Value *answer = builder.CreateCall(prepare_call(memcmp_func), { diff --git a/src/dump.c b/src/dump.c index 926df27e559bf..65bbe6608dca5 100644 --- a/src/dump.c +++ b/src/dump.c @@ -555,25 +555,27 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) return; } - size_t nf = jl_datatype_nfields(dt); - write_uint16(s, nf); write_int32(s, dt->size); int has_instance = !!(dt->instance != NULL); - write_uint8(s, dt->abstract | (dt->mutabl<<1) | (dt->pointerfree<<2) | (has_instance<<3) | + int has_layout = !!(dt->layout != NULL); + write_uint8(s, dt->abstract | (dt->mutabl<<1) | (has_layout<<2) | (has_instance<<3) | (dt->hastypevars<<4) | (dt->haswildcard<<5) | (dt->isleaftype<<6)); write_int32(s, dt->depth); - write_int8(s, dt->fielddesc_type); if (!dt->abstract) { write_uint16(s, dt->ninitialized); if (mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK) { write_int32(s, dt->uid); } } - if (nf > 0) { - write_int32(s, dt->alignment); - write_int8(s, dt->haspadding); - size_t fieldsize = jl_fielddesc_size(dt->fielddesc_type); - ios_write(s, jl_datatype_fields(dt), nf * fieldsize); + if (has_layout) { + size_t nf = dt->layout->nfields; + write_uint16(s, nf); + write_int8(s, dt->layout->fielddesc_type); + write_int32(s, dt->layout->alignment); + write_int8(s, dt->layout->haspadding); + write_int8(s, dt->layout->pointerfree); + size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type); + ios_write(s, (char*)(&dt->layout[1]), nf * fieldsize); } if (has_instance) @@ -581,9 +583,7 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) jl_serialize_value(s, dt->name); jl_serialize_value(s, dt->parameters); jl_serialize_value(s, dt->super); - - if (nf > 0) - jl_serialize_value(s, dt->types); + jl_serialize_value(s, dt->types); } static void jl_serialize_module(ios_t *s, jl_module_t *m) @@ -1177,11 +1177,9 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) backref_list.items[pos] = dtv; return dtv; } - uint16_t nf = read_uint16(s); size_t size = read_int32(s); uint8_t flags = read_uint8(s); uint8_t depth = read_int32(s); - uint8_t fielddesc_type = read_int8(s); jl_datatype_t *dt = NULL; if (tag == 2) dt = jl_int32_type; @@ -1192,7 +1190,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) else if (tag == 8) dt = jl_uint8_type; else if (tag == 0 || tag == 5) - dt = jl_new_uninitialized_datatype(nf, fielddesc_type); + dt = jl_new_uninitialized_datatype(); else assert(0); assert(tree_literal_values==NULL && mode != MODE_AST); @@ -1203,7 +1201,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->ditype = NULL; dt->abstract = flags&1; dt->mutabl = (flags>>1)&1; - dt->pointerfree = (flags>>2)&1; + int has_layout = (flags>>2)&1; + int has_instance = (flags>>3)&1; dt->hastypevars = (flags>>4)&1; dt->haswildcard = (flags>>5)&1; dt->isleaftype = (flags>>6)&1; @@ -1212,6 +1211,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->parameters = NULL; dt->name = NULL; dt->super = NULL; + dt->layout = NULL; if (!dt->abstract) { dt->ninitialized = read_uint16(s); dt->uid = mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK ? read_int32(s) : 0; @@ -1221,18 +1221,19 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->uid = 0; } - if (nf > 0) { - dt->alignment = read_int32(s); - dt->haspadding = read_int8(s); - size_t fieldsize = jl_fielddesc_size(fielddesc_type); - ios_read(s, jl_datatype_fields(dt), nf * fieldsize); - } - else { - dt->alignment = dt->size; - dt->haspadding = 0; - if (dt->alignment > MAX_ALIGN) - dt->alignment = MAX_ALIGN; - dt->types = jl_emptysvec; + if (has_layout) { + uint16_t nf = read_uint16(s); + uint8_t fielddesc_type = read_int8(s); + size_t fielddesc_size = nf > 0 ? jl_fielddesc_size(fielddesc_type) : 0; + struct _jl_datatype_layout_t *layout = (struct _jl_datatype_layout_t*)malloc( + sizeof(struct _jl_datatype_layout_t) + nf * fielddesc_size); + layout->nfields = nf; + layout->fielddesc_type = fielddesc_type; + layout->alignment = read_int32(s); + layout->haspadding = read_int8(s); + layout->pointerfree = read_int8(s); + ios_read(s, (char*)&layout[1], nf * fielddesc_size); + dt->layout = layout; } if (tag == 5) { @@ -1243,7 +1244,6 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->uid = -1; // mark that this type needs a new uid } - int has_instance = (flags >> 3) & 1; if (has_instance) { assert(mode != MODE_MODULE_POSTWORK); // there shouldn't be an instance on a type with uid = 0 dt->instance = jl_deserialize_value(s, &dt->instance); @@ -1255,11 +1255,9 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) jl_gc_wb(dt, dt->parameters); dt->super = (jl_datatype_t*)jl_deserialize_value(s, (jl_value_t**)&dt->super); jl_gc_wb(dt, dt->super); + dt->types = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&dt->types); + jl_gc_wb(dt, dt->types); - if (nf > 0) { - dt->types = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&dt->types); - jl_gc_wb(dt, dt->types); - } return (jl_value_t*)dt; } diff --git a/src/gc.c b/src/gc.c index d36dc9b03206a..c42614df2adbd 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1307,7 +1307,7 @@ static int push_root(jl_ptls_t ptls, jl_value_t *v, int d, int bits) bits = gc_setmark(ptls, v, sizeof(jl_weakref_t)); goto ret; } - if ((jl_is_datatype(vt) && ((jl_datatype_t*)vt)->pointerfree)) { + if ((jl_is_datatype(vt) && ((jl_datatype_t*)vt)->layout->pointerfree)) { int sz = jl_datatype_size(vt); bits = gc_setmark(ptls, v, sz); goto ret; @@ -1405,16 +1405,7 @@ static int push_root(jl_ptls_t ptls, jl_value_t *v, int d, int bits) // this check should not be needed but it helps catching corruptions early else if (jl_typeof(vt) == (jl_value_t*)jl_datatype_type) { jl_datatype_t *dt = (jl_datatype_t*)vt; - size_t dtsz; - if (dt == jl_datatype_type) { - size_t fieldsize = - jl_fielddesc_size(((jl_datatype_t*)v)->fielddesc_type); - dtsz = NWORDS(sizeof(jl_datatype_t) + - jl_datatype_nfields(v) * fieldsize) * sizeof(void*); - } - else { - dtsz = jl_datatype_size(dt); - } + size_t dtsz = jl_datatype_size(dt); bits = gc_setmark(ptls, v, dtsz); if (d >= MAX_MARK_DEPTH) goto queue_the_root; diff --git a/src/gf.c b/src/gf.c index b7bfdff383001..1d3c668b4be52 100644 --- a/src/gf.c +++ b/src/gf.c @@ -331,7 +331,7 @@ static int very_general_type(jl_value_t *t) jl_value_t *jl_nth_slot_type(jl_tupletype_t *sig, size_t i) { - size_t len = jl_datatype_nfields(sig); + size_t len = jl_field_count(sig); if (len == 0) return NULL; if (i < len-1) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index ffc6c4326f097..68e5df0626233 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -532,7 +532,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c // always fixed size nb = jl_datatype_size(bt); llvmt = staticeval_bitstype(bt); - alignment = ((jl_datatype_t*)bt)->alignment; + alignment = ((jl_datatype_t*)bt)->layout->alignment; } else { bt = v.typ; @@ -544,7 +544,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c } nb = jl_datatype_size(bt); llvmt = staticeval_bitstype(bt); - alignment = ((jl_datatype_t*)bt)->alignment; + alignment = ((jl_datatype_t*)bt)->layout->alignment; } Value *runtime_bt = boxed(bt_value, ctx); // XXX: emit type validity check on runtime_bt (bitstype of size nb) @@ -775,7 +775,7 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct Value *strct = emit_allocobj(ctx, size, literal_pointer_val((jl_value_t*)ety)); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, - LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); + LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment))); thePtr = builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1); prepare_call(builder.CreateMemCpy(builder.CreateBitCast(strct, T_pint8), thePtr, size, 1)->getCalledValue()); @@ -834,7 +834,7 @@ static jl_cgval_t emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, j assert(jl_is_datatype(ety)); uint64_t size = ((jl_datatype_t*)ety)->size; im1 = builder.CreateMul(im1, ConstantInt::get(T_size, - LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); + LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment))); prepare_call(builder.CreateMemCpy(builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1), data_pointer(val, ctx, T_pint8), size, 1)->getCalledValue()); } diff --git a/src/jltypes.c b/src/jltypes.c index 54285e7356d85..82ee4b074a2d5 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1742,7 +1742,7 @@ static int valid_type_param(jl_value_t *v) size_t l = jl_nparams(tt); for(i=0; i < l; i++) { jl_value_t *pi = jl_tparam(tt,i); - if (!(pi == (jl_value_t*)jl_symbol_type || jl_isbits(pi))) + if (!(pi == (jl_value_t*)jl_sym_type || jl_isbits(pi))) return 0; } return 1; @@ -2207,7 +2207,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i } // create and initialize new type - ndt = jl_new_uninitialized_datatype(istuple ? ntp : dt->nfields, 2); // TODO + ndt = jl_new_uninitialized_datatype(); // associate these parameters with the new type on // the stack, in case one of its field types references it. top.tt = (jl_datatype_t*)ndt; @@ -2226,8 +2226,6 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i ndt->struct_decl = NULL; ndt->ditype = NULL; ndt->size = 0; - ndt->alignment = 1; - ndt->pointerfree = 0; jl_precompute_memoized_dt(ndt); // assign uid as early as possible @@ -2245,8 +2243,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i if (!istuple && ndt->name->names == jl_emptysvec) { assert(ftypes == NULL || ftypes == jl_emptysvec); ndt->size = dt->size; - ndt->alignment = dt->alignment; - ndt->pointerfree = dt->pointerfree; + ndt->layout = dt->layout; ndt->types = jl_emptysvec; if (jl_is_datatype_make_singleton(ndt)) { ndt->instance = jl_gc_alloc(ptls, 0, ndt); @@ -2279,8 +2276,6 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i assert(ndt->name->names == jl_emptysvec); } } - if (tn == jl_array_typename) - assert(!ndt->pointerfree); if (istuple) ndt->ninitialized = ntp; else @@ -3466,13 +3461,13 @@ void jl_init_types(void) jl_ptls_t ptls = jl_get_ptls_states(); arraylist_new(&partial_inst, 0); // create base objects - jl_datatype_type = jl_new_uninitialized_datatype(11, 1); + jl_datatype_type = jl_new_uninitialized_datatype(); jl_set_typeof(jl_datatype_type, jl_datatype_type); - jl_typename_type = jl_new_uninitialized_datatype(8, 1); - jl_sym_type = jl_new_uninitialized_datatype(0, 1); + jl_typename_type = jl_new_uninitialized_datatype(); + jl_sym_type = jl_new_uninitialized_datatype(); jl_symbol_type = jl_sym_type; - jl_simplevector_type = jl_new_uninitialized_datatype(1, 1); - jl_methtable_type = jl_new_uninitialized_datatype(6, 1); + jl_simplevector_type = jl_new_uninitialized_datatype(); + jl_methtable_type = jl_new_uninitialized_datatype(); jl_nothing = jl_gc_alloc(ptls, 0, NULL); jl_emptysvec = (jl_svec_t*)jl_gc_alloc(ptls, sizeof(void*), @@ -3490,28 +3485,38 @@ void jl_init_types(void) jl_datatype_type->name->primary = (jl_value_t*)jl_datatype_type; jl_datatype_type->super = jl_type_type; jl_datatype_type->parameters = jl_emptysvec; - jl_datatype_type->name->names = jl_svec(11, jl_symbol("name"), + jl_datatype_type->name->names = jl_svec(17, + jl_symbol("name"), jl_symbol("super"), jl_symbol("parameters"), jl_symbol("types"), jl_symbol("instance"), + jl_symbol("layout"), jl_symbol("size"), + jl_symbol("ninitialized"), + jl_symbol("uid"), jl_symbol("abstract"), jl_symbol("mutable"), - jl_symbol("pointerfree"), - jl_symbol("ninitialized"), - jl_symbol("depth")); - jl_datatype_type->types = jl_svec(11, jl_typename_type, jl_type_type, - jl_simplevector_type, jl_simplevector_type, - jl_any_type, - jl_any_type, // size - jl_any_type, jl_any_type, jl_any_type, jl_any_type, jl_any_type); + jl_symbol("llvm::StructType"), + jl_symbol("llvm::DIType"), + jl_symbol("depth"), + jl_symbol("hastypevars"), + jl_symbol("haswildcard"), + jl_symbol("isleaftype")); + jl_datatype_type->types = jl_svec(17, + jl_typename_type, + jl_type_type, + jl_simplevector_type, + jl_simplevector_type, + jl_any_type, // instance + jl_any_type, jl_any_type, jl_any_type, jl_any_type, + jl_any_type, jl_any_type, jl_any_type, jl_any_type, + jl_any_type, jl_any_type, jl_any_type, jl_any_type); jl_datatype_type->instance = NULL; jl_datatype_type->uid = jl_assign_type_uid(); jl_datatype_type->struct_decl = NULL; jl_datatype_type->ditype = NULL; jl_datatype_type->abstract = 0; - jl_datatype_type->pointerfree = 0; // NOTE: types should not really be mutable, but the instance and // struct_decl fields are basically caches, which are mutated. jl_datatype_type->mutabl = 1; @@ -3534,7 +3539,6 @@ void jl_init_types(void) jl_typename_type->struct_decl = NULL; jl_typename_type->ditype = NULL; jl_typename_type->abstract = 0; - jl_typename_type->pointerfree = 0; jl_typename_type->mutabl = 1; jl_typename_type->ninitialized = 2; @@ -3553,7 +3557,6 @@ void jl_init_types(void) jl_methtable_type->struct_decl = NULL; jl_methtable_type->ditype = NULL; jl_methtable_type->abstract = 0; - jl_methtable_type->pointerfree = 0; jl_methtable_type->mutabl = 1; jl_methtable_type->ninitialized = 4; @@ -3570,7 +3573,6 @@ void jl_init_types(void) jl_sym_type->ditype = NULL; jl_sym_type->size = 0; jl_sym_type->abstract = 0; - jl_sym_type->pointerfree = 0; jl_sym_type->mutabl = 1; jl_sym_type->ninitialized = 0; @@ -3586,7 +3588,6 @@ void jl_init_types(void) jl_simplevector_type->struct_decl = NULL; jl_simplevector_type->ditype = NULL; jl_simplevector_type->abstract = 0; - jl_simplevector_type->pointerfree = 0; jl_simplevector_type->mutabl = 1; jl_simplevector_type->ninitialized = 1; @@ -3625,7 +3626,7 @@ void jl_init_types(void) jl_anytuple_type->parameters = jl_svec(1, jl_wrap_vararg((jl_value_t*)jl_any_type, (jl_value_t*)NULL)); //jl_anytuple_type->parameters = jl_svec(1, jl_wrap_vararg((jl_value_t*)NULL, (jl_value_t*)NULL)); jl_anytuple_type->types = jl_anytuple_type->parameters; - jl_anytuple_type->nfields = 1; + jl_anytuple_type->layout = NULL; jl_anytuple_type->hastypevars = 1; jl_anytuple_type->haswildcard = 1; jl_anytuple_type->isleaftype = 0; @@ -3724,8 +3725,9 @@ void jl_init_types(void) tv, jl_emptysvec, jl_emptysvec, 0, 1, 0); jl_array_typename = jl_array_type->name; - jl_array_type->pointerfree = 0; jl_array_type->ninitialized = 0; + static const struct _jl_datatype_layout_t _jl_array_layout = { 0, sizeof(void*), 0, 0, 0 }; + jl_array_type->layout = &_jl_array_layout; jl_array_any_type = (jl_value_t*)jl_apply_type((jl_value_t*)jl_array_type, @@ -3734,7 +3736,7 @@ void jl_init_types(void) jl_array_symbol_type = (jl_value_t*)jl_apply_type((jl_value_t*)jl_array_type, - jl_svec(2, jl_symbol_type, + jl_svec(2, jl_sym_type, jl_box_long(1))); jl_array_uint8_type = jl_apply_type((jl_value_t*)jl_array_type, @@ -3918,13 +3920,19 @@ void jl_init_types(void) jl_value_t *pointer_void = jl_apply_type((jl_value_t*)jl_pointer_type, jl_svec1(jl_void_type)); jl_voidpointer_type = (jl_datatype_t*)pointer_void; - jl_svecset(jl_datatype_type->types, 5, jl_int32_type); - jl_svecset(jl_datatype_type->types, 6, (jl_value_t*)jl_bool_type); - jl_svecset(jl_datatype_type->types, 7, (jl_value_t*)jl_bool_type); - jl_svecset(jl_datatype_type->types, 8, (jl_value_t*)jl_bool_type); - jl_svecset(jl_datatype_type->types, 9, jl_int32_type); - jl_svecset(jl_datatype_type->types, 10, jl_int32_type); - jl_svecset(jl_tvar_type->types, 3, (jl_value_t*)jl_bool_type); + jl_svecset(jl_datatype_type->types, 5, jl_voidpointer_type); + jl_svecset(jl_datatype_type->types, 6, jl_int32_type); + jl_svecset(jl_datatype_type->types, 7, jl_int32_type); + jl_svecset(jl_datatype_type->types, 8, jl_int32_type); + jl_svecset(jl_datatype_type->types, 9, jl_bool_type); + jl_svecset(jl_datatype_type->types, 10, jl_bool_type); + jl_svecset(jl_datatype_type->types, 11, jl_voidpointer_type); + jl_svecset(jl_datatype_type->types, 12, jl_voidpointer_type); + jl_svecset(jl_datatype_type->types, 13, jl_int32_type); + jl_svecset(jl_datatype_type->types, 14, jl_bool_type); + jl_svecset(jl_datatype_type->types, 15, jl_bool_type); + jl_svecset(jl_datatype_type->types, 16, jl_bool_type); + jl_svecset(jl_tvar_type->types, 3, jl_bool_type); jl_svecset(jl_simplevector_type->types, 0, jl_long_type); jl_svecset(jl_typename_type->types, 6, jl_long_type); jl_svecset(jl_methtable_type->types, 3, jl_long_type); @@ -3946,7 +3954,11 @@ void jl_init_types(void) jl_compute_field_offsets(jl_lambda_info_type); jl_compute_field_offsets(jl_typector_type); jl_compute_field_offsets(jl_simplevector_type); - jl_simplevector_type->pointerfree = 0; + jl_compute_field_offsets(jl_sym_type); + + // TODO: don't modify layout objects + ((struct _jl_datatype_layout_t*)jl_sym_type->layout)->pointerfree = 0; + ((struct _jl_datatype_layout_t*)jl_simplevector_type->layout)->pointerfree = 0; empty_sym = jl_symbol(""); call_sym = jl_symbol("call"); diff --git a/src/julia.h b/src/julia.h index 7c2431c73b1e4..788406a4b5324 100644 --- a/src/julia.h +++ b/src/julia.h @@ -305,24 +305,38 @@ typedef struct { jl_svec_t *types; } jl_uniontype_t; +// in little-endian, isptr is always the first bit, avoiding the need for a branch in computing isptr typedef struct { - uint8_t offset; // offset relative to data start, excluding type tag - uint8_t size:7; uint8_t isptr:1; + uint8_t size:7; + uint8_t offset; // offset relative to data start, excluding type tag } jl_fielddesc8_t; typedef struct { - uint16_t offset; // offset relative to data start, excluding type tag - uint16_t size:15; uint16_t isptr:1; + uint16_t size:15; + uint16_t offset; // offset relative to data start, excluding type tag } jl_fielddesc16_t; typedef struct { - uint32_t offset; // offset relative to data start, excluding type tag - uint32_t size:31; uint32_t isptr:1; + uint32_t size:31; + uint32_t offset; // offset relative to data start, excluding type tag } jl_fielddesc32_t; +struct _jl_datatype_layout_t { + uint32_t nfields; + uint32_t alignment : 28; // strictest alignment over all fields + uint32_t haspadding : 1; // has internal undefined bytes + uint32_t pointerfree : 1; // has any julia gc pointers + uint32_t fielddesc_type : 2; // 0 -> 8, 1 -> 16, 2 -> 32 + // union { + // jl_fielddesc8_t field8[]; + // jl_fielddesc16_t field16[]; + // jl_fielddesc32_t field32[]; + // }; +}; + typedef struct _jl_datatype_t { JL_DATA_TYPE jl_typename_t *name; @@ -330,30 +344,19 @@ typedef struct _jl_datatype_t { jl_svec_t *parameters; jl_svec_t *types; jl_value_t *instance; // for singletons - int32_t size; + const struct _jl_datatype_layout_t *layout; + int32_t size; // TODO: move to _jl_datatype_layout_t + int32_t ninitialized; + uint32_t uid; uint8_t abstract; uint8_t mutabl; - uint8_t pointerfree; - int32_t ninitialized; // memoized properties + void *struct_decl; //llvm::Type* + void *ditype; // llvm::MDNode* to be used as llvm::DIType(ditype) int32_t depth; int8_t hastypevars; // bound int8_t haswildcard; // unbound int8_t isleaftype; - // hidden fields: - uint32_t nfields; - uint32_t alignment : 29; // strictest alignment over all fields - uint32_t haspadding : 1; // has internal undefined bytes - uint32_t fielddesc_type : 2; // 0 -> 8, 1 -> 16, 2 -> 32 - uint32_t uid; - void *struct_decl; //llvm::Type* - void *ditype; // llvm::MDNode* to be used as llvm::DIType(ditype) - // Last field needs to be pointer size aligned - // union { - // jl_fielddesc8_t field8[]; - // jl_fielddesc16_t field16[]; - // jl_fielddesc32_t field32[]; - // }; } jl_datatype_t; typedef struct { @@ -770,8 +773,9 @@ STATIC_INLINE void jl_array_uint8_set(void *a, size_t i, uint8_t x) // struct type info #define jl_field_name(st,i) (jl_sym_t*)jl_svecref(((jl_datatype_t*)st)->name->names, (i)) #define jl_field_type(st,i) jl_svecref(((jl_datatype_t*)st)->types, (i)) +#define jl_field_count(st) jl_svec_len(((jl_datatype_t*)st)->types) #define jl_datatype_size(t) (((jl_datatype_t*)t)->size) -#define jl_datatype_nfields(t) (((jl_datatype_t*)(t))->nfields) +#define jl_datatype_nfields(t) (((jl_datatype_t*)(t))->layout->nfields) // inline version with strong type check to detect typos in a `->name` chain STATIC_INLINE char *jl_symbol_name_(jl_sym_t *s) @@ -780,40 +784,32 @@ STATIC_INLINE char *jl_symbol_name_(jl_sym_t *s) } #define jl_symbol_name(s) jl_symbol_name_(s) -#define jl_datatype_fields(d) ((char*)(d) + sizeof(jl_datatype_t)) - -#define DEFINE_FIELD_ACCESSORS(f) \ - static inline uint32_t jl_field_##f(jl_datatype_t *st, int i) \ - { \ - assert(i >= 0 && (size_t)i < jl_datatype_nfields(st)); \ - if (st->fielddesc_type == 0) { \ - return ((jl_fielddesc8_t*)jl_datatype_fields(st))[i].f; \ - } \ - else if (st->fielddesc_type == 1) { \ - return ((jl_fielddesc16_t*)jl_datatype_fields(st))[i].f; \ - } \ - else { \ - return ((jl_fielddesc32_t*)jl_datatype_fields(st))[i].f; \ - } \ - } \ - static inline void jl_field_set##f(jl_datatype_t *st, int i, \ - uint32_t val) \ - { \ - assert(i >= 0 && (size_t)i < jl_datatype_nfields(st)); \ - if (st->fielddesc_type == 0) { \ - ((jl_fielddesc8_t*)jl_datatype_fields(st))[i].f = val; \ - } \ - else if (st->fielddesc_type == 1) { \ - ((jl_fielddesc16_t*)jl_datatype_fields(st))[i].f = val; \ - } \ - else { \ - ((jl_fielddesc32_t*)jl_datatype_fields(st))[i].f = val; \ - } \ - } +#define jl_dt_layout_fields(d) ((const char*)(d) + sizeof(struct _jl_datatype_layout_t)) + +#define DEFINE_FIELD_ACCESSORS(f) \ + static inline uint32_t jl_field_##f(jl_datatype_t *st, int i) \ + { \ + const struct _jl_datatype_layout_t *ly = st->layout; \ + assert(i >= 0 && (size_t)i < ly->nfields); \ + if (ly->fielddesc_type == 0) { \ + return ((const jl_fielddesc8_t*)jl_dt_layout_fields(ly))[i].f; \ + } \ + else if (ly->fielddesc_type == 1) { \ + return ((const jl_fielddesc16_t*)jl_dt_layout_fields(ly))[i].f; \ + } \ + else { \ + return ((const jl_fielddesc32_t*)jl_dt_layout_fields(ly))[i].f; \ + } \ + } \ DEFINE_FIELD_ACCESSORS(offset) DEFINE_FIELD_ACCESSORS(size) -DEFINE_FIELD_ACCESSORS(isptr) +static inline int jl_field_isptr(jl_datatype_t *st, int i) +{ + const struct _jl_datatype_layout_t *ly = st->layout; + assert(i >= 0 && (size_t)i < ly->nfields); + return ((const jl_fielddesc8_t*)(jl_dt_layout_fields(ly) + (i << (ly->fielddesc_type + 1))))->isptr; +} static inline uint32_t jl_fielddesc_size(int8_t fielddesc_type) { @@ -836,7 +832,6 @@ static inline uint32_t jl_fielddesc_size(int8_t fielddesc_type) #define jl_is_svec(v) jl_typeis(v,jl_simplevector_type) #define jl_is_simplevector(v) jl_is_svec(v) #define jl_is_datatype(v) jl_typeis(v,jl_datatype_type) -#define jl_is_pointerfree(t) (((jl_datatype_t*)t)->pointerfree) #define jl_is_mutable(t) (((jl_datatype_t*)t)->mutabl) #define jl_is_mutable_datatype(t) (jl_is_datatype(t) && (((jl_datatype_t*)t)->mutabl)) #define jl_is_immutable(t) (!((jl_datatype_t*)t)->mutabl) @@ -881,23 +876,23 @@ static inline uint32_t jl_fielddesc_size(int8_t fielddesc_type) STATIC_INLINE int jl_is_bitstype(void *v) { return (jl_is_datatype(v) && jl_is_immutable(v) && + ((jl_datatype_t*)(v))->layout && jl_datatype_nfields(v) == 0 && - !((jl_datatype_t*)(v))->abstract && ((jl_datatype_t*)(v))->size > 0); } STATIC_INLINE int jl_is_structtype(void *v) { return (jl_is_datatype(v) && - (jl_datatype_nfields(v) > 0 || + (jl_field_count(v) > 0 || ((jl_datatype_t*)(v))->size == 0) && !((jl_datatype_t*)(v))->abstract); } STATIC_INLINE int jl_isbits(void *t) // corresponding to isbits() in julia { - return (jl_is_datatype(t) && !((jl_datatype_t*)t)->mutabl && - ((jl_datatype_t*)t)->pointerfree && !((jl_datatype_t*)t)->abstract); + return (jl_is_datatype(t) && ((jl_datatype_t*)t)->layout && + !((jl_datatype_t*)t)->mutabl && ((jl_datatype_t*)t)->layout->pointerfree); } STATIC_INLINE int jl_is_datatype_singleton(jl_datatype_t *d) @@ -1000,8 +995,6 @@ JL_DLLEXPORT jl_tvar_t *jl_new_typevar(jl_sym_t *name,jl_value_t *lb,jl_value_t JL_DLLEXPORT jl_value_t *jl_apply_type(jl_value_t *tc, jl_svec_t *params); JL_DLLEXPORT jl_tupletype_t *jl_apply_tuple_type(jl_svec_t *params); JL_DLLEXPORT jl_tupletype_t *jl_apply_tuple_type_v(jl_value_t **p, size_t np); -JL_DLLEXPORT jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields, - int8_t fielddesc_type); JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super, jl_svec_t *parameters, jl_svec_t *fnames, jl_svec_t *ftypes, diff --git a/src/julia_internal.h b/src/julia_internal.h index fbe97c63223e8..e2438768ca87e 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -263,6 +263,7 @@ jl_value_t *jl_type_intersection_matching(jl_value_t *a, jl_value_t *b, jl_svec_t **penv, jl_svec_t *tvars); jl_value_t *jl_apply_type_(jl_value_t *tc, jl_value_t **params, size_t n); jl_value_t *jl_instantiate_type_with(jl_value_t *t, jl_value_t **env, size_t n); +jl_datatype_t *jl_new_uninitialized_datatype(void); jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_datatype_t *super, jl_svec_t *parameters); void jl_precompute_memoized_dt(jl_datatype_t *dt); diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 7d5b2bc2e923c..1709ef7238644 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -44,7 +44,7 @@ JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i) else { if (!jl_is_datatype(ety)) jl_error("pointerref: invalid pointer"); - size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->alignment); + size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->layout->alignment); char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; return jl_new_bits(ety, pp); } @@ -63,7 +63,7 @@ JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t else { if (!jl_is_datatype(ety)) jl_error("pointerset: invalid pointer"); - size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->alignment); + size_t nb = LLT_ALIGN(jl_datatype_size(ety), ((jl_datatype_t*)ety)->layout->alignment); char *pp = (char*)jl_unbox_long(p) + (jl_unbox_long(i)-1)*nb; if (jl_typeof(x) != ety) jl_error("pointerset: type mismatch in assign"); diff --git a/src/sys.c b/src/sys.c index 257d5ec474228..afc61db38683e 100644 --- a/src/sys.c +++ b/src/sys.c @@ -605,14 +605,16 @@ JL_DLLEXPORT long jl_SC_CLK_TCK(void) JL_DLLEXPORT size_t jl_get_field_offset(jl_datatype_t *ty, int field) { - if (field > jl_datatype_nfields(ty) || field < 1) + if (ty->layout == NULL || field > jl_datatype_nfields(ty) || field < 1) jl_bounds_error_int((jl_value_t*)ty, field); return jl_field_offset(ty, field - 1); } JL_DLLEXPORT size_t jl_get_alignment(jl_datatype_t *ty) { - return ty->alignment; + if (ty->layout == NULL) + jl_error("non-leaf type doesn't have an alignment"); + return ty->layout->alignment; } // Takes a handle (as returned from dlopen()) and returns the absolute path to the image loaded diff --git a/src/typemap.c b/src/typemap.c index 10fbd30db9a0f..f55425f847f21 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -461,7 +461,7 @@ int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs, if (jl_typeof(map.unknown) == (jl_value_t*)jl_typemap_level_type) { jl_typemap_level_t *cache = map.node; jl_value_t *ty = NULL; - size_t l = jl_datatype_nfields(closure->type); + size_t l = jl_field_count(closure->type); if (closure->va && l <= offs + 1) { ty = closure->va; } @@ -535,13 +535,13 @@ int sigs_eq(jl_value_t *a, jl_value_t *b, int useenv) */ static jl_typemap_entry_t *jl_typemap_assoc_by_type_(jl_typemap_entry_t *ml, jl_tupletype_t *types, int8_t inexact, jl_svec_t **penv) { - size_t n = jl_datatype_nfields(types); + size_t n = jl_field_count(types); while (ml != (void*)jl_nothing) { - size_t lensig = jl_datatype_nfields(ml->sig); + size_t lensig = jl_field_count(ml->sig); if (lensig == n || (ml->va && lensig <= n+1)) { int resetenv = 0, ismatch = 1; if (ml->simplesig != (void*)jl_nothing) { - size_t lensimplesig = jl_datatype_nfields(ml->simplesig); + size_t lensimplesig = jl_field_count(ml->simplesig); int isva = lensimplesig > 0 && jl_is_vararg_type(jl_tparam(ml->simplesig, lensimplesig - 1)); if (lensig == n || (isva && lensimplesig <= n + 1)) ismatch = sig_match_by_type_simple(jl_svec_data(types->parameters), n, @@ -644,7 +644,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_ jl_typemap_level_t *cache = ml_or_cache.node; // called object is the primary key for constructors, otherwise first argument jl_value_t *ty = NULL; - size_t l = jl_datatype_nfields(types); + size_t l = jl_field_count(types); int isva = 0; // compute the type at offset `offs` into `types`, which may be a Vararg if (l <= offs + 1) { @@ -712,7 +712,7 @@ jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_valu // some manually-unrolled common special cases while (ml->simplesig == (void*)jl_nothing && ml->guardsigs == jl_emptysvec && ml->isleafsig) { // use a tight loop for a long as possible - if (n == jl_datatype_nfields(ml->sig) && jl_typeof(args[0]) == jl_tparam(ml->sig, 0)) { + if (n == jl_field_count(ml->sig) && jl_typeof(args[0]) == jl_tparam(ml->sig, 0)) { if (n == 1) return ml; if (n == 2) { @@ -735,10 +735,10 @@ jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_valu } while (ml != (void*)jl_nothing) { - size_t lensig = jl_datatype_nfields(ml->sig); + size_t lensig = jl_field_count(ml->sig); if (lensig == n || (ml->va && lensig <= n+1)) { if (ml->simplesig != (void*)jl_nothing) { - size_t lensimplesig = jl_datatype_nfields(ml->simplesig); + size_t lensimplesig = jl_field_count(ml->simplesig); int isva = lensimplesig > 0 && jl_is_vararg_type(jl_tparam(ml->simplesig, lensimplesig - 1)); if (lensig == n || (isva && lensimplesig <= n + 1)) { if (!sig_match_simple(args, n, jl_svec_data(ml->simplesig->parameters), isva, lensimplesig)) @@ -901,7 +901,7 @@ static int jl_typemap_array_insert_(jl_array_t **cache, jl_value_t *key, jl_type static void jl_typemap_level_insert_(jl_typemap_level_t *cache, jl_typemap_entry_t *newrec, int8_t offs, const struct jl_typemap_info *tparams) { - size_t l = jl_datatype_nfields(newrec->sig); + size_t l = jl_field_count(newrec->sig); // compute the type at offset `offs` into `sig`, which may be a Vararg jl_value_t *t1 = NULL; int isva = 0; @@ -991,7 +991,7 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par newrec->isleafsig = newrec->issimplesig && !newrec->va; // entirely leaf types don't need to be sorted JL_GC_PUSH1(&newrec); size_t i, l; - for (i = 0, l = jl_datatype_nfields(type); i < l && newrec->issimplesig; i++) { + for (i = 0, l = jl_field_count(type); i < l && newrec->issimplesig; i++) { jl_value_t *decl = jl_field_type(type, i); if (decl == (jl_value_t*)jl_datatype_type) newrec->isleafsig = 0; // Type{} may have a higher priority than DataType From 08815dc2c31f6668575d6e494daabef4230172ce Mon Sep 17 00:00:00 2001 From: Lyndon White Date: Tue, 5 Jul 2016 00:26:47 +0800 Subject: [PATCH 0221/1117] Fix documentation for `base` (round2) (#17209) Updated docs for base, with regard to there being no option to specify symbol lists --- base/docs/helpdb/Base.jl | 6 ++---- doc/stdlib/numbers.rst | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 1f89d95b136e6..ea72078c4ca94 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4206,11 +4206,9 @@ rand """ base(base, n, [pad]) -Convert an integer to a string in the given base, optionally specifying a number of digits -to pad to. The base can be specified as either an integer, or as a `UInt8` array of -character values to use as digit symbols. +Convert an integer to a string in the given base, optionally specifying a number of digits to pad to. """ -base +base(base, n, pad) """ BoundsError([a],[i]) diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index e2464460f3bcb..b367bb7692192 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -40,7 +40,7 @@ Data Formats .. Docstring generated from Julia source - Convert an integer to a string in the given base, optionally specifying a number of digits to pad to. The base can be specified as either an integer, or as a ``UInt8`` array of character values to use as digit symbols. + Convert an integer to a string in the given base, optionally specifying a number of digits to pad to. .. function:: digits([T], n, [base], [pad]) From e23f75d317d097d4bc2e64b863e17e513e4600e4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 2 Jul 2016 17:05:11 -0400 Subject: [PATCH 0222/1117] add a healthy dose of locks to generic functions also fix a couple of recently-introduced data-flow ordering bugs in typeinf fix #17059 fix #16524 --- base/inference.jl | 9 +- src/alloc.c | 2 + src/codegen.cpp | 101 ++++++++++------- src/dump.c | 7 +- src/gf.c | 259 ++++++++++++++++++++++++++++--------------- src/jltypes.c | 15 ++- src/julia.h | 5 + src/julia_internal.h | 7 +- src/julia_threads.h | 8 ++ src/task.c | 3 +- 10 files changed, 272 insertions(+), 144 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 446b033b4a075..2170e681a0a7d 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1560,6 +1560,9 @@ end function typeinf_ext(linfo::LambdaInfo) if isdefined(linfo, :def) # method lambda - infer this specialization via the method cache + if linfo.inferred && linfo.code !== nothing + return linfo + end (code, _t, inferred) = typeinf_edge(linfo.def, linfo.specTypes, linfo.sparam_vals, true, true, true, linfo) if inferred && code.inferred && linfo !== code # This case occurs when the IR for a function has been deleted. @@ -1574,12 +1577,12 @@ function typeinf_ext(linfo::LambdaInfo) linfo.pure = code.pure linfo.inlineable = code.inlineable ccall(:jl_set_lambda_rettype, Void, (Any, Any), linfo, code.rettype) - linfo.inferred = true - linfo.inInference = false if code.jlcall_api == 2 linfo.constval = code.constval linfo.jlcall_api = 2 end + linfo.inferred = true + linfo.inInference = false end return code else @@ -1946,6 +1949,7 @@ function finish(me::InferenceState) widen_all_consts!(me.linfo) ispure = me.linfo.pure + ccall(:jl_set_lambda_rettype, Void, (Any, Any), me.linfo, widenconst(me.bestguess)) if (isa(me.bestguess,Const) && me.bestguess.val !== nothing) || (isType(me.bestguess) && !has_typevars(me.bestguess.parameters[1],true)) @@ -1994,7 +1998,6 @@ function finish(me::InferenceState) me.linfo.inlineable = false end - ccall(:jl_set_lambda_rettype, Void, (Any, Any), me.linfo, widenconst(me.bestguess)) me.linfo.inferred = true me.linfo.inInference = false # finalize and record the linfo result diff --git a/src/alloc.c b/src/alloc.c index ca68721489705..7bb8b16367203 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -600,6 +600,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(void) m->isstaged = 0; m->needs_sparam_vals_ducttape = 2; m->traced = 0; + JL_MUTEX_INIT(&m->writelock); return m; } @@ -827,6 +828,7 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo #ifdef JL_GF_PROFILE mt->ncalls = 0; #endif + JL_MUTEX_INIT(&mt->writelock); return mt; } diff --git a/src/codegen.cpp b/src/codegen.cpp index 3d964a2781bfe..1f7d8d5ee7e91 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -807,12 +807,28 @@ void jl_dump_compiles(void *s) static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations); void jl_add_linfo_in_flight(StringRef name, jl_lambda_info_t *linfo, const DataLayout &DL); -// this is the implementation component of jl_compile_linfo -// which compiles li and adds the result to the jitlayers -static void to_function(jl_lambda_info_t *li) +// this generates llvm code for the lambda info +// and adds the result to the jitlayers +// (and the shadow module), but doesn't yet compile +// or generate object code for it +// objective: assign li->functionObject +extern "C" void jl_compile_linfo(jl_lambda_info_t *li) { - // setup global state + if (li->jlcall_api == 2) { + // delete code for functions reduced to a constant + jl_set_lambda_code_null(li); + return; + } + // grab the codegen lock and see if this needs to be compiled + if (li->functionObjectsDecls.functionObject != NULL) { + return; + } JL_LOCK(&codegen_lock); + if (li->functionObjectsDecls.functionObject != NULL) { + JL_UNLOCK(&codegen_lock); + return; + } + // setup global state assert(!li->inInference); li->inCompile = 1; BasicBlock *old = nested_compile ? builder.GetInsertBlock() : NULL; @@ -1016,21 +1032,6 @@ extern "C" void jl_generate_fptr(jl_lambda_info_t *li) JL_UNLOCK(&codegen_lock); // Might GC } -// this generates llvm code for the lambda info -// (and adds it to the shadow module), but doesn't yet compile -// or generate object code for it -extern "C" void jl_compile_linfo(jl_lambda_info_t *li) -{ - if (li->jlcall_api == 2) { - // delete code for functions reduced to a constant - jl_set_lambda_code_null(li); - } - else if (li->functionObjectsDecls.functionObject == NULL) { - // objective: assign li->functionObject - to_function(li); - } -} - static Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t *argt); // get the address of a C-callable entry point for a function extern "C" JL_DLLEXPORT @@ -1102,11 +1103,13 @@ void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) // for use in reflection from Julia. // this is paired with jl_dump_function_ir and jl_dump_function_asm in particular ways: // misuse will leak memory or cause read-after-free +extern "C" JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp); + extern "C" JL_DLLEXPORT void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) { - jl_lambda_info_t *linfo = NULL; - JL_GC_PUSH2(&linfo, &tt); + jl_lambda_info_t *linfo = NULL, *temp = NULL; + JL_GC_PUSH3(&linfo, &temp, &tt); if (tt != NULL) { linfo = jl_get_specialization1(tt); if (linfo == NULL) { @@ -1129,11 +1132,12 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) if (!getdeclarations) { if (linfo->code == jl_nothing) { // re-infer if we've deleted the code - // XXX: this only /partially/ corrupts linfo - // (by confusing the compiler about the - // validity of the code it already generated) - linfo = jl_type_infer(linfo, 0); - if (linfo->code == jl_nothing || linfo->inInference) { + // first copy the linfo to avoid corrupting it and + // confusing the compiler about the + // validity of the code it already generated + temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals); + jl_type_infer(temp, 0); + if (temp->code == jl_nothing || temp->inInference) { JL_GC_POP(); return NULL; } @@ -1141,21 +1145,21 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) // emit this function into a new module Function *f, *specf; jl_llvm_functions_t declarations; - std::unique_ptr m = emit_function(linfo, &declarations); + std::unique_ptr m = emit_function(temp ? temp : linfo, &declarations); finalize_gc_frame(m.get()); PM->run(*m.get()); f = (llvm::Function*)declarations.functionObject; specf = (llvm::Function*)declarations.specFunctionObject; // swap declarations for definitions and destroy declarations if (specf) { - Function *temp = cast(m->getNamedValue(specf->getName())); + Function *tempf = cast(m->getNamedValue(specf->getName())); delete specf; - specf = temp; + specf = tempf; } if (f) { - Function *temp = cast(m->getNamedValue(f->getName())); + Function *tempf = cast(m->getNamedValue(f->getName())); delete f; - f = temp; + f = tempf; } Function *specf_decl = (Function*)linfo->functionObjectsDecls.specFunctionObject; if (specf_decl) { @@ -1176,13 +1180,21 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) } if (linfo->jlcall_api == 2) { // normally we don't generate native code for these functions, so need an exception here - if (linfo->functionObjectsDecls.functionObject == NULL) - to_function(linfo); + // This leaks a bit of memory to cache the native code that we'll never actually need + if (linfo->functionObjectsDecls.functionObject == NULL) { + temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals); + jl_type_infer(temp, 0); + temp->jlcall_api = 0; + temp->constval = jl_nothing; + if (temp->code == jl_nothing || temp->inInference) { + JL_GC_POP(); + return NULL; + } + jl_compile_linfo(temp); + linfo->functionObjectsDecls = temp->functionObjectsDecls; + } jl_set_lambda_code_null(linfo); } - else { - jl_compile_linfo(linfo); - } Function *llvmf; if (!getwrapper && linfo->functionObjectsDecls.specFunctionObject != NULL) { llvmf = (Function*)linfo->functionObjectsDecls.specFunctionObject; @@ -1882,8 +1894,9 @@ static void jl_add_linfo_root(jl_lambda_info_t *li, jl_value_t *val) { if (jl_is_leaf_type(val) || jl_is_bool(val) || jl_is_symbol(val)) return; - JL_GC_PUSH1(&val); jl_method_t *m = li->def; + JL_GC_PUSH1(&val); + JL_LOCK(&m->writelock); if (m->roots == NULL) { m->roots = jl_alloc_vec_any(1); jl_gc_wb(m, m->roots); @@ -1893,12 +1906,14 @@ static void jl_add_linfo_root(jl_lambda_info_t *li, jl_value_t *val) size_t rlen = jl_array_dim0(m->roots); for(size_t i=0; i < rlen; i++) { if (jl_array_ptr_ref(m->roots,i) == val) { + JL_UNLOCK(&li->def->writelock); JL_GC_POP(); return; } } jl_array_ptr_1d_push(m->roots, val); } + JL_UNLOCK(&li->def->writelock); JL_GC_POP(); } @@ -3553,10 +3568,14 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t // infer it first, if necessary if (lam && lam->inInference) lam = NULL; // TODO: use emit_invoke framework to dispatch these - if (lam && (lam->code == jl_nothing || !lam->inferred)) - jl_type_infer(lam, 0); - if (lam && (lam->inInference || !lam->inferred)) - lam = NULL; // TODO: use emit_invoke framework to dispatch these + if (lam) { + if (lam->functionObjectsDecls.functionObject == NULL && lam->jlcall_api != 2) { + if (lam && (lam->code == jl_nothing || !lam->inferred)) + jl_type_infer(lam, 0); + if (lam && (lam->inInference || !lam->inferred)) + lam = NULL; // TODO: use emit_invoke framework to dispatch these + } + } if (lam != NULL) { jl_compile_linfo(lam); name = jl_symbol_name(lam->def->name); diff --git a/src/dump.c b/src/dump.c index 65bbe6608dca5..b1620f54b0d21 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1469,6 +1469,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_gc_wb(m, m->invokes.unknown); m->needs_sparam_vals_ducttape = read_int8(s); m->traced = 0; + JL_MUTEX_INIT(&m->writelock); return (jl_value_t*)m; } else if (vtag == (jl_value_t*)jl_lambda_info_type) { @@ -2086,7 +2087,8 @@ JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) JL_DLLEXPORT jl_array_t *jl_compress_ast(jl_lambda_info_t *li, jl_array_t *ast) { - JL_LOCK(&dump_lock); // Might GC + JL_LOCK(&li->def->writelock); // protect the roots array (Might GC) + JL_LOCK(&dump_lock); // protect global structures in this file (Might GC) assert(jl_is_lambda_info(li)); assert(jl_is_array(ast)); DUMP_MODES last_mode = mode; @@ -2117,12 +2119,14 @@ JL_DLLEXPORT jl_array_t *jl_compress_ast(jl_lambda_info_t *li, jl_array_t *ast) jl_gc_enable(en); mode = last_mode; JL_UNLOCK(&dump_lock); // Might GC + JL_UNLOCK(&li->def->writelock); // Might GC JL_GC_POP(); return v; } JL_DLLEXPORT jl_array_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_array_t *data) { + JL_LOCK(&li->def->writelock); // protect the roots array (Might GC) JL_LOCK(&dump_lock); // Might GC assert(jl_is_lambda_info(li)); assert(jl_is_array(data)); @@ -2144,6 +2148,7 @@ JL_DLLEXPORT jl_array_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_array_t *dat tree_enclosing_module = NULL; mode = last_mode; JL_UNLOCK(&dump_lock); // Might GC + JL_UNLOCK(&li->def->writelock); // Might GC JL_GC_POP(); return v; } diff --git a/src/gf.c b/src/gf.c index 1d3c668b4be52..61d4d105cf357 100644 --- a/src/gf.c +++ b/src/gf.c @@ -122,13 +122,17 @@ JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t // get or create the LambdaInfo for a specialization JL_DLLEXPORT jl_lambda_info_t *jl_specializations_get_linfo(jl_method_t *m, jl_tupletype_t *type, jl_svec_t *sparams) { + JL_LOCK(&m->writelock); jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(m->specializations, type, NULL, 1, /*subtype*/0, /*offs*/0); - if (sf && jl_is_lambda_info(sf->func.value) && ((jl_lambda_info_t*)sf->func.value)->code != jl_nothing) + if (sf && jl_is_lambda_info(sf->func.value) && ((jl_lambda_info_t*)sf->func.value)->code != jl_nothing) { + JL_UNLOCK(&m->writelock); return (jl_lambda_info_t*)sf->func.value; + } jl_lambda_info_t *li = jl_get_specialized(m, type, sparams); JL_GC_PUSH1(&li); // TODO: fuse lookup and insert steps jl_typemap_insert(&m->specializations, (jl_value_t*)m, type, jl_emptysvec, NULL, jl_emptysvec, (jl_value_t*)li, 0, &tfunc_cache, NULL); + JL_UNLOCK(&m->writelock); JL_GC_POP(); return li; } @@ -170,38 +174,6 @@ jl_value_t *jl_mk_builtin_func(const char *name, jl_fptr_t fptr) return f; } -jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method) -{ - // one unspecialized version of a function can be shared among all cached specializations - jl_method_t *def = method->def; - if (method->unspecialized_ducttape != NULL) - return method->unspecialized_ducttape; - if (method->sparam_syms != jl_emptysvec) { - if (def->needs_sparam_vals_ducttape == 2) { - jl_array_t *code = (jl_array_t*)def->lambda_template->code; - JL_GC_PUSH1(&code); - if (!jl_typeis(code, jl_array_any_type)) - code = jl_uncompress_ast(def->lambda_template, code); - size_t i, l = jl_array_len(code); - def->needs_sparam_vals_ducttape = 0; - for (i = 0; i < l; i++) { - if (jl_has_intrinsics(method, jl_array_ptr_ref(code, i), def->module)) { - def->needs_sparam_vals_ducttape = 1; - break; - } - } - JL_GC_POP(); - } - if (def->needs_sparam_vals_ducttape) { - method->unspecialized_ducttape = jl_get_specialized(def, method->specTypes, method->sparam_vals); - jl_gc_wb(method, method->unspecialized_ducttape); - method->unspecialized_ducttape->unspecialized_ducttape = method->unspecialized_ducttape; - return method->unspecialized_ducttape; - } - } - return def->lambda_template; -} - /* run type inference on lambda "li" for given argument types. if "li" has been inferred before but the IR was deleted, returns a @@ -243,6 +215,8 @@ JL_DLLEXPORT void jl_set_lambda_rettype(jl_lambda_info_t *li, jl_value_t *rettyp // changing rettype changes the llvm signature, // so clear all of the llvm state at the same time assert(li->inInference); + assert(!li->inferred || li->functionObjectsDecls.functionObject == NULL); // protect against some double-infer dataflow mistakes + assert(li->jlcall_api != 2); // protect against some double-infer dataflow mistakes li->rettype = rettype; jl_gc_wb(li, rettype); li->functionObjectsDecls.functionObject = NULL; @@ -250,6 +224,7 @@ JL_DLLEXPORT void jl_set_lambda_rettype(jl_lambda_info_t *li, jl_value_t *rettyp li->functionID = 0; li->specFunctionID = 0; li->jlcall_api = 0; + li->constval = jl_nothing; } JL_DLLEXPORT void jl_set_lambda_code_null(jl_lambda_info_t *li) @@ -317,7 +292,9 @@ JL_DLLEXPORT void jl_set_typeinf_func(jl_value_t *f) jl_reset_mt_caches(jl_main_module, unspec); size_t i, l; for (i = 0, l = jl_array_len(unspec); i < l; i++) { - jl_type_infer((jl_lambda_info_t*)jl_array_ptr_ref(unspec, i), 1); + jl_lambda_info_t *li = (jl_lambda_info_t*)jl_array_ptr_ref(unspec, i); + if (!li->inferred) + jl_type_infer(li, 1); } JL_GC_POP(); } @@ -631,6 +608,7 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_typemap_entry_t *m, jl_svec_t *sparams) { + // caller must hold the mt->writelock JL_LOCK(&codegen_lock); // Might GC jl_method_t *definition = m->func.method; jl_tupletype_t *decl = m->sig; @@ -824,6 +802,7 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca static jl_lambda_info_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t *tt, int cache, int inexact) { + // caller must hold the mt->writelock jl_typemap_entry_t *entry = NULL; jl_svec_t *env = jl_emptysvec; jl_method_t *func = NULL; @@ -843,10 +822,12 @@ static jl_lambda_info_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t * } sig = join_tsig(tt, entry->sig); jl_lambda_info_t *nf; - if (!cache) + if (!cache) { nf = jl_get_specialized(m, sig, env); - else + } + else { nf = cache_method(mt, &mt->cache, (jl_value_t*)mt, sig, tt, entry, env); + } JL_GC_POP(); return nf; } @@ -1064,9 +1045,9 @@ void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletyp jl_tupletype_t *type = method->sig; jl_svec_t *tvars = method->tvars; assert(jl_is_tuple_type(type)); - JL_SIGATOMIC_BEGIN(); jl_value_t *oldvalue = NULL; JL_GC_PUSH1(&oldvalue); + JL_LOCK(&mt->writelock); jl_typemap_entry_t *newentry = jl_typemap_insert(&mt->defs, (jl_value_t*)mt, type, tvars, simpletype, jl_emptysvec, (jl_value_t*)method, 0, &method_defs, &oldvalue); if (oldvalue) { @@ -1083,7 +1064,7 @@ void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletyp invalidate_conflicting(&mt->cache, (jl_value_t*)type, (jl_value_t*)mt, (jl_array_t*)oldvalue); JL_GC_POP(); update_max_args(mt, type); - JL_SIGATOMIC_END(); + JL_UNLOCK(&mt->writelock); } void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args) @@ -1155,13 +1136,24 @@ jl_lambda_info_t *jl_method_lookup_by_type(jl_methtable_t *mt, jl_tupletype_t *t int cache, int inexact) { jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(mt->cache, types, NULL, 0, 1, jl_cachearg_offset(mt)); - jl_lambda_info_t *sf; + if (entry) + return entry->func.linfo; + JL_LOCK(&mt->writelock); + entry = jl_typemap_assoc_by_type(mt->cache, types, NULL, 0, 1, jl_cachearg_offset(mt)); if (entry) { - sf = entry->func.linfo; + JL_UNLOCK(&mt->writelock); + return entry->func.linfo; + } + if (jl_is_leaf_type((jl_value_t*)types)) + cache = 1; + jl_lambda_info_t *sf = jl_mt_assoc_by_type(mt, types, cache, inexact); + if (cache) { + JL_UNLOCK(&mt->writelock); } else { - if (jl_is_leaf_type((jl_value_t*)types)) cache=1; - sf = jl_mt_assoc_by_type(mt, types, cache, inexact); + JL_GC_PUSH1(&sf); + JL_UNLOCK(&mt->writelock); + JL_GC_POP(); } return sf; } @@ -1173,25 +1165,77 @@ JL_DLLEXPORT int jl_method_exists(jl_methtable_t *mt, jl_tupletype_t *types) jl_lambda_info_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t nargs, int cache) { - jl_lambda_info_t *sf; jl_typemap_entry_t *entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt)); - if (entry == NULL) { - jl_tupletype_t *tt = arg_type_tuple(args, nargs); - JL_GC_PUSH1(&tt); - sf = jl_mt_assoc_by_type(mt, tt, cache, 0); - JL_GC_POP(); + if (entry) + return entry->func.linfo; + + JL_LOCK(&mt->writelock); + entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt)); + if (entry) { + JL_UNLOCK(&mt->writelock); + return entry->func.linfo; + } + jl_tupletype_t *tt = arg_type_tuple(args, nargs); + jl_lambda_info_t *sf = NULL; + JL_GC_PUSH2(&tt, &sf); + sf = jl_mt_assoc_by_type(mt, tt, cache, 0); + if (cache) { + JL_UNLOCK(&mt->writelock); } else { - sf = entry->func.linfo; + JL_GC_PUSH1(&sf); + JL_UNLOCK(&mt->writelock); + JL_GC_POP(); } - + JL_GC_POP(); return sf; } +static jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method) +{ + // one unspecialized version of a function can be shared among all cached specializations + jl_method_t *def = method->def; + if (method->unspecialized_ducttape != NULL) + return method->unspecialized_ducttape; + if (method->sparam_syms != jl_emptysvec) { + JL_LOCK(&def->writelock); + if (method->unspecialized_ducttape != NULL) { + JL_UNLOCK(&def->writelock); + return method->unspecialized_ducttape; + } + if (def->needs_sparam_vals_ducttape == 2) { + jl_array_t *code = (jl_array_t*)def->lambda_template->code; + JL_GC_PUSH1(&code); + if (!jl_typeis(code, jl_array_any_type)) + code = jl_uncompress_ast(def->lambda_template, code); + size_t i, l = jl_array_len(code); + def->needs_sparam_vals_ducttape = 0; + for (i = 0; i < l; i++) { + if (jl_has_intrinsics(method, jl_array_ptr_ref(code, i), def->module)) { + def->needs_sparam_vals_ducttape = 1; + break; + } + } + JL_GC_POP(); + } + if (def->needs_sparam_vals_ducttape) { + method->unspecialized_ducttape = jl_get_specialized(def, method->specTypes, method->sparam_vals); + jl_gc_wb(method, method->unspecialized_ducttape); + method->unspecialized_ducttape->unspecialized_ducttape = method->unspecialized_ducttape; + } + JL_UNLOCK(&def->writelock); + if (method->unspecialized_ducttape != NULL) + return method->unspecialized_ducttape; + } + return def->lambda_template; +} + JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int include_ambiguous); jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) { + if (li->jlcall_api == 2) + return li; if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF || jl_options.compile_enabled == JL_OPTIONS_COMPILE_MIN) { // copy fptr from the template method definition @@ -1199,7 +1243,8 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) if (def && !def->isstaged) { li->fptr = def->lambda_template->fptr; li->jlcall_api = def->lambda_template->jlcall_api; - if (li->fptr != NULL) + li->constval = def->lambda_template->constval; + if (li->fptr != NULL || li->jlcall_api == 2) return li; } if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF) { @@ -1208,20 +1253,27 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) jl_printf(JL_STDERR, " sysimg may not have been built with --compile=all\n"); } } - if (li->functionObjectsDecls.functionObject == NULL) { + if (li->functionObjectsDecls.functionObject != NULL) + return li; + if (li->def) { + JL_LOCK(&li->def->writelock); + if (li->functionObjectsDecls.functionObject != NULL) { + JL_UNLOCK(&li->def->writelock); + return li; + } if (li->inInference || li->inCompile) { // if inference is running on this function, get a copy // of the function to be compiled without inference and run. assert(li->def != NULL); li = jl_get_unspecialized(li); } - else { + else if (li->jlcall_api != 2) { if (li->code == jl_nothing || (!li->inferred && li->def != NULL && jl_symbol_name(li->def->name)[0] != '@')) { // don't bother with typeinf on macros or toplevel thunks jl_type_infer(li, 0); } - if (li->functionObjectsDecls.functionObject == NULL) { + if (li->functionObjectsDecls.functionObject == NULL && li->jlcall_api != 2) { if (li->inInference || li->inCompile || li->code == jl_nothing) { // if inference is running on this function, get a copy // of the function to be compiled without inference and run. @@ -1230,10 +1282,13 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) } } } - if (li->functionObjectsDecls.functionObject == NULL) { // check again, because jl_type_infer may have compiled it - jl_compile_linfo(li); - } } + assert(!li->inInference && !li->inCompile && li->code != jl_nothing || li->jlcall_api == 2); + if (li->functionObjectsDecls.functionObject == NULL) { // check again, because jl_type_infer may have compiled it + jl_compile_linfo(li); + } + if (li->def) + JL_UNLOCK(&li->def->writelock); return li; } @@ -1283,7 +1338,7 @@ JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) jl_lambda_info_t *li = jl_get_specialization1(types); if (li == NULL) return 0; - if (li->functionObjectsDecls.functionObject == NULL) { + if (li->functionObjectsDecls.functionObject == NULL && li->jlcall_api != 2) { assert(!li->inCompile); if (li->inInference) return 0; @@ -1513,6 +1568,9 @@ static void _compile_all_deq(jl_array_t *found) else linfo = jl_specializations_get_linfo(m, ml->sig, jl_emptysvec); + if (linfo->jlcall_api == 2) + continue; + // infer this function now, if necessary if (!linfo->inferred || linfo->code == jl_nothing) jl_type_infer(linfo, 1); @@ -1534,6 +1592,7 @@ static void _compile_all_deq(jl_array_t *found) templ->functionID = linfo->functionID; templ->specFunctionID = linfo->specFunctionID; templ->jlcall_api = linfo->jlcall_api; + templ->constval = linfo->constval; jl_gc_wb(templ, linfo); } } @@ -1547,7 +1606,7 @@ static int _compile_all_enq(jl_typemap_entry_t *ml, void *env) jl_array_t *found = (jl_array_t*)env; // method definition -- compile template field jl_method_t *m = ml->func.method; - if (!m->lambda_template->functionID) { + if (m->lambda_template->functionID == 0 && m->lambda_template->jlcall_api != 2) { // found a lambda that still needs to be compiled jl_array_ptr_1d_push(found, (jl_value_t*)ml); } @@ -1901,15 +1960,23 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) } jl_lambda_info_t *mfunc = NULL; - jl_tupletype_t *tt = NULL; - if (entry == NULL) { - // cache miss case - tt = arg_type_tuple(args, nargs); - // if running inference overwrites this particular method, it becomes - // unreachable from the method table, so root mfunc. - JL_GC_PUSH2(&tt, &mfunc); - mfunc = jl_mt_assoc_by_type(mt, tt, 1, 0); - + if (entry) { + mfunc = entry->func.linfo; + } + else { + JL_LOCK(&mt->writelock); + entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt)); + if (entry) { + mfunc = entry->func.linfo; + } + else { + // cache miss case + jl_tupletype_t *tt = arg_type_tuple(args, nargs); + JL_GC_PUSH1(&tt); + mfunc = jl_mt_assoc_by_type(mt, tt, 1, 0); + JL_GC_POP(); + } + JL_UNLOCK(&mt->writelock); if (mfunc == NULL) { #ifdef JL_TRACE if (error_en) @@ -1919,12 +1986,8 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) jl_method_error((jl_function_t*)args[0], args, nargs); // unreachable } - jl_value_t *res = jl_call_method_internal(mfunc, args, nargs); - JL_GC_POP(); - return verify_type(res); } - mfunc = entry->func.linfo; #ifdef JL_TRACE if (traceen) jl_printf(JL_STDOUT, " at %s:%d\n", jl_symbol_name(mfunc->file), mfunc->line); @@ -1973,28 +2036,38 @@ jl_value_t *jl_gf_invoke(jl_tupletype_t *types0, jl_value_t **args, size_t nargs // now we have found the matching definition. // next look for or create a specialization of this definition. + jl_method_t *method = entry->func.method; jl_lambda_info_t *mfunc = NULL; jl_typemap_entry_t *tm = NULL; - if (entry->func.method->invokes.unknown != NULL) - tm = jl_typemap_assoc_exact(entry->func.method->invokes, args, nargs, jl_cachearg_offset(mt)); - if (tm == NULL) { - tt = arg_type_tuple(args, nargs); - if (entry->tvars != jl_emptysvec) { - jl_value_t *ti = - jl_lookup_match((jl_value_t*)tt, (jl_value_t*)entry->sig, &tpenv, entry->tvars); - assert(ti != (jl_value_t*)jl_bottom_type); - (void)ti; + if (method->invokes.unknown != NULL) + tm = jl_typemap_assoc_exact(method->invokes, args, nargs, jl_cachearg_offset(mt)); + if (tm) { + mfunc = tm->func.linfo; + } + else { + JL_LOCK(&method->writelock); + if (method->invokes.unknown != NULL) + tm = jl_typemap_assoc_exact(method->invokes, args, nargs, jl_cachearg_offset(mt)); + if (tm) { + mfunc = tm->func.linfo; } - sig = join_tsig(tt, entry->sig); - jl_method_t *func = entry->func.method; + else { + tt = arg_type_tuple(args, nargs); + if (entry->tvars != jl_emptysvec) { + jl_value_t *ti = + jl_lookup_match((jl_value_t*)tt, (jl_value_t*)entry->sig, &tpenv, entry->tvars); + assert(ti != (jl_value_t*)jl_bottom_type); + (void)ti; + } + sig = join_tsig(tt, entry->sig); + jl_method_t *func = entry->func.method; - if (func->invokes.unknown == NULL) - func->invokes.unknown = jl_nothing; + if (func->invokes.unknown == NULL) + func->invokes.unknown = jl_nothing; - mfunc = cache_method(mt, &func->invokes, entry->func.value, sig, tt, entry, tpenv); - } - else { - mfunc = tm->func.linfo; + mfunc = cache_method(mt, &func->invokes, entry->func.value, sig, tt, entry, tpenv); + } + JL_UNLOCK(&method->writelock); } JL_GC_POP(); return jl_call_method_internal(mfunc, args, nargs); @@ -2033,8 +2106,12 @@ JL_DLLEXPORT jl_function_t *jl_get_kwsorter(jl_typename_t *tn) { jl_methtable_t *mt = tn->mt; if (!mt->kwsorter) { - mt->kwsorter = jl_new_generic_function_with_supertype(tn->name, mt->module, jl_function_type, 1); - jl_gc_wb(mt, mt->kwsorter); + JL_LOCK(&mt->writelock); + if (!mt->kwsorter) { + mt->kwsorter = jl_new_generic_function_with_supertype(tn->name, mt->module, jl_function_type, 1); + jl_gc_wb(mt, mt->kwsorter); + } + JL_UNLOCK(&mt->writelock); } return mt->kwsorter; } diff --git a/src/jltypes.c b/src/jltypes.c index 82ee4b074a2d5..5ead0237a1c9d 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3547,11 +3547,12 @@ void jl_init_types(void) jl_methtable_type->name->mt = jl_new_method_table(jl_methtable_type->name->name, ptls->current_module); jl_methtable_type->super = jl_any_type; jl_methtable_type->parameters = jl_emptysvec; - jl_methtable_type->name->names = jl_svec(6, jl_symbol("name"), jl_symbol("defs"), + jl_methtable_type->name->names = jl_svec(8, jl_symbol("name"), jl_symbol("defs"), jl_symbol("cache"), jl_symbol("max_args"), - jl_symbol("kwsorter"), jl_symbol("module")); - jl_methtable_type->types = jl_svec(6, jl_sym_type, jl_any_type, jl_any_type, jl_any_type, - jl_any_type, jl_any_type); + jl_symbol("kwsorter"), jl_symbol("module"), + jl_symbol(""), jl_symbol("")); + jl_methtable_type->types = jl_svec(8, jl_sym_type, jl_any_type, jl_any_type, jl_any_type/*jl_long*/, + jl_any_type, jl_any_type/*module*/, jl_any_type/*long*/, jl_any_type/*int32*/); jl_methtable_type->uid = jl_assign_type_uid(); jl_methtable_type->instance = NULL; jl_methtable_type->struct_decl = NULL; @@ -3936,6 +3937,12 @@ void jl_init_types(void) jl_svecset(jl_simplevector_type->types, 0, jl_long_type); jl_svecset(jl_typename_type->types, 6, jl_long_type); jl_svecset(jl_methtable_type->types, 3, jl_long_type); +#ifdef __LP64__ + jl_svecset(jl_methtable_type->types, 6, jl_int64_type); // unsigned long +#else + jl_svecset(jl_methtable_type->types, 6, jl_int32_type); // DWORD +#endif + jl_svecset(jl_methtable_type->types, 7, jl_int32_type); // uint32_t jl_svecset(jl_lambda_info_type->types, 21, jl_voidpointer_type); jl_svecset(jl_lambda_info_type->types, 22, jl_voidpointer_type); jl_svecset(jl_lambda_info_type->types, 23, jl_voidpointer_type); diff --git a/src/julia.h b/src/julia.h index 788406a4b5324..f1349b4aa1c01 100644 --- a/src/julia.h +++ b/src/julia.h @@ -228,6 +228,10 @@ typedef struct _jl_method_t { // 0 = no, 1 = yes, 2 = not yet known uint8_t needs_sparam_vals_ducttape; uint8_t traced; + +// hidden fields: + // lock for modifications to the method + jl_mutex_t writelock; } jl_method_t; // This holds data for a single executable function body: @@ -437,6 +441,7 @@ typedef struct _jl_methtable_t { #ifdef JL_GF_PROFILE int ncalls; #endif + jl_mutex_t writelock; } jl_methtable_t; typedef struct { diff --git a/src/julia_internal.h b/src/julia_internal.h index e2438768ca87e..3f6afb6c9bfb7 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -160,11 +160,10 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li); JL_DLLEXPORT void jl_set_lambda_code_null(jl_lambda_info_t *li); // invoke (compiling if necessary) the jlcall function pointer for a method -jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method); STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_value_t **args, uint32_t nargs) { jl_lambda_info_t *mfptr = meth; - if (__unlikely(mfptr->fptr == NULL)) { + if (__unlikely(mfptr->fptr == NULL && mfptr->jlcall_api != 2)) { mfptr = jl_compile_for_dispatch(mfptr); if (!mfptr->fptr) jl_generate_fptr(mfptr); @@ -173,8 +172,10 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val return mfptr->fptr(args[0], &args[1], nargs-1); else if (mfptr->jlcall_api == 1) return ((jl_fptr_sparam_t)mfptr->fptr)(meth->sparam_vals, args[0], &args[1], nargs-1); - else + else if (mfptr->jlcall_api == 2) return meth->constval; + else + abort(); } jl_tupletype_t *jl_argtype_with_function(jl_function_t *f, jl_tupletype_t *types); diff --git a/src/julia_threads.h b/src/julia_threads.h index b4d07e797d65d..6b8c9ea9a306a 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -477,8 +477,15 @@ static inline void jl_mutex_unlock(jl_mutex_t *lock) JL_SIGATOMIC_END(); } +static inline void jl_mutex_init(jl_mutex_t *lock) +{ + lock->owner = 0; + lock->count = 0; +} + // Locks #ifdef JULIA_ENABLE_THREADING +#define JL_MUTEX_INIT(m) jl_mutex_init(m) #define JL_LOCK(m) jl_mutex_lock(m) #define JL_UNLOCK(m) jl_mutex_unlock(m) #define JL_LOCK_NOGC(m) jl_mutex_lock_nogc(m) @@ -488,6 +495,7 @@ static inline void jl_mutex_check_type(jl_mutex_t *m) { (void)m; } +#define JL_MUTEX_INIT(m) jl_mutex_check_type(m) #define JL_LOCK(m) do { \ JL_SIGATOMIC_BEGIN(); \ jl_gc_enable_finalizers(jl_get_ptls_states(), 0); \ diff --git a/src/task.c b/src/task.c index bcadd9497f777..607bc0a879d63 100644 --- a/src/task.c +++ b/src/task.c @@ -97,7 +97,6 @@ static int mangle_pointers; static void _probe_arch(void) { - jl_ptls_t ptls = jl_get_ptls_states(); struct _probe_data p; memset(p.probe_env, 0, sizeof(jl_jmp_buf)); memset(p.probe_sameAR, 0, sizeof(jl_jmp_buf)); @@ -112,10 +111,12 @@ static void _probe_arch(void) boundlow(&p); #if defined(__linux__) && defined(__i386__) + jl_ptls_t ptls = jl_get_ptls_states(); char **s = (char**)p.ref_probe; mangle_pointers = !(s[4] > ptls->stack_lo && s[4] < ptls->stack_hi); #elif defined(__linux__) && defined(__x86_64__) + jl_ptls_t ptls = jl_get_ptls_states(); char **s = (char**)p.ref_probe; mangle_pointers = !(s[6] > ptls->stack_lo && s[6] < ptls->stack_hi); From 7a15162777c42dd303aef1336a320a88304e611d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 4 Jul 2016 12:56:53 -0400 Subject: [PATCH 0223/1117] refactor JL_GF_PROFILE to be more usable in current build system --- src/alloc.c | 3 --- src/gf.c | 3 ++- src/julia.h | 3 --- src/options.h | 3 +-- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 7bb8b16367203..d754d53f01aae 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -825,9 +825,6 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo mt->cache.unknown = jl_nothing; mt->max_args = 0; mt->kwsorter = NULL; -#ifdef JL_GF_PROFILE - mt->ncalls = 0; -#endif JL_MUTEX_INIT(&mt->writelock); return mt; } diff --git a/src/gf.c b/src/gf.c index 61d4d105cf357..8eff3cc31b412 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1807,6 +1807,7 @@ STATIC_INLINE int sig_match_fast(jl_value_t **args, jl_value_t **sig, size_t i, jl_typemap_entry_t *call_cache[N_CALL_CACHE]; static uint8_t pick_which[N_CALL_CACHE]; #ifdef JL_GF_PROFILE +size_t ncalls; void call_cache_stats() { int pick_which_stat[4] = {0, 0, 0, 0}; int i, count = 0; @@ -1830,7 +1831,7 @@ void call_cache_stats() { JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) { #ifdef JL_GF_PROFILE - mt->ncalls++; + ncalls++; #endif #ifdef JL_TRACE int traceen = trace_en; //&& ((char*)&mt < jl_stack_hi-6000000); diff --git a/src/julia.h b/src/julia.h index f1349b4aa1c01..113675a27749a 100644 --- a/src/julia.h +++ b/src/julia.h @@ -438,9 +438,6 @@ typedef struct _jl_methtable_t { intptr_t max_args; // max # of non-vararg arguments in a signature jl_value_t *kwsorter; // keyword argument sorter function jl_module_t *module; // used for incremental serialization to locate original binding -#ifdef JL_GF_PROFILE - int ncalls; -#endif jl_mutex_t writelock; } jl_methtable_t; diff --git a/src/options.h b/src/options.h index ec185e9db7c64..ac641af99351b 100644 --- a/src/options.h +++ b/src/options.h @@ -81,8 +81,7 @@ // sites). this generally prints too much output to be useful. //#define JL_TRACE -// count generic (not inlined or specialized) calls to each function. recorded -// in the `ncalls` field of jl_methtable_t. +// profile generic (not inlined or specialized) calls to each function //#define JL_GF_PROFILE From 04ecd642c3c05f1181e041fcfe2682b714aa8153 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 4 Jul 2016 15:05:08 -0500 Subject: [PATCH 0224/1117] Prevent mapslices from mutating the original array --- NEWS.md | 5 ++++- base/abstractarray.jl | 6 ++++-- base/statistics.jl | 3 +-- test/arrayops.jl | 6 ++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 12db8fff3237e..19d0c2a421e31 100644 --- a/NEWS.md +++ b/NEWS.md @@ -84,7 +84,9 @@ Breaking changes If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of the desired shape ([#4211]). - * `mapslices` will always pass a view, so passing mutating functions will mutate the underlying array ([#16260]) + * `mapslices` now re-uses temporary storage. Recipient functions + that expect input slices to be persistent should copy data to + other storage ([#17266]). * Local variables and arguments are represented in lowered code as numbered `Slot` objects instead of as symbols ([#15609]). @@ -279,3 +281,4 @@ Deprecated or removed [#16481]: https://github.com/JuliaLang/julia/issues/16481 [#16731]: https://github.com/JuliaLang/julia/issues/16731 [#16972]: https://github.com/JuliaLang/julia/issues/16972 +[#17266]: https://github.com/JuliaLang/julia/issues/17266 diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 52f55c6b30493..fec3620e2a2bb 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1417,7 +1417,8 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector) idx[d] = Colon() end - r1 = f(view(A, idx...)) + Aslice = A[idx...] + r1 = f(Aslice) # determine result size and allocate Rsize = copy(dimsA) @@ -1449,7 +1450,8 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector) for i in 1:nidx idx[otherdims[i]] = ridx[otherdims[i]] = I.I[i] end - R[ridx...] = f(view(A, idx...)) + _unsafe_getindex!(Aslice, A, idx...) + R[ridx...] = f(Aslice) end end diff --git a/base/statistics.jl b/base/statistics.jl index 9fff861eda492..764a8fa70a394 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -492,8 +492,7 @@ end median!{T}(v::AbstractArray{T}) = median!(vec(v)) median{T}(v::AbstractArray{T}) = median!(copy!(Array(T, length(v)), v)) -median!{T}(v::AbstractArray{T}, region) = mapslices(median!, v, region) -median{T}(v::AbstractArray{T}, region) = median!(copy(v), region) +median{T}(v::AbstractArray{T}, region) = mapslices(median!, v, region) # for now, use the R/S definition of quantile; may want variants later # see ?quantile in R -- this is type 7 diff --git a/test/arrayops.jl b/test/arrayops.jl index 66bb95f096563..16a96faae6801 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -843,6 +843,12 @@ let n3a = mapslices(x-> ones(1,6), c, [2,3]) @test size(n1a) == (1,6,4) && size(n2a) == (1,3,6) && size(n3a) == (2,1,6) @test size(n1) == (6,1,4) && size(n2) == (6,3,1) && size(n3) == (2,6,1) + + # mutating functions + o = ones(3, 4) + m = mapslices(x->fill!(x, 0), o, 2) + @test m == zeros(3, 4) + @test o == ones(3, 4) end From f1c69fa232abccf2a6bb58f90c8ded6a530c8ff0 Mon Sep 17 00:00:00 2001 From: Jay Weisskopf Date: Mon, 4 Jul 2016 16:24:22 -0600 Subject: [PATCH 0225/1117] Fix spelling from "Arugment" to "Argument" --- base/docs/helpdb/Base.jl | 2 +- doc/stdlib/strings.rst | 3 +-- test/cmdlineargs.jl | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index ea72078c4ca94..b88280d21e73e 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -8262,7 +8262,7 @@ filt! ascii(s::AbstractString) Convert a string to `String` type and check that it contains only ASCII data, otherwise -throwing an `ArugmentError` indicating the position of the first non-ASCII byte. +throwing an `ArgumentError` indicating the position of the first non-ASCII byte. """ ascii(s) diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index e5bb975d8745a..3b98d8a73b0b2 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -80,7 +80,7 @@ .. Docstring generated from Julia source - Convert a string to ``String`` type and check that it contains only ASCII data, otherwise throwing an ``ArugmentError`` indicating the position of the first non-ASCII byte. + Convert a string to ``String`` type and check that it contains only ASCII data, otherwise throwing an ``ArgumentError`` indicating the position of the first non-ASCII byte. .. function:: @r_str -> Regex @@ -500,4 +500,3 @@ .. Docstring generated from Julia source Create a string from the address of a NUL-terminated UTF-32 string. A copy is made; the pointer can be safely freed. If ``length`` is specified, the string does not have to be NUL-terminated. - diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 05a698b067571..16b3c3c94117d 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -183,7 +183,7 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` @test readchomp(`$exename -E "Bool(Base.JLOptions().can_inline)"`) == "true" @test readchomp(`$exename --inline=yes -E "Bool(Base.JLOptions().can_inline)"`) == "true" @test readchomp(`$exename --inline=no -E "Bool(Base.JLOptions().can_inline)"`) == "false" - # --inline takes yes/no as arugment + # --inline takes yes/no as argument @test !success(`$exename --inline=false`) # --fast-math @@ -196,7 +196,7 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` @test parse(Int,readchomp(`$exename --math-mode=fast -E "Int(Base.JLOptions().fast_math)"`)) == JL_OPTIONS_FAST_MATH_ON end - # --worker takes default / custom as arugment (default/custom arguments tested in test/parallel.jl, test/examples.jl) + # --worker takes default / custom as argument (default/custom arguments tested in test/parallel.jl, test/examples.jl) @test !success(`$exename --worker=true`) escape(str) = replace(str, "\\", "\\\\") From 98a9b39ed8196b8d7a31135a0a1f4331e32b1b55 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Mon, 4 Jul 2016 20:39:17 -0700 Subject: [PATCH 0226/1117] MSVC compatibility fixes add missing third argument to jl_gc_alloc use alloca instead of variable length array --- src/alloc.c | 2 +- src/julia_internal.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index d754d53f01aae..4075d716e12fc 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -991,7 +991,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) uint64_t max_size = max_offset >> 1; uint32_t nfields = jl_svec_len(st->types); - jl_fielddesc32_t desc[nfields]; + jl_fielddesc32_t* desc = (jl_fielddesc32_t*) alloca(nfields * sizeof(jl_fielddesc32_t)); int haspadding = 0; for (size_t i = 0; i < nfields; i++) { diff --git a/src/julia_internal.h b/src/julia_internal.h index 3f6afb6c9bfb7..23fd66c500327 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -143,7 +143,7 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc(jl_ptls_t ptls, size_t sz, void *ty); (__builtin_constant_p(sz) ? jl_gc_alloc_(ptls, sz, ty) : \ (jl_gc_alloc)(ptls, sz, ty)) #else -# define jl_gc_alloc(ptls, sz) jl_gc_alloc_(ptls, sz, ty) +# define jl_gc_alloc(ptls, sz, ty) jl_gc_alloc_(ptls, sz, ty) #endif #define jl_buff_tag ((uintptr_t)0x4eade800) From 9f564b42bb352da08ce5965f7f6d1d9f1adfce5b Mon Sep 17 00:00:00 2001 From: Ben Arthur Date: Tue, 5 Jul 2016 09:27:56 -0400 Subject: [PATCH 0227/1117] Dict completions in REPL (#17017) --- base/REPLCompletions.jl | 43 ++++++++++++++++++++++++++++-- test/replcompletions.jl | 58 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/base/REPLCompletions.jl b/base/REPLCompletions.jl index 207be534a0520..473ab5ad28750 100644 --- a/base/REPLCompletions.jl +++ b/base/REPLCompletions.jl @@ -386,12 +386,51 @@ function bslash_completions(string, pos) return (false, (String[], 0:-1, false)) end +function dict_identifier_key(str,tag) + if tag === :string + str_close = str*"\"" + elseif tag === :cmd + str_close = str*"`" + else + str_close = str + end + + frange, end_of_indentifier = find_start_brace(str_close, c_start='[', c_end=']') + isempty(frange) && return (nothing, nothing, nothing) + identifier = Symbol(str[frange[1]:end_of_indentifier]) + isdefined(Main,identifier) || return (nothing, nothing, nothing) + begin_of_key = findnext(x->!in(x,whitespace_chars), str, end_of_indentifier+2) + begin_of_key==0 && return (identifier, nothing, nothing) + partial_key = str[begin_of_key:end] + main_id = getfield(Main,identifier) + typeof(main_id) <: Associative && length(main_id)<1e6 || return (identifier, nothing, nothing) + main_id, partial_key, begin_of_key +end + function completions(string, pos) # First parse everything up to the current position partial = string[1:pos] inc_tag = Base.syntax_deprecation_warnings(false) do Base.incomplete_tag(parse(partial, raise=false)) end + + # if completing a key in a Dict + identifier, partial_key, loc = dict_identifier_key(partial,inc_tag) + if identifier != nothing + if partial_key != nothing + matches = [] + for key in keys(identifier) + rkey = repr(key) + startswith(rkey,partial_key) && push!(matches,rkey) + end + length(matches)==1 && (length(string) <= pos || string[pos+1] != ']') && (matches[1]*="]") + length(matches)>0 && return sort(matches), loc:pos, true + else + return String[], 0:-1, false + end + end + + # otherwise... if inc_tag in [:cmd, :string] m = match(r"[\t\n\r\"'`@\$><=;|&\{]| (?!\\)", reverse(partial)) startpos = nextind(partial, reverseind(partial, m.offset)) @@ -413,7 +452,7 @@ function completions(string, pos) # Make sure that only bslash_completions is working on strings inc_tag==:string && return String[], 0:-1, false - if inc_tag == :other && should_method_complete(partial) + if inc_tag == :other && should_method_complete(partial) frange, method_name_end = find_start_brace(partial) ex = Base.syntax_deprecation_warnings(false) do parse(partial[frange] * ")", raise=false) @@ -484,7 +523,7 @@ function completions(string, pos) elseif c==']' c_start='['; c_end=']' end - frange, end_off_indentifier = find_start_brace(string[1:prevind(string, i)], c_start=c_start, c_end=c_end) + frange, end_of_indentifier = find_start_brace(string[1:prevind(string, i)], c_start=c_start, c_end=c_end) startpos = start(frange) i = prevind(string, startpos) elseif c in ["\'\"\`"...] diff --git a/test/replcompletions.jl b/test/replcompletions.jl index 0bc56bb4cc11b..8abd0c35a01f1 100644 --- a/test/replcompletions.jl +++ b/test/replcompletions.jl @@ -661,3 +661,61 @@ c, r, res = test_complete(s) s = "CompletionFoo.tuple." c, r, res = test_complete(s) @test isempty(c) + +# test Dicts +test_dict = Dict("abc"=>1, "abcd"=>10, :bar=>2, :bar2=>9, Base=>3, contains=>4, `ls`=>5, + 66=>7, 67=>8, ("q",3)=>11, "α"=>12, :α=>13) +s="test_dict[\"ab" +c,r = test_complete(s) +@test c == Any["\"abc\"","\"abcd\""] +s="test_dict[\"abcd" +c,r = test_complete(s) +@test c == Any["\"abcd\"]"] +s="test_dict[ \"abcd" # leading whitespace +c,r = test_complete(s) +@test c == Any["\"abcd\"]"] +s="test_dict[\"abcd]" # trailing close bracket +c,r = completions(s,endof(s)-1) +@test c == Any["\"abcd\""] +s="test_dict[:b" +c,r = test_complete(s) +@test c == Any[":bar",":bar2"] +s="test_dict[:bar2" +c,r = test_complete(s) +@test c == Any[":bar2]"] +s="test_dict[Ba" +c,r = test_complete(s) +@test c == Any["Base]"] +s="test_dict[co" +c,r = test_complete(s) +@test c == Any["contains]"] +s="test_dict[`l" +c,r = test_complete(s) +@test c == Any["`ls`]"] +s="test_dict[6" +c,r = test_complete(s) +@test c == Any["66","67"] +s="test_dict[66" +c,r = test_complete(s) +@test c == Any["66]"] +s="test_dict[(" +c,r = test_complete(s) +@test c == Any["(\"q\",3)]"] +s="test_dict[\"\\alp" +c,r = test_complete(s) +@test c == String["\\alpha"] +s="test_dict[\"\\alpha" +c,r = test_complete(s) +@test c == String["α"] +s="test_dict[\"α" +c,r = test_complete(s) +@test c == Any["\"α\"]"] +s="test_dict[:\\alp" +c,r = test_complete(s) +@test c == String["\\alpha"] +s="test_dict[:\\alpha" +c,r = test_complete(s) +@test c == String["α"] +s="test_dict[:α" +c,r = test_complete(s) +@test c == Any[":α]"] From 341efb4a2d9ecef0b5b40322708e4478beea8aba Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Tue, 5 Jul 2016 06:53:13 -0700 Subject: [PATCH 0228/1117] Mention arm and powerpc architectures in NEWS. [ci skip] --- NEWS.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/NEWS.md b/NEWS.md index 19d0c2a421e31..d332f885826bc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -32,6 +32,15 @@ New language features packages, but now Julia provides an API for writing generic algorithms for arbitrary indexing schemes ([#16260]). +New architectures +----------------- + + This release greatly improves support for ARM, and introduces support for Power. + + * [ARM](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Aarm) ([#14194], [#14519], [#16645], [#16621]) + + * [Power](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Apower) ([#16455], [#16404]) + Language changes ---------------- @@ -255,10 +264,12 @@ Deprecated or removed [#13824]: https://github.com/JuliaLang/julia/issues/13824 [#13897]: https://github.com/JuliaLang/julia/issues/13897 [#14114]: https://github.com/JuliaLang/julia/issues/14114 +[#14194]: https://github.com/JuliaLang/julia/issues/14194 [#14243]: https://github.com/JuliaLang/julia/issues/14243 [#14413]: https://github.com/JuliaLang/julia/issues/14413 [#14424]: https://github.com/JuliaLang/julia/issues/14424 [#14469]: https://github.com/JuliaLang/julia/issues/14469 +[#14519]: https://github.com/JuliaLang/julia/issues/14519 [#14759]: https://github.com/JuliaLang/julia/issues/14759 [#14798]: https://github.com/JuliaLang/julia/issues/14798 [#15032]: https://github.com/JuliaLang/julia/issues/15032 @@ -278,7 +289,11 @@ Deprecated or removed [#16260]: https://github.com/JuliaLang/julia/issues/16260 [#16362]: https://github.com/JuliaLang/julia/issues/16362 [#16403]: https://github.com/JuliaLang/julia/issues/16403 +[#16404]: https://github.com/JuliaLang/julia/issues/16404 +[#16455]: https://github.com/JuliaLang/julia/issues/16455 [#16481]: https://github.com/JuliaLang/julia/issues/16481 +[#16621]: https://github.com/JuliaLang/julia/issues/16621 +[#16645]: https://github.com/JuliaLang/julia/issues/16645 [#16731]: https://github.com/JuliaLang/julia/issues/16731 [#16972]: https://github.com/JuliaLang/julia/issues/16972 [#17266]: https://github.com/JuliaLang/julia/issues/17266 From 4c26bf4c26ae097355fbd1911388c5b639e24051 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 29 Jun 2016 16:46:40 +0200 Subject: [PATCH 0229/1117] Address space-preserving bitcast. This replaces manual calls to IRBuilder.CreateBitcast with calls to emit_bitcast, which preserves the address space of a pointer when casting it to a different type. This is useful for GPU programming, where address spaces are used to denote different kinds of memory. --- src/ccall.cpp | 18 +++++----- src/cgutils.cpp | 86 +++++++++++++++++++++++++++------------------- src/codegen.cpp | 14 ++++---- src/intrinsics.cpp | 30 ++++++++-------- 4 files changed, 81 insertions(+), 67 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 984faf61dd717..63dbdb782ac52 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -166,7 +166,7 @@ Value *llvm_type_rewrite(Value *v, Type *from_type, Type *target_type, // simple integer and float widening & conversion cases if (from_type->getPrimitiveSizeInBits() > 0 && target_type->getPrimitiveSizeInBits() == from_type->getPrimitiveSizeInBits()) { - return builder.CreateBitCast(v, target_type); + return emit_bitcast(v, target_type); } if (target_type->isFloatingPointTy() && from_type->isFloatingPointTy()) { if (target_type->getPrimitiveSizeInBits() > from_type->getPrimitiveSizeInBits()) @@ -274,7 +274,7 @@ static Value *julia_to_native(Type *to, bool toboxed, jl_value_t *jlto, const jl ai->setAlignment(16); prepare_call( builder.CreateMemCpy(ai, data_pointer(jvinfo, ctx, T_pint8), nbytes, sizeof(void*))->getCalledValue()); // minimum gc-alignment in julia is pointer size - return builder.CreateBitCast(ai, to); + return emit_bitcast(ai, to); } } // emit maybe copy @@ -293,7 +293,7 @@ static Value *julia_to_native(Type *to, bool toboxed, jl_value_t *jlto, const jl AllocaInst *ai = builder.CreateAlloca(T_int8, nbytes); ai->setAlignment(16); prepare_call(builder.CreateMemCpy(ai, data_pointer(jvinfo, ctx, T_pint8), nbytes, sizeof(void*))->getCalledValue()); // minimum gc-alignment in julia is pointer size - Value *p2 = builder.CreateBitCast(ai, to); + Value *p2 = emit_bitcast(ai, to); builder.CreateBr(afterBB); builder.SetInsertPoint(afterBB); PHINode *p = builder.CreatePHI(to, 2); @@ -1189,7 +1189,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(!(jl_is_expr(argi) && ((jl_expr_t*)argi)->head == amp_sym)); jl_cgval_t ary = emit_expr(argi, ctx); JL_GC_POP(); - return mark_or_box_ccall_result(builder.CreateBitCast(emit_arrayptr(ary, ctx), lrt), + return mark_or_box_ccall_result(emit_bitcast(emit_arrayptr(ary, ctx), lrt), retboxed, args[2], rt, static_rt, ctx); } if (fptr == (void(*)(void))&jl_value_ptr || @@ -1226,7 +1226,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) ary = emit_unbox(largty, emit_expr(argi, ctx), tti); } JL_GC_POP(); - return mark_or_box_ccall_result(builder.CreateBitCast(ary, lrt), + return mark_or_box_ccall_result(emit_bitcast(ary, lrt), retboxed, args[2], rt, static_rt, ctx); } if (JL_CPU_WAKE_NOOP && @@ -1266,7 +1266,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(nargt == 0); JL_GC_POP(); return mark_or_box_ccall_result( - builder.CreateBitCast(ctx->ptlsStates, lrt), + emit_bitcast(ctx->ptlsStates, lrt), retboxed, args[2], rt, static_rt, ctx); } if (fptr == &jl_sigatomic_begin || @@ -1355,7 +1355,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) emit_expr(args[6], ctx); emit_expr(args[8], ctx); JL_GC_POP(); - return mark_or_box_ccall_result(builder.CreateBitCast(llvmf, lrt), + return mark_or_box_ccall_result(emit_bitcast(llvmf, lrt), retboxed, args[2], rt, static_rt, ctx); } JL_CATCH { @@ -1415,7 +1415,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) // XXX: result needs a GC root here if result->getType() == T_pjlvalue result = sret_val.V; } - argvals[0] = builder.CreateBitCast(result, fargt_sig.at(0)); + argvals[0] = emit_bitcast(result, fargt_sig.at(0)); sretboxed = sret_val.isboxed; } @@ -1595,7 +1595,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(newst.typ != NULL && "Type was not concrete"); assert(newst.isboxed); // copy the data from the return value to the new struct - tbaa_decorate(newst.tbaa, builder.CreateAlignedStore(result, builder.CreateBitCast(newst.V, prt->getPointerTo()), 16)); // julia gc is aligned 16 + tbaa_decorate(newst.tbaa, builder.CreateAlignedStore(result, emit_bitcast(newst.V, prt->getPointerTo()), 16)); // julia gc is aligned 16 return newst; } else if (jlrt != prt) { diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 7eef89a7e1e51..696b1da0fc2ea 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -283,6 +283,21 @@ static Value *literal_pointer_val(jl_binding_t *p) return julia_gv("jl_bnd#", p->name, p->owner, p); } +// bitcast a value, but preserve its address space when dealing with pointer types +static Value *emit_bitcast(Value *v, Type *jl_value) +{ + if (isa(jl_value) && + v->getType()->getPointerAddressSpace() != jl_value->getPointerAddressSpace()) { + // Cast to the proper address space + Type *jl_value_addr = + PointerType::get(cast(jl_value)->getElementType(), + v->getType()->getPointerAddressSpace()); + return builder.CreateBitCast(v, jl_value_addr); + } else { + return builder.CreateBitCast(v, jl_value); + } +} + static Value *julia_binding_gv(Value *bv) { return builder. @@ -295,7 +310,7 @@ static Value *julia_binding_gv(jl_binding_t *b) // emit a literal_pointer_val to the value field of a jl_binding_t // binding->value are prefixed with * Value *bv = imaging_mode ? - builder.CreateBitCast(julia_gv("*", b->name, b->owner, b), T_ppjlvalue) : + emit_bitcast(julia_gv("*", b->name, b->owner, b), T_ppjlvalue) : literal_static_pointer_val(b,T_ppjlvalue); return julia_binding_gv(bv); } @@ -468,13 +483,13 @@ static bool deserves_sret(jl_value_t *dt, Type *T) static Value *emit_nthptr_addr(Value *v, ssize_t n) { - return builder.CreateGEP(builder.CreateBitCast(v, T_ppjlvalue), + return builder.CreateGEP(emit_bitcast(v, T_ppjlvalue), ConstantInt::get(T_size, n)); } static Value *emit_nthptr_addr(Value *v, Value *idx) { - return builder.CreateGEP(builder.CreateBitCast(v, T_ppjlvalue), idx); + return builder.CreateGEP(emit_bitcast(v, T_ppjlvalue), idx); } static Value *emit_nthptr(Value *v, ssize_t n, MDNode *tbaa) @@ -488,14 +503,14 @@ static Value *emit_nthptr_recast(Value *v, Value *idx, MDNode *tbaa, Type *ptype { // p = (jl_value_t**)v; *(ptype)&p[n] Value *vptr = emit_nthptr_addr(v, idx); - return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); + return tbaa_decorate(tbaa,builder.CreateLoad(emit_bitcast(vptr,ptype), false)); } static Value *emit_nthptr_recast(Value *v, ssize_t n, MDNode *tbaa, Type *ptype) { // p = (jl_value_t**)v; *(ptype)&p[n] Value *vptr = emit_nthptr_addr(v, n); - return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); + return tbaa_decorate(tbaa,builder.CreateLoad(emit_bitcast(vptr,ptype), false)); } static Value *emit_typeptr_addr(Value *p) @@ -541,20 +556,19 @@ static Value *emit_typeof_boxed(const jl_cgval_t &p, jl_codectx_t *ctx) static Value *emit_datatype_types(Value *dt) { return tbaa_decorate(tbaa_const, builder. - CreateLoad(builder. - CreateBitCast(builder. - CreateGEP(builder.CreateBitCast(dt, T_pint8), - ConstantInt::get(T_size, offsetof(jl_datatype_t, types))), - T_ppjlvalue))); + CreateLoad(emit_bitcast(builder. + CreateGEP(emit_bitcast(dt, T_pint8), + ConstantInt::get(T_size, offsetof(jl_datatype_t, types))), + T_ppjlvalue))); } static Value *emit_datatype_nfields(Value *dt) { Value *nf = tbaa_decorate(tbaa_const, builder.CreateLoad( tbaa_decorate(tbaa_const, builder.CreateLoad( - builder.CreateBitCast( + emit_bitcast( builder.CreateGEP( - builder.CreateBitCast(dt, T_pint8), + emit_bitcast(dt, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, types))), T_pint32->getPointerTo()))))); #ifdef _P64 @@ -566,18 +580,17 @@ static Value *emit_datatype_nfields(Value *dt) static Value *emit_datatype_size(Value *dt) { Value *size = tbaa_decorate(tbaa_const, builder. - CreateLoad(builder. - CreateBitCast(builder. - CreateGEP(builder.CreateBitCast(dt, T_pint8), - ConstantInt::get(T_size, offsetof(jl_datatype_t, size))), - T_pint32))); + CreateLoad(emit_bitcast(builder. + CreateGEP(emit_bitcast(dt, T_pint8), + ConstantInt::get(T_size, offsetof(jl_datatype_t, size))), + T_pint32))); return size; } static Value *emit_datatype_mutabl(Value *dt) { Value *mutabl = tbaa_decorate(tbaa_const, builder. - CreateLoad(builder.CreateGEP(builder.CreateBitCast(dt, T_pint8), + CreateLoad(builder.CreateGEP(emit_bitcast(dt, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, mutabl))))); return builder.CreateTrunc(mutabl, T_int1); } @@ -585,7 +598,7 @@ static Value *emit_datatype_mutabl(Value *dt) static Value *emit_datatype_abstract(Value *dt) { Value *abstract = tbaa_decorate(tbaa_const, builder. - CreateLoad(builder.CreateGEP(builder.CreateBitCast(dt, T_pint8), + CreateLoad(builder.CreateGEP(emit_bitcast(dt, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, abstract))))); return builder.CreateTrunc(abstract, T_int1); } @@ -861,6 +874,7 @@ static jl_cgval_t typed_load(Value *ptr, Value *idx_0based, jl_value_t *jltype, if (type_is_ghost(elty)) return ghostValue(jltype); Value *data; + // TODO: preserving_pointercast? if (ptr->getType()->getContainedType(0) != elty) data = builder.CreatePointerCast(ptr, PointerType::get(elty, 0)); else @@ -907,7 +921,7 @@ static void typed_store(Value *ptr, Value *idx_0based, const jl_cgval_t &rhs, } Value *data; if (ptr->getType()->getContainedType(0) != elty) - data = builder.CreateBitCast(ptr, PointerType::get(elty, 0)); + data = emit_bitcast(ptr, PointerType::get(elty, 0)); else data = ptr; Instruction *store = builder.CreateAlignedStore(r, builder.CreateGEP(data, idx_0based), isboxed ? alignment : julia_alignment(r, jltype, alignment)); @@ -1003,7 +1017,7 @@ static Value *data_pointer(const jl_cgval_t &x, jl_codectx_t *ctx, Type *astype { Value *data = x.constant ? boxed(x, ctx) : x.V; if (data->getType() != astype) - data = builder.CreateBitCast(data, astype); + data = emit_bitcast(data, astype); return data; } @@ -1087,11 +1101,11 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, Value *fldv = NULL; if (strct.isboxed) { Value *addr = - builder.CreateGEP(builder.CreateBitCast(boxed(strct, ctx), T_pint8), + builder.CreateGEP(emit_bitcast(boxed(strct, ctx), T_pint8), ConstantInt::get(T_size, jl_field_offset(jt,idx))); MDNode *tbaa = strct.tbaa; if (jl_field_isptr(jt, idx)) { - Value *fldv = tbaa_decorate(tbaa, builder.CreateLoad(builder.CreateBitCast(addr, T_ppjlvalue))); + Value *fldv = tbaa_decorate(tbaa, builder.CreateLoad(emit_bitcast(addr, T_ppjlvalue))); if (idx >= (unsigned)jt->ninitialized) null_pointer_check(fldv, ctx); jl_cgval_t ret = mark_julia_type(fldv, true, jfty, ctx, true); @@ -1187,7 +1201,7 @@ static Value *emit_arraylen_prim(const jl_cgval_t &tinfo, jl_codectx_t *ctx) #ifdef LLVM37 nullptr, #endif - builder.CreateBitCast(t,jl_parray_llvmt), + emit_bitcast(t,jl_parray_llvmt), 1); //index (not offset) of length field in jl_parray_llvmt MDNode *tbaa = arraytype_constshape(ty) ? tbaa_const : tbaa_arraylen; @@ -1227,7 +1241,7 @@ static Value *emit_arrayptr(const jl_cgval_t &tinfo, jl_codectx_t *ctx) #ifdef LLVM37 nullptr, #endif - builder.CreateBitCast(t,jl_parray_llvmt), + emit_bitcast(t,jl_parray_llvmt), 0); //index (not offset) of data field in jl_parray_llvmt MDNode *tbaa = arraytype_constshape(tinfo.typ) ? tbaa_const : tbaa_arrayptr; @@ -1262,14 +1276,14 @@ static Value *emit_arrayflags(const jl_cgval_t &tinfo, jl_codectx_t *ctx) #ifdef LLVM37 nullptr, #endif - builder.CreateBitCast(t, jl_parray_llvmt), + emit_bitcast(t, jl_parray_llvmt), arrayflag_field); return tbaa_decorate(tbaa_arrayflags, builder.CreateLoad(addr)); } static void assign_arrayvar(jl_arrayvar_t &av, const jl_cgval_t &ainfo, jl_codectx_t *ctx) { - tbaa_decorate(tbaa_arrayptr,builder.CreateStore(builder.CreateBitCast(emit_arrayptr(ainfo, ctx), + tbaa_decorate(tbaa_arrayptr,builder.CreateStore(emit_bitcast(emit_arrayptr(ainfo, ctx), av.dataptr->getType()->getContainedType(0)), av.dataptr)); builder.CreateStore(emit_arraylen_prim(ainfo, ctx), av.len); @@ -1351,7 +1365,7 @@ static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, static Value *init_bits_value(Value *newv, Value *v, MDNode *tbaa) { // newv should already be tagged - tbaa_decorate(tbaa, builder.CreateAlignedStore(v, builder.CreateBitCast(newv, PointerType::get(v->getType(),0)), sizeof(void*))); // min alignment in julia's gc is pointer-aligned + tbaa_decorate(tbaa, builder.CreateAlignedStore(v, emit_bitcast(newv, PointerType::get(v->getType(),0)), sizeof(void*))); // min alignment in julia's gc is pointer-aligned return newv; } static Value *as_value(Type *t, const jl_cgval_t&); @@ -1570,7 +1584,7 @@ static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt) int osize; int end_offset; int offset = jl_gc_classify_pools(static_size, &osize, &end_offset); - Value *ptls_ptr = builder.CreateBitCast(ctx->ptlsStates, T_pint8); + Value *ptls_ptr = emit_bitcast(ctx->ptlsStates, T_pint8); Value *v; if (offset < 0) { Value *args[] = {ptls_ptr, @@ -1597,7 +1611,7 @@ static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, // if ptr is NULL this emits a write barrier _back_ static void emit_write_barrier(jl_codectx_t *ctx, Value *parent, Value *ptr) { - Value *parenttag = builder.CreateBitCast(emit_typeptr_addr(parent), T_psize); + Value *parenttag = emit_bitcast(emit_typeptr_addr(parent), T_psize); Value *parent_type = tbaa_decorate(tbaa_tag, builder.CreateLoad(parenttag)); Value *parent_bits = builder.CreateAnd(parent_type, 3); @@ -1611,11 +1625,11 @@ static void emit_write_barrier(jl_codectx_t *ctx, Value *parent, Value *ptr) builder.CreateCondBr(parent_old_marked, barrier_may_trigger, cont); builder.SetInsertPoint(barrier_may_trigger); - Value *ptr_mark_bit = builder.CreateAnd(tbaa_decorate(tbaa_tag, builder.CreateLoad(builder.CreateBitCast(emit_typeptr_addr(ptr), T_psize))), 1); + Value *ptr_mark_bit = builder.CreateAnd(tbaa_decorate(tbaa_tag, builder.CreateLoad(emit_bitcast(emit_typeptr_addr(ptr), T_psize))), 1); Value *ptr_not_marked = builder.CreateICmpEQ(ptr_mark_bit, ConstantInt::get(T_size, 0)); builder.CreateCondBr(ptr_not_marked, barrier_trigger, cont); builder.SetInsertPoint(barrier_trigger); - builder.CreateCall(prepare_call(queuerootfun), builder.CreateBitCast(parent, T_pjlvalue)); + builder.CreateCall(prepare_call(queuerootfun), emit_bitcast(parent, T_pjlvalue)); builder.CreateBr(cont); ctx->f->getBasicBlockList().push_back(cont); builder.SetInsertPoint(cont); @@ -1645,7 +1659,7 @@ static void emit_setfield(jl_datatype_t *sty, const jl_cgval_t &strct, size_t id jl_value_t *jfty = jl_svecref(sty->types, idx0); if (jl_field_isptr(sty, idx0)) { Value *r = boxed(rhs, ctx, false); // don't need a temporary gcroot since it'll be rooted by strct (but should ensure strct is rooted via mark_gc_use) - tbaa_decorate(strct.tbaa, builder.CreateStore(r, builder.CreateBitCast(addr, T_ppjlvalue))); + tbaa_decorate(strct.tbaa, builder.CreateStore(r, emit_bitcast(addr, T_ppjlvalue))); if (wb && strct.isboxed) emit_checked_write_barrier(ctx, boxed(strct, ctx), r); mark_gc_use(strct); } @@ -1754,7 +1768,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg tbaa_decorate(strctinfo.tbaa, builder.CreateStore( V_null, builder.CreatePointerCast( - builder.CreateGEP(builder.CreateBitCast(strct, T_pint8), + builder.CreateGEP(emit_bitcast(strct, T_pint8), ConstantInt::get(T_size, jl_field_offset(sty,i))), T_ppjlvalue))); } @@ -1795,7 +1809,7 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg static Value *emit_exc_in_transit(jl_codectx_t *ctx) { - Value *pexc_in_transit = builder.CreateBitCast(ctx->ptlsStates, T_ppjlvalue); + Value *pexc_in_transit = emit_bitcast(ctx->ptlsStates, T_ppjlvalue); Constant *offset = ConstantInt::getSigned(T_int32, offsetof(jl_tls_states_t, exception_in_transit) / sizeof(void*)); return builder.CreateGEP(pexc_in_transit, ArrayRef(offset), "jl_exception_in_transit"); } @@ -1820,7 +1834,7 @@ static void emit_signal_fence(void) static Value *emit_defer_signal(jl_codectx_t *ctx) { - Value *ptls = builder.CreateBitCast(ctx->ptlsStates, + Value *ptls = emit_bitcast(ctx->ptlsStates, PointerType::get(T_sigatomic, 0)); Constant *offset = ConstantInt::getSigned(T_int32, offsetof(jl_tls_states_t, defer_signal) / sizeof(sig_atomic_t)); return builder.CreateGEP(ptls, ArrayRef(offset), "jl_defer_signal"); diff --git a/src/codegen.cpp b/src/codegen.cpp index 1f7d8d5ee7e91..b8d820e952e39 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2406,9 +2406,9 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, Value *own_ptr; if (jl_is_long(ndp)) { own_ptr = tbaa_decorate(tbaa_const, builder.CreateLoad( - builder.CreateBitCast( + emit_bitcast( builder.CreateConstGEP1_32( - builder.CreateBitCast(aryv, T_pint8), + emit_bitcast(aryv, T_pint8), jl_array_data_owner_offset(nd)), T_ppjlvalue))); } @@ -2567,7 +2567,7 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, Value *types_len = emit_datatype_nfields(tyv); Value *idx = emit_unbox(T_size, emit_expr(args[2], ctx), (jl_value_t*)jl_long_type); emit_bounds_check(ty, (jl_value_t*)jl_datatype_type, idx, types_len, ctx); - Value *fieldtyp = tbaa_decorate(tbaa_const, builder.CreateLoad(builder.CreateGEP(builder.CreateBitCast(types_svec, T_ppjlvalue), idx))); + Value *fieldtyp = tbaa_decorate(tbaa_const, builder.CreateLoad(builder.CreateGEP(emit_bitcast(types_svec, T_ppjlvalue), idx))); *ret = mark_julia_type(fieldtyp, true, expr_type(expr, ctx), ctx); JL_GC_POP(); return true; @@ -2879,7 +2879,7 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, PHINode *p = builder.CreatePHI(T_pjlvalue, 2); p->addIncoming(cachedval, currentbb); p->addIncoming(bval, not_found); - return julia_binding_gv(builder.CreateBitCast(p, T_ppjlvalue)); + return julia_binding_gv(emit_bitcast(p, T_ppjlvalue)); } if (b->deprecated) cg_bdw(b, ctx); } @@ -2907,7 +2907,7 @@ static jl_cgval_t emit_sparam(size_t i, jl_codectx_t *ctx) } assert(ctx->spvals_ptr != NULL); Value *bp = builder.CreateConstInBoundsGEP1_32(LLVM37_param(T_pjlvalue) - builder.CreateBitCast(ctx->spvals_ptr, T_ppjlvalue), + emit_bitcast(ctx->spvals_ptr, T_ppjlvalue), i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); return mark_julia_type(tbaa_decorate(tbaa_const, builder.CreateLoad(bp)), true, jl_any_type, ctx); } @@ -3654,7 +3654,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t if (jlfunc_sret) { // fuse the two sret together, or emit an alloca to hold it if (sret) - result = builder.CreateBitCast(sretPtr, theFptr->getFunctionType()->getParamType(0)); + result = emit_bitcast(sretPtr, theFptr->getFunctionType()->getParamType(0)); else result = builder.CreateAlloca(theFptr->getFunctionType()->getParamType(0)->getContainedType(0)); args.push_back(result); @@ -3732,7 +3732,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t literal_pointer_val((jl_value_t*)jargty)); tbaa_decorate(jl_is_mutable(jargty) ? tbaa_mutab : tbaa_immut, builder.CreateAlignedStore(val, - builder.CreateBitCast(mem, val->getType()->getPointerTo()), + emit_bitcast(mem, val->getType()->getPointerTo()), 16)); // julia's gc gives 16-byte aligned addresses inputarg = mark_julia_type(mem, true, jargty, &ctx); } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 68e5df0626233..9126629855481 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -86,7 +86,7 @@ static Value *FP(Value *v) { if (v->getType()->isFloatingPointTy()) return v; - return builder.CreateBitCast(v, FT(v->getType())); + return emit_bitcast(v, FT(v->getType())); } // convert float type to same-size int type @@ -129,7 +129,7 @@ static Value *JL_INT(Value *v) return v; if (t->isPointerTy()) return builder.CreatePtrToInt(v, JL_INTT(t)); - return builder.CreateBitCast(v, JL_INTT(t)); + return emit_bitcast(v, JL_INTT(t)); } static Value *uint_cnvt(Type *to, Value *x) @@ -302,7 +302,7 @@ static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt, Value *d Value *p = x.constant ? literal_pointer_val(x.constant) : x.V; Type *ptype = (to == T_int1 ? T_pint8 : to->getPointerTo()); if (p->getType() != ptype) - p = builder.CreateBitCast(p, ptype); + p = emit_bitcast(p, ptype); Value *unboxed = NULL; if (to == T_int1) @@ -500,7 +500,7 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx else if (!vxt->isPointerTy() && llvmt->isPointerTy()) vx = builder.CreateIntToPtr(vx, llvmt); else - vx = builder.CreateBitCast(vx, llvmt); + vx = emit_bitcast(vx, llvmt); } if (jl_is_leaf_type(bt)) @@ -603,7 +603,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c else if (!vxt->isPointerTy() && llvmt->isPointerTy()) vx = builder.CreateIntToPtr(vx, llvmt); else - vx = builder.CreateBitCast(vx, llvmt); + vx = emit_bitcast(vx, llvmt); } return mark_julia_type(vx, false, bt, ctx); @@ -762,7 +762,7 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct if (ety == (jl_value_t*)jl_any_type) return mark_julia_type( builder.CreateLoad(builder.CreateGEP( - builder.CreateBitCast(thePtr, T_ppjlvalue), + emit_bitcast(thePtr, T_ppjlvalue), im1)), true, ety, ctx); @@ -776,8 +776,8 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct literal_pointer_val((jl_value_t*)ety)); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment))); - thePtr = builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1); - prepare_call(builder.CreateMemCpy(builder.CreateBitCast(strct, T_pint8), + thePtr = builder.CreateGEP(emit_bitcast(thePtr, T_pint8), im1); + prepare_call(builder.CreateMemCpy(emit_bitcast(strct, T_pint8), thePtr, size, 1)->getCalledValue()); return mark_julia_type(strct, true, ety, ctx); } @@ -835,7 +835,7 @@ static jl_cgval_t emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, j uint64_t size = ((jl_datatype_t*)ety)->size; im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment))); - prepare_call(builder.CreateMemCpy(builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1), + prepare_call(builder.CreateMemCpy(builder.CreateGEP(emit_bitcast(thePtr, T_pint8), im1), data_pointer(val, ctx, T_pint8), size, 1)->getCalledValue()); } else { @@ -1131,7 +1131,7 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, r = emit_untyped_intrinsic(f, x, y, z, nargs, ctx, (jl_datatype_t**)&newtyp); if (!newtyp && r->getType() != x->getType()) // cast back to the exact original type (e.g. float vs. int) before remarking as a julia type - r = builder.CreateBitCast(r, x->getType()); + r = emit_bitcast(r, x->getType()); if (r->getType() == T_int1) r = builder.CreateZExt(r, T_int8); return mark_julia_type(r, false, newtyp ? newtyp : xinfo.typ, ctx); @@ -1424,11 +1424,11 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, x); #else Type *intt = JL_INTT(x->getType()); - Value *bits = builder.CreateBitCast(FP(x), intt); + Value *bits = emit_bitcast(FP(x), intt); Value *absbits = builder.CreateAnd(bits, ConstantInt::get(intt, APInt::getSignedMaxValue(((IntegerType*)intt)->getBitWidth()))); - return builder.CreateBitCast(absbits, x->getType()); + return emit_bitcast(absbits, x->getType()); #endif } case copysign_float: @@ -1436,8 +1436,8 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, x = FP(x); fy = FP(y); Type *intt = JL_INTT(x->getType()); - Value *bits = builder.CreateBitCast(x, intt); - Value *sbits = builder.CreateBitCast(fy, intt); + Value *bits = emit_bitcast(x, intt); + Value *sbits = emit_bitcast(fy, intt); unsigned nb = ((IntegerType*)intt)->getBitWidth(); APInt notsignbit = APInt::getSignedMaxValue(nb); APInt signbit0(nb, 0); signbit0.setBit(nb-1); @@ -1448,7 +1448,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, builder.CreateAnd(sbits, ConstantInt::get(intt, signbit0))); - return builder.CreateBitCast(rbits, x->getType()); + return emit_bitcast(rbits, x->getType()); } case flipsign_int: { From b25842a9d752ef2d629664b9b17006a8253662d4 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Mon, 4 Jul 2016 23:35:30 +0200 Subject: [PATCH 0230/1117] Add admonition syntax to markdown parser Syntax: !!! {category} "{optional title}" {nested markdown blocks} `category` can be any single lower case word. When no `title` is provided the title is taken from the `category`, but with the first letter capitalised. Any markdown syntax can be nested within an admonition block. Renderers for all output formats have also been implemented. --- base/docs/utils.jl | 1 + base/markdown/Common/Common.jl | 2 +- base/markdown/Common/block.jl | 58 +++++++ base/markdown/GitHub/GitHub.jl | 2 +- base/markdown/Julia/Julia.jl | 2 +- base/markdown/render/html.jl | 9 + base/markdown/render/latex.jl | 10 ++ base/markdown/render/plain.jl | 10 ++ base/markdown/render/rst.jl | 10 ++ base/markdown/render/terminal/render.jl | 10 ++ test/markdown.jl | 216 ++++++++++++++++++++++++ 11 files changed, 327 insertions(+), 3 deletions(-) diff --git a/base/docs/utils.jl b/base/docs/utils.jl index 0224ffda1991f..a6caf2da11709 100644 --- a/base/docs/utils.jl +++ b/base/docs/utils.jl @@ -368,6 +368,7 @@ stripmd(x::AbstractString) = x # base case stripmd(x::Void) = " " stripmd(x::Vector) = string(map(stripmd, x)...) stripmd(x::Markdown.BlockQuote) = "$(stripmd(x.content))" +stripmd(x::Markdown.Admonition) = "$(stripmd(x.content))" stripmd(x::Markdown.Bold) = "$(stripmd(x.text))" stripmd(x::Markdown.Code) = "$(stripmd(x.code))" stripmd{N}(x::Markdown.Header{N}) = stripmd(x.text) diff --git a/base/markdown/Common/Common.jl b/base/markdown/Common/Common.jl index c9fd42bb86695..edb630bcf79b8 100644 --- a/base/markdown/Common/Common.jl +++ b/base/markdown/Common/Common.jl @@ -3,7 +3,7 @@ include("block.jl") include("inline.jl") -@flavor common [list, indentcode, blockquote, hashheader, horizontalrule, +@flavor common [list, indentcode, blockquote, admonition, hashheader, horizontalrule, paragraph, linebreak, escapes, inline_code, diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index 17d4ed0294a1a..dd91e783c1f3d 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -150,6 +150,64 @@ function blockquote(stream::IO, block::MD) end end +# ----------- +# Admonitions +# ----------- + +type Admonition + category::String + title::String + content::Vector +end + +@breaking true -> +function admonition(stream::IO, block::MD) + withstream(stream) do + # Admonition syntax: + # + # !!! category "optional explicit title within double quotes" + # Any number of other indented markdown elements. + # + # This is the second paragraph. + # + startswith(stream, "!!! ") || return false + # Extract the category of admonition and its title: + category, title = + let untitled = r"^([a-z]+)$", # !!! + titled = r"^([a-z]+) \"(.*)\"$", # !!! "" + line = strip(readline(stream)) + if ismatch(untitled, line) + m = match(untitled, line) + # When no title is provided we use CATEGORY_NAME, capitalising it. + m.captures[1], ucfirst(m.captures[1]) + elseif ismatch(titled, line) + m = match(titled, line) + # To have a blank TITLE provide an explicit empty string as TITLE. + m.captures[1], m.captures[2] + else + # Admonition header is invalid so we give up parsing here and move + # on to the next parser. + return false + end + end + # Consume the following indented (4 spaces) block. + buffer = IOBuffer() + while !eof(stream) + if startswith(stream, " ") + write(buffer, readline(stream)) + elseif blankline(stream) + write(buffer, '\n') + else + break + end + end + # Parse the nested block as markdown and create a new Admonition block. + nested = parse(takebuf_string(buffer), flavor = config(block)) + push!(block, Admonition(category, title, nested.content)) + return true + end +end + # ––––– # Lists # ––––– diff --git a/base/markdown/GitHub/GitHub.jl b/base/markdown/GitHub/GitHub.jl index 8093e8ed4b834..ffd6e0b5b103a 100644 --- a/base/markdown/GitHub/GitHub.jl +++ b/base/markdown/GitHub/GitHub.jl @@ -58,7 +58,7 @@ function github_paragraph(stream::IO, md::MD) return true end -@flavor github [list, indentcode, blockquote, fencedcode, hashheader, +@flavor github [list, indentcode, blockquote, admonition, fencedcode, hashheader, github_table, github_paragraph, linebreak, escapes, en_dash, inline_code, asterisk_bold, diff --git a/base/markdown/Julia/Julia.jl b/base/markdown/Julia/Julia.jl index 48b9c195cd6f2..f588d47456801 100644 --- a/base/markdown/Julia/Julia.jl +++ b/base/markdown/Julia/Julia.jl @@ -8,7 +8,7 @@ include("interp.jl") @flavor julia [blocktex, blockinterp, hashheader, list, indentcode, fencedcode, - blockquote, github_table, horizontalrule, setextheader, paragraph, + blockquote, admonition, github_table, horizontalrule, setextheader, paragraph, linebreak, escapes, tex, interp, en_dash, inline_code, asterisk_bold, asterisk_italic, image, footnote, link] diff --git a/base/markdown/render/html.jl b/base/markdown/render/html.jl index f7d662eb2dda3..fa2dad364ae55 100644 --- a/base/markdown/render/html.jl +++ b/base/markdown/render/html.jl @@ -88,6 +88,15 @@ function html(io::IO, md::BlockQuote) end end +function html(io::IO, md::Admonition) + withtag(io, :div, :class => "admonition $(md.category)") do + withtag(io, :p, :class => "admonition-title") do + print(io, md.title) + end + html(io, md.content) + end +end + function html(io::IO, md::List) withtag(io, md.ordered ? :ol : :ul) do for item in md.items diff --git a/base/markdown/render/latex.jl b/base/markdown/render/latex.jl index efc9ad2e774c8..20d682f657ccf 100644 --- a/base/markdown/render/latex.jl +++ b/base/markdown/render/latex.jl @@ -58,6 +58,16 @@ function latex(io::IO, md::BlockQuote) end end +function latex(io::IO, md::Admonition) + wrapblock(io, "quote") do + wrapinline(io, "textbf") do + print(io, md.category) + end + println(io, "\n\n", md.title, "\n") + latex(io, md.content) + end +end + function latex(io::IO, md::List) env = md.ordered ? "enumerate" : "itemize" wrapblock(io, env) do diff --git a/base/markdown/render/plain.jl b/base/markdown/render/plain.jl index 2b1d6d058545d..85d0acf01a6a0 100644 --- a/base/markdown/render/plain.jl +++ b/base/markdown/render/plain.jl @@ -49,6 +49,16 @@ function plain(io::IO, q::BlockQuote) println(io) end +function plain(io::IO, md::Admonition) + s = sprint(buf -> plain(buf, md.content)) + title = md.title == ucfirst(md.category) ? "" : " \"$(md.title)\"" + println(io, "!!! ", md.category, title) + for line in split(rstrip(s), "\n") + println(io, isempty(line) ? "" : " ", line) + end + println(io) +end + function plain(io::IO, md::HorizontalRule) println(io, "-" ^ 3) end diff --git a/base/markdown/render/rst.jl b/base/markdown/render/rst.jl index ba0638cd2216c..370dc8fe56601 100644 --- a/base/markdown/render/rst.jl +++ b/base/markdown/render/rst.jl @@ -52,6 +52,16 @@ function rst(io::IO, q::BlockQuote) println(io) end +function rst(io::IO, md::Admonition) + s = sprint(buf -> rst(buf, md.content)) + title = md.title == ucfirst(md.category) ? "" : md.title + println(io, ".. ", md.category, "::", isempty(title) ? "" : " $title") + for line in split(rstrip(s), "\n") + println(io, isempty(line) ? "" : " ", line) + end + println(io) +end + function rst(io::IO, md::HorizontalRule) println(io, "–" ^ 5) end diff --git a/base/markdown/render/terminal/render.jl b/base/markdown/render/terminal/render.jl index 035e8cc27bf5b..243a3fa3d3274 100644 --- a/base/markdown/render/terminal/render.jl +++ b/base/markdown/render/terminal/render.jl @@ -30,6 +30,16 @@ function term(io::IO, md::BlockQuote, columns) end end +function term(io::IO, md::Admonition, columns) + print(io, " "^margin, "| ") + with_output_format(:bold, print, io, isempty(md.title) ? md.category : md.title) + println(io, "\n", " "^margin, "|") + s = sprint(io -> term(io, md.content, columns - 10)) + for line in split(rstrip(s), "\n") + println(io, " "^margin, "|", line) + end +end + function term(io::IO, md::List, columns) for (i, point) in enumerate(md.items) print(io, " "^2margin, md.ordered ? "$i. " : "• ") diff --git a/test/markdown.jl b/test/markdown.jl index d30ef1fa2367a..65b1db2a0c193 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -463,3 +463,219 @@ let t_1 = "`code` ``math`` ```code``` ````math```` `````code`````", LaTeX("math at end of string"), ])) end + +# Admonitions. + +let t_1 = + """ + # Foo + + !!! note + + !!! warning "custom title" + + ## Bar + + !!! danger "" + + !!! + """, + t_2 = + """ + !!! note + foo bar baz + + !!! warning "custom title" + - foo + - bar + - baz + + foo bar baz + + !!! danger "" + + ``` + foo + ``` + + bar + + # baz + """, + m_1 = Markdown.parse(t_1), + m_2 = Markdown.parse(t_2) + + # Content Tests. + + @test isa(m_1.content[2], Markdown.Admonition) + @test m_1.content[2].category == "note" + @test m_1.content[2].title == "Note" + @test m_1.content[2].content == [] + + @test isa(m_1.content[3], Markdown.Admonition) + @test m_1.content[3].category == "warning" + @test m_1.content[3].title == "custom title" + @test m_1.content[3].content == [] + + @test isa(m_1.content[5], Markdown.Admonition) + @test m_1.content[5].category == "danger" + @test m_1.content[5].title == "" + @test m_1.content[5].content == [] + + @test isa(m_1.content[6], Markdown.Paragraph) + + @test isa(m_2.content[1], Markdown.Admonition) + @test m_2.content[1].category == "note" + @test m_2.content[1].title == "Note" + @test isa(m_2.content[1].content[1], Markdown.Paragraph) + + @test isa(m_2.content[2], Markdown.Admonition) + @test m_2.content[2].category == "warning" + @test m_2.content[2].title == "custom title" + @test isa(m_2.content[2].content[1], Markdown.List) + @test isa(m_2.content[2].content[2], Markdown.Paragraph) + + @test isa(m_2.content[3], Markdown.Admonition) + @test m_2.content[3].category == "danger" + @test m_2.content[3].title == "" + @test isa(m_2.content[3].content[1], Markdown.Code) + @test isa(m_2.content[3].content[2], Markdown.Code) + @test isa(m_2.content[3].content[3], Markdown.Header{1}) + + # Rendering Tests. + + let out = Markdown.plain(m_1), + expected = + """ + # Foo + + !!! note + \n\n + !!! warning "custom title" + \n\n + ## Bar + + !!! danger "" + \n\n + !!! + """ + @test out == expected + end + let out = Markdown.rst(m_1), + expected = + """ + Foo + *** + \n + .. note:: + \n\n + .. warning:: custom title + \n\n + Bar + === + \n + .. danger:: + \n\n + !!! + """ + @test out == expected + end + let out = Markdown.latex(m_1), + expected = + """ + \\section{Foo} + \\begin{quote} + \\textbf{note} + + Note + + \\end{quote} + \\begin{quote} + \\textbf{warning} + + custom title + + \\end{quote} + \\subsection{Bar} + \\begin{quote} + \\textbf{danger} + \n\n + \\end{quote} + !!! + """ + @test out == expected + end + let out = Markdown.html(m_1), + expected = + """ + <h1>Foo</h1> + <div class="admonition note"><p class="admonition-title">Note</p></div> + <div class="admonition warning"><p class="admonition-title">custom title</p></div> + <h2>Bar</h2> + <div class="admonition danger"><p class="admonition-title"></p></div> + <p>!!!</p> + """ + @test out == expected + end + + let out = Markdown.plain(m_2), + expected = + """ + !!! note + foo bar baz + + + !!! warning "custom title" + * foo + * bar + * baz + + foo bar baz + + + !!! danger "" + ``` + foo + ``` + + ``` + bar + ``` + + # baz + + """ + @test out == expected + end + let out = Markdown.rst(m_2), + expected = + """ + .. note:: + foo bar baz + + + .. warning:: custom title + * foo + * bar + * baz + + foo bar baz + + + .. danger:: + .. code-block:: julia + + foo + + .. code-block:: julia + + bar + + baz + *** + + """ + @test out == expected + end +end + From 3798395d6fb102175300ea9154ca16dba733eaa5 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Mon, 4 Jul 2016 23:40:35 +0200 Subject: [PATCH 0231/1117] Use `!!! note` admonitions in docstrings Replace all `**notes**` occurrences with proper notes blocks. --- base/docs/helpdb/Base.jl | 98 ++++++++++++++++++++-------------------- base/linalg/arnoldi.jl | 47 ++++++++++--------- base/sparse/cholmod.jl | 52 ++++++++++----------- doc/stdlib/file.rst | 4 +- doc/stdlib/linalg.rst | 72 ++++++++++++++--------------- doc/stdlib/math.rst | 26 +++++------ 6 files changed, 147 insertions(+), 152 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index b88280d21e73e..40ff6c720d1f2 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -366,10 +366,9 @@ isfile Creates a symbolic link to `target` with the name `link`. -**note** - -This function raises an error under operating systems that do not support soft symbolic -links, such as Windows XP. +!!! note + This function raises an error under operating systems that do not support + soft symbolic links, such as Windows XP. """ symlink @@ -2356,25 +2355,25 @@ julia> round(pi, 3, 2) 3.125 ``` -**note** - -Rounding to specified digits in bases other than 2 can be inexact when operating on binary -floating point numbers. For example, the `Float64` value represented by `1.15` is actually -*less* than 1.15, yet will be rounded to 1.2. +!!! note + Rounding to specified digits in bases other than 2 can be inexact when + operating on binary floating point numbers. For example, the `Float64` + value represented by `1.15` is actually *less* than 1.15, yet will be + rounded to 1.2. -```jldoctest -julia> x = 1.15 -1.15 + ```jldoctest + julia> x = 1.15 + 1.15 -julia> @sprintf "%.20f" x -"1.14999999999999991118" + julia> @sprintf "%.20f" x + "1.14999999999999991118" -julia> x < 115//100 -true + julia> x < 115//100 + true -julia> round(x, 1) -1.2 -``` + julia> round(x, 1) + 1.2 + ``` """ round(T::Type, x) @@ -4433,13 +4432,13 @@ julia> gcdx(240, 46) (2,-9,47) ``` -**note** - -Bézout coefficients are *not* uniquely defined. `gcdx` returns the minimal Bézout -coefficients that are computed by the extended Euclid algorithm. (Ref: D. Knuth, TAoCP, 2/e, -p. 325, Algorithm X.) These coefficients `u` and `v` are minimal in the sense that -``|u| < |\\frac y d`` and ``|v| < |\\frac x d``. Furthermore, the signs of `u` and `v` are -chosen so that `d` is positive. +!!! note + Bézout coefficients are *not* uniquely defined. `gcdx` returns the minimal + Bézout coefficients that are computed by the extended Euclid algorithm. + (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients `u` + and `v` are minimal in the sense that ``|u| < |\\frac y d`` and ``|v| < + |\\frac x d``. Furthermore, the signs of `u` and `v` are chosen so that `d` + is positive. """ gcdx @@ -8658,37 +8657,38 @@ Multiplication with respect to either thin or full `Q` is allowed, i.e. both `F[ and `F[:Q]*A` are supported. A `Q` matrix can be converted into a regular matrix with [`full`](:func:`full`) which has a named argument `thin`. -**note** +!!! note + `qrfact` returns multiple types because LAPACK uses several representations + that minimize the memory storage requirements of products of Householder + elementary reflectors, so that the `Q` and `R` matrices can be stored + compactly rather as two separate dense matrices. -`qrfact` returns multiple types because LAPACK uses several representations that minimize -the memory storage requirements of products of Householder elementary reflectors, so that -the `Q` and `R` matrices can be stored compactly rather as two separate dense matrices. + The data contained in `QR` or `QRPivoted` can be used to construct the + `QRPackedQ` type, which is a compact representation of the rotation matrix: -The data contained in `QR` or `QRPivoted` can be used to construct the `QRPackedQ` type, -which is a compact representation of the rotation matrix: + ```math + Q = \\prod_{i=1}^{\\min(m,n)} (I - \\tau_i v_i v_i^T) + ``` -```math -Q = \\prod_{i=1}^{\\min(m,n)} (I - \\tau_i v_i v_i^T) -``` - -where ``\\tau_i`` is the scale factor and ``v_i`` is the projection vector associated with -the ``i^{th}`` Householder elementary reflector. + where ``\\tau_i`` is the scale factor and ``v_i`` is the projection vector + associated with the ``i^{th}`` Householder elementary reflector. -The data contained in `QRCompactWY` can be used to construct the `QRCompactWYQ` type, -which is a compact representation of the rotation matrix - -```math -Q = I + Y T Y^T -``` + The data contained in `QRCompactWY` can be used to construct the + `QRCompactWYQ` type, which is a compact representation of the rotation + matrix -where `Y` is ``m \\times r`` lower trapezoidal and `T` is ``r \\times r`` upper -triangular. The *compact WY* representation [^Schreiber1989] is not to be confused with the -older, *WY* representation [^Bischof1987]. (The LAPACK documentation uses `V` in lieu of `Y`.) + ```math + Q = I + Y T Y^T + ``` -[^Bischof1987]: C Bischof and C Van Loan, "The WY representation for products of Householder matrices", SIAM J Sci Stat Comput 8 (1987), s2-s13. [doi:10.1137/0908009](http://dx.doi.org/10.1137/0908009) + where `Y` is ``m \\times r`` lower trapezoidal and `T` is ``r \\times r`` + upper triangular. The *compact WY* representation [^Schreiber1989] is not + to be confused with the older, *WY* representation [^Bischof1987]. (The + LAPACK documentation uses `V` in lieu of `Y`.) -[^Schreiber1989]: R Schreiber and C Van Loan, "A storage-efficient WY representation for products of Householder transformations", SIAM J Sci Stat Comput 10 (1989), 53-57. [doi:10.1137/0910005](http://dx.doi.org/10.1137/0910005) + [^Bischof1987]: C Bischof and C Van Loan, "The WY representation for products of Householder matrices", SIAM J Sci Stat Comput 8 (1987), s2-s13. [doi:10.1137/0908009](http://dx.doi.org/10.1137/0908009) + [^Schreiber1989]: R Schreiber and C Van Loan, "A storage-efficient WY representation for products of Householder transformations", SIAM J Sci Stat Comput 10 (1989), 53-57. [doi:10.1137/0910005](http://dx.doi.org/10.1137/0910005) """ qrfact(A,?) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index 138ef65ed2ac3..fba8d84227a46 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -50,30 +50,29 @@ The following keyword arguments are supported: iterations `niter` and the number of matrix vector multiplications `nmult`, as well as the final residual vector `resid`. -**note** - -The `sigma` and `which` keywords interact: the description of eigenvalues searched for by -`which` do _not_ necessarily refer to the eigenvalues of `A`, but rather the linear operator -constructed by the specification of the iteration mode implied by `sigma`. - -| `sigma` | iteration mode | `which` refers to eigenvalues of | -|:----------------|:---------------------------------|:---------------------------------| -| `nothing` | ordinary (forward) | ``A`` | -| real or complex | inverse with level shift `sigma` | ``(A - \\sigma I )^{-1}`` | - -**note** - -Although `tol` has a default value, the best choice depends strongly on the -matrix `A`. We recommend that users _always_ specify a value for `tol` which -suits their specific needs. - -For details of how the errors in the computed eigenvalues are estimated, see: - -* B. N. Parlett, "The Symmetric Eigenvalue Problem", SIAM: Philadelphia, 2/e - (1998), Ch. 13.2, "Accessing Accuracy in Lanczos Problems", pp. 290-292 ff. -* R. B. Lehoucq and D. C. Sorensen, "Deflation Techniques for an Implicitly - Restarted Arnoldi Iteration", SIAM Journal on Matrix Analysis and - Applications (1996), 17(4), 789–821. doi:10.1137/S0895479895281484 +!!! note + The `sigma` and `which` keywords interact: the description of eigenvalues + searched for by `which` do _not_ necessarily refer to the eigenvalues of + `A`, but rather the linear operator constructed by the specification of the + iteration mode implied by `sigma`. + + | `sigma` | iteration mode | `which` refers to eigenvalues of | + |:----------------|:---------------------------------|:---------------------------------| + | `nothing` | ordinary (forward) | ``A`` | + | real or complex | inverse with level shift `sigma` | ``(A - \\sigma I )^{-1}`` | + +!!! note + Although `tol` has a default value, the best choice depends strongly on the + matrix `A`. We recommend that users _always_ specify a value for `tol` + which suits their specific needs. + + For details of how the errors in the computed eigenvalues are estimated, see: + + * B. N. Parlett, "The Symmetric Eigenvalue Problem", SIAM: Philadelphia, 2/e + (1998), Ch. 13.2, "Accessing Accuracy in Lanczos Problems", pp. 290-292 ff. + * R. B. Lehoucq and D. C. Sorensen, "Deflation Techniques for an Implicitly + Restarted Arnoldi Iteration", SIAM Journal on Matrix Analysis and + Applications (1996), 17(4), 789–821. doi:10.1137/S0895479895281484 """ eigs(A; kwargs...) = eigs(A, I; kwargs...) eigs{T<:BlasFloat}(A::AbstractMatrix{T}, ::UniformScaling; kwargs...) = _eigs(A, I; kwargs...) diff --git a/base/sparse/cholmod.jl b/base/sparse/cholmod.jl index 3d0b57e30a3ca..5dd273ceb7e95 100644 --- a/base/sparse/cholmod.jl +++ b/base/sparse/cholmod.jl @@ -1264,12 +1264,11 @@ factorization `F`. `A` must be a `SparseMatrixCSC`, `Symmetric{SparseMatrixCSC}` or `Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't have the type tag, it must still be symmetric or Hermitian. -** Note ** - -This method uses the CHOLMOD library from SuiteSparse, which only supports -doubles or complex doubles. Input matrices not of those element types will be -converted to `SparseMatrixCSC{Float64}` or `SparseMatrixCSC{Complex128}` as -appropriate. +!!! note + This method uses the CHOLMOD library from SuiteSparse, which only supports + doubles or complex doubles. Input matrices not of those element types will + be converted to `SparseMatrixCSC{Float64}` or `SparseMatrixCSC{Complex128}` + as appropriate. """ cholfact!{T<:Real}(F::Factor, A::Union{SparseMatrixCSC{T}, SparseMatrixCSC{Complex{T}}, @@ -1319,15 +1318,14 @@ Setting optional `shift` keyword argument computes the factorization of it should be a permutation of `1:size(A,1)` giving the ordering to use (instead of CHOLMOD's default AMD ordering). -** Note ** - -This method uses the CHOLMOD library from SuiteSparse, which only supports -doubles or complex doubles. Input matrices not of those element types will be -converted to `SparseMatrixCSC{Float64}` or `SparseMatrixCSC{Complex128}` as -appropriate. +!!! note + This method uses the CHOLMOD library from SuiteSparse, which only supports + doubles or complex doubles. Input matrices not of those element types will + be converted to `SparseMatrixCSC{Float64}` or `SparseMatrixCSC{Complex128}` + as appropriate. -Many other functions from CHOLMOD are wrapped but not exported from the -`Base.SparseArrays.CHOLMOD` module. + Many other functions from CHOLMOD are wrapped but not exported from the + `Base.SparseArrays.CHOLMOD` module. """ cholfact{T<:Real}(A::Union{SparseMatrixCSC{T}, SparseMatrixCSC{Complex{T}}, Symmetric{T,SparseMatrixCSC{T,SuiteSparse_long}}, @@ -1361,12 +1359,11 @@ Compute the ``LDL'`` factorization of `A`, reusing the symbolic factorization `F `Hermitian{SparseMatrixCSC}`. Note that even if `A` doesn't have the type tag, it must still be symmetric or Hermitian. -** Note ** - -This method uses the CHOLMOD library from SuiteSparse, which only supports -doubles or complex doubles. Input matrices not of those element types will be -converted to `SparseMatrixCSC{Float64}` or `SparseMatrixCSC{Complex128}` as -appropriate. +!!! note + This method uses the CHOLMOD library from SuiteSparse, which only supports + doubles or complex doubles. Input matrices not of those element types will + be converted to `SparseMatrixCSC{Float64}` or `SparseMatrixCSC{Complex128}` + as appropriate. """ ldltfact!{T<:Real}(F::Factor, A::Union{SparseMatrixCSC{T}, SparseMatrixCSC{Complex{T}}, @@ -1417,15 +1414,14 @@ Setting optional `shift` keyword argument computes the factorization of it should be a permutation of `1:size(A,1)` giving the ordering to use (instead of CHOLMOD's default AMD ordering). -** Note ** - -This method uses the CHOLMOD library from SuiteSparse, which only supports -doubles or complex doubles. Input matrices not of those element types will be -converted to `SparseMatrixCSC{Float64}` or `SparseMatrixCSC{Complex128}` as -appropriate. +!!! note + This method uses the CHOLMOD library from SuiteSparse, which only supports + doubles or complex doubles. Input matrices not of those element types will + be converted to `SparseMatrixCSC{Float64}` or `SparseMatrixCSC{Complex128}` + as appropriate. -Many other functions from CHOLMOD are wrapped but not exported from the -`Base.SparseArrays.CHOLMOD` module. + Many other functions from CHOLMOD are wrapped but not exported from the + `Base.SparseArrays.CHOLMOD` module. """ ldltfact{T<:Real}(A::Union{SparseMatrixCSC{T},SparseMatrixCSC{Complex{T}}, Symmetric{T,SparseMatrixCSC{T,SuiteSparse_long}}, diff --git a/doc/stdlib/file.rst b/doc/stdlib/file.rst index 29a298728e3fc..b10ca641e2935 100644 --- a/doc/stdlib/file.rst +++ b/doc/stdlib/file.rst @@ -65,9 +65,9 @@ Creates a symbolic link to ``target`` with the name ``link``\ . - **note** + .. note:: + This function raises an error under operating systems that do not support soft symbolic links, such as Windows XP. - This function raises an error under operating systems that do not support soft symbolic links, such as Windows XP. .. function:: readlink(path) -> AbstractString diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 627f85bc26c1a..55b7c5296403e 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -326,11 +326,11 @@ Linear algebra functions in Julia are largely implemented by calling functions f Setting optional ``shift`` keyword argument computes the factorization of ``A+shift*I`` instead of ``A``\ . If the ``perm`` argument is nonempty, it should be a permutation of ``1:size(A,1)`` giving the ordering to use (instead of CHOLMOD's default AMD ordering). - ** Note ** + .. note:: + This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. - This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. + Many other functions from CHOLMOD are wrapped but not exported from the ``Base.SparseArrays.CHOLMOD`` module. - Many other functions from CHOLMOD are wrapped but not exported from the ``Base.SparseArrays.CHOLMOD`` module. .. function:: cholfact!(F::Factor, A; shift = 0.0) -> CHOLMOD.Factor @@ -338,9 +338,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the Cholesky (:math:`LL'`\ ) factorization of ``A``\ , reusing the symbolic factorization ``F``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . Note that even if ``A`` doesn't have the type tag, it must still be symmetric or Hermitian. - ** Note ** + .. note:: + This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. - This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. .. function:: cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky @@ -396,11 +396,11 @@ Linear algebra functions in Julia are largely implemented by calling functions f Setting optional ``shift`` keyword argument computes the factorization of ``A+shift*I`` instead of ``A``\ . If the ``perm`` argument is nonempty, it should be a permutation of ``1:size(A,1)`` giving the ordering to use (instead of CHOLMOD's default AMD ordering). - ** Note ** + .. note:: + This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. - This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. + Many other functions from CHOLMOD are wrapped but not exported from the ``Base.SparseArrays.CHOLMOD`` module. - Many other functions from CHOLMOD are wrapped but not exported from the ``Base.SparseArrays.CHOLMOD`` module. .. function:: ldltfact!(F::Factor, A; shift = 0.0) -> CHOLMOD.Factor @@ -408,9 +408,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the :math:`LDL'` factorization of ``A``\ , reusing the symbolic factorization ``F``\ . ``A`` must be a ``SparseMatrixCSC``\ , ``Symmetric{SparseMatrixCSC}``\ , or ``Hermitian{SparseMatrixCSC}``\ . Note that even if ``A`` doesn't have the type tag, it must still be symmetric or Hermitian. - ** Note ** + .. note:: + This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. - This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. .. function:: ldltfact!(::SymTridiagonal) -> LDLt @@ -498,29 +498,29 @@ Linear algebra functions in Julia are largely implemented by calling functions f Multiplication with respect to either thin or full ``Q`` is allowed, i.e. both ``F[:Q]*F[:R]`` and ``F[:Q]*A`` are supported. A ``Q`` matrix can be converted into a regular matrix with :func:`full` which has a named argument ``thin``\ . - **note** + .. note:: + ``qrfact`` returns multiple types because LAPACK uses several representations that minimize the memory storage requirements of products of Householder elementary reflectors, so that the ``Q`` and ``R`` matrices can be stored compactly rather as two separate dense matrices. - ``qrfact`` returns multiple types because LAPACK uses several representations that minimize the memory storage requirements of products of Householder elementary reflectors, so that the ``Q`` and ``R`` matrices can be stored compactly rather as two separate dense matrices. + The data contained in ``QR`` or ``QRPivoted`` can be used to construct the ``QRPackedQ`` type, which is a compact representation of the rotation matrix: - The data contained in ``QR`` or ``QRPivoted`` can be used to construct the ``QRPackedQ`` type, which is a compact representation of the rotation matrix: + .. math:: - .. math:: + Q = \prod_{i=1}^{\min(m,n)} (I - \tau_i v_i v_i^T) - Q = \prod_{i=1}^{\min(m,n)} (I - \tau_i v_i v_i^T) + where :math:`\tau_i` is the scale factor and :math:`v_i` is the projection vector associated with the :math:`i^{th}` Householder elementary reflector. - where :math:`\tau_i` is the scale factor and :math:`v_i` is the projection vector associated with the :math:`i^{th}` Householder elementary reflector. + The data contained in ``QRCompactWY`` can be used to construct the ``QRCompactWYQ`` type, which is a compact representation of the rotation matrix - The data contained in ``QRCompactWY`` can be used to construct the ``QRCompactWYQ`` type, which is a compact representation of the rotation matrix + .. math:: - .. math:: + Q = I + Y T Y^T - Q = I + Y T Y^T + where ``Y`` is :math:`m \times r` lower trapezoidal and ``T`` is :math:`r \times r` upper triangular. The *compact WY* representation [Schreiber1989]_ is not to be confused with the older, *WY* representation [Bischof1987]_. (The LAPACK documentation uses ``V`` in lieu of ``Y``\ .) - where ``Y`` is :math:`m \times r` lower trapezoidal and ``T`` is :math:`r \times r` upper triangular. The *compact WY* representation [Schreiber1989]_ is not to be confused with the older, *WY* representation [Bischof1987]_. (The LAPACK documentation uses ``V`` in lieu of ``Y``\ .) + .. [Bischof1987] C Bischof and C Van Loan, "The WY representation for products of Householder matrices", SIAM J Sci Stat Comput 8 (1987), s2-s13. `doi:10.1137/0908009 <http://dx.doi.org/10.1137/0908009>`_ - .. [Bischof1987] C Bischof and C Van Loan, "The WY representation for products of Householder matrices", SIAM J Sci Stat Comput 8 (1987), s2-s13. `doi:10.1137/0908009 <http://dx.doi.org/10.1137/0908009>`_ + .. [Schreiber1989] R Schreiber and C Van Loan, "A storage-efficient WY representation for products of Householder transformations", SIAM J Sci Stat Comput 10 (1989), 53-57. `doi:10.1137/0910005 <http://dx.doi.org/10.1137/0910005>`_ - .. [Schreiber1989] R Schreiber and C Van Loan, "A storage-efficient WY representation for products of Householder transformations", SIAM J Sci Stat Comput 10 (1989), 53-57. `doi:10.1137/0910005 <http://dx.doi.org/10.1137/0910005>`_ .. function:: qrfact(A) -> SPQR.Factorization @@ -1337,26 +1337,26 @@ Linear algebra functions in Julia are largely implemented by calling functions f ``eigs`` returns the ``nev`` requested eigenvalues in ``d``\ , the corresponding Ritz vectors ``v`` (only if ``ritzvec=true``\ ), the number of converged eigenvalues ``nconv``\ , the number of iterations ``niter`` and the number of matrix vector multiplications ``nmult``\ , as well as the final residual vector ``resid``\ . - **note** + .. note:: + The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalues of ``A``\ , but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``\ . - The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalues of ``A``\ , but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``\ . + +-----------------+------------------------------------+------------------------------------+ + | ``sigma`` | iteration mode | ``which`` refers to eigenvalues of | + +=================+====================================+====================================+ + | ``nothing`` | ordinary (forward) | :math:`A` | + +-----------------+------------------------------------+------------------------------------+ + | real or complex | inverse with level shift ``sigma`` | :math:`(A - \sigma I )^{-1}` | + +-----------------+------------------------------------+------------------------------------+ - +-----------------+------------------------------------+------------------------------------+ - | ``sigma`` | iteration mode | ``which`` refers to eigenvalues of | - +=================+====================================+====================================+ - | ``nothing`` | ordinary (forward) | :math:`A` | - +-----------------+------------------------------------+------------------------------------+ - | real or complex | inverse with level shift ``sigma`` | :math:`(A - \sigma I )^{-1}` | - +-----------------+------------------------------------+------------------------------------+ - **note** + .. note:: + Although ``tol`` has a default value, the best choice depends strongly on the matrix ``A``\ . We recommend that users _always_ specify a value for ``tol`` which suits their specific needs. - Although ``tol`` has a default value, the best choice depends strongly on the matrix ``A``\ . We recommend that users _always_ specify a value for ``tol`` which suits their specific needs. + For details of how the errors in the computed eigenvalues are estimated, see: - For details of how the errors in the computed eigenvalues are estimated, see: + * B. N. Parlett, "The Symmetric Eigenvalue Problem", SIAM: Philadelphia, 2/e (1998), Ch. 13.2, "Accessing Accuracy in Lanczos Problems", pp. 290-292 ff. + * R. B. Lehoucq and D. C. Sorensen, "Deflation Techniques for an Implicitly Restarted Arnoldi Iteration", SIAM Journal on Matrix Analysis and Applications (1996), 17(4), 789–821. doi:10.1137/S0895479895281484 - * B. N. Parlett, "The Symmetric Eigenvalue Problem", SIAM: Philadelphia, 2/e (1998), Ch. 13.2, "Accessing Accuracy in Lanczos Problems", pp. 290-292 ff. - * R. B. Lehoucq and D. C. Sorensen, "Deflation Techniques for an Implicitly Restarted Arnoldi Iteration", SIAM Journal on Matrix Analysis and Applications (1996), 17(4), 789–821. doi:10.1137/S0895479895281484 .. function:: eigs(A, B; nev=6, ncv=max(20,2*nev+1), which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 970f1e8e75b58..79782b1cf9e49 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -993,23 +993,23 @@ Mathematical Functions julia> round(pi, 3, 2) 3.125 - **note** + .. note:: + Rounding to specified digits in bases other than 2 can be inexact when operating on binary floating point numbers. For example, the ``Float64`` value represented by ``1.15`` is actually *less* than 1.15, yet will be rounded to 1.2. - Rounding to specified digits in bases other than 2 can be inexact when operating on binary floating point numbers. For example, the ``Float64`` value represented by ``1.15`` is actually *less* than 1.15, yet will be rounded to 1.2. + .. doctest:: - .. doctest:: + julia> x = 1.15 + 1.15 - julia> x = 1.15 - 1.15 + julia> @sprintf "%.20f" x + "1.14999999999999991118" - julia> @sprintf "%.20f" x - "1.14999999999999991118" + julia> x < 115//100 + true - julia> x < 115//100 - true + julia> round(x, 1) + 1.2 - julia> round(x, 1) - 1.2 .. data:: RoundingMode @@ -1375,9 +1375,9 @@ Mathematical Functions julia> gcdx(240, 46) (2,-9,47) - **note** + .. note:: + Bézout coefficients are *not* uniquely defined. ``gcdx`` returns the minimal Bézout coefficients that are computed by the extended Euclid algorithm. (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients ``u`` and ``v`` are minimal in the sense that :math:`|u| < |\frac y d` and :math:`|v| < |\frac x d`\ . Furthermore, the signs of ``u`` and ``v`` are chosen so that ``d`` is positive. - Bézout coefficients are *not* uniquely defined. ``gcdx`` returns the minimal Bézout coefficients that are computed by the extended Euclid algorithm. (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients ``u`` and ``v`` are minimal in the sense that :math:`|u| < |\frac y d` and :math:`|v| < |\frac x d`\ . Furthermore, the signs of ``u`` and ``v`` are chosen so that ``d`` is positive. .. function:: ispow2(n) -> Bool From a4a350e5572be0f6a21e67a121386b89c4a4dfaf Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Tue, 5 Jul 2016 16:35:42 +0200 Subject: [PATCH 0232/1117] Move modified docstrings inline --- base/docs/helpdb/Base.jl | 163 --------------------------------------- base/file.jl | 10 +++ base/floatfuncs.jl | 57 ++++++++++++++ base/intfuncs.jl | 25 ++++++ base/linalg/qr.jl | 65 ++++++++++++++++ doc/stdlib/strings.rst | 1 + 6 files changed, 158 insertions(+), 163 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 40ff6c720d1f2..70ffd3ca1c168 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -361,17 +361,6 @@ Returns `true` if `path` is a regular file, `false` otherwise. """ isfile -""" - symlink(target, link) - -Creates a symbolic link to `target` with the name `link`. - -!!! note - This function raises an error under operating systems that do not support - soft symbolic links, such as Windows XP. -""" -symlink - """ task_local_storage(symbol) @@ -2319,64 +2308,6 @@ For matrices or vectors ``A`` and ``B``, calculates ``A / Bᴴ``. """ A_rdiv_Bc -""" - round([T,] x, [digits, [base]], [r::RoundingMode]) - -`round(x)` rounds `x` to an integer value according to the default rounding mode (see -[`rounding`](:func:`rounding`)), returning a value of the same type as `x`. By default -([`RoundNearest`](:obj:`RoundNearest`)), this will round to the nearest integer, with ties -(fractional values of 0.5) being rounded to the even integer. - -```jldoctest -julia> round(1.7) -2.0 - -julia> round(1.5) -2.0 - -julia> round(2.5) -2.0 -``` - -The optional [`RoundingMode`](:obj:`RoundingMode`) argument will change how the number gets -rounded. - -`round(T, x, [r::RoundingMode])` converts the result to type `T`, throwing an -[`InexactError`](:exc:`InexactError`) if the value is not representable. - -`round(x, digits)` rounds to the specified number of digits after the decimal place (or -before if negative). `round(x, digits, base)` rounds using a base other than 10. - -```jldoctest -julia> round(pi, 2) -3.14 - -julia> round(pi, 3, 2) -3.125 -``` - -!!! note - Rounding to specified digits in bases other than 2 can be inexact when - operating on binary floating point numbers. For example, the `Float64` - value represented by `1.15` is actually *less* than 1.15, yet will be - rounded to 1.2. - - ```jldoctest - julia> x = 1.15 - 1.15 - - julia> @sprintf "%.20f" x - "1.14999999999999991118" - - julia> x < 115//100 - true - - julia> round(x, 1) - 1.2 - ``` -""" -round(T::Type, x) - """ round(z, RoundingModeReal, RoundingModeImaginary) @@ -4415,33 +4346,6 @@ Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x)``. """ hankelh1 -""" - gcdx(x,y) - -Computes the greatest common (positive) divisor of `x` and `y` and their Bézout -coefficients, i.e. the integer coefficients `u` and `v` that satisfy -``ux+vy = d = gcd(x,y)``. - -```jldoctest -julia> gcdx(12, 42) -(6,-3,1) -``` - -```jldoctest -julia> gcdx(240, 46) -(2,-9,47) -``` - -!!! note - Bézout coefficients are *not* uniquely defined. `gcdx` returns the minimal - Bézout coefficients that are computed by the extended Euclid algorithm. - (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients `u` - and `v` are minimal in the sense that ``|u| < |\\frac y d`` and ``|v| < - |\\frac x d``. Furthermore, the signs of `u` and `v` are chosen so that `d` - is positive. -""" -gcdx - """ rem(x, y) %(x, y) @@ -8626,73 +8530,6 @@ Converts the endianness of a value from Network byte order (big-endian) to that """ ntoh -""" - qrfact(A [,pivot=Val{false}]) -> F - -Computes the QR factorization of `A`. The return type of `F` depends on the element type of -`A` and whether pivoting is specified (with `pivot==Val{true}`). - -| Return type | `eltype(A)` | `pivot` | Relationship between `F` and `A` | -|:--------------|:----------------|:-------------|:---------------------------------| -| `QR` | not `BlasFloat` | either | `A==F[:Q]*F[:R]` | -| `QRCompactWY` | `BlasFloat` | `Val{false}` | `A==F[:Q]*F[:R]` | -| `QRPivoted` | `BlasFloat` | `Val{true}` | `A[:,F[:p]]==F[:Q]*F[:R]` | - -`BlasFloat` refers to any of: `Float32`, `Float64`, `Complex64` or `Complex128`. - -The individual components of the factorization `F` can be accessed by indexing: - -| Component | Description | `QR` | `QRCompactWY` | `QRPivoted` | -|:----------|:------------------------------------------|:----------------|:-------------------|:----------------| -| `F[:Q]` | `Q` (orthogonal/unitary) part of `QR` | ✓ (`QRPackedQ`) | ✓ (`QRCompactWYQ`) | ✓ (`QRPackedQ`) | -| `F[:R]` | `R` (upper right triangular) part of `QR` | ✓ | ✓ | ✓ | -| `F[:p]` | pivot `Vector` | | | ✓ | -| `F[:P]` | (pivot) permutation `Matrix` | | | ✓ | - -The following functions are available for the `QR` objects: `size`, `\\`. When `A` is -rectangular, `\\` will return a least squares solution and if the solution is not unique, -the one with smallest norm is returned. - -Multiplication with respect to either thin or full `Q` is allowed, i.e. both `F[:Q]*F[:R]` -and `F[:Q]*A` are supported. A `Q` matrix can be converted into a regular matrix with -[`full`](:func:`full`) which has a named argument `thin`. - -!!! note - `qrfact` returns multiple types because LAPACK uses several representations - that minimize the memory storage requirements of products of Householder - elementary reflectors, so that the `Q` and `R` matrices can be stored - compactly rather as two separate dense matrices. - - The data contained in `QR` or `QRPivoted` can be used to construct the - `QRPackedQ` type, which is a compact representation of the rotation matrix: - - ```math - Q = \\prod_{i=1}^{\\min(m,n)} (I - \\tau_i v_i v_i^T) - ``` - - where ``\\tau_i`` is the scale factor and ``v_i`` is the projection vector - associated with the ``i^{th}`` Householder elementary reflector. - - The data contained in `QRCompactWY` can be used to construct the - `QRCompactWYQ` type, which is a compact representation of the rotation - matrix - - ```math - Q = I + Y T Y^T - ``` - - where `Y` is ``m \\times r`` lower trapezoidal and `T` is ``r \\times r`` - upper triangular. The *compact WY* representation [^Schreiber1989] is not - to be confused with the older, *WY* representation [^Bischof1987]. (The - LAPACK documentation uses `V` in lieu of `Y`.) - - [^Bischof1987]: C Bischof and C Van Loan, "The WY representation for products of Householder matrices", SIAM J Sci Stat Comput 8 (1987), s2-s13. [doi:10.1137/0908009](http://dx.doi.org/10.1137/0908009) - - [^Schreiber1989]: R Schreiber and C Van Loan, "A storage-efficient WY representation for products of Householder transformations", SIAM J Sci Stat Comput 10 (1989), 53-57. [doi:10.1137/0910005](http://dx.doi.org/10.1137/0910005) -""" -qrfact(A,?) - - """ qrfact(A) -> SPQR.Factorization diff --git a/base/file.jl b/base/file.jl index cf11560eb745b..97aa73a31aee3 100644 --- a/base/file.jl +++ b/base/file.jl @@ -422,6 +422,16 @@ end if is_windows() const UV_FS_SYMLINK_JUNCTION = 0x0002 end + +""" + symlink(target, link) + +Creates a symbolic link to `target` with the name `link`. + +!!! note + This function raises an error under operating systems that do not support + soft symbolic links, such as Windows XP. +""" function symlink(p::AbstractString, np::AbstractString) @static if is_windows() if Sys.windows_version() < Sys.WINDOWS_VISTA_VER diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index 3630501c43863..fa25bb6bd6058 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -48,6 +48,63 @@ end @vectorize_1arg Number isinf @vectorize_1arg Number isfinite +""" + round([T,] x, [digits, [base]], [r::RoundingMode]) + +`round(x)` rounds `x` to an integer value according to the default rounding mode (see +[`rounding`](:func:`rounding`)), returning a value of the same type as `x`. By default +([`RoundNearest`](:obj:`RoundNearest`)), this will round to the nearest integer, with ties +(fractional values of 0.5) being rounded to the even integer. + +```jldoctest +julia> round(1.7) +2.0 + +julia> round(1.5) +2.0 + +julia> round(2.5) +2.0 +``` + +The optional [`RoundingMode`](:obj:`RoundingMode`) argument will change how the number gets +rounded. + +`round(T, x, [r::RoundingMode])` converts the result to type `T`, throwing an +[`InexactError`](:exc:`InexactError`) if the value is not representable. + +`round(x, digits)` rounds to the specified number of digits after the decimal place (or +before if negative). `round(x, digits, base)` rounds using a base other than 10. + +```jldoctest +julia> round(pi, 2) +3.14 + +julia> round(pi, 3, 2) +3.125 +``` + +!!! note + Rounding to specified digits in bases other than 2 can be inexact when + operating on binary floating point numbers. For example, the `Float64` + value represented by `1.15` is actually *less* than 1.15, yet will be + rounded to 1.2. + + ```jldoctest + julia> x = 1.15 + 1.15 + + julia> @sprintf "%.20f" x + "1.14999999999999991118" + + julia> x < 115//100 + true + + julia> round(x, 1) + 1.2 + ``` +""" +round(T::Type, x) round(x::Real, ::RoundingMode{:ToZero}) = trunc(x) round(x::Real, ::RoundingMode{:Up}) = ceil(x) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index f7b3eee752ce5..f7da37713690c 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -48,6 +48,31 @@ gcd{T<:Integer}(abc::AbstractArray{T}) = reduce(gcd,abc) lcm{T<:Integer}(abc::AbstractArray{T}) = reduce(lcm,abc) # return (gcd(a,b),x,y) such that ax+by == gcd(a,b) +""" + gcdx(x,y) + +Computes the greatest common (positive) divisor of `x` and `y` and their Bézout +coefficients, i.e. the integer coefficients `u` and `v` that satisfy +``ux+vy = d = gcd(x,y)``. + +```jldoctest +julia> gcdx(12, 42) +(6,-3,1) +``` + +```jldoctest +julia> gcdx(240, 46) +(2,-9,47) +``` + +!!! note + Bézout coefficients are *not* uniquely defined. `gcdx` returns the minimal + Bézout coefficients that are computed by the extended Euclid algorithm. + (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients `u` + and `v` are minimal in the sense that ``|u| < |\\frac y d`` and ``|v| < + |\\frac x d``. Furthermore, the signs of `u` and `v` are chosen so that `d` + is positive. +""" function gcdx{T<:Integer}(a::T, b::T) s0, s1 = one(T), zero(T) t0, t1 = s1, s0 diff --git a/base/linalg/qr.jl b/base/linalg/qr.jl index 46899f0e7bc21..51f37e39946d8 100644 --- a/base/linalg/qr.jl +++ b/base/linalg/qr.jl @@ -93,6 +93,71 @@ qrfact!{T<:BlasFloat}(A::StridedMatrix{T}) = qrfact!(A, Val{false}) qrfact!(A::StridedMatrix, ::Type{Val{false}}) = qrfactUnblocked!(A) qrfact!(A::StridedMatrix, ::Type{Val{true}}) = qrfactPivotedUnblocked!(A) qrfact!(A::StridedMatrix) = qrfact!(A, Val{false}) + +""" + qrfact(A [,pivot=Val{false}]) -> F + +Computes the QR factorization of `A`. The return type of `F` depends on the element type of +`A` and whether pivoting is specified (with `pivot==Val{true}`). + +| Return type | `eltype(A)` | `pivot` | Relationship between `F` and `A` | +|:--------------|:----------------|:-------------|:---------------------------------| +| `QR` | not `BlasFloat` | either | `A==F[:Q]*F[:R]` | +| `QRCompactWY` | `BlasFloat` | `Val{false}` | `A==F[:Q]*F[:R]` | +| `QRPivoted` | `BlasFloat` | `Val{true}` | `A[:,F[:p]]==F[:Q]*F[:R]` | + +`BlasFloat` refers to any of: `Float32`, `Float64`, `Complex64` or `Complex128`. + +The individual components of the factorization `F` can be accessed by indexing: + +| Component | Description | `QR` | `QRCompactWY` | `QRPivoted` | +|:----------|:------------------------------------------|:----------------|:-------------------|:----------------| +| `F[:Q]` | `Q` (orthogonal/unitary) part of `QR` | ✓ (`QRPackedQ`) | ✓ (`QRCompactWYQ`) | ✓ (`QRPackedQ`) | +| `F[:R]` | `R` (upper right triangular) part of `QR` | ✓ | ✓ | ✓ | +| `F[:p]` | pivot `Vector` | | | ✓ | +| `F[:P]` | (pivot) permutation `Matrix` | | | ✓ | + +The following functions are available for the `QR` objects: `size`, `\\`. When `A` is +rectangular, `\\` will return a least squares solution and if the solution is not unique, +the one with smallest norm is returned. + +Multiplication with respect to either thin or full `Q` is allowed, i.e. both `F[:Q]*F[:R]` +and `F[:Q]*A` are supported. A `Q` matrix can be converted into a regular matrix with +[`full`](:func:`full`) which has a named argument `thin`. + +!!! note + `qrfact` returns multiple types because LAPACK uses several representations + that minimize the memory storage requirements of products of Householder + elementary reflectors, so that the `Q` and `R` matrices can be stored + compactly rather as two separate dense matrices. + + The data contained in `QR` or `QRPivoted` can be used to construct the + `QRPackedQ` type, which is a compact representation of the rotation matrix: + + ```math + Q = \\prod_{i=1}^{\\min(m,n)} (I - \\tau_i v_i v_i^T) + ``` + + where ``\\tau_i`` is the scale factor and ``v_i`` is the projection vector + associated with the ``i^{th}`` Householder elementary reflector. + + The data contained in `QRCompactWY` can be used to construct the + `QRCompactWYQ` type, which is a compact representation of the rotation + matrix + + ```math + Q = I + Y T Y^T + ``` + + where `Y` is ``m \\times r`` lower trapezoidal and `T` is ``r \\times r`` + upper triangular. The *compact WY* representation [^Schreiber1989] is not + to be confused with the older, *WY* representation [^Bischof1987]. (The + LAPACK documentation uses `V` in lieu of `Y`.) + + [^Bischof1987]: C Bischof and C Van Loan, "The WY representation for products of Householder matrices", SIAM J Sci Stat Comput 8 (1987), s2-s13. [doi:10.1137/0908009](http://dx.doi.org/10.1137/0908009) + + [^Schreiber1989]: R Schreiber and C Van Loan, "A storage-efficient WY representation for products of Householder transformations", SIAM J Sci Stat Comput 10 (1989), 53-57. [doi:10.1137/0910005](http://dx.doi.org/10.1137/0910005) +""" function qrfact{T}(A::AbstractMatrix{T}, arg) AA = similar(A, typeof(zero(T)/norm(one(T))), size(A)) copy!(AA, A) diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index 3b98d8a73b0b2..0817c94f0fa56 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -500,3 +500,4 @@ .. Docstring generated from Julia source Create a string from the address of a NUL-terminated UTF-32 string. A copy is made; the pointer can be safely freed. If ``length`` is specified, the string does not have to be NUL-terminated. + From 612807d1b989d5024a64366dad40f4585da1a019 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sun, 3 Jul 2016 23:03:33 -0400 Subject: [PATCH 0233/1117] drop functionID parameter from LambdaInfo no point in tracking this at all times when we only needed the result of the computation exactly once and it's easier to just compute it then --- src/alloc.c | 2 -- src/codegen.cpp | 5 ----- src/dump.c | 6 ++---- src/gf.c | 22 +++++++++++----------- src/jitlayers.cpp | 7 ++++--- src/jltypes.c | 8 +++----- src/julia.h | 2 -- src/julia_internal.h | 1 + 8 files changed, 21 insertions(+), 32 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 4075d716e12fc..6cabae7dd437d 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -405,8 +405,6 @@ JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info_uninit(void) li->compile_traced = 0; li->functionObjectsDecls.functionObject = NULL; li->functionObjectsDecls.specFunctionObject = NULL; - li->functionID = 0; - li->specFunctionID = 0; li->specTypes = NULL; li->unspecialized_ducttape = NULL; li->inferred = 0; diff --git a/src/codegen.cpp b/src/codegen.cpp index b8d820e952e39..3f50138997ec1 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -878,11 +878,6 @@ extern "C" void jl_compile_linfo(jl_lambda_info_t *li) // success. add the result to the execution engine now jl_finalize_module(std::move(m), !toplevel); - if (!toplevel) { - li->functionID = jl_assign_functionID(f, 0); - if (specf) - li->specFunctionID = jl_assign_functionID(specf, 1); - } // mark the pointer calling convention li->jlcall_api = (f->getFunctionType() == jl_func_sig ? 0 : 1); diff --git a/src/dump.c b/src/dump.c index b1620f54b0d21..6e169b06e7149 100644 --- a/src/dump.c +++ b/src/dump.c @@ -878,8 +878,8 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_serialize_value(s, li->constval); jl_serialize_fptr(s, li->fptr); // save functionObject pointers - write_int32(s, li->functionID); - write_int32(s, li->specFunctionID); + write_int32(s, jl_assign_functionID(li->functionObjectsDecls.functionObject)); + write_int32(s, jl_assign_functionID(li->functionObjectsDecls.specFunctionObject)); write_int8(s, li->jlcall_api); } else if (jl_typeis(v, jl_module_type)) { @@ -1507,8 +1507,6 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t li->inInference = 0; li->inCompile = 0; li->fptr = jl_deserialize_fptr(s); - li->functionID = 0; - li->specFunctionID = 0; int32_t cfunc_llvm, func_llvm; func_llvm = read_int32(s); cfunc_llvm = read_int32(s); diff --git a/src/gf.c b/src/gf.c index 8eff3cc31b412..658bf7a33e9b0 100644 --- a/src/gf.c +++ b/src/gf.c @@ -221,8 +221,6 @@ JL_DLLEXPORT void jl_set_lambda_rettype(jl_lambda_info_t *li, jl_value_t *rettyp jl_gc_wb(li, rettype); li->functionObjectsDecls.functionObject = NULL; li->functionObjectsDecls.specFunctionObject = NULL; - li->functionID = 0; - li->specFunctionID = 0; li->jlcall_api = 0; li->constval = jl_nothing; } @@ -1579,21 +1577,19 @@ static void _compile_all_deq(jl_array_t *found) // this is necessary because many intrinsics try to call static_eval and thus are not compilable unspecialized int complete = _compile_all_union(ml->sig, ml->tvars); if (complete) { - if (!templ->functionID) - // indicate that this method doesn't need a functionID because it was fully covered above - templ->functionID = -1; + if (templ->fptr == NULL) + // indicate that this method doesn't need to be compiled, because it was fully covered above + templ->fptr = (jl_fptr_t)(uintptr_t)-1; } else { jl_compile_linfo(linfo); - assert(linfo->functionID > 0); + assert(linfo->functionObjectsDecls.functionObject != NULL || linfo->jlcall_api == 2); if (linfo != templ) { // copy the function pointer back to the lambda_template templ->functionObjectsDecls = linfo->functionObjectsDecls; - templ->functionID = linfo->functionID; - templ->specFunctionID = linfo->specFunctionID; templ->jlcall_api = linfo->jlcall_api; templ->constval = linfo->constval; - jl_gc_wb(templ, linfo); + if (templ->constval) jl_gc_wb(templ, templ->constval); } } } @@ -1606,7 +1602,9 @@ static int _compile_all_enq(jl_typemap_entry_t *ml, void *env) jl_array_t *found = (jl_array_t*)env; // method definition -- compile template field jl_method_t *m = ml->func.method; - if (m->lambda_template->functionID == 0 && m->lambda_template->jlcall_api != 2) { + if (m->lambda_template->functionObjectsDecls.functionObject == NULL && + m->lambda_template->jlcall_api != 2 && + m->lambda_template->fptr == NULL) { // found a lambda that still needs to be compiled jl_array_ptr_1d_push(found, (jl_value_t*)ml); } @@ -1663,7 +1661,9 @@ static void jl_compile_all_defs(void) static int _precompile_enq_tfunc(jl_typemap_entry_t *l, void *closure) { - if (jl_is_lambda_info(l->func.value) && !l->func.linfo->functionID) + if (jl_is_lambda_info(l->func.value) && + l->func.linfo->functionObjectsDecls.functionObject == NULL && + l->func.linfo->jlcall_api != 2) jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)l->sig); return 1; } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 37d3061ef086a..c0eeb33aabd72 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1140,13 +1140,14 @@ void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sys imaging_mode = false; } -static int32_t jl_assign_functionID(Function *functionObject, int specsig) +extern "C" int32_t jl_assign_functionID(void *function) { // give the function an index in the constant lookup table - if (!imaging_mode) + assert(imaging_mode); + if (function == NULL) return 0; jl_sysimg_fvars.push_back(ConstantExpr::getBitCast( - shadow_output->getNamedValue(functionObject->getName()), + shadow_output->getNamedValue(((Function*)function)->getName()), T_pvoidfunc)); return jl_sysimg_fvars.size(); } diff --git a/src/jltypes.c b/src/jltypes.c index 5ead0237a1c9d..0d7f2c1e98040 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3830,7 +3830,7 @@ void jl_init_types(void) jl_lambda_info_type = jl_new_datatype(jl_symbol("LambdaInfo"), jl_any_type, jl_emptysvec, - jl_svec(26, + jl_svec(24, jl_symbol("rettype"), jl_symbol("sparam_syms"), jl_symbol("sparam_vals"), @@ -3853,9 +3853,8 @@ void jl_init_types(void) jl_symbol("jlcall_api"), jl_symbol(""), jl_symbol("fptr"), - jl_symbol(""), jl_symbol(""), jl_symbol(""), jl_symbol("")), - jl_svec(26, + jl_svec(24, jl_any_type, jl_simplevector_type, jl_simplevector_type, @@ -3878,8 +3877,7 @@ void jl_init_types(void) jl_uint8_type, jl_bool_type, jl_any_type, - jl_any_type, jl_any_type, - jl_int32_type, jl_int32_type), + jl_any_type, jl_any_type), 0, 1, 7); jl_svecset(jl_lambda_info_type->types, 9, jl_lambda_info_type); jl_svecset(jl_method_type->types, 8, jl_lambda_info_type); diff --git a/src/julia.h b/src/julia.h index 113675a27749a..65dca8c35e612 100644 --- a/src/julia.h +++ b/src/julia.h @@ -268,8 +268,6 @@ typedef struct _jl_lambda_info_t { // with the same name as the generated functions for this linfo, suitable // for referencing in LLVM IR jl_llvm_functions_t functionObjectsDecls; - int32_t functionID; // index that this function will have in the codegen table - int32_t specFunctionID; // index that this specFunction will have in the codegen table } jl_lambda_info_t; // all values are callable as Functions diff --git a/src/julia_internal.h b/src/julia_internal.h index 23fd66c500327..f6c19a073fe51 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -400,6 +400,7 @@ static inline void jl_set_gc_and_wait(void) void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len); int32_t jl_get_llvm_gv(jl_value_t *p); +int32_t jl_assign_functionID(/*llvm::Function*/void *function); // the first argument to jl_idtable_rehash is used to return a value // make sure it is rooted if it is used after the function returns void jl_idtable_rehash(jl_array_t **pa, size_t newsz); From b5c8f96923eb44c91641c4da3e588ead4d31fcd3 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 4 Jul 2016 14:06:39 -0400 Subject: [PATCH 0234/1117] handle reloading of incremental files correctly while generating integrated output --- src/dump.c | 15 +++++++++++---- src/init.c | 14 +++++++++----- src/julia.h | 1 - src/julia_internal.h | 3 +-- src/module.c | 28 ---------------------------- src/toplevel.c | 28 +++++++++++++++++++++++++++- 6 files changed, 48 insertions(+), 41 deletions(-) diff --git a/src/dump.c b/src/dump.c index 6e169b06e7149..562ee1340870e 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1867,14 +1867,21 @@ static jl_array_t *jl_finalize_deserializer(ios_t *f, arraylist_t *tracee_list) return init_order; } -void jl_init_restored_modules(jl_array_t *init_order) +static void jl_init_restored_modules(jl_array_t *init_order) { if (!init_order) return; - int i; - for(i=0; i < jl_array_len(init_order); i++) { + int i, l = jl_array_len(init_order); + for (i = 0; i < l; i++) { jl_value_t *mod = jl_array_ptr_ref(init_order, i); - jl_module_run_initializer((jl_module_t*)mod); + if (!jl_generating_output() || jl_options.incremental) { + jl_module_run_initializer((jl_module_t*)mod); + } + else { + if (jl_module_init_order == NULL) + jl_module_init_order = jl_alloc_vec_any(0); + jl_array_ptr_1d_push(jl_module_init_order, mod); + } } } diff --git a/src/init.c b/src/init.c index dbef9f61be2fd..7ebb1488afff9 100644 --- a/src/init.c +++ b/src/init.c @@ -702,11 +702,15 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_gc_enable(1); - if (jl_options.image_file && (!jl_generating_output() || jl_options.incremental)) { - jl_array_t *temp = jl_module_init_order; - JL_GC_PUSH1(&temp); + if (jl_options.image_file && (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { + jl_array_t *init_order = jl_module_init_order; + JL_GC_PUSH1(&init_order); jl_module_init_order = NULL; - jl_init_restored_modules(temp); + int i, l = jl_array_len(init_order); + for (i = 0; i < l; i++) { + jl_value_t *mod = jl_array_ptr_ref(init_order, i); + jl_module_run_initializer((jl_module_t*)mod); + } JL_GC_POP(); } @@ -742,7 +746,7 @@ static void julia_save(void) int i, l = jl_array_len(worklist); for (i = 0; i < l; i++) { jl_value_t *m = jl_arrayref(worklist, i); - if (jl_module_get_initializer((jl_module_t*)m)) { + if (jl_get_global((jl_module_t*)m, jl_symbol("__init__"))) { jl_array_ptr_1d_push(jl_module_init_order, m); } } diff --git a/src/julia.h b/src/julia.h index 65dca8c35e612..174164f0eea02 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1213,7 +1213,6 @@ STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name) { return (jl_function_t*)jl_get_global(m, jl_symbol(name)); } -JL_DLLEXPORT void jl_module_run_initializer(jl_module_t *m); int jl_is_submodule(jl_module_t *child, jl_module_t *parent); // eq hash tables diff --git a/src/julia_internal.h b/src/julia_internal.h index f6c19a073fe51..5f44d1da1e73a 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -303,6 +303,7 @@ jl_value_t *jl_nth_slot_type(jl_tupletype_t *sig, size_t i); void jl_compute_field_offsets(jl_datatype_t *st); jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims, int isunboxed, int elsz); +void jl_module_run_initializer(jl_module_t *m); extern jl_array_t *jl_module_init_order; extern union jl_typemap_t jl_cfunction_list; @@ -325,7 +326,6 @@ void jl_init_stack_limits(int ismaster); void jl_init_root_task(void *stack, size_t ssize); void jl_init_serializer(void); void jl_gc_init(void); -void jl_init_restored_modules(jl_array_t *init_order); void jl_init_signal_async(void); void jl_init_debuginfo(void); void jl_init_runtime_ccall(void); @@ -409,7 +409,6 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); int jl_has_call_ambiguities(jl_tupletype_t *types, jl_method_t *m); -jl_function_t *jl_module_get_initializer(jl_module_t *m); uint32_t jl_module_next_counter(jl_module_t *m); void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig); jl_tupletype_t *arg_type_tuple(jl_value_t **args, size_t nargs); diff --git a/src/module.c b/src/module.c index 77856cf2796ad..93405223f8d29 100644 --- a/src/module.c +++ b/src/module.c @@ -583,34 +583,6 @@ JL_DLLEXPORT jl_sym_t *jl_module_name(jl_module_t *m) { return m->name; } JL_DLLEXPORT jl_module_t *jl_module_parent(jl_module_t *m) { return m->parent; } JL_DLLEXPORT uint64_t jl_module_uuid(jl_module_t *m) { return m->uuid; } -jl_function_t *jl_module_get_initializer(jl_module_t *m) -{ - jl_value_t *f = jl_get_global(m, jl_symbol("__init__")); - if (f == NULL /*|| !jl_is_function(f)*/) - return NULL; - return (jl_function_t*)f; -} - -JL_DLLEXPORT void jl_module_run_initializer(jl_module_t *m) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - jl_function_t *f = jl_module_get_initializer(m); - if (f == NULL) - return; - JL_TRY { - jl_apply(&f, 1); - } - JL_CATCH { - if (jl_initerror_type == NULL) { - jl_rethrow(); - } - else { - jl_rethrow_other(jl_new_struct(jl_initerror_type, m->name, - ptls->exception_in_transit)); - } - } -} - int jl_is_submodule(jl_module_t *child, jl_module_t *parent) { while (1) { diff --git a/src/toplevel.c b/src/toplevel.c index cbf7aa381dfd5..4ea1cb8972a8c 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -68,6 +68,32 @@ JL_DLLEXPORT jl_module_t *jl_new_main_module(void) return old_main; } +static jl_function_t *jl_module_get_initializer(jl_module_t *m) +{ + return (jl_function_t*)jl_get_global(m, jl_symbol("__init__")); +} + +void jl_module_run_initializer(jl_module_t *m) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + jl_function_t *f = jl_module_get_initializer(m); + if (f == NULL) + return; + JL_TRY { + jl_apply(&f, 1); + } + JL_CATCH { + if (jl_initerror_type == NULL) { + jl_rethrow(); + } + else { + jl_rethrow_other(jl_new_struct(jl_initerror_type, m->name, + ptls->exception_in_transit)); + } + } +} + + // load time init procedure: in build mode, only record order static void jl_module_load_time_initialize(jl_module_t *m) { @@ -77,7 +103,7 @@ static void jl_module_load_time_initialize(jl_module_t *m) jl_module_init_order = jl_alloc_vec_any(0); jl_array_ptr_1d_push(jl_module_init_order, (jl_value_t*)m); jl_function_t *f = jl_module_get_initializer(m); - if (f) { + if (f != NULL) { jl_value_t *tt = jl_is_type(f) ? (jl_value_t*)jl_wrap_Type(f) : jl_typeof(f); JL_GC_PUSH1(&tt); tt = (jl_value_t*)jl_apply_tuple_type_v(&tt, 1); From c6134abb8d93740840b86d4bedaac25078274611 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 4 Jul 2016 14:25:31 -0400 Subject: [PATCH 0235/1117] handle missing-code more correctly previously, we might overwrite jlcall_api from the template, but fail to realize we already had a functionObject with a different calling convention already emitted (but not yet compiled into a fptr) or we might do the reverse --- src/codegen.cpp | 6 ++++-- src/gf.c | 24 ++++++++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 3f50138997ec1..d2e3a9fc9d7fd 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -876,10 +876,12 @@ extern "C" void jl_compile_linfo(jl_lambda_info_t *li) jl_add_linfo_in_flight((specf ? specf : f)->getName(), li, DL); } - // success. add the result to the execution engine now - jl_finalize_module(std::move(m), !toplevel); // mark the pointer calling convention li->jlcall_api = (f->getFunctionType() == jl_func_sig ? 0 : 1); + li->fptr = NULL; + + // success. add the result to the execution engine now + jl_finalize_module(std::move(m), !toplevel); // if not inlineable, code won't be needed again if (JL_DELETE_NON_INLINEABLE && diff --git a/src/gf.c b/src/gf.c index 658bf7a33e9b0..d2d5ef7e5aad4 100644 --- a/src/gf.c +++ b/src/gf.c @@ -221,8 +221,7 @@ JL_DLLEXPORT void jl_set_lambda_rettype(jl_lambda_info_t *li, jl_value_t *rettyp jl_gc_wb(li, rettype); li->functionObjectsDecls.functionObject = NULL; li->functionObjectsDecls.specFunctionObject = NULL; - li->jlcall_api = 0; - li->constval = jl_nothing; + li->constval = NULL; } JL_DLLEXPORT void jl_set_lambda_code_null(jl_lambda_info_t *li) @@ -1239,11 +1238,21 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) // copy fptr from the template method definition jl_method_t *def = li->def; if (def && !def->isstaged) { - li->fptr = def->lambda_template->fptr; - li->jlcall_api = def->lambda_template->jlcall_api; - li->constval = def->lambda_template->constval; - if (li->fptr != NULL || li->jlcall_api == 2) + if (def->lambda_template->jlcall_api == 2) { + li->functionObjectsDecls.functionObject = NULL; + li->functionObjectsDecls.specFunctionObject = NULL; + li->jlcall_api = 2; + li->constval = def->lambda_template->constval; + jl_gc_wb(li, li->constval); return li; + } + if (def->lambda_template->fptr) { + li->functionObjectsDecls.functionObject = NULL; + li->functionObjectsDecls.specFunctionObject = NULL; + li->fptr = def->lambda_template->fptr; + li->jlcall_api = def->lambda_template->jlcall_api; + return li; + } } if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF) { jl_printf(JL_STDERR, "code missing for "); @@ -1590,6 +1599,9 @@ static void _compile_all_deq(jl_array_t *found) templ->jlcall_api = linfo->jlcall_api; templ->constval = linfo->constval; if (templ->constval) jl_gc_wb(templ, templ->constval); + templ->rettype = linfo->rettype; + jl_gc_wb(templ, templ->rettype); + templ->fptr = NULL; } } } From 84749e084fc132148f8f43b52612c645439c90b6 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 4 Jul 2016 18:19:08 -0400 Subject: [PATCH 0236/1117] add static handling of Ref to ccall in compile-all mode --- src/ccall.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/ccall.cpp b/src/ccall.cpp index 63dbdb782ac52..adfced1fe94fa 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1062,6 +1062,11 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) rt = (jl_value_t*)jl_any_type; static_rt = true; } + else if (jl_is_typevar(jl_tparam0(rtt_)) && jl_is_abstract_ref_type(((jl_tvar_t*)jl_tparam0(rtt_))->ub)) { + // `Ref{T}` used as return type just returns T (from a jl_value_t*) + rt = (jl_value_t*)jl_any_type; + static_rt = true; + } } if (rt == NULL) { if (jl_is_expr(args[2])) { @@ -1079,6 +1084,13 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) // substitute Ptr{Void} for statically-unknown pointer type rt = (jl_value_t*)jl_voidpointer_type; } + else if (rtexpr->head == call_sym && jl_expr_nargs(rtexpr) == 3 && + static_eval(jl_exprarg(rtexpr, 0), ctx, true, false) == jl_builtin_apply_type && + static_eval(jl_exprarg(rtexpr, 1), ctx, true, false) == (jl_value_t*)jl_ref_type) { + // `Ref{T}` used as return type just returns T (from a jl_value_t*) + rt = (jl_value_t*)jl_any_type; + static_rt = true; + } } } if (rt == NULL) { From c0d9e22f9931c10e988c2b939ae38f817a741fc2 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 5 Jul 2016 13:07:41 -0400 Subject: [PATCH 0237/1117] test for double-inference dataflow errors in type-inference --- base/inference.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index 2170e681a0a7d..4c8d2c0ff8bc7 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1145,6 +1145,8 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState) t = abstract_eval(e.args[1], vtypes, sv) elseif is(e.head,:inert) return abstract_eval_constant(e.args[1]) + elseif is(e.head,:invoke) + error("type inference data-flow error: tried to double infer a function") else t = Any end @@ -1480,7 +1482,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr if isa(code, LambdaInfo) && code.code !== nothing # reuse the existing code object linfo = code - @assert typeseq(linfo.specTypes, atypes) + @assert typeseq(linfo.specTypes, atypes) && !code.inferred elseif method.isstaged if !isleaftype(atypes) # don't call staged functions on abstract types. From a8ca0ef4b24b9d5c3967d6f185ddcf6f8bb0d7bc Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Mon, 6 Jun 2016 17:32:19 +0200 Subject: [PATCH 0238/1117] Add align argument to pointerref and -set intrinsics. --- base/boot.jl | 4 +-- base/inference.jl | 6 ++-- base/pointer.jl | 8 ++--- src/codegen.cpp | 9 ++++-- src/intrinsics.cpp | 64 ++++++++++++++++++++++++---------------- src/intrinsics.h | 4 +-- src/julia_internal.h | 4 +-- src/runtime_intrinsics.c | 10 +++++-- 8 files changed, 65 insertions(+), 44 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index ba95bdf70a32a..170815e85a7e0 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -352,8 +352,8 @@ type CoreSTDOUT <: IO end type CoreSTDERR <: IO end const STDOUT = CoreSTDOUT() const STDERR = CoreSTDERR() -io_pointer(::CoreSTDOUT) = Intrinsics.pointerref(Intrinsics.cglobal(:jl_uv_stdout, Ptr{Void}), 1) -io_pointer(::CoreSTDERR) = Intrinsics.pointerref(Intrinsics.cglobal(:jl_uv_stderr, Ptr{Void}), 1) +io_pointer(::CoreSTDOUT) = Intrinsics.pointerref(Intrinsics.cglobal(:jl_uv_stdout, Ptr{Void}), 1, 1) +io_pointer(::CoreSTDERR) = Intrinsics.pointerref(Intrinsics.cglobal(:jl_uv_stderr, Ptr{Void}), 1, 1) unsafe_write(io::IO, x::Ptr{UInt8}, nb::UInt) = (ccall(:jl_uv_puts, Void, (Ptr{Void}, Ptr{UInt8}, UInt), io_pointer(io), x, nb); nb) diff --git a/base/inference.jl b/base/inference.jl index 4c8d2c0ff8bc7..7747386288874 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -302,12 +302,12 @@ add_tfunc(Core._expr, 1, IInf, (args...)->Expr) add_tfunc(applicable, 1, IInf, (f, args...)->Bool) add_tfunc(Core.Intrinsics.arraylen, 1, 1, x->Int) add_tfunc(arraysize, 2, 2, (a,d)->Int) -add_tfunc(pointerref, 2, 2, - function (a,i) +add_tfunc(pointerref, 3, 3, + function (a,i,align) a = widenconst(a) isa(a,DataType) && a<:Ptr && isa(a.parameters[1],Union{Type,TypeVar}) ? a.parameters[1] : Any end) -add_tfunc(pointerset, 3, 3, (a,v,i)->a) +add_tfunc(pointerset, 4, 4, (a,v,i,align)->a) function typeof_tfunc(t::ANY) if isa(t,Const) diff --git a/base/pointer.jl b/base/pointer.jl index 30f82b18fbee9..ef7289d9133ce 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -60,11 +60,9 @@ end unsafe_wrap{N,I<:Integer}(Atype::Type, p::Ptr, dims::NTuple{N,I}, own::Bool=false) = unsafe_wrap(Atype, p, convert(Tuple{Vararg{Int}}, dims), own) -unsafe_load(p::Ptr,i::Integer) = pointerref(p, Int(i)) -unsafe_load(p::Ptr) = unsafe_load(p, 1) -unsafe_store!(p::Ptr{Any}, x::ANY, i::Integer) = pointerset(p, x, Int(i)) -unsafe_store!{T}(p::Ptr{T}, x, i::Integer) = pointerset(p, convert(T,x), Int(i)) -unsafe_store!{T}(p::Ptr{T}, x) = pointerset(p, convert(T,x), 1) +unsafe_load(p::Ptr, i::Integer=1) = pointerref(p, Int(i), 1) +unsafe_store!(p::Ptr{Any}, x::ANY, i::Integer=1) = pointerset(p, x, Int(i), 1) +unsafe_store!{T}(p::Ptr{T}, x, i::Integer=1) = pointerset(p, convert(T,x), Int(i), 1) # unsafe pointer to string conversions (don't make a copy, unlike unsafe_string) # (Cstring versions are in c.jl) diff --git a/src/codegen.cpp b/src/codegen.cpp index d2e3a9fc9d7fd..f627251616f6a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -430,6 +430,7 @@ static Function *jlcall_frame_func; static std::vector<Type *> two_pvalue_llvmt; static std::vector<Type *> three_pvalue_llvmt; +static std::vector<Type *> four_pvalue_llvmt; static std::map<jl_fptr_t, Function*> builtin_func_map; @@ -5125,6 +5126,10 @@ static void init_julia_llvm_env(Module *m) three_pvalue_llvmt.push_back(T_pjlvalue); three_pvalue_llvmt.push_back(T_pjlvalue); three_pvalue_llvmt.push_back(T_pjlvalue); + four_pvalue_llvmt.push_back(T_pjlvalue); + four_pvalue_llvmt.push_back(T_pjlvalue); + four_pvalue_llvmt.push_back(T_pjlvalue); + four_pvalue_llvmt.push_back(T_pjlvalue); V_null = Constant::getNullValue(T_pjlvalue); std::vector<Type*> ftargs(0); @@ -5363,11 +5368,11 @@ static void init_julia_llvm_env(Module *m) "jl_get_binding_or_error", m); add_named_global(jlgetbindingorerror_func, &jl_get_binding_or_error); - jlpref_func = Function::Create(FunctionType::get(T_pjlvalue, two_pvalue_llvmt, false), + jlpref_func = Function::Create(FunctionType::get(T_pjlvalue, three_pvalue_llvmt, false), Function::ExternalLinkage, "jl_pointerref", m); - jlpset_func = Function::Create(FunctionType::get(T_pjlvalue, three_pvalue_llvmt, false), + jlpset_func = Function::Create(FunctionType::get(T_pjlvalue, four_pvalue_llvmt, false), Function::ExternalLinkage, "jl_pointerset", m); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 9126629855481..639a279893fca 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -18,7 +18,12 @@ static void jl_init_intrinsic_functions_codegen(Module *m) std::vector<Type *> args3(0); \ args3.push_back(T_pjlvalue); \ args3.push_back(T_pjlvalue); \ - args3.push_back(T_pjlvalue); + args3.push_back(T_pjlvalue); \ + std::vector<Type *> args4(0); \ + args4.push_back(T_pjlvalue); \ + args4.push_back(T_pjlvalue); \ + args4.push_back(T_pjlvalue); \ + args4.push_back(T_pjlvalue); #define ADD_I(name, nargs) do { \ Function *func = Function::Create(FunctionType::get(T_pjlvalue, args##nargs, false), \ @@ -723,14 +728,15 @@ static jl_cgval_t emit_checked_fptoui(jl_value_t *targ, jl_value_t *x, jl_codect return mark_julia_type(ans, false, jlto, ctx); } -static jl_cgval_t emit_runtime_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ctx) +static jl_cgval_t emit_runtime_pointerref(jl_value_t *e, jl_value_t *i, jl_value_t *align, jl_codectx_t *ctx) { jl_cgval_t parg = emit_expr(e, ctx); Value *iarg = boxed(emit_expr(i, ctx), ctx); + Value *alignarg = boxed(emit_expr(align, ctx), ctx); #ifdef LLVM37 - Value *ret = builder.CreateCall(prepare_call(jlpref_func), { boxed(parg, ctx), iarg }); + Value *ret = builder.CreateCall(prepare_call(jlpref_func), { boxed(parg, ctx), iarg, alignarg }); #else - Value *ret = builder.CreateCall2(prepare_call(jlpref_func), boxed(parg, ctx), iarg); + Value *ret = builder.CreateCall3(prepare_call(jlpref_func), boxed(parg, ctx), iarg, alignarg); #endif jl_value_t *ety; if (jl_is_cpointer_type(parg.typ)) { @@ -742,28 +748,32 @@ static jl_cgval_t emit_runtime_pointerref(jl_value_t *e, jl_value_t *i, jl_codec return mark_julia_type(ret, true, ety, ctx); } -static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ctx) +static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_value_t *align, jl_codectx_t *ctx) { jl_value_t *aty = expr_type(e, ctx); if (!jl_is_cpointer_type(aty)) - return emit_runtime_pointerref(e, i, ctx); + return emit_runtime_pointerref(e, i, align, ctx); //jl_error("pointerref: expected pointer type as first argument"); jl_value_t *ety = jl_tparam0(aty); if (jl_is_typevar(ety)) - return emit_runtime_pointerref(e, i, ctx); + return emit_runtime_pointerref(e, i, align, ctx); //jl_error("pointerref: invalid pointer"); if (expr_type(i, ctx) != (jl_value_t*)jl_long_type) - return emit_runtime_pointerref(e, i, ctx); + return emit_runtime_pointerref(e, i, align, ctx); //jl_error("pointerref: invalid index type"); + jl_cgval_t align_val = emit_expr(align, ctx); + if (align_val.constant == NULL || !jl_is_long(align_val.constant)) + return emit_runtime_pointerref(e, i, align, ctx); + //jl_error("pointerref: invalid or non-statically evaluatable alignment") Value *thePtr = auto_unbox(e,ctx); Value *idx = emit_unbox(T_size, emit_expr(i, ctx), (jl_value_t*)jl_long_type); Value *im1 = builder.CreateSub(idx, ConstantInt::get(T_size, 1)); if (!jl_isbits(ety)) { if (ety == (jl_value_t*)jl_any_type) return mark_julia_type( - builder.CreateLoad(builder.CreateGEP( + builder.CreateAlignedLoad(builder.CreateGEP( emit_bitcast(thePtr, T_ppjlvalue), - im1)), + im1), jl_unbox_long(align_val.constant)), true, ety, ctx); if (!jl_is_structtype(ety) || jl_is_array_type(ety) || !jl_is_leaf_type(ety)) { @@ -781,33 +791,33 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct thePtr, size, 1)->getCalledValue()); return mark_julia_type(strct, true, ety, ctx); } - // TODO: alignment? - return typed_load(thePtr, im1, ety, ctx, tbaa_data, 1); + return typed_load(thePtr, im1, ety, ctx, tbaa_data, jl_unbox_long(align_val.constant)); } -static jl_cgval_t emit_runtime_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, jl_codectx_t *ctx) +static jl_cgval_t emit_runtime_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, jl_value_t *align, jl_codectx_t *ctx) { jl_cgval_t parg = emit_expr(e, ctx); - Value *iarg = boxed(emit_expr(i, ctx), ctx); Value *xarg = boxed(emit_expr(x, ctx), ctx); + Value *iarg = boxed(emit_expr(i, ctx), ctx); + Value *alignarg = boxed(emit_expr(align, ctx), ctx); #ifdef LLVM37 - builder.CreateCall(prepare_call(jlpset_func), { boxed(parg, ctx), xarg, iarg }); + builder.CreateCall(prepare_call(jlpset_func), { boxed(parg, ctx), xarg, iarg, alignarg }); #else - builder.CreateCall3(prepare_call(jlpset_func), boxed(parg, ctx), xarg, iarg); + builder.CreateCall3(prepare_call(jlpset_func), boxed(parg, ctx), xarg, iarg, alignarg); #endif return parg; } // e[i] = x -static jl_cgval_t emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, jl_codectx_t *ctx) +static jl_cgval_t emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, jl_value_t *align, jl_codectx_t *ctx) { jl_value_t *aty = expr_type(e, ctx); if (!jl_is_cpointer_type(aty)) - return emit_runtime_pointerset(e, x, i, ctx); + return emit_runtime_pointerset(e, x, i, align, ctx); //jl_error("pointerset: expected pointer type as first argument"); jl_value_t *ety = jl_tparam0(aty); if (jl_is_typevar(ety)) - return emit_runtime_pointerset(e, x, i, ctx); + return emit_runtime_pointerset(e, x, i, align, ctx); //jl_error("pointerset: invalid pointer"); jl_value_t *xty = expr_type(x, ctx); jl_cgval_t val; @@ -818,8 +828,12 @@ static jl_cgval_t emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, j emit_typecheck(val, ety, "pointerset: type mismatch in assign", ctx); } if (expr_type(i, ctx) != (jl_value_t*)jl_long_type) - return emit_runtime_pointerset(e, x, i, ctx); + return emit_runtime_pointerset(e, x, i, align, ctx); //jl_error("pointerset: invalid index type"); + jl_cgval_t align_val = emit_expr(align, ctx); + if (align_val.constant == NULL || !jl_is_long(align_val.constant)) + return emit_runtime_pointerset(e, x, i, align, ctx); + //jl_error("pointerset: invalid or non-statically evaluatable alignment") Value *idx = emit_unbox(T_size, emit_expr(i, ctx),(jl_value_t*)jl_long_type); Value *im1 = builder.CreateSub(idx, ConstantInt::get(T_size, 1)); Value *thePtr = auto_unbox(e,ctx); @@ -836,14 +850,14 @@ static jl_cgval_t emit_pointerset(jl_value_t *e, jl_value_t *x, jl_value_t *i, j im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->layout->alignment))); prepare_call(builder.CreateMemCpy(builder.CreateGEP(emit_bitcast(thePtr, T_pint8), im1), - data_pointer(val, ctx, T_pint8), size, 1)->getCalledValue()); + data_pointer(val, ctx, T_pint8), size, jl_unbox_long(align_val.constant))->getCalledValue()); } else { if (!emitted) { val = emit_expr(x, ctx); } - // TODO: alignment? - typed_store(thePtr, im1, val, ety, ctx, tbaa_data, NULL, 1); + assert(jl_is_datatype(ety)); + typed_store(thePtr, im1, val, ety, ctx, tbaa_data, NULL, jl_unbox_long(align_val.constant)); } return mark_julia_type(thePtr, false, aty, ctx); } @@ -959,9 +973,9 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, return mark_julia_type(r, true, (jl_value_t*)jl_any_type, ctx); #else case pointerref: - return emit_pointerref(args[1], args[2], ctx); + return emit_pointerref(args[1], args[2], args[3], ctx); case pointerset: - return emit_pointerset(args[1], args[2], args[3], ctx); + return emit_pointerset(args[1], args[2], args[3], args[4], ctx); case box: return generic_box(args[1], args[2], ctx); case unbox: diff --git a/src/intrinsics.h b/src/intrinsics.h index 665b8ffd1bdf6..eb72f3244e816 100644 --- a/src/intrinsics.h +++ b/src/intrinsics.h @@ -97,8 +97,8 @@ ADD_I(powi_llvm, 2) \ ALIAS(sqrt_llvm_fast, sqrt_llvm) \ /* pointer access */ \ - ADD_I(pointerref, 2) \ - ADD_I(pointerset, 3) \ + ADD_I(pointerref, 3) \ + ADD_I(pointerset, 4) \ /* c interface */ \ ALIAS(ccall, ccall) \ ALIAS(cglobal, cglobal) \ diff --git a/src/julia_internal.h b/src/julia_internal.h index 5f44d1da1e73a..c61f2911faf88 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -515,8 +515,8 @@ extern JL_DLLEXPORT jl_value_t *jl_segv_exception; const char *jl_intrinsic_name(int f); JL_DLLEXPORT jl_value_t *jl_reinterpret(jl_value_t *ty, jl_value_t *v); -JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i); -JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i); +JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t *align); +JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *align, jl_value_t *i); JL_DLLEXPORT jl_value_t *jl_neg_int(jl_value_t *a); JL_DLLEXPORT jl_value_t *jl_add_int(jl_value_t *a, jl_value_t *b); diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 1709ef7238644..cebd18b9c1230 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -32,10 +32,12 @@ JL_DLLEXPORT jl_value_t *jl_reinterpret(jl_value_t *ty, jl_value_t *v) } // run time version of pointerref intrinsic (warning: i is not rooted) -JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i) +JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t *align) { JL_TYPECHK(pointerref, pointer, p); - JL_TYPECHK(pointerref, long, i); + JL_TYPECHK(pointerref, long, i) + JL_TYPECHK(pointerref, long, align); + // TODO: alignment jl_value_t *ety = jl_tparam0(jl_typeof(p)); if (ety == (jl_value_t*)jl_any_type) { jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*)); @@ -51,10 +53,12 @@ JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i) } // run time version of pointerset intrinsic (warning: x is not gc-rooted) -JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i) +JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t *i, jl_value_t *align) { JL_TYPECHK(pointerset, pointer, p); JL_TYPECHK(pointerset, long, i); + JL_TYPECHK(pointerref, long, align); + // TODO: alignment jl_value_t *ety = jl_tparam0(jl_typeof(p)); if (ety == (jl_value_t*)jl_any_type) { jl_value_t **pp = (jl_value_t**)(jl_unbox_long(p) + (jl_unbox_long(i)-1)*sizeof(void*)); From ba836e97f7cd615cd84ac073c668a740f7272890 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Tue, 5 Jul 2016 13:52:02 -0500 Subject: [PATCH 0239/1117] Loosen the signature on indices passed to _flatten. Fixes #17275. --- base/multidimensional.jl | 13 +++++++------ test/arrayops.jl | 3 +++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 52b078a89c5bd..838101efd8328 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -27,10 +27,11 @@ CartesianIndex{N}(index::NTuple{N,Integer}) = CartesianIndex{N}(index) (::Type{CartesianIndex{N}}){N}(index::Integer...) = CartesianIndex{N}(index) (::Type{CartesianIndex{N}}){N}() = CartesianIndex{N}(()) # Un-nest passed CartesianIndexes -CartesianIndex(index::Union{Integer, CartesianIndex}...) = CartesianIndex(_flatten((), index...)) -_flatten(out) = out -@inline _flatten(out, i::Integer, I...) = _flatten((out..., i), I...) -@inline _flatten(out, i::CartesianIndex, I...) = _flatten((out..., i.I...), I...) +CartesianIndex(index::Union{Integer, CartesianIndex}...) = CartesianIndex(flatten(index)) +Base.@pure flatten(I) = (_flatten(I...)...,) +Base.@pure _flatten() = () +Base.@pure _flatten(i, I...) = (i, _flatten(I...)...) +Base.@pure _flatten(i::CartesianIndex, I...) = (i.I..., _flatten(I...)...) CartesianIndex(index::Tuple{Vararg{Union{Integer, CartesianIndex}}}) = CartesianIndex(index...) # length @@ -374,10 +375,10 @@ end end @propagate_inbounds function _getindex{T,N}(l::LinearIndexing, A::AbstractArray{T,N}, I::Union{Real,AbstractArray,Colon,CartesianIndex}...) - getindex(A, IteratorsMD._flatten((), I...)...) + getindex(A, IteratorsMD.flatten(I)...) end @propagate_inbounds function _setindex!{T,N}(l::LinearIndexing, A::AbstractArray{T,N}, v, I::Union{Real,AbstractArray,Colon,CartesianIndex}...) - setindex!(A, v, IteratorsMD._flatten((), I...)...) + setindex!(A, v, IteratorsMD.flatten(I)...) end ## diff --git a/test/arrayops.jl b/test/arrayops.jl index 16a96faae6801..c7e0356e00b37 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1228,6 +1228,9 @@ a[1,CartesianIndex{2}(3,4)] = -2 @test a[CartesianIndex{1}(2),3,CartesianIndex{1}(4)] == 44 a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] = -3 @test a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] == -3 +@test a[:, :, CartesianIndex((1,))] == a[:,:,1] +@test a[CartesianIndex((1,)), [1,2], :] == a[1,[1,2],:] +@test a[CartesianIndex((2,)), 3:4, :] == a[2,3:4,:] a = view(zeros(3, 4, 5), :, :, :) a[CartesianIndex{3}(2,3,3)] = -1 From 2ed8a3d740664ddab6ef1fb7d128bc5220a1a4ff Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 22 Jun 2016 21:03:26 -0400 Subject: [PATCH 0240/1117] fallback anticodegen compile to use interpreter, rather than error --- src/anticodegen.c | 11 +++++++++-- src/julia.h | 2 ++ src/julia_internal.h | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/anticodegen.c b/src/anticodegen.c index 8e19c2989323d..4ee97cb5ad86e 100644 --- a/src/anticodegen.c +++ b/src/anticodegen.c @@ -14,8 +14,6 @@ void jl_dump_objfile(char *fname, int jit_model, const char *sysimg_data, size_t int32_t jl_get_llvm_gv(jl_value_t *p) UNAVAILABLE void jl_write_malloc_log(void) UNAVAILABLE void jl_write_coverage_data(void) UNAVAILABLE -void jl_generate_fptr(jl_lambda_info_t *li) UNAVAILABLE -void jl_compile_linfo(jl_lambda_info_t *li) UNAVAILABLE JL_DLLEXPORT void jl_clear_malloc_data(void) UNAVAILABLE JL_DLLEXPORT void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE @@ -48,3 +46,12 @@ void jl_register_fptrs(uint64_t sysimage_base, void **fptrs, jl_lambda_info_t ** { (void)sysimage_base; (void)fptrs; (void)linfos; (void)n; } + +void jl_compile_linfo(jl_lambda_info_t *li) { } + +jl_value_t *jl_interpret_call(jl_lambda_info_t *lam, jl_value_t **args, uint32_t nargs); +void jl_generate_fptr(jl_lambda_info_t *li) +{ + li->fptr = (jl_fptr_t)&jl_interpret_call; + li->jlcall_api = 3; +} diff --git a/src/julia.h b/src/julia.h index 174164f0eea02..f7b03bef4cfe4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -168,6 +168,7 @@ STATIC_INLINE int jl_array_ndimwords(uint32_t ndims) } typedef struct _jl_datatype_t jl_tupletype_t; +struct _jl_lambda_info_t; // TypeMap is an implicitly defined type // that can consist of any of the following nodes: @@ -187,6 +188,7 @@ union jl_typemap_t { // This defines the default ABI used by compiled julia functions. typedef jl_value_t *(*jl_fptr_t)(jl_value_t*, jl_value_t**, uint32_t); typedef jl_value_t *(*jl_fptr_sparam_t)(jl_svec_t*, jl_value_t*, jl_value_t**, uint32_t); +typedef jl_value_t *(*jl_fptr_linfo_t)(struct _jl_lambda_info_t*, jl_value_t**, uint32_t, jl_svec_t*); typedef struct _jl_llvm_functions_t { void *functionObject; // jlcall llvm Function diff --git a/src/julia_internal.h b/src/julia_internal.h index c61f2911faf88..105b3641fdf47 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -174,6 +174,8 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val return ((jl_fptr_sparam_t)mfptr->fptr)(meth->sparam_vals, args[0], &args[1], nargs-1); else if (mfptr->jlcall_api == 2) return meth->constval; + else if (mfptr->jlcall_api == 3) + return ((jl_fptr_linfo_t)mfptr->fptr)(mfptr, &args[0], nargs, meth->sparam_vals); else abort(); } From 64a34ee03d16d6c796c5ad26237b1cd971d37b2d Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 19 May 2016 03:37:28 -0400 Subject: [PATCH 0241/1117] reduce TypeMap memory wastage the TypeMap structure is usually very sparsely populated switching to an OrderedDict implementation allows faster iteration (the filled cells don't have gaps) and the index array can use a small memory footprint --- src/dump.c | 39 ++----- src/gf.c | 24 ++-- src/jltypes.c | 20 +++- src/julia.h | 8 +- src/typemap.c | 297 +++++++++++++++++++++++++++++++------------------- 5 files changed, 233 insertions(+), 155 deletions(-) diff --git a/src/dump.c b/src/dump.c index 562ee1340870e..d4af9d6c1703d 100644 --- a/src/dump.c +++ b/src/dump.c @@ -938,38 +938,17 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) // perform some compression on the typemap levels // (which will need to be rehashed during deserialization anyhow) jl_typemap_level_t *node = (jl_typemap_level_t*)v; - size_t i, l; assert( // make sure this type has the expected ordering offsetof(jl_typemap_level_t, arg1) == 0 * sizeof(jl_value_t*) && - offsetof(jl_typemap_level_t, targ) == 1 * sizeof(jl_value_t*) && - offsetof(jl_typemap_level_t, linear) == 2 * sizeof(jl_value_t*) && - offsetof(jl_typemap_level_t, any) == 3 * sizeof(jl_value_t*) && - offsetof(jl_typemap_level_t, key) == 4 * sizeof(jl_value_t*) && - sizeof(jl_typemap_level_t) == 5 * sizeof(jl_value_t*)); - if (node->arg1 != (void*)jl_nothing) { - jl_array_t *a = jl_alloc_vec_any(0); - for (i = 0, l = jl_array_len(node->arg1); i < l; i++) { - jl_value_t *d = jl_array_ptr_ref(node->arg1, i); - if (d != NULL && d != jl_nothing) - jl_array_ptr_1d_push(a, d); - } - jl_serialize_value(s, a); - } - else { - jl_serialize_value(s, jl_nothing); - } - if (node->targ != (void*)jl_nothing) { - jl_array_t *a = jl_alloc_vec_any(0); - for (i = 0, l = jl_array_len(node->targ); i < l; i++) { - jl_value_t *d = jl_array_ptr_ref(node->targ, i); - if (d != NULL && d != jl_nothing) - jl_array_ptr_1d_push(a, d); - } - jl_serialize_value(s, a); - } - else { - jl_serialize_value(s, jl_nothing); - } + offsetof(jl_typemap_level_t, targ) == 2 * sizeof(jl_value_t*) && + offsetof(jl_typemap_level_t, linear) == 4 * sizeof(jl_value_t*) && + offsetof(jl_typemap_level_t, any) == 5 * sizeof(jl_value_t*) && + offsetof(jl_typemap_level_t, key) == 6 * sizeof(jl_value_t*) && + sizeof(jl_typemap_level_t) == 7 * sizeof(jl_value_t*)); + jl_serialize_value(s, jl_nothing); + jl_serialize_value(s, node->arg1.values); + jl_serialize_value(s, jl_nothing); + jl_serialize_value(s, node->targ.values); jl_serialize_value(s, node->linear); jl_serialize_value(s, node->any.unknown); jl_serialize_value(s, node->key); diff --git a/src/gf.c b/src/gf.c index d2d5ef7e5aad4..7e1222f55f818 100644 --- a/src/gf.c +++ b/src/gf.c @@ -981,19 +981,23 @@ static void invalidate_conflicting(union jl_typemap_t *pml, jl_value_t *type, jl jl_typemap_entry_t **pl; if (jl_typeof(pml->unknown) == (jl_value_t*)jl_typemap_level_type) { jl_typemap_level_t *cache = pml->node; - if (cache->arg1 != (void*)jl_nothing) { - for(int i=0; i < jl_array_len(cache->arg1); i++) { - union jl_typemap_t *pl = &((union jl_typemap_t*)jl_array_data(cache->arg1))[i]; - if (pl->unknown && pl->unknown != jl_nothing) { - invalidate_conflicting(pl, type, (jl_value_t*)cache->arg1, shadowed); + if (cache->arg1.values != (void*)jl_nothing) { + size_t i, l = jl_array_len(cache->arg1.values); + union jl_typemap_t *d = (union jl_typemap_t*)jl_array_data(cache->arg1.values); + for (i = 0; i < l; i++) { + union jl_typemap_t *pl = &d[i]; + if (pl->unknown != jl_nothing) { + invalidate_conflicting(pl, type, (jl_value_t*)cache->arg1.values, shadowed); } } } - if (cache->targ != (void*)jl_nothing) { - for(int i=0; i < jl_array_len(cache->targ); i++) { - union jl_typemap_t *pl = &((union jl_typemap_t*)jl_array_data(cache->targ))[i]; - if (pl->unknown && pl->unknown != jl_nothing) { - invalidate_conflicting(pl, type, (jl_value_t*)cache->targ, shadowed); + if (cache->targ.values != (void*)jl_nothing) { + size_t i, l = jl_array_len(cache->targ.values); + union jl_typemap_t *d = (union jl_typemap_t*)jl_array_data(cache->targ.values); + for (i = 0; i < l; i++) { + union jl_typemap_t *pl = &d[i]; + if (pl->unknown != jl_nothing) { + invalidate_conflicting(pl, type, (jl_value_t*)cache->targ.values, shadowed); } } } diff --git a/src/jltypes.c b/src/jltypes.c index 0d7f2c1e98040..3aefaeb42ad59 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3678,9 +3678,23 @@ void jl_init_types(void) jl_typemap_level_type = jl_new_datatype(jl_symbol("TypeMapLevel"), jl_any_type, jl_emptysvec, - jl_svec(5, jl_symbol("arg1"), jl_symbol("targ"), jl_symbol("list"), jl_symbol("any"), jl_symbol("key")), - jl_svec(5, jl_any_type, jl_any_type, jl_any_type, jl_any_type, jl_any_type), - 0, 1, 4); + jl_svec(7, + jl_symbol("index_arg1"), + jl_symbol("arg1"), + jl_symbol("index_targ"), + jl_symbol("targ"), + jl_symbol("list"), + jl_symbol("any"), + jl_symbol("key")), + jl_svec(7, + jl_any_type, + jl_any_type, + jl_any_type, + jl_any_type, + jl_any_type, + jl_any_type, + jl_any_type), + 0, 1, 6); jl_typemap_entry_type = jl_new_datatype(jl_symbol("TypeMapEntry"), jl_any_type, jl_emptysvec, diff --git a/src/julia.h b/src/julia.h index f7b03bef4cfe4..d2be17938910b 100644 --- a/src/julia.h +++ b/src/julia.h @@ -420,10 +420,14 @@ typedef struct _jl_typemap_entry_t { // one level in a TypeMap tree // indexed by key if it is a sublevel in an array +struct jl_ordereddict_t { + jl_array_t *indexes; // Array{Int{8,16,32}} + jl_array_t *values; // Array{union jl_typemap_t} +}; typedef struct _jl_typemap_level_t { JL_DATA_TYPE - jl_array_t *arg1; // Array{union jl_typemap_t} - jl_array_t *targ; // Array{union jl_typemap_t} + struct jl_ordereddict_t arg1; + struct jl_ordereddict_t targ; jl_typemap_entry_t *linear; // union jl_typemap_t (but no more levels) union jl_typemap_t any; // type at offs is Any jl_value_t *key; // [nullable] diff --git a/src/typemap.c b/src/typemap.c index f55425f847f21..466eb26d0556f 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -171,29 +171,96 @@ static inline int sig_match_simple(jl_value_t **args, size_t n, jl_value_t **sig // ----- MethodCache helper functions ----- // +static inline size_t jl_intref(const jl_array_t *arr, size_t idx) +{ + jl_value_t *el = jl_tparam0(jl_typeof(arr)); + if (el == (jl_value_t*)jl_uint8_type) + return ((uint8_t*)jl_array_data(arr))[idx]; + else if (el == (jl_value_t*)jl_uint16_type) + return ((uint16_t*)jl_array_data(arr))[idx]; + else if (el == (jl_value_t*)jl_uint32_type) + return ((uint32_t*)jl_array_data(arr))[idx]; + else + abort(); +} + +static inline void jl_intset(const jl_array_t *arr, size_t idx, size_t val) +{ + jl_value_t *el = jl_tparam0(jl_typeof(arr)); + if (el == (jl_value_t*)jl_uint8_type) + ((uint8_t*)jl_array_data(arr))[idx] = val; + else if (el == (jl_value_t*)jl_uint16_type) + ((uint16_t*)jl_array_data(arr))[idx] = val; + else if (el == (jl_value_t*)jl_uint32_type) + ((uint32_t*)jl_array_data(arr))[idx] = val; + else + abort(); +} + +static inline size_t jl_max_int(const jl_array_t *arr) +{ + jl_value_t *el = jl_tparam0(jl_typeof(arr)); + if (el == (jl_value_t*)jl_uint8_type) + return 0xFF; + else if (el == (jl_value_t*)jl_uint16_type) + return 0xFFFF; + else if (el == (jl_value_t*)jl_uint32_type) + return 0xFFFFFFFF; + else + abort(); +} + + +static jl_array_t *jl_alloc_int_1d(size_t np, size_t len) +{ + jl_value_t *ty; + if (np < 0xFF) { + ty = jl_array_uint8_type; + } + else if (np < 0xFFFF) { + static jl_value_t *int16 = NULL; + if (int16 == NULL) + int16 = jl_apply_array_type(jl_uint16_type, 1); + ty = int16; + } + else { + assert(np < 0x7FFFFFFF); + static jl_value_t *int32 = NULL; + if (int32 == NULL) + int32 = jl_apply_array_type(jl_uint32_type, 1); + ty = int32; + } + jl_array_t *a = jl_alloc_array_1d(ty, len); + memset(a->data, 0, len * a->elsize); + return a; +} + static inline -union jl_typemap_t mtcache_hash_lookup(jl_array_t *a, jl_value_t *ty, int8_t tparam, int8_t offs) +union jl_typemap_t mtcache_hash_lookup(const struct jl_ordereddict_t *a, jl_value_t *ty, int8_t tparam, int8_t offs) { uintptr_t uid = ((jl_datatype_t*)ty)->uid; union jl_typemap_t ml; ml.unknown = jl_nothing; if (!uid) return ml; - ml.unknown = jl_array_ptr_ref(a, uid & (a->nrows-1)); - if (ml.unknown != NULL && ml.unknown != jl_nothing) { + size_t idx = jl_intref(a->indexes, uid & (a->indexes->nrows-1)); + if (idx > 0) { + ml.unknown = jl_array_ptr_ref(a->values, idx - 1); + if (ml.unknown == jl_nothing) + return ml; jl_value_t *t; if (jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_level_type) { t = ml.node->key; } else { + assert(jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_entry_type); t = jl_field_type(ml.leaf->sig, offs); if (tparam) t = jl_tparam0(t); } - if (t == ty) - return ml; + if (t != ty) + ml.unknown = jl_nothing; } - ml.unknown = jl_nothing; return ml; } @@ -210,66 +277,65 @@ static inline unsigned int next_power_of_two(unsigned int val) return val; } -static void mtcache_rehash(jl_array_t **pa, jl_value_t *parent, int8_t tparam, int8_t offs) +static void mtcache_rehash(struct jl_ordereddict_t *pa, size_t newlen, jl_value_t *parent, int8_t tparam, int8_t offs) { - size_t i, len = jl_array_len(*pa); - size_t newlen = next_power_of_two(len) * 2; - jl_value_t **d = (jl_value_t**)jl_array_data(*pa); - jl_array_t *n = jl_alloc_vec_any(newlen); - for (i = 1; i <= len; i++) { + size_t i, nval = jl_array_len(pa->values); + jl_array_t *n = jl_alloc_int_1d(nval + 1, newlen); + for (i = 1; i <= nval; i++) { union jl_typemap_t ml; - ml.unknown = d[i - 1]; - if (ml.unknown != NULL && ml.unknown != jl_nothing) { - jl_value_t *t; - if (jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_level_type) { - t = ml.node->key; - } - else { - t = jl_field_type(ml.leaf->sig, offs); - if (tparam) - t = jl_tparam0(t); - } - uintptr_t uid = ((jl_datatype_t*)t)->uid; - size_t idx = uid & (newlen - 1); - if (((jl_value_t**)n->data)[idx] == NULL) { - ((jl_value_t**)n->data)[idx] = ml.unknown; - } - else { - // hash collision: start over after doubling the size again - i = 0; - newlen *= 2; - n = jl_alloc_vec_any(newlen); - } + ml.unknown = jl_array_ptr_ref(pa->values, i - 1); + if (ml.unknown == jl_nothing) + continue; + jl_value_t *t; + if (jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_level_type) { + t = ml.node->key; + } + else { + assert(jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_entry_type); + t = jl_field_type(ml.leaf->sig, offs); + if (tparam) + t = jl_tparam0(t); + } + uintptr_t uid = ((jl_datatype_t*)t)->uid; + size_t newi = uid & (newlen - 1); + if (jl_intref(n, newi) == 0) { + jl_intset(n, newi, i); + } + else { + // hash collision: start over after doubling the size again + i = 0; + newlen *= 2; + n = jl_alloc_int_1d(nval + 1, newlen); } } - *pa = n; + pa->indexes = n; jl_gc_wb(parent, n); } // Recursively rehash a TypeMap (for example, after deserialization) void jl_typemap_rehash(union jl_typemap_t ml, int8_t offs); -void jl_typemap_rehash_array(jl_array_t **pa, jl_value_t *parent, int8_t tparam, int8_t offs) { - size_t i, len = (*pa)->nrows; +void jl_typemap_rehash_array(struct jl_ordereddict_t *pa, jl_value_t *parent, int8_t tparam, int8_t offs) { + size_t i, len = jl_array_len(pa->values); for (i = 0; i < len; i++) { union jl_typemap_t ml; - ml.unknown = jl_array_ptr_ref(*pa, i); + ml.unknown = jl_array_ptr_ref(pa->values, i); assert(ml.unknown != NULL); jl_typemap_rehash(ml, offs+1); } - mtcache_rehash(pa, parent, tparam, offs); + mtcache_rehash(pa, 4 * next_power_of_two(len), parent, tparam, offs); } void jl_typemap_rehash(union jl_typemap_t ml, int8_t offs) { if (jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_level_type) { - if (ml.node->targ != (void*)jl_nothing) + if (ml.node->targ.values != (void*)jl_nothing) jl_typemap_rehash_array(&ml.node->targ, ml.unknown, 1, offs); - if (ml.node->arg1 != (void*)jl_nothing) + if (ml.node->arg1.values != (void*)jl_nothing) jl_typemap_rehash_array(&ml.node->arg1, ml.unknown, 0, offs); jl_typemap_rehash(ml.node->any, offs+1); } } -static union jl_typemap_t *mtcache_hash_bp(jl_array_t **pa, jl_value_t *ty, - int8_t tparam, int8_t offs, jl_value_t *parent) +static union jl_typemap_t *mtcache_hash_bp(struct jl_ordereddict_t *pa, jl_value_t *ty, + int8_t tparam, int8_t offs, jl_value_t *parent) { if (jl_is_datatype(ty)) { uintptr_t uid = ((jl_datatype_t*)ty)->uid; @@ -277,29 +343,39 @@ static union jl_typemap_t *mtcache_hash_bp(jl_array_t **pa, jl_value_t *ty, // be careful not to put non-leaf types or DataType/TypeConstructor in the cache here, // since they should have a lower priority and need to go into the sorted list return NULL; - if (*pa == (void*)jl_nothing) { - *pa = jl_alloc_vec_any(INIT_CACHE_SIZE); - jl_gc_wb(parent, *pa); + if (pa->values == (void*)jl_nothing) { + pa->indexes = jl_alloc_int_1d(0, INIT_CACHE_SIZE); + jl_gc_wb(parent, pa->indexes); + pa->values = jl_alloc_vec_any(0); + jl_gc_wb(parent, pa->values); } while (1) { - union jl_typemap_t *pml = &((union jl_typemap_t*)jl_array_data(*pa))[uid & ((*pa)->nrows-1)]; - union jl_typemap_t ml = *pml; - if (ml.unknown == NULL || ml.unknown == jl_nothing) { - pml->unknown = jl_nothing; - return pml; + size_t slot = uid & (pa->indexes->nrows - 1); + size_t idx = jl_intref(pa->indexes, slot); + if (idx == 0) { + jl_array_ptr_1d_push(pa->values, jl_nothing); + idx = jl_array_len(pa->values); + if (idx > jl_max_int(pa->indexes)) + mtcache_rehash(pa, jl_array_len(pa->indexes), parent, tparam, offs); + jl_intset(pa->indexes, slot, idx); + return &((union jl_typemap_t*)jl_array_data(pa->values))[idx - 1]; } + union jl_typemap_t *pml = &((union jl_typemap_t*)jl_array_data(pa->values))[idx - 1]; + if (pml->unknown == jl_nothing) + return pml; jl_value_t *t; - if (jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_level_type) { - t = ml.node->key; + if (jl_typeof(pml->unknown) == (jl_value_t*)jl_typemap_level_type) { + t = pml->node->key; } else { - t = jl_field_type(ml.leaf->sig, offs); + assert(jl_typeof(pml->unknown) == (jl_value_t*)jl_typemap_entry_type); + t = jl_field_type(pml->leaf->sig, offs); if (tparam) t = jl_tparam0(t); } if (t == ty) return pml; - mtcache_rehash(pa, parent, tparam, offs); + mtcache_rehash(pa, jl_array_len(pa->indexes) * 2, parent, tparam, offs); } } return NULL; @@ -338,14 +414,13 @@ jl_value_t *jl_lookup_match(jl_value_t *a, jl_value_t *b, jl_svec_t **penv, return ti; } -static int jl_typemap_array_visitor(jl_array_t *a, jl_typemap_visitor_fptr fptr, void *closure) +static int jl_typemap_array_visitor(struct jl_ordereddict_t *a, jl_typemap_visitor_fptr fptr, void *closure) { - size_t i, l = jl_array_len(a); - jl_value_t **data = (jl_value_t**)jl_array_data(a); + size_t i, l = jl_array_len(a->values); + union jl_typemap_t *data = (union jl_typemap_t*)jl_array_data(a->values); for(i=0; i < l; i++) { - if (data[i] != NULL) - if (!jl_typemap_visitor(((union jl_typemap_t*)data)[i], fptr, closure)) - return 0; + if (!jl_typemap_visitor(data[i], fptr, closure)) + return 0; } return 1; } @@ -364,11 +439,11 @@ static int jl_typemap_node_visitor(jl_typemap_entry_t *ml, jl_typemap_visitor_fp int jl_typemap_visitor(union jl_typemap_t cache, jl_typemap_visitor_fptr fptr, void *closure) { if (jl_typeof(cache.unknown) == (jl_value_t*)jl_typemap_level_type) { - if (cache.node->targ != (void*)jl_nothing) - if (!jl_typemap_array_visitor(cache.node->targ, fptr, closure)) + if (cache.node->targ.values != (void*)jl_nothing) + if (!jl_typemap_array_visitor(&cache.node->targ, fptr, closure)) return 0; - if (cache.node->arg1 != (void*)jl_nothing) - if (!jl_typemap_array_visitor(cache.node->arg1, fptr, closure)) + if (cache.node->arg1.values != (void*)jl_nothing) + if (!jl_typemap_array_visitor(&cache.node->arg1, fptr, closure)) return 0; if (!jl_typemap_node_visitor(cache.node->linear, fptr, closure)) return 0; @@ -386,34 +461,34 @@ int is_cache_leaf(jl_value_t *ty) return (jl_is_datatype(ty) && ((jl_datatype_t*)ty)->uid != 0 && !is_kind(ty)); } -static int jl_typemap_intersection_array_visitor(jl_array_t *a, jl_value_t *ty, int tparam, +static int jl_typemap_intersection_array_visitor(struct jl_ordereddict_t *a, jl_value_t *ty, int tparam, int offs, struct typemap_intersection_env *closure) { - size_t i, l = jl_array_len(a); - jl_value_t **data = (jl_value_t**)jl_array_data(a); + size_t i, l = jl_array_len(a->values); + union jl_typemap_t *data = (union jl_typemap_t*)jl_array_data(a->values); for (i = 0; i < l; i++) { - union jl_typemap_t ml = ((union jl_typemap_t*)data)[i]; - if (ml.unknown != NULL && ml.unknown != jl_nothing) { - jl_value_t *t; - if (jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_level_type) { - t = ml.node->key; - } - else { - t = jl_field_type(ml.leaf->sig, offs); - if (tparam) - t = jl_tparam0(t); - } - if (ty == (jl_value_t*)jl_any_type || // easy case: Any always matches - (tparam ? // need to compute `ty <: Type{t}` - (jl_is_uniontype(ty) || // punt on Union{...} right now - jl_typeof(t) == ty || // deal with kinds (e.g. ty == DataType && t == Type{t}) - (jl_is_type_type(ty) && (jl_is_typevar(jl_tparam0(ty)) ? - jl_subtype(t, ((jl_tvar_t*)jl_tparam0(ty))->ub, 0) : // deal with ty == Type{<:T} - jl_subtype(t, jl_tparam0(ty), 0)))) // deal with ty == Type{T{#<:T}} - : jl_subtype(t, ty, 0))) // `t` is a leaftype, so intersection test becomes subtype - if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) - return 0; + union jl_typemap_t ml = data[i]; + if (ml.unknown == jl_nothing) + continue; + jl_value_t *t; + if (jl_typeof(ml.unknown) == (jl_value_t*)jl_typemap_level_type) { + t = ml.node->key; } + else { + t = jl_field_type(ml.leaf->sig, offs); + if (tparam) + t = jl_tparam0(t); + } + if (ty == (jl_value_t*)jl_any_type || // easy case: Any always matches + (tparam ? // need to compute `ty <: Type{t}` + (jl_is_uniontype(ty) || // punt on Union{...} right now + jl_typeof(t) == ty || // deal with kinds (e.g. ty == DataType && t == Type{t}) + (jl_is_type_type(ty) && (jl_is_typevar(jl_tparam0(ty)) ? + jl_subtype(t, ((jl_tvar_t*)jl_tparam0(ty))->ub, 0) : // deal with ty == Type{<:T} + jl_subtype(t, jl_tparam0(ty), 0)))) // deal with ty == Type{T{#<:T}} + : jl_subtype(t, ty, 0))) // `t` is a leaftype, so intersection test becomes subtype + if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) + return 0; } return 1; } @@ -469,12 +544,12 @@ int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs, ty = jl_tparam(closure->type, offs); } if (ty) { - if (cache->targ != (void*)jl_nothing) { + if (cache->targ.values != (void*)jl_nothing) { jl_value_t *typetype = jl_is_type_type(ty) ? jl_tparam0(ty) : NULL; if (typetype && !jl_has_typevars(typetype)) { if (is_cache_leaf(typetype)) { // direct lookup of leaf types - union jl_typemap_t ml = mtcache_hash_lookup(cache->targ, typetype, 1, offs); + union jl_typemap_t ml = mtcache_hash_lookup(&cache->targ, typetype, 1, offs); if (ml.unknown != jl_nothing) { if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) return 0; } @@ -484,20 +559,20 @@ int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs, // else an array scan is required to check subtypes // first, fast-path: optimized pre-intersection test to see if `ty` could intersect with any Type if (typetype || jl_type_intersection((jl_value_t*)jl_type_type, ty) != jl_bottom_type) - if (!jl_typemap_intersection_array_visitor(cache->targ, ty, 1, offs, closure)) return 0; + if (!jl_typemap_intersection_array_visitor(&cache->targ, ty, 1, offs, closure)) return 0; } } - if (cache->arg1 != (void*)jl_nothing) { + if (cache->arg1.values != (void*)jl_nothing) { if (is_cache_leaf(ty)) { // direct lookup of leaf types - union jl_typemap_t ml = mtcache_hash_lookup(cache->arg1, ty, 0, offs); + union jl_typemap_t ml = mtcache_hash_lookup(&cache->arg1, ty, 0, offs); if (ml.unknown != jl_nothing) { if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) return 0; } } else { // else an array scan is required to check subtypes - if (!jl_typemap_intersection_array_visitor(cache->arg1, ty, 0, offs, closure)) return 0; + if (!jl_typemap_intersection_array_visitor(&cache->arg1, ty, 0, offs, closure)) return 0; } } } @@ -668,10 +743,10 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_ ty = NULL; } if (ty) { - if (cache->targ != (void*)jl_nothing && jl_is_type_type(ty)) { + if (jl_is_type_type(ty)) { jl_value_t *a0 = jl_tparam0(ty); - if (cache->targ != (void*)jl_nothing && jl_is_datatype(a0)) { - union jl_typemap_t ml = mtcache_hash_lookup(cache->targ, a0, 1, offs); + if (cache->targ.values != (void*)jl_nothing && jl_is_datatype(a0)) { + union jl_typemap_t ml = mtcache_hash_lookup(&cache->targ, a0, 1, offs); if (ml.unknown != jl_nothing) { jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, types, penv, subtype_inexact__sigseq_useenv, subtype, offs+1); @@ -680,8 +755,8 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_ } if (!subtype && is_cache_leaf(a0)) return NULL; } - if (cache->arg1 != (void*)jl_nothing && jl_is_datatype(ty)) { - union jl_typemap_t ml = mtcache_hash_lookup(cache->arg1, ty, 0, offs); + if (cache->arg1.values != (void*)jl_nothing && jl_is_datatype(ty)) { + union jl_typemap_t ml = mtcache_hash_lookup(&cache->arg1, ty, 0, offs); if (ml.unknown != jl_nothing) { jl_typemap_entry_t *li = jl_typemap_assoc_by_type(ml, types, penv, subtype_inexact__sigseq_useenv, subtype, offs+1); @@ -787,13 +862,13 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v jl_value_t *a1 = args[offs]; jl_value_t *ty = (jl_value_t*)jl_typeof(a1); assert(jl_is_datatype(ty)); - if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (void*)jl_nothing) { - union jl_typemap_t ml_or_cache = mtcache_hash_lookup(cache->targ, a1, 1, offs); + if (ty == (jl_value_t*)jl_datatype_type && cache->targ.values != (void*)jl_nothing) { + union jl_typemap_t ml_or_cache = mtcache_hash_lookup(&cache->targ, a1, 1, offs); jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1); if (ml) return ml; } - if (cache->arg1 != (void*)jl_nothing) { - union jl_typemap_t ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs); + if (cache->arg1.values != (void*)jl_nothing) { + union jl_typemap_t ml_or_cache = mtcache_hash_lookup(&cache->arg1, ty, 0, offs); jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1); if (ml) return ml; } @@ -833,8 +908,10 @@ static jl_typemap_level_t *jl_new_typemap_level(void) cache->key = NULL; cache->linear = (jl_typemap_entry_t*)jl_nothing; cache->any.unknown = jl_nothing; - cache->arg1 = (jl_array_t*)jl_nothing; - cache->targ = (jl_array_t*)jl_nothing; + cache->targ.indexes = (jl_array_t*)jl_nothing; + cache->targ.values = (jl_array_t*)jl_nothing; + cache->arg1.indexes = (jl_array_t*)jl_nothing; + cache->arg1.values = (jl_array_t*)jl_nothing; return cache; } @@ -888,13 +965,13 @@ static void jl_typemap_insert_generic(union jl_typemap_t *pml, jl_value_t *paren jl_typemap_list_insert_(&pml->leaf, parent, newrec, tparams); } -static int jl_typemap_array_insert_(jl_array_t **cache, jl_value_t *key, jl_typemap_entry_t *newrec, - jl_value_t *parent, int8_t tparam, int8_t offs, - const struct jl_typemap_info *tparams) +static int jl_typemap_array_insert_(struct jl_ordereddict_t *cache, jl_value_t *key, jl_typemap_entry_t *newrec, + jl_value_t *parent, int8_t tparam, int8_t offs, + const struct jl_typemap_info *tparams) { union jl_typemap_t *pml = mtcache_hash_bp(cache, key, tparam, offs, (jl_value_t*)parent); if (pml) - jl_typemap_insert_generic(pml, (jl_value_t*)*cache, newrec, key, offs+1, tparams); + jl_typemap_insert_generic(pml, (jl_value_t*)cache->values, newrec, key, offs+1, tparams); return pml != NULL; } From 146502b59a1b43b2b8c1b0dbe0a01f89eaff7b6d Mon Sep 17 00:00:00 2001 From: Alexey Stukalov <astukalov@gmail.com> Date: Tue, 5 Jul 2016 23:24:06 +0200 Subject: [PATCH 0242/1117] BLAS.gemm!(): check that A and B sizes match --- base/linalg/blas.jl | 9 +++++---- test/blas.jl | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/base/linalg/blas.jl b/base/linalg/blas.jl index 4e3b8422577a8..3e022375fb4b3 100644 --- a/base/linalg/blas.jl +++ b/base/linalg/blas.jl @@ -960,10 +960,11 @@ for (gemm, elty) in # error("gemm!: BLAS module requires contiguous matrix columns") # end # should this be checked on every call? m = size(A, transA == 'N' ? 1 : 2) - k = size(A, transA == 'N' ? 2 : 1) + ka = size(A, transA == 'N' ? 2 : 1) + kb = size(B, transB == 'N' ? 1 : 2) n = size(B, transB == 'N' ? 2 : 1) - if m != size(C,1) || n != size(C,2) - throw(DimensionMismatch("A has size ($m,$k), B has size ($k,$n), C has size $(size(C))")) + if ka != kb || m != size(C,1) || n != size(C,2) + throw(DimensionMismatch("A has size ($m,$ka), B has size ($kb,$n), C has size $(size(C))")) end ccall((@blasfunc($gemm), libblas), Void, (Ptr{UInt8}, Ptr{UInt8}, Ptr{BlasInt}, Ptr{BlasInt}, @@ -971,7 +972,7 @@ for (gemm, elty) in Ptr{$elty}, Ptr{BlasInt}, Ptr{$elty}, Ptr{$elty}, Ptr{BlasInt}), &transA, &transB, &m, &n, - &k, &alpha, A, &max(1,stride(A,2)), + &ka, &alpha, A, &max(1,stride(A,2)), B, &max(1,stride(B,2)), &beta, C, &max(1,stride(C,2))) C diff --git a/test/blas.jl b/test/blas.jl index f7a4684c40abc..8b8cfeec7e25b 100644 --- a/test/blas.jl +++ b/test/blas.jl @@ -265,6 +265,10 @@ for elty in [Float32, Float64, Complex64, Complex128] @test all(BLAS.gemm('N', 'N', I4, U4) .== U4) @test all(BLAS.gemm('N', 'T', I4, U4) .== L4) @test_throws DimensionMismatch BLAS.gemm!('N','N', one(elty), I4, I4, elm1, eye(elty,5)) + @test_throws DimensionMismatch BLAS.gemm!('N','N', one(elty), I43, I4, elm1, I4) + @test_throws DimensionMismatch BLAS.gemm!('T','N', one(elty), I43, I4, elm1, I43) + @test_throws DimensionMismatch BLAS.gemm!('N','T', one(elty), I43, I43, elm1, I43) + @test_throws DimensionMismatch BLAS.gemm!('T','T', one(elty), I43, I43, elm1, I43') # gemm compared to (sy)(he)rk if eltype(elm1)<:Complex From 2ea706ec31472417ae2f02c230dc92e7759c8eda Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Tue, 5 Jul 2016 17:36:26 -0400 Subject: [PATCH 0243/1117] Fix compiler warnings --- src/alloc.c | 2 +- src/gf.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 6cabae7dd437d..50d42703424a4 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -860,7 +860,7 @@ jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_datatype_t *super, return dt; } -jl_datatype_t *jl_new_uninitialized_datatype() +jl_datatype_t *jl_new_uninitialized_datatype(void) { jl_ptls_t ptls = jl_get_ptls_states(); jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ptls, sizeof(jl_datatype_t), jl_datatype_type); diff --git a/src/gf.c b/src/gf.c index 7e1222f55f818..2c33823599904 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1294,7 +1294,8 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) } } } - assert(!li->inInference && !li->inCompile && li->code != jl_nothing || li->jlcall_api == 2); + assert(!li->inInference && !li->inCompile && + (li->code != jl_nothing || li->jlcall_api == 2)); if (li->functionObjectsDecls.functionObject == NULL) { // check again, because jl_type_infer may have compiled it jl_compile_linfo(li); } From 7af0f1b6396ed702702cb9dcca5cc1e1225cf1d7 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat <nalimilan@club.fr> Date: Wed, 6 Jul 2016 02:27:08 +0200 Subject: [PATCH 0244/1117] Make endof() robust to invalid UTF-8 (#17276) When an invalid string contains only continuation bytes, endof() tried to index the underlying array at position 0. Instead of relying on bounds checking, explicitly check for > 0. Returning 0 when only continuation bytes where encountered is consistent with the definition of endof(), which gives the last valid index. This also allows removing the i == 0 check. The new code appears to be slightly faster than the old one. --- base/strings/string.jl | 3 +-- test/strings/basic.jl | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/base/strings/string.jl b/base/strings/string.jl index 57dabebe6925a..47eb243b21647 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -36,8 +36,7 @@ const utf8_trailing = [ function endof(s::String) d = s.data i = length(d) - i == 0 && return i - while is_valid_continuation(d[i]) + @inbounds while i > 0 && is_valid_continuation(d[i]) i -= 1 end i diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 940eae91afc7a..f8a63850a0253 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -477,3 +477,7 @@ foobaz(ch) = reinterpret(Char, typemax(UInt32)) @test typeof(ascii(GenericString("Hello, world"))) == String @test_throws ArgumentError ascii("Hello, ∀") @test_throws ArgumentError ascii(GenericString("Hello, ∀")) + +# issue #17271: endof() doesn't throw an error even with invalid strings +@test endof(String(b"\x90")) == 0 +@test endof(String(b"\xce")) == 1 From 1b540ba470d68fba44b698ee91e1fb3179d48ccc Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Wed, 25 May 2016 17:24:42 -0400 Subject: [PATCH 0245/1117] remove UTF-16 and UTF-32 string types and functions --- base/deprecated.jl | 10 -- base/docs/helpdb/Base.jl | 52 ------- base/exports.jl | 4 - base/replutil.jl | 12 +- base/serialize.jl | 5 +- base/strings/io.jl | 19 +++ base/strings/string.jl | 103 +------------- base/sysimg.jl | 1 - base/test.jl | 1 + base/unicode/checkstring.jl | 238 ------------------------------- base/unicode/types.jl | 34 ----- base/unicode/unicode.jl | 6 - base/unicode/utf16.jl | 275 ------------------------------------ base/unicode/utf32.jl | 195 ------------------------- test/serialize.jl | 4 +- test/strings/basic.jl | 68 +-------- test/strings/io.jl | 70 --------- test/strings/types.jl | 44 +++--- test/strings/util.jl | 2 +- test/unicode.jl | 4 - test/unicode/utf16.jl | 23 --- test/unicode/utf32.jl | 258 --------------------------------- test/unicode/utf8.jl | 6 +- test/unicode/utf8proc.jl | 2 +- 24 files changed, 69 insertions(+), 1367 deletions(-) delete mode 100644 base/unicode/checkstring.jl delete mode 100644 base/unicode/types.jl delete mode 100644 base/unicode/unicode.jl delete mode 100644 base/unicode/utf16.jl delete mode 100644 base/unicode/utf32.jl delete mode 100644 test/unicode/utf16.jl delete mode 100644 test/unicode/utf32.jl diff --git a/base/deprecated.jl b/base/deprecated.jl index b22c1a3cc6523..dc862dc46eca6 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -488,16 +488,6 @@ end end ) -if sizeof(Cwchar_t) == 2 - @deprecate_binding WString UTF16String - @deprecate_binding wstring utf16 - utf16(s::Cwstring) = utf16(convert(Ptr{Cwchar_t}, s)) -elseif sizeof(Cwchar_t) == 4 - @deprecate_binding WString UTF32String - @deprecate_binding wstring utf32 - utf32(s::Cwstring) = utf32(convert(Ptr{Cwchar_t}, s)) -end - @deprecate ==(x::Char, y::Integer) UInt32(x) == y @deprecate ==(x::Integer, y::Char) x == UInt32(y) @deprecate isless(x::Char, y::Integer) UInt32(x) < y diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 70ffd3ca1c168..f8af1543b5d68 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -95,32 +95,6 @@ Get the step size of a [`Range`](:obj:`Range`) object. """ step -""" - utf32(s) - -Create a UTF-32 string from a byte array, array of `Char` or `UInt32`, or any other string -type. (Conversions of byte arrays check for a byte-order marker in the first four bytes, and -do not include it in the resulting string.) - -Note that the resulting `UTF32String` data is terminated by the NUL codepoint (32-bit zero), -which is not treated as a character in the string (so that it is mostly invisible in Julia); -this allows the string to be passed directly to external functions requiring NUL-terminated -data. This NUL is appended automatically by the `utf32(s)` conversion function. If you have -a `Char` or `UInt32` array `A` that is already NUL-terminated UTF-32 data, then you can -instead use `UTF32String(A)` to construct the string without making a copy of the data and -treating the NUL as a terminator rather than as part of the string. -""" -utf32(s) - -""" - utf32(::Union{Ptr{Char},Ptr{UInt32},Ptr{Int32}} [, length]) - -Create a string from the address of a NUL-terminated UTF-32 string. A copy is made; the -pointer can be safely freed. If `length` is specified, the string does not have to be -NUL-terminated. -""" -utf32(::Union{Ptr{Char},Ptr{UInt32},Ptr{Int32}}, length=?) - """ takebuf_array(b::IOBuffer) @@ -3620,32 +3594,6 @@ Compute ``\\sin(\\pi x) / (\\pi x)`` if ``x \\neq 0``, and ``1`` if ``x = 0``. """ sinc -""" - utf16(s) - -Create a UTF-16 string from a byte array, array of `UInt16`, or any other string type. (Data -must be valid UTF-16. Conversions of byte arrays check for a byte-order marker in the first -two bytes, and do not include it in the resulting string.) - -Note that the resulting `UTF16String` data is terminated by the NUL codepoint (16-bit zero), -which is not treated as a character in the string (so that it is mostly invisible in Julia); -this allows the string to be passed directly to external functions requiring NUL-terminated -data. This NUL is appended automatically by the `utf16(s)` conversion function. If you have -a `UInt16` array `A` that is already NUL-terminated valid UTF-16 data, then you can instead -use `UTF16String(A)` to construct the string without making a copy of the data and treating -the NUL as a terminator rather than as part of the string. -""" -utf16(s) - -""" - utf16(::Union{Ptr{UInt16},Ptr{Int16}} [, length]) - -Create a string from the address of a NUL-terminated UTF-16 string. A copy is made; the -pointer can be safely freed. If `length` is specified, the string does not have to be -NUL-terminated. -""" -utf16(::Union{Ptr{UInt16},Ptr{Int16}}, length=?) - """ median(v[, region]) diff --git a/base/exports.jl b/base/exports.jl index 000d2b4a96d2f..5af15b345eb31 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -119,8 +119,6 @@ export Tridiagonal, UnitRange, UpperTriangular, - UTF16String, - UTF32String, Val, VecOrMat, Vector, @@ -878,8 +876,6 @@ export ucfirst, unescape_string, uppercase, - utf16, - utf32, warn, # random numbers diff --git a/base/replutil.jl b/base/replutil.jl index 229aa877f657c..edbc2863e53a9 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -233,7 +233,6 @@ end showerror(io::IO, ::DivideError) = print(io, "DivideError: integer division error") showerror(io::IO, ::StackOverflowError) = print(io, "StackOverflowError:") showerror(io::IO, ::UndefRefError) = print(io, "UndefRefError: access to undefined reference") -showerror(io::IO, ex::UndefVarError) = print(io, "UndefVarError: $(ex.var) not defined") showerror(io::IO, ::EOFError) = print(io, "EOFError: read end of file") showerror(io::IO, ex::ErrorException) = print(io, ex.msg) showerror(io::IO, ex::KeyError) = print(io, "KeyError: key $(repr(ex.key)) not found") @@ -241,6 +240,17 @@ showerror(io::IO, ex::InterruptException) = print(io, "InterruptException:") showerror(io::IO, ex::ArgumentError) = print(io, "ArgumentError: $(ex.msg)") showerror(io::IO, ex::AssertionError) = print(io, "AssertionError: $(ex.msg)") +function showerror(io::IO, ex::UndefVarError) + if ex.var in [:UTF16String, :UTF32String, :WString, :utf16, :utf32, :wstring] + return showerror(io, ErrorException(""" + `$(ex.var)` has been moved to the package LegacyStrings.jl: + Run Pkg.add("LegacyStrings") to install LegacyStrings on Julia v0.5-; + Then do `using LegacyStrings` to get `$(ex.var)`. + """)) + end + print(io, "UndefVarError: $(ex.var) not defined") +end + function showerror(io::IO, ex::MethodError) # ex.args is a tuple type if it was thrown from `invoke` and is # a tuple of the arguments otherwise. diff --git a/base/serialize.jl b/base/serialize.jl index adb0048310034..debf09797391f 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -21,8 +21,7 @@ const TAGS = Any[ Symbol, Tuple, Expr, # dummy entries, intentionally shadowed by earlier ones LineNumberNode, Slot, LabelNode, GotoNode, QuoteNode, :reserved23 #=was TopNode=#, TypeVar, Core.Box, LambdaInfo, - Module, #=UndefRefTag=#Symbol, Task, String, - UTF16String, UTF32String, Float16, + Module, #=UndefRefTag=#Symbol, Task, String, Float16, SimpleVector, #=BackrefTag=#Symbol, Method, GlobalRef, (), Bool, Any, :Any, Bottom, :reserved21, :reserved22, Type, @@ -42,7 +41,7 @@ const TAGS = Any[ 28, 29, 30, 31, 32 ] -const ser_version = 3 # do not make changes without bumping the version #! +const ser_version = 4 # do not make changes without bumping the version #! const NTAGS = length(TAGS) diff --git a/base/strings/io.jl b/base/strings/io.jl index 199fe0eefc785..151909d6405e4 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -324,3 +324,22 @@ function unindent(str::AbstractString, indent::Int; tabwidth=8) end takebuf_string(buf) end + +function convert(::Type{String}, chars::AbstractVector{Char}) + sprint(length(chars), io->begin + state = start(chars) + while !done(chars, state) + c, state = next(chars, state) + if '\ud7ff' < c && c + 1024 < '\ue000' + d, state = next(chars, state) + if '\ud7ff' < d - 1024 && d < '\ue000' + c = Char(0x10000 + ((UInt32(c) & 0x03ff) << 10) | (UInt32(d) & 0x03ff)) + else + write(io, c) + c = d + end + end + write(io, c) + end + end) +end diff --git a/base/strings/string.jl b/base/strings/string.jl index 47eb243b21647..c9637b746ef91 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -33,6 +33,8 @@ const utf8_trailing = [ ## required core functionality ## +is_valid_continuation(c) = ((c & 0xc0) == 0x80) + function endof(s::String) d = s.data i = length(d) @@ -239,109 +241,10 @@ function reverse(s::String) String(buf) end -## outputting UTF-8 strings ## - write(io::IO, s::String) = write(io, s.data) pointer(x::String) = pointer(x.data) pointer(x::String, i::Integer) = pointer(x.data)+(i-1) -## transcoding to UTF-8 ## - convert(::Type{String}, s::String) = s - -function convert(::Type{String}, dat::Vector{UInt8}) - # handle zero length string quickly - isempty(dat) && return empty_utf8 - # get number of bytes to allocate - len, flags, num4byte, num3byte, num2byte = unsafe_checkstring(dat) - if (flags & (UTF_LONG | UTF_SURROGATE)) == 0 - len = sizeof(dat) - @inbounds return String(copy!(Vector{UInt8}(len), 1, dat, 1, len)) - end - # Copy, but eliminate over-long encodings and surrogate pairs - len += num2byte + num3byte*2 + num4byte*3 - buf = Vector{UInt8}(len) - out = 0 - pos = 0 - @inbounds while out < len - ch::UInt32 = dat[pos += 1] - # Handle ASCII characters - if ch <= 0x7f - buf[out += 1] = ch - # Handle overlong < 0x100 - elseif ch < 0xc2 - buf[out += 1] = ((ch & 3) << 6) | (dat[pos += 1] & 0x3f) - # Handle 0x100-0x7ff - elseif ch < 0xe0 - buf[out += 1] = ch - buf[out += 1] = dat[pos += 1] - elseif ch != 0xed - buf[out += 1] = ch - buf[out += 1] = dat[pos += 1] - buf[out += 1] = dat[pos += 1] - # Copy 4-byte encoded value - ch >= 0xf0 && (buf[out += 1] = dat[pos += 1]) - # Handle surrogate pairs - else - ch = dat[pos += 1] - if ch < 0xa0 # not surrogate pairs - buf[out += 1] = 0xed - buf[out += 1] = ch - buf[out += 1] = dat[pos += 1] - else - # Pick up surrogate pairs (CESU-8 format) - ch = ((((((ch & 0x3f) << 6) | (dat[pos + 1] & 0x3f)) << 10) - + (((dat[pos + 3] & 0x3f)%UInt32 << 6) | (dat[pos + 4] & 0x3f))) - - 0x01f0c00) - pos += 4 - output_utf8_4byte!(buf, out, ch) - out += 4 - end - end - end - String(buf) -end - -""" -Converts an already validated vector of `UInt16` or `UInt32` to a `String` - -Input Arguments: - -* `dat` Vector of code units (`UInt16` or `UInt32`), explicit `\0` is not converted -* `len` length of output in bytes - -Returns: - -* `String` -""" -function encode_to_utf8{T<:Union{UInt16, UInt32}}(::Type{T}, dat, len) - buf = Vector{UInt8}(len) - out = 0 - pos = 0 - @inbounds while out < len - ch::UInt32 = dat[pos += 1] - # Handle ASCII characters - if ch <= 0x7f - buf[out += 1] = ch - # Handle 0x80-0x7ff - elseif ch < 0x800 - buf[out += 1] = 0xc0 | (ch >>> 6) - buf[out += 1] = 0x80 | (ch & 0x3f) - # Handle 0x10000-0x10ffff (if input is UInt32) - elseif ch > 0xffff # this is only for T == UInt32, should not be generated for UInt16 - output_utf8_4byte!(buf, out, ch) - out += 4 - # Handle surrogate pairs - elseif is_surrogate_codeunit(ch) - output_utf8_4byte!(buf, out, get_supplementary(ch, dat[pos += 1])) - out += 4 - # Handle 0x800-0xd7ff, 0xe000-0xffff UCS-2 characters - else - buf[out += 1] = 0xe0 | ((ch >>> 12) & 0x3f) - buf[out += 1] = 0x80 | ((ch >>> 6) & 0x3f) - buf[out += 1] = 0x80 | (ch & 0x3f) - end - end - String(buf) -end +convert(::Type{String}, v::Vector{UInt8}) = String(v) diff --git a/base/sysimg.jl b/base/sysimg.jl index 022c4a9dac16c..85dfeaf5ef75e 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -145,7 +145,6 @@ include("iobuffer.jl") include("char.jl") include("intfuncs.jl") include("strings/strings.jl") -include("unicode/unicode.jl") include("parse.jl") include("shell.jl") include("regex.jl") diff --git a/base/test.jl b/base/test.jl index b0382f3284f5c..f9790636f6906 100644 --- a/base/test.jl +++ b/base/test.jl @@ -1009,5 +1009,6 @@ end Base.convert(::Type{GenericString}, s::AbstractString) = GenericString(s) Base.endof(s::GenericString) = endof(s.string) Base.next(s::GenericString, i::Int) = next(s.string, i) +Base.reverseind(s::GenericString, i::Integer) = reverseind(s.string, i) end # module diff --git a/base/unicode/checkstring.jl b/base/unicode/checkstring.jl deleted file mode 100644 index 72e9eb9d31062..0000000000000 --- a/base/unicode/checkstring.jl +++ /dev/null @@ -1,238 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -## Functions to check validity of UTF-8, UTF-16, and UTF-32 encoded strings, -# and also to return information necessary to convert to other encodings - -## Return flags for check_string function - -const UTF_LONG = 1 ##< Long encodings are present -const UTF_LATIN1 = 2 ##< characters in range 0x80-0xFF present -const UTF_UNICODE2 = 4 ##< characters in range 0x100-0x7ff present -const UTF_UNICODE3 = 8 ##< characters in range 0x800-0xd7ff, 0xe000-0xffff -const UTF_UNICODE4 = 16 ##< non-BMP characters present -const UTF_SURROGATE = 32 ##< surrogate pairs present - -## Get a UTF-8 continuation byte, give error if invalid, return updated character value -@inline function get_continuation(ch::UInt32, byt::UInt8, pos) - if !is_valid_continuation(byt) - throw(UnicodeError(UTF_ERR_CONT, pos, byt)) - end - (ch << 6) | (byt & 0x3f) -end - -""" -Validates and calculates number of characters in a UTF-8,UTF-16 or UTF-32 encoded vector/string - -Warning: this function does not check the bounds of the start or end positions -Use `checkstring` to make sure the bounds are checked - -Input Arguments: - -* `dat` UTF-8 (`Vector{UInt8}`), UTF-16 (`Vector{UInt16}`) or UTF-32 (`Vector{UInt32}`, `AbstractString`) encoded string - -Optional Input Arguments: - -* `pos` start position (defaults to 1) -* `endpos` end position (defaults to `endof(dat)`) - -Keyword Arguments: - -* `accept_long_null` = `true` # Modified UTF-8 (`\\0` represented as `b\"\\xc0\\x80\"`) -* `accept_surrogates` = `true` # `CESU-8` -* `accept_long_char` = `false` # Accept arbitrary long encodings - -Returns: - -* (total characters, flags, 4-byte, 3-byte, 2-byte) - -Throws: - -* `UnicodeError` -""" -function unsafe_checkstring end - -function unsafe_checkstring(dat::AbstractVector{UInt8}, - pos = 1, - endpos = endof(dat) - ; - accept_long_null = true, - accept_surrogates = true, - accept_long_char = false) - local byt::UInt8, ch::UInt32, surr::UInt32 - flags::UInt = 0 - totalchar = num2byte = num3byte = num4byte = 0 - @inbounds while pos <= endpos - ch, pos = next(dat, pos) - totalchar += 1 - if ch > 0x7f - # Check UTF-8 encoding - if ch < 0xe0 - # 2-byte UTF-8 sequence (i.e. characters 0x80-0x7ff) - (pos > endpos) && throw(UnicodeError(UTF_ERR_SHORT, pos, ch)) - byt, pos = next(dat, pos) - ch = get_continuation(ch & 0x3f, byt, pos) - if ch > 0x7f - num2byte += 1 - flags |= (ch > 0xff) ? UTF_UNICODE2 : UTF_LATIN1 - elseif accept_long_char - flags |= UTF_LONG - elseif (ch == 0) && accept_long_null - flags |= UTF_LONG - else - throw(UnicodeError(UTF_ERR_LONG, pos, ch)) - end - elseif ch < 0xf0 - # 3-byte UTF-8 sequence (i.e. characters 0x800-0xffff) - (pos + 1 > endpos) && throw(UnicodeError(UTF_ERR_SHORT, pos, ch)) - byt, pos = next(dat, pos) - ch = get_continuation(ch & 0x0f, byt, pos) - byt, pos = next(dat, pos) - ch = get_continuation(ch, byt, pos) - # check for surrogate pairs, make sure correct - if is_surrogate_codeunit(ch) - !is_surrogate_lead(ch) && throw(UnicodeError(UTF_ERR_NOT_LEAD, pos-2, ch)) - # next character *must* be a trailing surrogate character - (pos + 2 > endpos) && throw(UnicodeError(UTF_ERR_MISSING_SURROGATE, pos-2, ch)) - byt, pos = next(dat, pos) - (byt != 0xed) && throw(UnicodeError(UTF_ERR_NOT_TRAIL, pos, byt)) - byt, pos = next(dat, pos) - surr = get_continuation(0x0000d, byt, pos) - byt, pos = next(dat, pos) - surr = get_continuation(surr, byt, pos) - !is_surrogate_trail(surr) && throw(UnicodeError(UTF_ERR_NOT_TRAIL, pos-2, surr)) - !accept_surrogates && throw(UnicodeError(UTF_ERR_SURROGATE, pos-2, surr)) - flags |= UTF_SURROGATE - num4byte += 1 - elseif ch > 0x07ff - num3byte += 1 - elseif accept_long_char - flags |= UTF_LONG - num2byte += 1 - else - throw(UnicodeError(UTF_ERR_LONG, pos-2, ch)) - end - elseif ch < 0xf5 - # 4-byte UTF-8 sequence (i.e. characters > 0xffff) - (pos + 2 > endpos) && throw(UnicodeError(UTF_ERR_SHORT, pos, ch)) - byt, pos = next(dat, pos) - ch = get_continuation(ch & 0x07, byt, pos) - byt, pos = next(dat, pos) - ch = get_continuation(ch, byt, pos) - byt, pos = next(dat, pos) - ch = get_continuation(ch, byt, pos) - if ch > 0x10ffff - throw(UnicodeError(UTF_ERR_INVALID, pos-3, ch)) - elseif ch > 0xffff - num4byte += 1 - elseif is_surrogate_codeunit(ch) - throw(UnicodeError(UTF_ERR_SURROGATE, pos-3, ch)) - elseif accept_long_char - # This is an overly long encoded character - flags |= UTF_LONG - if ch > 0x7ff - num3byte += 1 - elseif ch > 0x7f - num2byte += 1 - end - else - throw(UnicodeError(UTF_ERR_LONG, pos-2, ch)) - end - else - throw(UnicodeError(UTF_ERR_INVALID, pos, ch)) - end - end - end - num3byte != 0 && (flags |= UTF_UNICODE3) - num4byte != 0 && (flags |= UTF_UNICODE4) - return totalchar, flags, num4byte, num3byte, num2byte -end - -typealias AbstractString1632{Tel<:Union{UInt16,UInt32}} Union{AbstractVector{Tel}, AbstractString} - -function unsafe_checkstring( - dat::AbstractString1632, - pos = 1, - endpos = endof(dat) - ; - accept_long_null = true, - accept_surrogates = true, - accept_long_char = false) - local ch::UInt32 - flags::UInt = 0 - totalchar = num2byte = num3byte = num4byte = 0 - @inbounds while pos <= endpos - ch, pos = next(dat, pos) - totalchar += 1 - if ch > 0x7f - if ch < 0x100 - num2byte += 1 - flags |= UTF_LATIN1 - elseif ch < 0x800 - num2byte += 1 - flags |= UTF_UNICODE2 - elseif ch > 0x0ffff - (ch > 0x10ffff) && throw(UnicodeError(UTF_ERR_INVALID, pos, ch)) - num4byte += 1 - elseif !is_surrogate_codeunit(ch) - num3byte += 1 - elseif is_surrogate_lead(ch) - pos > endpos && throw(UnicodeError(UTF_ERR_MISSING_SURROGATE, pos, ch)) - # next character *must* be a trailing surrogate character - ch, pos = next(dat, pos) - !is_surrogate_trail(ch) && throw(UnicodeError(UTF_ERR_NOT_TRAIL, pos, ch)) - num4byte += 1 - if !(typeof(dat) <: AbstractVector{UInt16}) - !accept_surrogates && throw(UnicodeError(UTF_ERR_SURROGATE, pos, ch)) - flags |= UTF_SURROGATE - end - else - throw(UnicodeError(UTF_ERR_NOT_LEAD, pos, ch)) - end - end - end - num3byte != 0 && (flags |= UTF_UNICODE3) - num4byte != 0 && (flags |= UTF_UNICODE4) - return totalchar, flags, num4byte, num3byte, num2byte -end - -""" -Validates and calculates number of characters in a UTF-8,UTF-16 or UTF-32 encoded vector/string - -This function checks the bounds of the start and end positions -Use `unsafe_checkstring` to avoid that overhead if the bounds have already been checked - -Input Arguments: - -* `dat` UTF-8 (`Vector{UInt8}`), UTF-16 (`Vector{UInt16}`) or UTF-32 (`Vector{UInt32}`, `AbstractString`) encoded string - -Optional Input Arguments: - -* `startpos` start position (defaults to 1) -* `endpos` end position (defaults to `endof(dat)`) - -Keyword Arguments: - -* `accept_long_null` = `true` # Modified UTF-8 (`\\0` represented as `b\"\\xc0\\x80\"`) -* `accept_surrogates` = `true` # `CESU-8` -* `accept_long_char` = `false` # Accept arbitrary long encodings - -Returns: - -* (total characters, flags, 4-byte, 3-byte, 2-byte) - -Throws: - -* `UnicodeError` -""" -function checkstring end - -# No need to check bounds if using defaults -checkstring(dat; kwargs...) = unsafe_checkstring(dat, 1, endof(dat); kwargs...) - -# Make sure that beginning and end positions are bounds checked -function checkstring(dat, startpos, endpos = endof(dat); kwargs...) - checkbounds(dat,startpos) - checkbounds(dat,endpos) - endpos < startpos && throw(ArgumentError("End position ($endpos) is less than start position ($startpos)")) - unsafe_checkstring(dat, startpos, endpos; kwargs...) -end diff --git a/base/unicode/types.jl b/base/unicode/types.jl deleted file mode 100644 index 61f69d6a6638d..0000000000000 --- a/base/unicode/types.jl +++ /dev/null @@ -1,34 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -##\brief Base UTF16String type, has 16-bit NULL termination word after data, native byte order -# -# \throws UnicodeError - -immutable UTF16String <: AbstractString - data::Vector{UInt16} # includes 16-bit NULL termination after string chars - function UTF16String(data::Vector{UInt16}) - if length(data) < 1 || data[end] != 0 - throw(UnicodeError(UTF_ERR_NULL_16_TERMINATE, 0, 0)) - end - new(data) - end -end - -##\brief Base UTF32String type, has 32-bit NULL termination word after data, native byte order -# -# \throws UnicodeError - -immutable UTF32String <: DirectIndexString - data::Vector{UInt32} # includes 32-bit NULL termination after string chars - - function UTF32String(data::Vector{UInt32}) - if length(data) < 1 || data[end] != 0 - throw(UnicodeError(UTF_ERR_NULL_32_TERMINATE, 0, 0)) - end - new(data) - end -end -UTF32String(data::Vector{Char}) = UTF32String(reinterpret(UInt32, data)) - -isvalid{T<:Union{String,UTF16String,UTF32String}}(str::T) = isvalid(T, str.data) -isvalid{T<:Union{String,UTF16String,UTF32String}}(::Type{T}, str::T) = isvalid(T, str.data) diff --git a/base/unicode/unicode.jl b/base/unicode/unicode.jl deleted file mode 100644 index a8bd91e1a5a81..0000000000000 --- a/base/unicode/unicode.jl +++ /dev/null @@ -1,6 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -include("unicode/types.jl") -include("unicode/checkstring.jl") -include("unicode/utf16.jl") -include("unicode/utf32.jl") diff --git a/base/unicode/utf16.jl b/base/unicode/utf16.jl deleted file mode 100644 index 21cb4059056e5..0000000000000 --- a/base/unicode/utf16.jl +++ /dev/null @@ -1,275 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -# Quickly copy and set trailing \0 -@inline function fast_utf_copy{S <: Union{UTF16String, UTF32String}, T <: Union{UInt16, UInt32}}( - ::Type{S}, ::Type{T}, len, dat, flag::Bool=false) - S(setindex!(copy!(Vector{T}(len+1), 1, dat, 1, flag ? len : len+1), 0, len+1)) -end - -# Get rest of character ch from 3-byte UTF-8 sequence in dat -@inline function get_utf8_3byte(dat, pos, ch) - @inbounds return ((ch & 0xf) << 12) | (UInt32(dat[pos-1] & 0x3f) << 6) | (dat[pos] & 0x3f) -end -# Get rest of character ch from 4-byte UTF-8 sequence in dat -@inline function get_utf8_4byte(dat, pos, ch) - @inbounds return (((ch & 0x7) << 18) - | (UInt32(dat[pos-2] & 0x3f) << 12) - | (UInt32(dat[pos-1] & 0x3f) << 6) - | (dat[pos] & 0x3f)) -end - -# Output a character as a 4-byte UTF-8 sequence -@inline function output_utf8_4byte!(buf, out, ch) - @inbounds begin - buf[out + 1] = 0xf0 | (ch >>> 18) - buf[out + 2] = 0x80 | ((ch >>> 12) & 0x3f) - buf[out + 3] = 0x80 | ((ch >>> 6) & 0x3f) - buf[out + 4] = 0x80 | (ch & 0x3f) - end -end - -const empty_utf16 = UTF16String(UInt16[0]) - -function length(s::UTF16String) - d = s.data - len = length(d) - 1 - len == 0 && return 0 - cnum = 0 - for i = 1:len - @inbounds cnum += !is_surrogate_trail(d[i]) - end - cnum -end - -function endof(s::UTF16String) - d = s.data - i = length(d) - 1 - i == 0 && return i - return is_surrogate_codeunit(d[i]) ? i-1 : i -end - -get_supplementary(lead::Unsigned, trail::Unsigned) = (UInt32(lead-0xd7f7)<<10 + trail) - -function next(s::UTF16String, i::Int) - ch = s.data[i] - !is_surrogate_codeunit(ch) && return (Char(ch), i+1) - # check length, account for terminating \0 - i >= (length(s.data)-1) && throw(UnicodeError(UTF_ERR_MISSING_SURROGATE, i, UInt32(ch))) - !is_surrogate_lead(ch) && throw(UnicodeError(UTF_ERR_NOT_LEAD, i, ch)) - ct = s.data[i+1] - !is_surrogate_trail(ct) && throw((UTF_ERR_NOT_TRAIL, i, ch)) - Char(get_supplementary(ch, ct)), i+2 -end - -function reverseind(s::UTF16String, i::Integer) - j = length(s.data) - i - return is_surrogate_trail(s.data[j]) ? j-1 : j -end - -lastidx(s::UTF16String) = length(s.data) - 1 # s.data includes NULL terminator - -function reverse(s::UTF16String) - d = s.data - out = similar(d) - out[end] = 0 # NULL termination - n = length(d) - @inbounds for i = 1:n-1 - ch = d[n-i] - if is_surrogate_lead(ch) - out[i],out[i-1] = out[i-1],ch - else - out[i] = ch - end - end - UTF16String(out) -end - -sizeof(s::UTF16String) = sizeof(s.data) - sizeof(UInt16) - -function isvalid(::Type{UTF16String}, data::AbstractArray{UInt16}) - i = 1 - n = length(data) # this may include NULL termination; that's okay - @inbounds while i < n # check for unpaired surrogates - if is_surrogate_lead(data[i]) && is_surrogate_trail(data[i+1]) - i += 2 - elseif is_surrogate_codeunit(data[i]) - return false - else - i += 1 - end - end - return i > n || !is_surrogate_codeunit(data[i]) -end - -function convert(::Type{UTF16String}, str::AbstractString) - len, flags, num4byte = unsafe_checkstring(str) - buf = Vector{UInt16}(len+num4byte+1) - out = 0 - @inbounds for ch in str - c = UInt32(ch) - if c < 0x10000 - buf[out += 1] = UInt16(c) - else - # output surrogate pair - buf[out += 1] = UInt16(0xd7c0 + (c >>> 10)) - buf[out += 1] = UInt16(0xdc00 + (c & 0x3ff)) - end - end - @inbounds buf[out + 1] = 0 # NULL termination - UTF16String(buf) -end - -function convert(::Type{UTF16String}, str::String) - dat = str.data - # handle zero length string quickly - sizeof(dat) == 0 && return empty_utf16 - # Check that is correct UTF-8 encoding and get number of words needed - len, flags, num4byte = unsafe_checkstring(dat) - len += num4byte - buf = Vector{UInt16}(len+1) - @inbounds buf[len+1] = 0 - # Optimize case where no characters > 0x7f - flags == 0 && @inbounds return UTF16String(copy!(buf, dat)) - out = 0 - pos = 0 - @inbounds while out < len - ch::UInt32 = dat[pos += 1] - # Handle ASCII characters - if ch <= 0x7f - buf[out += 1] = ch - # Handle range 0x80-0x7ff - elseif ch < 0xe0 - buf[out += 1] = ((ch & 0x1f) << 6) | (dat[pos += 1] & 0x3f) - # Handle range 0x800-0xffff - elseif ch < 0xf0 - pos += 2 - buf[out += 1] = get_utf8_3byte(dat, pos, ch) - # Handle range 0x10000-0x10ffff - else - pos += 3 - ch = get_utf8_4byte(dat, pos, ch) - # output surrogate pair - buf[out += 1] = UInt16(0xd7c0 + (ch >>> 10)) - buf[out += 1] = UInt16(0xdc00 + (ch & 0x3ff)) - end - end - UTF16String(buf) -end - -function convert(::Type{String}, str::UTF16String) - dat = str.data - len = sizeof(dat) >>> 1 - # handle zero length string quickly - len <= 1 && return empty_utf8 - # get number of bytes to allocate - len, flags, num4byte, num3byte, num2byte = unsafe_checkstring(dat, 1, len-1) - flags == 0 && @inbounds return String(copy!(Vector{UInt8}(len), 1, dat, 1, len)) - return encode_to_utf8(UInt16, dat, len + num2byte + num3byte*2 + num4byte*3) -end - -""" -Converts an already validated UTF-32 encoded vector of `UInt32` to a `UTF16String` - -Input Arguments: - -* `dat` `Vector{UInt32}` of UTF-32 encoded data -* `len` length of output in 16-bit words - -Returns: - -* `UTF16String` -""" -function encode_to_utf16(dat, len) - buf = Vector{UInt16}(len) - @inbounds buf[len] = 0 # NULL termination - out = 0 - pos = 0 - @inbounds while out < len - ch = UInt32(dat[pos += 1]) - if ch > 0xffff - # Output surrogate pair for 0x10000-0x10ffff - buf[out += 1] = 0xd7c0 + (ch >>> 10) - ch = 0xdc00 + (ch & 0x3ff) - end - buf[out += 1] = ch - end - UTF16String(buf) -end - -convert(::Type{Vector{UInt16}}, str::UTF16String) = str.data -convert(::Type{Array{UInt16}}, str::UTF16String) = str.data - -convert(::Type{UTF16String}, str::UTF16String) = str - -unsafe_convert{T<:Union{Int16,UInt16}}(::Type{Ptr{T}}, s::UTF16String) = - convert(Ptr{T}, pointer(s)) - -convert(T::Type{UTF16String}, data::AbstractArray{UInt16}) = - convert(T, reshape(data, length(data))) - -convert(T::Type{UTF16String}, data::AbstractArray{Int16}) = - convert(T, reinterpret(UInt16, data)) - -function convert(::Type{UTF16String}, dat::AbstractVector{UInt16}) - len, flags, num4byte = unsafe_checkstring(dat) - @inbounds return fast_utf_copy(UTF16String, UInt16, len+num4byte, dat, true) -end - -function convert(T::Type{UTF16String}, bytes::AbstractArray{UInt8}) - isempty(bytes) && return UTF16String(UInt16[0]) - isodd(length(bytes)) && throw(UnicodeError(UTF_ERR_ODD_BYTES_16, length(bytes), 0)) - data = reinterpret(UInt16, bytes) - # check for byte-order mark (BOM): - if data[1] == 0xfeff # native byte order - d = Array{UInt16}(length(data)) - copy!(d,1, data,2, length(data)-1) - elseif data[1] == 0xfffe # byte-swapped - d = Array{UInt16}(length(data)) - for i = 2:length(data) - d[i-1] = bswap(data[i]) - end - else - d = Array{UInt16}(length(data) + 1) - copy!(d,1, data,1, length(data)) # assume native byte order - end - d[end] = 0 # NULL terminate - !isvalid(UTF16String, d) && throw(UnicodeError(UTF_ERR_INVALID_16,0,0)) - UTF16String(d) -end - -utf16(x) = convert(UTF16String, x) -utf16(p::Ptr{UInt16}, len::Integer) = utf16(unsafe_wrap(Array, p, len)) -utf16(p::Ptr{Int16}, len::Integer) = utf16(convert(Ptr{UInt16}, p), len) -function utf16(p::Union{Ptr{UInt16}, Ptr{Int16}}) - len = 0 - while unsafe_load(p, len+1) != 0; len += 1; end - utf16(p, len) -end - -function map(fun, str::UTF16String) - buf = UInt16[] - sizehint!(buf, length(str.data)) - for ch in str - c2 = fun(ch) - if !isa(c2, Char) - throw(UnicodeError(UTF_ERR_MAP_CHAR, 0, 0)) - end - uc = UInt32(c2) - if uc < 0x10000 - if is_surrogate_codeunit(UInt16(uc)) - throw(UnicodeError(UTF_ERR_INVALID_CHAR, 0, uc)) - end - push!(buf, UInt16(uc)) - elseif uc <= 0x10ffff - push!(buf, UInt16(0xd7c0 + (uc >> 10))) - push!(buf, UInt16(0xdc00 + (uc & 0x3ff))) - else - throw(UnicodeError(UTF_ERR_INVALID_CHAR, 0, uc)) - end - end - push!(buf, 0) - UTF16String(buf) -end - -cconvert(::Type{Cwstring}, v::Vector{UInt16}) = transcode(Cwchar_t, v) -cconvert(::Type{Cwstring}, s::UTF16String) = transcode(Cwchar_t, s.data) diff --git a/base/unicode/utf32.jl b/base/unicode/utf32.jl deleted file mode 100644 index 9250e9310909e..0000000000000 --- a/base/unicode/utf32.jl +++ /dev/null @@ -1,195 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -# UTF-32 basic functions -next(s::UTF32String, i::Int) = (Char(s.data[i]), i+1) -endof(s::UTF32String) = length(s.data) - 1 -length(s::UTF32String) = length(s.data) - 1 - -reverse(s::UTF32String) = UTF32String(reverse!(copy(s.data), 1, length(s))) - -sizeof(s::UTF32String) = sizeof(s.data) - sizeof(UInt32) - -const empty_utf32 = UTF32String(UInt32[0]) - -convert(::Type{UTF32String}, c::Char) = UTF32String(UInt32[c, 0]) -convert(::Type{UTF32String}, s::UTF32String) = s - -function convert(::Type{UTF32String}, str::AbstractString) - len, flags = unsafe_checkstring(str) - buf = Vector{UInt32}(len+1) - out = 0 - @inbounds for ch in str ; buf[out += 1] = ch ; end - @inbounds buf[out + 1] = 0 # NULL termination - UTF32String(buf) -end - -function convert(::Type{String}, str::UTF32String) - dat = str.data - len = sizeof(dat) >>> 2 - # handle zero length string quickly - len <= 1 && return empty_utf8 - # get number of bytes to allocate - len, flags, num4byte, num3byte, num2byte = unsafe_checkstring(dat, 1, len-1) - flags == 0 && @inbounds return String(copy!(Vector{UInt8}(len), 1, dat, 1, len)) - return encode_to_utf8(UInt32, dat, len + num2byte + num3byte*2 + num4byte*3) -end - -function convert(::Type{UTF32String}, str::String) - dat = str.data - # handle zero length string quickly - sizeof(dat) == 0 && return empty_utf32 - # Validate UTF-8 encoding, and get number of words to create - len, flags = unsafe_checkstring(dat) - # Optimize case where no characters > 0x7f - flags == 0 && @inbounds return fast_utf_copy(UTF32String, UInt32, len, dat, true) - # has multi-byte UTF-8 sequences - buf = Vector{UInt32}(len+1) - @inbounds buf[len+1] = 0 # NULL termination - local ch::UInt32, surr::UInt32 - out = 0 - pos = 0 - @inbounds while out < len - ch = dat[pos += 1] - # Handle ASCII characters - if ch <= 0x7f - buf[out += 1] = ch - # Handle range 0x80-0x7ff - elseif ch < 0xe0 - buf[out += 1] = ((ch & 0x1f) << 6) | (dat[pos += 1] & 0x3f) - # Handle range 0x800-0xffff - elseif ch < 0xf0 - pos += 2 - ch = get_utf8_3byte(dat, pos, ch) - # Handle surrogate pairs (should have been encoded in 4 bytes) - if is_surrogate_lead(ch) - # Build up 32-bit character from ch and trailing surrogate in next 3 bytes - pos += 3 - surr = ((UInt32(dat[pos-2] & 0xf) << 12) - | (UInt32(dat[pos-1] & 0x3f) << 6) - | (dat[pos] & 0x3f)) - ch = get_supplementary(ch, surr) - end - buf[out += 1] = ch - # Handle range 0x10000-0x10ffff - else - pos += 3 - buf[out += 1] = get_utf8_4byte(dat, pos, ch) - end - end - UTF32String(buf) -end - -function convert(::Type{UTF32String}, str::UTF16String) - dat = str.data - len = sizeof(dat) - # handle zero length string quickly (account for trailing \0) - len <= 2 && return empty_utf32 - # get number of words to create - len, flags, num4byte = unsafe_checkstring(dat, 1, len>>>1) - # No surrogate pairs, do optimized copy - (flags & UTF_UNICODE4) == 0 && @inbounds return UTF32String(copy!(Vector{Char}(len), dat)) - local ch::UInt32 - buf = Vector{UInt32}(len) - out = 0 - pos = 0 - @inbounds while out < len - ch = dat[pos += 1] - # check for surrogate pair - if is_surrogate_lead(ch) ; ch = get_supplementary(ch, dat[pos += 1]) ; end - buf[out += 1] = ch - end - UTF32String(buf) -end - -function convert(::Type{UTF16String}, str::UTF32String) - dat = str.data - len = sizeof(dat) - # handle zero length string quickly - len <= 4 && return empty_utf16 - # get number of words to allocate - len, flags, num4byte = unsafe_checkstring(dat, 1, len>>>2) - # optimized path, no surrogates - num4byte == 0 && @inbounds return UTF16String(copy!(Vector{UInt16}(len), dat)) - return encode_to_utf16(dat, len + num4byte) -end - -function convert(::Type{UTF32String}, dat::AbstractVector{UInt32}) - @inbounds return fast_utf_copy(UTF32String, UInt32, length(dat), dat, true) -end - -convert(::Type{UTF32String}, data::AbstractVector{Int32}) = - convert(UTF32String, reinterpret(UInt32, convert(Vector{T}, data))) - -convert(::Type{UTF32String}, data::AbstractVector{Char}) = - convert(UTF32String, map(UInt32, data)) - -convert{T<:AbstractString, S<:Union{UInt32,Char,Int32}}(::Type{T}, v::AbstractVector{S}) = - convert(T, utf32(v)) - -convert(::Type{Vector{UInt32}}, str::UTF32String) = str.data -convert(::Type{Array{UInt32}}, str::UTF32String) = str.data - -unsafe_convert{T<:Union{UInt32,Int32,Char}}(::Type{Ptr{T}}, s::UTF32String) = - convert(Ptr{T}, pointer(s)) - -function convert(T::Type{UTF32String}, bytes::AbstractArray{UInt8}) - isempty(bytes) && return empty_utf32 - length(bytes) & 3 != 0 && throw(UnicodeError(UTF_ERR_ODD_BYTES_32,0,0)) - data = reinterpret(UInt32, bytes) - # check for byte-order mark (BOM): - if data[1] == 0x0000feff # native byte order - d = Array{UInt32}(length(data)) - copy!(d,1, data, 2, length(data)-1) - elseif data[1] == 0xfffe0000 # byte-swapped - d = Array{UInt32}(length(data)) - for i = 2:length(data) - @inbounds d[i-1] = bswap(data[i]) - end - else - d = Array{UInt32}(length(data) + 1) - copy!(d, 1, data, 1, length(data)) # assume native byte order - end - d[end] = 0 # NULL terminate - UTF32String(d) -end - -cconvert(::Type{Cwstring}, v::Vector{UInt32}) = transcode(Cwchar_t, v) -cconvert(::Type{Cwstring}, s::UTF32String) = transcode(Cwchar_t, s.data) - -function isvalid(::Type{UTF32String}, str::Union{Vector{UInt32}, Vector{Char}}) - for c in str - @inbounds if !isvalid(Char, UInt32(c)) ; return false ; end - end - return true -end -isvalid(str::Vector{Char}) = isvalid(UTF32String, str) - -utf32(x) = convert(UTF32String, x) - -utf32(p::Ptr{UInt32}, len::Integer) = utf32(unsafe_wrap(Array, p, len)) -utf32(p::Union{Ptr{Char}, Ptr{Int32}}, len::Integer) = utf32(convert(Ptr{UInt32}, p), len) -function utf32(p::Union{Ptr{UInt32}, Ptr{Char}, Ptr{Int32}}) - len = 0 - while unsafe_load(p, len+1) != 0; len += 1; end - utf32(p, len) -end - -function map(f, s::UTF32String) - d = s.data - out = similar(d) - out[end] = 0 - - @inbounds for i = 1:(length(d)-1) - c2 = f(Char(d[i])) - if !isa(c2, Char) - throw(UnicodeError(UTF_ERR_MAP_CHAR, 0, 0)) - end - out[i] = (c2::Char) - end - UTF32String(out) -end - -pointer(x::Union{UTF16String,UTF32String}) = pointer(x.data) -pointer(x::Union{UTF16String,UTF32String}, i::Integer) = pointer(x)+(i-1)*sizeof(eltype(x.data)) -pointer{T<:Union{UTF16String,UTF32String}}(x::SubString{T}) = pointer(x.string.data) + x.offset*sizeof(eltype(x.string.data)) -pointer{T<:Union{UTF16String,UTF32String}}(x::SubString{T}, i::Integer) = pointer(x.string.data) + (x.offset + (i-1))*sizeof(eltype(x.string.data)) diff --git a/test/serialize.jl b/test/serialize.jl index c7c9e7e3d3c34..67a7249c2b91c 100644 --- a/test/serialize.jl +++ b/test/serialize.jl @@ -4,8 +4,8 @@ using Base.Test # Check that serializer hasn't gone out-of-frame @test Serializer.sertag(Symbol) == 2 -@test Serializer.sertag(()) == 46 -@test Serializer.sertag(false) == 122 +@test Serializer.sertag(()) == 44 +@test Serializer.sertag(false) == 120 function create_serialization_stream(f::Function) s = IOBuffer() diff --git a/test/strings/basic.jl b/test/strings/basic.jl index f8a63850a0253..5d62ddb81a842 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -246,28 +246,11 @@ let p = cstrdup("hello") Libc.free(p) end -# issue # 11389: Vector{UInt32} was copied with UTF32String, unlike Vector{Char} -a = UInt32[48,0] -b = UTF32String(a) -@test b == "0" -a[1] = 65 -@test b == "A" -c = Char['0','\0'] -d = UTF32String(c) -@test d == "0" -c[1] = 'A' -@test d == "A" - # iteration @test [c for c in "ḟøøƀäṙ"] == ['ḟ', 'ø', 'ø', 'ƀ', 'ä', 'ṙ'] @test [i for i in eachindex("ḟøøƀäṙ")] == [1, 4, 6, 8, 10, 12] @test [x for x in enumerate("ḟøøƀäṙ")] == [(1, 'ḟ'), (2, 'ø'), (3, 'ø'), (4, 'ƀ'), (5, 'ä'), (6, 'ṙ')] -# Issue #11140 -@test isvalid(utf32("a")) == true -@test isvalid(utf32("\x00")) == true -@test isvalid(UTF32String, UInt32[0xd800,0]) == false - # test all edge conditions for (val, pass) in ( (0, true), (0xd7ff, true), @@ -306,39 +289,9 @@ for (val, pass) in ( ) @test isvalid(String, val) == pass end -for (val, pass) in ( - (UInt16[0x0000], true), - (UInt16[0xd7ff,0], true), - (UInt16[0xd800,0], false), - (UInt16[0xdfff,0], false), - (UInt16[0xe000,0], true), - (UInt16[0xffff,0], true), - (UInt16[0xd800,0xdc00,0], true), - (UInt16[0xdbff,0xdfff,0], true), - (UInt16[0xd800,0x0100,0], false), - (UInt16[0xdc00,0x0100,0], false), - (UInt16[0xdc00,0xd800,0], false) - ) - @test isvalid(UTF16String, val) == pass -end -for (val, pass) in ( - (UInt32[0x0000], true), - (UInt32[0xd7ff,0], true), - (UInt32[0xd800,0], false), - (UInt32[0xdfff,0], false), - (UInt32[0xe000,0], true), - (UInt32[0xffff,0], true), - (UInt32[0x100000,0], true), - (UInt32[0x10ffff,0], true), - (UInt32[0x110000,0], false), - ) - @test isvalid(UTF32String, val) == pass -end # Issue #11203 @test isvalid(String, UInt8[]) == true -@test isvalid(UTF16String,UInt16[]) == true -@test isvalid(UTF32String,UInt32[]) == true # Check UTF-8 characters # Check ASCII range (true), @@ -411,21 +364,6 @@ end # 11482 -# isvalid -let s = "abcdef", u8 = "abcdef\uff", u16 = utf16(u8), u32 = utf32(u8), - bad32 = utf32(UInt32[65,0x110000]), badch = Char[0x110000][1] - - @test !isvalid(bad32) - @test !isvalid(badch) - @test isvalid(s) - @test isvalid(u8) - @test isvalid(u16) - @test isvalid(u32) - @test isvalid(String, u8) - @test isvalid(UTF16String, u16) - @test isvalid(UTF32String, u32) -end - # lower and upper @test uppercase("aBc") == "ABC" @test uppercase('A') == 'A' @@ -458,9 +396,9 @@ str = "abcdef\uff\uffff\u10ffffABCDEF" foomap(ch) = (ch > Char(65)) foobar(ch) = Char(0xd800) foobaz(ch) = reinterpret(Char, typemax(UInt32)) -@test_throws UnicodeError map(foomap, utf16(str)) -@test_throws UnicodeError map(foobar, utf16(str)) -@test_throws UnicodeError map(foobaz, utf16(str)) +@test_throws ArgumentError map(foomap, GenericString(str)) +@test map(foobar, GenericString(str)) == String(repeat(b"\ud800", outer=[17])) +@test map(foobaz, GenericString(str)) == String(repeat(b"\ufffd", outer=[17])) @test "a".*["b","c"] == ["ab","ac"] @test ["b","c"].*"a" == ["ba","ca"] diff --git a/test/strings/io.jl b/test/strings/io.jl index 41df929335f39..1ab6be0035c44 100644 --- a/test/strings/io.jl +++ b/test/strings/io.jl @@ -136,76 +136,6 @@ end @test "\x0f" == unescape_string("\\x0f") @test "\x0F" == unescape_string("\\x0F") -extrapath = is_windows() ? joinpath(JULIA_HOME,"..","Git","usr","bin")*";" : "" -withenv("PATH" => extrapath * ENV["PATH"]) do -if !success(`iconv --version`) - warn("iconv not found, skipping unicode tests!") - is_windows() && warn("Use WinRPM.install(\"win_iconv\") to run these tests") -else - # Create unicode test data directory - unicodedir = mktempdir() - - # Use perl to generate the primary data - primary_encoding = "UTF-32BE" - primary_path = replace(joinpath(unicodedir, primary_encoding*".unicode"),"\\","\\\\\\\\") - run(`perl -e " - $$fname = \"$primary_path\"; - open(UNICODEF, \">\", \"$$fname\") or die \"can\'t open $$fname: $$!\"; - binmode(UNICODEF); - print UNICODEF pack \"N*\", 0xfeff, 0..0xd7ff, 0xe000..0x10ffff; - close(UNICODEF);"` ) - - # Use iconv to generate the other data - for encoding in ["UTF-32LE", "UTF-16BE", "UTF-16LE", "UTF-8"] - output_path = joinpath(unicodedir, encoding*".unicode") - f = Base.Filesystem.open(output_path,Base.JL_O_WRONLY|Base.JL_O_CREAT,Base.S_IRUSR | Base.S_IWUSR | Base.S_IRGRP | Base.S_IROTH) - run(pipeline(`iconv -f $primary_encoding -t $encoding $primary_path`, f)) - Base.Filesystem.close(f) - end - - f=open(joinpath(unicodedir,"UTF-32LE.unicode")) - str1 = utf32(read(f, UInt32, 1112065)[2:end]) - close(f) - - f=open(joinpath(unicodedir,"UTF-8.unicode")) - str2 = String(read(f, UInt8, 4382595)[4:end]) - close(f) - @test str1 == str2 - - @test str1 == utf16(read(joinpath(unicodedir,"UTF-16LE.unicode"), - UInt16, 2160641)[2:end]) - - @test str1 == utf16(read(joinpath(unicodedir,"UTF-16LE.unicode"), - UInt8, 2160641*2)) - - @test str1 == utf16(read(joinpath(unicodedir,"UTF-16BE.unicode"), - UInt8, 2160641*2)) - - - @test str1 == utf32(read(joinpath(unicodedir,"UTF-32LE.unicode"), - UInt8, 1112065*4)) - - @test str1 == utf32(read(joinpath(unicodedir,"UTF-32BE.unicode"), - UInt8, 1112065*4)) - - - str1 = "∀ ε > 0, ∃ δ > 0: |x-y| < δ ⇒ |f(x)-f(y)| < ε" - str2 = UTF32String(UInt32[ - 8704, 32, 949, 32, 62, 32, 48, 44, 32, 8707, 32, - 948, 32, 62, 32, 48, 58, 32, 124, 120, 45, 121, 124, - 32, 60, 32, 948, 32, 8658, 32, 124, 102, 40, 120, - 41, 45, 102, 40, 121, 41, 124, 32, 60, 32, 949 - ,0]) - @test str1 == str2 - - # Cleanup unicode data - for encoding in ["UTF-32BE", "UTF-32LE", "UTF-16BE", "UTF-16LE", "UTF-8"] - rm(joinpath(unicodedir,encoding*".unicode")) - end - rm(unicodedir) -end -end - # Tests of join() @test join([]) == "" @test join(["a"],"?") == "a" diff --git a/test/strings/types.jl b/test/strings/types.jl index 48991654a7a81..0a5aba56d48ad 100644 --- a/test/strings/types.jl +++ b/test/strings/types.jl @@ -163,27 +163,29 @@ rs = RevString("foobar") @test rsplit(RevString("ailuj"),'l') == ["ju","ia"] @test parse(Float64,RevString("64")) === 46.0 -# reverseind -for T in (String, UTF16String, UTF32String) - for prefix in ("", "abcd", "\U0001d6a4\U0001d4c1", "\U0001d6a4\U0001d4c1c", " \U0001d6a4\U0001d4c1") - for suffix in ("", "abcde", "\U0001d4c1β\U0001d6a4", "\U0001d4c1β\U0001d6a4c", " \U0001d4c1β\U0001d6a4") - for c in ('X', 'δ', '\U0001d6a5') - s = convert(T, string(prefix, c, suffix)) - ri = search(reverse(s), c) - @test reverse(s) == RevString(s) - @test c == s[reverseind(s, ri)] == reverse(s)[ri] - s = RevString(s) - ri = search(reverse(s), c) - @test c == s[reverseind(s, ri)] == reverse(s)[ri] - s = convert(T, string(prefix, prefix, c, suffix, suffix)) - pre = convert(T, prefix) - sb = SubString(s, nextind(pre, endof(pre)), endof(convert(T, string(prefix, prefix, c, suffix)))) - ri = search(reverse(sb), c) - @test c == sb[reverseind(sb, ri)] == reverse(sb)[ri] - end - end - end -end +# # reverseind +# for T in (String, GenericString) +# for prefix in ("", "abcd", "\U0001d6a4\U0001d4c1", "\U0001d6a4\U0001d4c1c", " \U0001d6a4\U0001d4c1") +# for suffix in ("", "abcde", "\U0001d4c1β\U0001d6a4", "\U0001d4c1β\U0001d6a4c", " \U0001d4c1β\U0001d6a4") +# for c in ('X', 'δ', '\U0001d6a5') +# @show (T,prefix,suffix,c) +# s = convert(T, string(prefix, c, suffix)) +# r = convert(T, String(reverse(s))) +# ri = search(r, c) +# @test r == RevString(s) +# @test c == s[reverseind(s, ri)] == r[ri] +# s = RevString(s) +# ri = search(r, c) +# @test c == s[reverseind(s, ri)] == r[ri] +# s = convert(T, string(prefix, prefix, c, suffix, suffix)) +# pre = convert(T, prefix) +# sb = SubString(s, nextind(pre, endof(pre)), endof(convert(T, string(prefix, prefix, c, suffix)))) +# ri = search(reverse(sb), c) +# @test c == sb[reverseind(sb, ri)] == reverse(sb)[ri] +# end +# end +# end +# end ## Repeat strings ## diff --git a/test/strings/util.jl b/test/strings/util.jl index d52189554f320..1aeb5b86e143d 100644 --- a/test/strings/util.jl +++ b/test/strings/util.jl @@ -21,7 +21,7 @@ for s in ("", " ", " abc", "abc ", " abc "), f in (lstrip, rstrip, strip) fs = f(s) - for T = (String, UTF16String, UTF32String) + for T = (String, GenericString) t = convert(T,s) ft = f(t) @test s == t diff --git a/test/unicode.jl b/test/unicode.jl index 21f3dd7d48fb4..18666448ef169 100644 --- a/test/unicode.jl +++ b/test/unicode.jl @@ -1,9 +1,5 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license include("unicode/UnicodeError.jl") -include("unicode/types.jl") -include("unicode/checkstring.jl") include("unicode/utf8.jl") -include("unicode/utf16.jl") -include("unicode/utf32.jl") include("unicode/utf8proc.jl") diff --git a/test/unicode/utf16.jl b/test/unicode/utf16.jl deleted file mode 100644 index 1c8e31cdece98..0000000000000 --- a/test/unicode/utf16.jl +++ /dev/null @@ -1,23 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -# UTF16 -u8 = "\U10ffff\U1d565\U1d7f6\U00066\U2008a" -u16 = utf16(u8) -@test sizeof(u16) == 18 -@test length(u16.data) == 10 && u16.data[end] == 0 -@test length(u16) == 5 -@test String(u16) == u8 -@test collect(u8) == collect(u16) -@test u8 == utf16(u16.data[1:end-1]) == utf16(copy!(Array(UInt8, 18), 1, reinterpret(UInt8, u16.data), 1, 18)) -@test u8 == utf16(pointer(u16)) == utf16(convert(Ptr{Int16}, pointer(u16))) -@test_throws UnicodeError utf16(utf32(Char(0x120000))) -@test_throws UnicodeError utf16(UInt8[1,2,3]) - -# Add tests for full coverage -@test convert(UTF16String, "test") == "test" -@test convert(UTF16String, u16) == u16 -@test convert(UTF16String, UInt16[[0x65, 0x66] [0x67, 0x68]]) == "efgh" -@test convert(UTF16String, Int16[[0x65, 0x66] [0x67, 0x68]]) == "efgh" -@test map(lowercase, utf16("TEST\U1f596")) == "test\U1f596" -@test typeof(Base.unsafe_convert(Ptr{UInt16}, utf16("test"))) == Ptr{UInt16} - diff --git a/test/unicode/utf32.jl b/test/unicode/utf32.jl deleted file mode 100644 index f4455271be054..0000000000000 --- a/test/unicode/utf32.jl +++ /dev/null @@ -1,258 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -# UTF32 -u8 = "\U10ffff\U1d565\U1d7f6\U00066\U2008a" -u32 = utf32(u8) -@test sizeof(u32) == 20 -@test length(u32.data) == 6 && u32.data[end] == 0 -@test length(u32) == 5 -@test String(u32) == u8 -@test collect(u8) == collect(u32) -@test u8 == utf32(u32.data[1:end-1]) == utf32(copy!(Array{UInt8}(20), 1, reinterpret(UInt8, u32.data), 1, 20)) -@test u8 == utf32(pointer(u32)) == utf32(convert(Ptr{Int32}, pointer(u32))) -@test_throws UnicodeError utf32(UInt8[1,2,3]) - -# issue #11551 (#11004,#10959) -function tstcvt(strUTF8::String, strUTF16::UTF16String, strUTF32::UTF32String) - @test utf16(strUTF8) == strUTF16 - @test utf32(strUTF8) == strUTF32 - @test String(strUTF16) == strUTF8 - @test utf32(strUTF16) == strUTF32 - @test String(strUTF32) == strUTF8 - @test utf16(strUTF32) == strUTF16 -end - -# Create some ASCII, UTF8, UTF16, and UTF32 strings - -strAscii = "abcdefgh" -strA_UTF8 = ("abcdefgh\uff")[1:8] -strL_UTF8 = "abcdef\uff\uff" -str2_UTF8 = "abcd\uff\uff\u7ff\u7ff" -str3_UTF8 = "abcd\uff\uff\u7fff\u7fff" -str4_UTF8 = "abcd\uff\u7ff\u7fff\U7ffff" -strS_UTF8 = String(b"abcd\xc3\xbf\xdf\xbf\xe7\xbf\xbf\xed\xa0\x80\xed\xb0\x80") -strC_UTF8 = String(b"abcd\xc3\xbf\xdf\xbf\xe7\xbf\xbf\U10000") -strz_UTF8 = String(b"abcd\xc3\xbf\xdf\xbf\xe7\xbf\xbf\0") -strZ = b"abcd\xc3\xbf\xdf\xbf\xe7\xbf\xbf\xc0\x80" - -strA_UTF16 = utf16(strA_UTF8) -strL_UTF16 = utf16(strL_UTF8) -str2_UTF16 = utf16(str2_UTF8) -str3_UTF16 = utf16(str3_UTF8) -str4_UTF16 = utf16(str4_UTF8) -strS_UTF16 = utf16(strS_UTF8) - -strA_UTF32 = utf32(strA_UTF8) -strL_UTF32 = utf32(strL_UTF8) -str2_UTF32 = utf32(str2_UTF8) -str3_UTF32 = utf32(str3_UTF8) -str4_UTF32 = utf32(str4_UTF8) -strS_UTF32 = utf32(strS_UTF8) - -@test String(strAscii) == strAscii -@test utf16(strAscii) == strAscii -@test utf32(strAscii) == strAscii - -tstcvt(strA_UTF8,strA_UTF16,strA_UTF32) -tstcvt(strL_UTF8,strL_UTF16,strL_UTF32) -tstcvt(str2_UTF8,str2_UTF16,str2_UTF32) -tstcvt(str3_UTF8,str3_UTF16,str3_UTF32) -tstcvt(str4_UTF8,str4_UTF16,str4_UTF32) - -# Test converting surrogate pairs -@test utf16(strS_UTF8) == strC_UTF8 -@test utf32(strS_UTF8) == strC_UTF8 -@test String(strS_UTF16) == strC_UTF8 -@test utf32(strS_UTF16) == strC_UTF8 -@test String(strS_UTF32) == strC_UTF8 -@test utf16(strS_UTF32) == strC_UTF8 - -# Test converting overlong \0 -@test convert(String, strZ) == strz_UTF8 -@test utf16(String(strZ)) == strz_UTF8 -@test utf32(String(strZ)) == strz_UTF8 - -# Test invalid sequences - -strval(::Type{String}, dat) = dat -strval(::Union{Type{UTF16String},Type{UTF32String}}, dat) = String(dat) - -byt = 0x0 -for T in (String, UTF16String, UTF32String) - try - # Continuation byte not after lead - for byt in 0x80:0xbf - @test_throws UnicodeError convert(T, strval(T, UInt8[byt])) - end - - # Test lead bytes - for byt in 0xc0:0xff - # Single lead byte at end of string - @test_throws UnicodeError convert(T, strval(T, UInt8[byt])) - # Lead followed by non-continuation character < 0x80 - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0])) - # Lead followed by non-continuation character > 0xbf - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0xc0])) - end - - # Test overlong 2-byte - for byt in 0x81:0xbf - @test_throws UnicodeError convert(T, strval(T, UInt8[0xc0,byt])) - end - for byt in 0x80:0xbf - @test_throws UnicodeError convert(T, strval(T, UInt8[0xc1,byt])) - end - - # Test overlong 3-byte - for byt in 0x80:0x9f - @test_throws UnicodeError convert(T, strval(T, UInt8[0xe0,byt,0x80])) - end - - # Test overlong 4-byte - for byt in 0x80:0x8f - @test_throws UnicodeError convert(T, strval(T, UInt8[0xef,byt,0x80,0x80])) - end - - # Test 4-byte > 0x10ffff - for byt in 0x90:0xbf - @test_throws UnicodeError convert(T, strval(T, UInt8[0xf4,byt,0x80,0x80])) - end - for byt in 0xf5:0xf7 - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0x80,0x80])) - end - - # Test 5-byte - for byt in 0xf8:0xfb - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0x80,0x80,0x80])) - end - - # Test 6-byte - for byt in 0xfc:0xfd - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0x80,0x80,0x80,0x80])) - end - - # Test 7-byte - @test_throws UnicodeError convert(T, strval(T, UInt8[0xfe,0x80,0x80,0x80,0x80,0x80,0x80])) - - # Three and above byte sequences - for byt in 0xe0:0xef - # Lead followed by only 1 continuation byte - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80])) - # Lead ended by non-continuation character < 0x80 - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0])) - # Lead ended by non-continuation character > 0xbf - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0xc0])) - end - - # 3-byte encoded surrogate character(s) - # Single surrogate - @test_throws UnicodeError convert(T, strval(T, UInt8[0xed,0xa0,0x80])) - # Not followed by surrogate - @test_throws UnicodeError convert(T, strval(T, UInt8[0xed,0xa0,0x80,0xed,0x80,0x80])) - # Trailing surrogate first - @test_throws UnicodeError convert(T, strval(T, UInt8[0xed,0xb0,0x80,0xed,0xb0,0x80])) - # Followed by lead surrogate - @test_throws UnicodeError convert(T, strval(T, UInt8[0xed,0xa0,0x80,0xed,0xa0,0x80])) - - # Four byte sequences - for byt in 0xf0:0xf4 - # Lead followed by only 2 continuation bytes - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0x80])) - # Lead followed by non-continuation character < 0x80 - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0x80,0])) - # Lead followed by non-continuation character > 0xbf - @test_throws UnicodeError convert(T, strval(T, UInt8[byt,0x80,0x80,0xc0])) - end - catch exp - println("Error checking $T: $byt") - throw(exp) - end -end - -# 12268 -for (fun, S, T) in ((utf16, UInt16, UTF16String), (utf32, UInt32, UTF32String)) - # AbstractString - str = "abcd\0\uff\u7ff\u7fff\U7ffff" - tst = SubString(convert(T,str),4) - cmp = Char['d','\0','\uff','\u7ff','\u7fff','\U7ffff'] - cmp32 = UInt32['d','\0','\uff','\u7ff','\u7fff','\U7ffff','\0'] - cmp16 = UInt16[0x0064,0x0000,0x00ff,0x07ff,0x7fff,0xd9bf,0xdfff,0x0000] - x = fun(tst) - cmpx = (S == UInt16 ? cmp16 : cmp32) - @test typeof(tst) == SubString{T} - @test convert(T, tst) == str[4:end] - @test convert(Vector{Char}, x) == cmp - # Vector{T} / Array{T} - @test convert(Vector{S}, x) == cmpx - @test convert(Array{S}, x) == cmpx - # Embedded nul checking - @test Base.containsnul(x) - @test Base.containsnul(tst) - # map - @test_throws UnicodeError map(islower, x) - @test_throws ArgumentError map(islower, tst) - # SubArray conversion - subarr = view(cmp, 1:6) - @test convert(T, subarr) == str[4:end] -end - -# Char to UTF32String -@test utf32('\U7ffff') == utf32("\U7ffff") -@test convert(UTF32String, '\U7ffff') == utf32("\U7ffff") - -@test isvalid(UTF32String, Char['d','\uff','\u7ff','\u7fff','\U7ffff']) -@test reverse(utf32("abcd \uff\u7ff\u7fff\U7ffff")) == utf32("\U7ffff\u7fff\u7ff\uff dcba") - -# Test pointer() functions -let str = ascii("this ") - u8 = String(str) - u16 = utf16(str) - u32 = utf32(str) - pa = pointer(str) - p8 = pointer(u8) - p16 = pointer(u16) - p32 = pointer(u32) - @test typeof(pa) == Ptr{UInt8} - @test unsafe_load(pa,1) == 0x74 - @test typeof(p8) == Ptr{UInt8} - @test unsafe_load(p8,1) == 0x74 - @test typeof(p16) == Ptr{UInt16} - @test unsafe_load(p16,1) == 0x74 - @test typeof(p32) == Ptr{UInt32} - @test unsafe_load(p32,1) == 0x74 - pa = pointer(str, 2) - p8 = pointer(u8, 2) - p16 = pointer(u16, 2) - p32 = pointer(u32, 2) - @test typeof(pa) == Ptr{UInt8} - @test unsafe_load(pa,1) == 0x68 - @test typeof(p8) == Ptr{UInt8} - @test unsafe_load(p8,1) == 0x68 - @test typeof(p16) == Ptr{UInt16} - @test unsafe_load(p16,1) == 0x68 - @test typeof(p32) == Ptr{UInt32} - @test unsafe_load(p32,1) == 0x68 - s8 = SubString{String}(u8, 3, 5) - s16 = SubString{UTF16String}(u16, 3, 5) - s32 = SubString{UTF32String}(u32, 3, 5) - p8 = pointer(s8) - p16 = pointer(s16) - p32 = pointer(s32) - @test typeof(p8) == Ptr{UInt8} - @test unsafe_load(p8,1) == 0x69 - @test typeof(p16) == Ptr{UInt16} - @test unsafe_load(p16,1) == 0x69 - @test typeof(p32) == Ptr{UInt32} - @test unsafe_load(p32,1) == 0x69 - p8 = pointer(s8, 2) - p16 = pointer(s16, 2) - p32 = pointer(s32, 2) - @test typeof(p8) == Ptr{UInt8} - @test unsafe_load(p8,1) == 0x73 - @test typeof(p16) == Ptr{UInt16} - @test unsafe_load(p16,1) == 0x73 - @test typeof(p32) == Ptr{UInt32} - @test unsafe_load(p32,1) == 0x73 -end - -@test isvalid(Char['f','o','o','b','a','r']) diff --git a/test/unicode/utf8.jl b/test/unicode/utf8.jl index 8827642ba9c3a..0a4665146b697 100644 --- a/test/unicode/utf8.jl +++ b/test/unicode/utf8.jl @@ -3,9 +3,9 @@ ## Test for CESU-8 sequences let ch = 0x10000 - for hichar = 0xd800:0xdbff - for lochar = 0xdc00:0xdfff - @test convert(String, String(Char[hichar, lochar]).data) == string(Char(ch)) + for hi = 0xd800:0xdbff + for lo = 0xdc00:0xdfff + @test convert(String, String(Char[hi, lo]).data) == string(Char(ch)) ch += 1 end end diff --git a/test/unicode/utf8proc.jl b/test/unicode/utf8proc.jl index 0f1e3b0727e26..3078f05fce080 100644 --- a/test/unicode/utf8proc.jl +++ b/test/unicode/utf8proc.jl @@ -234,7 +234,7 @@ let grphtest = (("b\u0300lahβlahb\u0302láh", ["b\u0300","l","a","h", "\U1d4c1\u0300"]), ("x",["x"]), ("abc",["a","b","c"])) - for T in (String,utf16,utf32) + for T in (String,GenericString) for nf in (:NFC, :NFD) for (s, g) in grphtest s_ = T(normalize_string(s, nf)) From b63c919c553514766b4d9bdadc6efe748460b361 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat <nalimilan@club.fr> Date: Sat, 2 Jul 2016 23:12:39 +0200 Subject: [PATCH 0246/1117] Add reverseind() for AbstractString, re-enable tests with GenericString reverse() for GenericString/AbstractString returns a RevString, whose indexing behavior is very different from a reverse()'d String which is returned for String. Thus, calling reverseind() on the underlying String object is not correct for GenericString. Add a generic but O(n) method for AbstractString and use it for GenericString. --- base/strings/string.jl | 4 +--- base/strings/types.jl | 7 +++---- base/test.jl | 1 - test/strings/types.jl | 47 +++++++++++++++++++++--------------------- 4 files changed, 28 insertions(+), 31 deletions(-) diff --git a/base/strings/string.jl b/base/strings/string.jl index c9637b746ef91..6395c03eac5c7 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -104,7 +104,7 @@ function first_utf8_byte(ch::Char) end function reverseind(s::String, i::Integer) - j = lastidx(s) + 1 - i + j = length(s.data) + 1 - i d = s.data while is_valid_continuation(d[j]) j -= 1 @@ -116,8 +116,6 @@ end sizeof(s::String) = sizeof(s.data) -lastidx(s::String) = length(s.data) - isvalid(s::String, i::Integer) = (1 <= i <= endof(s.data)) && !is_valid_continuation(s.data[i]) diff --git a/base/strings/types.jl b/base/strings/types.jl index fe3470bac5721..b98ea229f57ab 100644 --- a/base/strings/types.jl +++ b/base/strings/types.jl @@ -118,12 +118,11 @@ reverse(s::RevString) = s.string ## reverse an index i so that reverse(s)[i] == s[reverseind(s,i)] +reverseind(s::AbstractString, i) = chr2ind(s, length(s) + 1 - ind2chr(reverse(s), i)) reverseind(s::Union{DirectIndexString,SubString{DirectIndexString}}, i::Integer) = length(s) + 1 - i reverseind(s::RevString, i::Integer) = endof(s) - i + 1 -lastidx(s::AbstractString) = nextind(s, endof(s)) - 1 -lastidx(s::DirectIndexString) = length(s) -reverseind(s::SubString, i::Integer) = - reverseind(s.string, lastidx(s.string)-s.offset-s.endof+i) - s.offset +reverseind(s::SubString{String}, i::Integer) = + reverseind(s.string, nextind(s.string, endof(s.string))-s.offset-s.endof+i-1) - s.offset ## efficient representation of repeated strings ## diff --git a/base/test.jl b/base/test.jl index f9790636f6906..b0382f3284f5c 100644 --- a/base/test.jl +++ b/base/test.jl @@ -1009,6 +1009,5 @@ end Base.convert(::Type{GenericString}, s::AbstractString) = GenericString(s) Base.endof(s::GenericString) = endof(s.string) Base.next(s::GenericString, i::Int) = next(s.string, i) -Base.reverseind(s::GenericString, i::Integer) = reverseind(s.string, i) end # module diff --git a/test/strings/types.jl b/test/strings/types.jl index 0a5aba56d48ad..10e8ca406682b 100644 --- a/test/strings/types.jl +++ b/test/strings/types.jl @@ -163,29 +163,30 @@ rs = RevString("foobar") @test rsplit(RevString("ailuj"),'l') == ["ju","ia"] @test parse(Float64,RevString("64")) === 46.0 -# # reverseind -# for T in (String, GenericString) -# for prefix in ("", "abcd", "\U0001d6a4\U0001d4c1", "\U0001d6a4\U0001d4c1c", " \U0001d6a4\U0001d4c1") -# for suffix in ("", "abcde", "\U0001d4c1β\U0001d6a4", "\U0001d4c1β\U0001d6a4c", " \U0001d4c1β\U0001d6a4") -# for c in ('X', 'δ', '\U0001d6a5') -# @show (T,prefix,suffix,c) -# s = convert(T, string(prefix, c, suffix)) -# r = convert(T, String(reverse(s))) -# ri = search(r, c) -# @test r == RevString(s) -# @test c == s[reverseind(s, ri)] == r[ri] -# s = RevString(s) -# ri = search(r, c) -# @test c == s[reverseind(s, ri)] == r[ri] -# s = convert(T, string(prefix, prefix, c, suffix, suffix)) -# pre = convert(T, prefix) -# sb = SubString(s, nextind(pre, endof(pre)), endof(convert(T, string(prefix, prefix, c, suffix)))) -# ri = search(reverse(sb), c) -# @test c == sb[reverseind(sb, ri)] == reverse(sb)[ri] -# end -# end -# end -# end +# reverseind +for T in (String, GenericString) + for prefix in ("", "abcd", "\U0001d6a4\U0001d4c1", "\U0001d6a4\U0001d4c1c", " \U0001d6a4\U0001d4c1") + for suffix in ("", "abcde", "\U0001d4c1β\U0001d6a4", "\U0001d4c1β\U0001d6a4c", " \U0001d4c1β\U0001d6a4") + for c in ('X', 'δ', '\U0001d6a5') + s = convert(T, string(prefix, c, suffix)) + r = reverse(s) + ri = search(r, c) + @test r == RevString(s) + @test c == s[reverseind(s, ri)] == r[ri] + s = RevString(s) + r = reverse(s) + ri = search(r, c) + @test c == s[reverseind(s, ri)] == r[ri] + s = convert(T, string(prefix, prefix, c, suffix, suffix)) + pre = convert(T, prefix) + sb = SubString(s, nextind(pre, endof(pre)), endof(convert(T, string(prefix, prefix, c, suffix)))) + r = reverse(sb) + ri = search(r, c) + @test c == sb[reverseind(sb, ri)] == r[ri] + end + end + end +end ## Repeat strings ## From dc5d873883050154d27235cd5b8df82d9b2a7d51 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 5 Jul 2016 20:24:24 -0700 Subject: [PATCH 0247/1117] Delete the now-unused UTF_ERR constants --- base/strings/errors.jl | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/base/strings/errors.jl b/base/strings/errors.jl index b1b694df7fdf2..c0ca38ff28db2 100644 --- a/base/strings/errors.jl +++ b/base/strings/errors.jl @@ -3,23 +3,7 @@ ## Error messages for Unicode / UTF support const UTF_ERR_SHORT = "invalid UTF-8 sequence starting at index <<1>> (0x<<2>> missing one or more continuation bytes)" -const UTF_ERR_CONT = "invalid UTF-8 sequence starting at index <<1>> (0x<<2>> is not a continuation byte)" -const UTF_ERR_LONG = "invalid UTF-8 sequence, overlong encoding starting at index <<1>> (0x<<2>>)" -const UTF_ERR_NOT_LEAD = "not a leading Unicode surrogate code unit at index <<1>> (0x<<2>>)" -const UTF_ERR_NOT_TRAIL = "not a trailing Unicode surrogate code unit at index <<1>> (0x<<2>>)" -const UTF_ERR_NOT_SURROGATE = "not a valid Unicode surrogate code unit at index <<1>> (0x<<2>>)" -const UTF_ERR_MISSING_SURROGATE = "missing trailing Unicode surrogate code unit after index <<1>> (0x<<2>>)" -const UTF_ERR_INVALID = "invalid Unicode character starting at index <<1>> (0x<<2>> > 0x10ffff)" -const UTF_ERR_SURROGATE = "surrogate encoding not allowed in UTF-8 or UTF-32, at index <<1>> (0x<<2>>)" -const UTF_ERR_NULL_16_TERMINATE = "UTF16String data must be NULL-terminated" -const UTF_ERR_NULL_32_TERMINATE = "UTF32String data must be NULL-terminated" -const UTF_ERR_ODD_BYTES_16 = "UTF16String can't have odd number of bytes <<1>>" -const UTF_ERR_ODD_BYTES_32 = "UTF32String must have multiple of 4 bytes <<1>>" -const UTF_ERR_INVALID_CHAR = "invalid Unicode character (0x<<2>> > 0x10ffff)" -const UTF_ERR_INVALID_8 = "invalid UTF-8 data" -const UTF_ERR_INVALID_16 = "invalid UTF-16 data" const UTF_ERR_INVALID_INDEX = "invalid character index" -const UTF_ERR_MAP_CHAR = "map(f,s::AbstractString) requires f to return Char; try map(f,collect(s)) or a comprehension instead" type UnicodeError <: Exception errmsg::AbstractString ##< A UTF_ERR_ message From e4c2df3276bf756ffd589486c287920faac8d2d8 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 5 Jul 2016 20:31:31 -0700 Subject: [PATCH 0248/1117] Doc update for utf16 and utf32 removal --- base/docs/helpdb/Base.jl | 12 +++++------- doc/manual/strings.rst | 9 +++------ doc/stdlib/strings.rst | 33 ++------------------------------- 3 files changed, 10 insertions(+), 44 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index f8af1543b5d68..a45a26f0c5225 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -8856,19 +8856,17 @@ vecnorm """ isvalid(value) -> Bool -Returns `true` if the given value is valid for its type, which currently can be one of -`Char`, `String`, `UTF16String`, or `UTF32String`. +Returns `true` if the given value is valid for its type, which currently can be either +`Char` or `String`. """ isvalid(value) """ isvalid(T, value) -> Bool -Returns `true` if the given value is valid for that type. Types currently can be `Char`, -`String`, `UTF16String`, or `UTF32String` Values for `Char` can be of -type `Char` or `UInt32` Values for `String` can be of that type, or -`Vector{UInt8}` Values for `UTF16String` can be `UTF16String` or `Vector{UInt16}` Values for -`UTF32String` can be `UTF32String`, `Vector{Char}` or `Vector{UInt32}` +Returns `true` if the given value is valid for that type. Types currently can +be either `Char` or `String`. Values for `Char` can be of type `Char` or `UInt32`. +Values for `String` can be of that type, or `Vector{UInt8}`. """ isvalid(T,value) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 36180463ba46a..45e556f45f99a 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -349,12 +349,9 @@ exception handling required: <BLANKLINE> y -UTF-8 is not the only encoding that Julia supports, and adding support -for new encodings is quite easy. In particular, Julia also provides -:obj:`UTF16String` and :obj:`UTF32String` types, constructed by -:func:`utf16` and :func:`utf32` respectively, for UTF-16 and -UTF-32 encodings. Additional discussion of other encodings and how to -implement support for them is beyond the scope of this document for +Julia uses UTF-8 encoding by default, and support for new encodings can +be added by packages. Additional discussion of other encodings and how +to implement support for them is beyond the scope of this document for the time being. For further discussion of UTF-8 encoding issues, see the section below on `byte array literals <#Byte+Array+Literals>`_, which goes into some greater detail. diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index 0817c94f0fa56..40ec3e2d69347 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -143,13 +143,13 @@ .. Docstring generated from Julia source - Returns ``true`` if the given value is valid for its type, which currently can be one of ``Char``\ , ``String``\ , ``UTF16String``\ , or ``UTF32String``\ . + Returns ``true`` if the given value is valid for its type, which currently can be either ``Char`` or ``String``\ . .. function:: isvalid(T, value) -> Bool .. Docstring generated from Julia source - Returns ``true`` if the given value is valid for that type. Types currently can be ``Char``\ , ``String``\ , ``UTF16String``\ , or ``UTF32String`` Values for ``Char`` can be of type ``Char`` or ``UInt32`` Values for ``String`` can be of that type, or ``Vector{UInt8}`` Values for ``UTF16String`` can be ``UTF16String`` or ``Vector{UInt16}`` Values for ``UTF32String`` can be ``UTF32String``\ , ``Vector{Char}`` or ``Vector{UInt32}`` + Returns ``true`` if the given value is valid for that type. Types currently can be either ``Char`` or ``String``\ . Values for ``Char`` can be of type ``Char`` or ``UInt32``\ . Values for ``String`` can be of that type, or ``Vector{UInt8}``\ . .. function:: isvalid(str, i) @@ -472,32 +472,3 @@ .. Docstring generated from Julia source General unescaping of traditional C and Unicode escape sequences. Reverse of :func:`escape_string`\ . See also :func:`unescape_string`\ . - -.. function:: utf16(s) - - .. Docstring generated from Julia source - - Create a UTF-16 string from a byte array, array of ``UInt16``\ , or any other string type. (Data must be valid UTF-16. Conversions of byte arrays check for a byte-order marker in the first two bytes, and do not include it in the resulting string.) - - Note that the resulting ``UTF16String`` data is terminated by the NUL codepoint (16-bit zero), which is not treated as a character in the string (so that it is mostly invisible in Julia); this allows the string to be passed directly to external functions requiring NUL-terminated data. This NUL is appended automatically by the ``utf16(s)`` conversion function. If you have a ``UInt16`` array ``A`` that is already NUL-terminated valid UTF-16 data, then you can instead use ``UTF16String(A)`` to construct the string without making a copy of the data and treating the NUL as a terminator rather than as part of the string. - -.. function:: utf16(::Union{Ptr{UInt16},Ptr{Int16}} [, length]) - - .. Docstring generated from Julia source - - Create a string from the address of a NUL-terminated UTF-16 string. A copy is made; the pointer can be safely freed. If ``length`` is specified, the string does not have to be NUL-terminated. - -.. function:: utf32(s) - - .. Docstring generated from Julia source - - Create a UTF-32 string from a byte array, array of ``Char`` or ``UInt32``\ , or any other string type. (Conversions of byte arrays check for a byte-order marker in the first four bytes, and do not include it in the resulting string.) - - Note that the resulting ``UTF32String`` data is terminated by the NUL codepoint (32-bit zero), which is not treated as a character in the string (so that it is mostly invisible in Julia); this allows the string to be passed directly to external functions requiring NUL-terminated data. This NUL is appended automatically by the ``utf32(s)`` conversion function. If you have a ``Char`` or ``UInt32`` array ``A`` that is already NUL-terminated UTF-32 data, then you can instead use ``UTF32String(A)`` to construct the string without making a copy of the data and treating the NUL as a terminator rather than as part of the string. - -.. function:: utf32(::Union{Ptr{Char},Ptr{UInt32},Ptr{Int32}} [, length]) - - .. Docstring generated from Julia source - - Create a string from the address of a NUL-terminated UTF-32 string. A copy is made; the pointer can be safely freed. If ``length`` is specified, the string does not have to be NUL-terminated. - From 388b92e820567b0beaa958cce3fdfdc009ac5e53 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 5 Jul 2016 20:21:50 -0700 Subject: [PATCH 0249/1117] Delete test/unicode/types.jl and test/unicode/checkstring.jl since their code in base has been removed --- test/unicode/checkstring.jl | 175 ------------------------------------ test/unicode/types.jl | 11 --- 2 files changed, 186 deletions(-) delete mode 100644 test/unicode/checkstring.jl delete mode 100644 test/unicode/types.jl diff --git a/test/unicode/checkstring.jl b/test/unicode/checkstring.jl deleted file mode 100644 index 8fd9c81554b7a..0000000000000 --- a/test/unicode/checkstring.jl +++ /dev/null @@ -1,175 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -# 11575 -# Test invalid sequences - -byt = 0x0 # Needs to be defined outside the try block! -try - # Continuation byte not after lead - for byt in 0x80:0xbf - @test_throws UnicodeError Base.checkstring(UInt8[byt]) - end - - # Test lead bytes - for byt in 0xc0:0xff - # Single lead byte at end of string - @test_throws UnicodeError Base.checkstring(UInt8[byt]) - # Lead followed by non-continuation character < 0x80 - @test_throws UnicodeError Base.checkstring(UInt8[byt,0]) - # Lead followed by non-continuation character > 0xbf - @test_throws UnicodeError Base.checkstring(UInt8[byt,0xc0]) - end - - # Test overlong 2-byte - for byt in 0x81:0xbf - @test_throws UnicodeError Base.checkstring(UInt8[0xc0,byt]) - end - for byt in 0x80:0xbf - @test_throws UnicodeError Base.checkstring(UInt8[0xc1,byt]) - end - - # Test overlong 3-byte - for byt in 0x80:0x9f - @test_throws UnicodeError Base.checkstring(UInt8[0xe0,byt,0x80]) - end - - # Test overlong 4-byte - for byt in 0x80:0x8f - @test_throws UnicodeError Base.checkstring(UInt8[0xef,byt,0x80,0x80]) - end - - # Test 4-byte > 0x10ffff - for byt in 0x90:0xbf - @test_throws UnicodeError Base.checkstring(UInt8[0xf4,byt,0x80,0x80]) - end - for byt in 0xf5:0xf7 - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80,0x80,0x80]) - end - - # Test 5-byte - for byt in 0xf8:0xfb - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80,0x80,0x80,0x80]) - end - - # Test 6-byte - for byt in 0xfc:0xfd - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80,0x80,0x80,0x80,0x80]) - end - - # Test 7-byte - @test_throws UnicodeError Base.checkstring(UInt8[0xfe,0x80,0x80,0x80,0x80,0x80,0x80]) - - # Three and above byte sequences - for byt in 0xe0:0xef - # Lead followed by only 1 continuation byte - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80]) - # Lead ended by non-continuation character < 0x80 - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80,0]) - # Lead ended by non-continuation character > 0xbf - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80,0xc0]) - end - - # 3-byte encoded surrogate character(s) - # Single surrogate - @test_throws UnicodeError Base.checkstring(UInt8[0xed,0xa0,0x80]) - # Not followed by surrogate - @test_throws UnicodeError Base.checkstring(UInt8[0xed,0xa0,0x80,0xed,0x80,0x80]) - # Trailing surrogate first - @test_throws UnicodeError Base.checkstring(UInt8[0xed,0xb0,0x80,0xed,0xb0,0x80]) - # Followed by lead surrogate - @test_throws UnicodeError Base.checkstring(UInt8[0xed,0xa0,0x80,0xed,0xa0,0x80]) - - # Four byte sequences - for byt in 0xf0:0xf4 - # Lead followed by only 2 continuation bytes - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80,0x80]) - # Lead followed by non-continuation character < 0x80 - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80,0x80,0]) - # Lead followed by non-continuation character > 0xbf - @test_throws UnicodeError Base.checkstring(UInt8[byt,0x80,0x80,0xc0]) - end - - # Long encoding of 0x01 - @test_throws UnicodeError convert(String, b"\xf0\x80\x80\x80") - # Test ends of long encoded surrogates - @test_throws UnicodeError convert(String, b"\xf0\x8d\xa0\x80") - @test_throws UnicodeError convert(String, b"\xf0\x8d\xbf\xbf") - # Long encodings - @test_throws UnicodeError Base.checkstring(b"\xf0\x80\x80\x80") - @test Base.checkstring(b"\xc0\x81"; accept_long_char=true) == (1,0x1,0,0,0) - @test Base.checkstring(b"\xf0\x80\x80\x80"; accept_long_char=true) == (1,0x1,0,0,0) -catch exp; - println("Error testing checkstring: $byt, $exp") - throw(exp) -end - -# Surrogates -@test_throws UnicodeError Base.checkstring(UInt16[0xd800]) -@test_throws UnicodeError Base.checkstring(UInt16[0xdc00]) -@test_throws UnicodeError Base.checkstring(UInt16[0xdc00,0xd800]) - -# Surrogates in UTF-32 -@test_throws UnicodeError Base.checkstring(UInt32[0xd800]) -@test_throws UnicodeError Base.checkstring(UInt32[0xdc00]) -@test_throws UnicodeError Base.checkstring(UInt32[0xdc00,0xd800]) - -# Characters > 0x10ffff -@test_throws UnicodeError Base.checkstring(UInt32[0x110000]) - -# Test starting and different position -@test Base.checkstring(UInt32[0x110000, 0x1f596], 2) == (1,0x10,1,0,0) - -# Test valid sequences -for (seq, res) in ( - (UInt8[0x0], (1,0,0,0,0)), # Nul byte, beginning of ASCII range - (UInt8[0x7f], (1,0,0,0,0)), # End of ASCII range - (UInt8[0xc0,0x80], (1,1,0,0,0)), # Long encoded Nul byte (Modified UTF-8, Java) - (UInt8[0xc2,0x80], (1,2,0,0,1)), # \u80, beginning of Latin1 range - (UInt8[0xc3,0xbf], (1,2,0,0,1)), # \uff, end of Latin1 range - (UInt8[0xc4,0x80], (1,4,0,0,1)), # \u100, beginning of non-Latin1 2-byte range - (UInt8[0xdf,0xbf], (1,4,0,0,1)), # \u7ff, end of non-Latin1 2-byte range - (UInt8[0xe0,0xa0,0x80], (1,8,0,1,0)), # \u800, beginning of 3-byte range - (UInt8[0xed,0x9f,0xbf], (1,8,0,1,0)), # \ud7ff, end of first part of 3-byte range - (UInt8[0xee,0x80,0x80], (1,8,0,1,0)), # \ue000, beginning of second part of 3-byte range - (UInt8[0xef,0xbf,0xbf], (1,8,0,1,0)), # \uffff, end of 3-byte range - (UInt8[0xf0,0x90,0x80,0x80],(1,16,1,0,0)), # \U10000, beginning of 4-byte range - (UInt8[0xf4,0x8f,0xbf,0xbf],(1,16,1,0,0)), # \U10ffff, end of 4-byte range - (UInt8[0xed,0xa0,0x80,0xed,0xb0,0x80], (1,0x30,1,0,0)), # Overlong \U10000, (CESU-8) - (UInt8[0xed,0xaf,0xbf,0xed,0xbf,0xbf], (1,0x30,1,0,0)), # Overlong \U10ffff, (CESU-8) - (UInt16[0x0000], (1,0,0,0,0)), # Nul byte, beginning of ASCII range - (UInt16[0x007f], (1,0,0,0,0)), # End of ASCII range - (UInt16[0x0080], (1,2,0,0,1)), # Beginning of Latin1 range - (UInt16[0x00ff], (1,2,0,0,1)), # End of Latin1 range - (UInt16[0x0100], (1,4,0,0,1)), # Beginning of non-Latin1 2-byte range - (UInt16[0x07ff], (1,4,0,0,1)), # End of non-Latin1 2-byte range - (UInt16[0x0800], (1,8,0,1,0)), # Beginning of 3-byte range - (UInt16[0xd7ff], (1,8,0,1,0)), # End of first part of 3-byte range - (UInt16[0xe000], (1,8,0,1,0)), # Beginning of second part of 3-byte range - (UInt16[0xffff], (1,8,0,1,0)), # End of 3-byte range - (UInt16[0xd800,0xdc00], (1,16,1,0,0)), # \U10000, beginning of 4-byte range - (UInt16[0xdbff,0xdfff], (1,16,1,0,0)), # \U10ffff, end of 4-byte range - (UInt32[0x0000], (1,0,0,0,0)), # Nul byte, beginning of ASCII range - (UInt32[0x007f], (1,0,0,0,0)), # End of ASCII range - (UInt32[0x0080], (1,2,0,0,1)), # Beginning of Latin1 range - (UInt32[0x00ff], (1,2,0,0,1)), # End of Latin1 range - (UInt32[0x0100], (1,4,0,0,1)), # Beginning of non-Latin1 2-byte range - (UInt32[0x07ff], (1,4,0,0,1)), # End of non-Latin1 2-byte range - (UInt32[0x0800], (1,8,0,1,0)), # Beginning of 3-byte range - (UInt32[0xd7ff], (1,8,0,1,0)), # End of first part of 3-byte range - (UInt32[0xe000], (1,8,0,1,0)), # Beginning of second part of 3-byte range - (UInt32[0xffff], (1,8,0,1,0)), # End of 3-byte range - (UInt32[0x10000], (1,16,1,0,0)), # \U10000, beginning of 4-byte range - (UInt32[0x10ffff], (1,16,1,0,0)), # \U10ffff, end of 4-byte range - (UInt32[0xd800,0xdc00], (1,0x30,1,0,0)),# Overlong \U10000, (CESU-8) - (UInt32[0xdbff,0xdfff], (1,0x30,1,0,0)))# Overlong \U10ffff, (CESU-8) - @test Base.checkstring(seq) == res -end - -# Test bounds checking -@test_throws BoundsError Base.checkstring(b"abcdef", -10) -@test_throws BoundsError Base.checkstring(b"abcdef", 0) -@test_throws BoundsError Base.checkstring(b"abcdef", 7) -@test_throws BoundsError Base.checkstring(b"abcdef", 3, -10) -@test_throws BoundsError Base.checkstring(b"abcdef", 3, 0) -@test_throws BoundsError Base.checkstring(b"abcdef", 3, 7) -@test_throws ArgumentError Base.checkstring(b"abcdef", 3, 1) diff --git a/test/unicode/types.jl b/test/unicode/types.jl deleted file mode 100644 index 919de34cc1786..0000000000000 --- a/test/unicode/types.jl +++ /dev/null @@ -1,11 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -nullstring16 = UInt16[] -badstring16 = UInt16[0x0065] -@test_throws UnicodeError UTF16String(nullstring16) -@test_throws UnicodeError UTF16String(badstring16) - -nullstring32 = UInt32[] -badstring32 = UInt32['a'] -@test_throws UnicodeError UTF32String(nullstring32) -@test_throws UnicodeError UTF32String(badstring32) From 0f6a90b44bf2b8b94eeef2da72b3b7d63d2bfee8 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 5 Jul 2016 20:41:30 -0700 Subject: [PATCH 0250/1117] One more doc update about utf-16 etc in calling-c-and-fortran-code --- doc/manual/calling-c-and-fortran-code.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/calling-c-and-fortran-code.rst b/doc/manual/calling-c-and-fortran-code.rst index 84e3b35ae4010..cc4fdea8a935e 100644 --- a/doc/manual/calling-c-and-fortran-code.rst +++ b/doc/manual/calling-c-and-fortran-code.rst @@ -487,10 +487,10 @@ C name Standard Julia Alias Julia Base Type For ``wchar_t*`` arguments, the Julia type should be ``Cwstring`` (if the C routine expects a NUL-terminated string) or ``Ptr{Cwchar_t}`` otherwise. Note - also that ASCII, UTF-8, UTF-16, and UTF-32 string data in Julia is internally - NUL-terminated, so it can be passed to C functions expecting NUL-terminated - data without making a copy (but using the ``Cwstring`` type will cause an - error to be thrown if the string itself contains NUL characters). + also that UTF-8 string data in Julia is internally NUL-terminated, so it can + be passed to C functions expecting NUL-terminated data without making a copy + (but using the ``Cwstring`` type will cause an error to be thrown if the string + itself contains NUL characters). .. note:: From 326a2a810e62689d6d8d1eaaa4151f8eb4af6bbc Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 6 Jul 2016 00:14:04 -0400 Subject: [PATCH 0251/1117] fix LLVM assertion failure on VecElement of aggregate --- src/cgutils.cpp | 9 ++++----- test/vecelement.jl | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 696b1da0fc2ea..d4c7f6b130b51 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1727,13 +1727,12 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg if (init_as_value) { if (lt->isVectorTy()) strct = builder.CreateInsertElement(strct, fval, ConstantInt::get(T_int32,idx)); + else if (jl_is_vecelement_type(ty)) + strct = fval; // VecElement type comes unwrapped in LLVM. else if (lt->isAggregateType()) strct = builder.CreateInsertValue(strct, fval, ArrayRef<unsigned>(&idx,1)); - else { - // Must be a VecElement type, which comes unwrapped in LLVM. - assert(jl_is_vecelement_type(ty)); - strct = fval; - } + else + assert(false); } } idx++; diff --git a/test/vecelement.jl b/test/vecelement.jl index d22984590bfbb..10ea4c5d8e899 100644 --- a/test/vecelement.jl +++ b/test/vecelement.jl @@ -63,3 +63,5 @@ a = Vector{Gr{2,Float64}}(2) a[2] = Gr(1.0, Bunch((VecElement(2.0), VecElement(3.0))), 4.0) a[1] = Gr(5.0, Bunch((VecElement(6.0), VecElement(7.0))), 8.0) @test a[2] == Gr(1.0, Bunch((VecElement(2.0), VecElement(3.0))), 4.0) + +@test isa(VecElement((1,2)), VecElement{Tuple{Int,Int}}) From 133e29d2f77e39bbccc54de1ea6a23ea9185a937 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 5 Jul 2016 21:25:30 -0700 Subject: [PATCH 0252/1117] Remove rmath-julia --- DISTRIBUTING.md | 15 +------- LICENSE.md | 5 --- Makefile | 2 +- README.md | 2 -- contrib/mac/macports.make | 1 - contrib/vagrant/Vagrantfile | 2 +- contrib/windows/msys_build.sh | 2 +- deps/Makefile | 11 +++--- deps/Versions.make | 1 - deps/checksums/Rmath-julia-0.1.tar.gz/md5 | 1 - deps/checksums/Rmath-julia-0.1.tar.gz/sha512 | 1 - deps/rmath-julia.mk | 38 -------------------- 12 files changed, 8 insertions(+), 73 deletions(-) delete mode 100644 deps/checksums/Rmath-julia-0.1.tar.gz/md5 delete mode 100644 deps/checksums/Rmath-julia-0.1.tar.gz/sha512 delete mode 100644 deps/rmath-julia.mk diff --git a/DISTRIBUTING.md b/DISTRIBUTING.md index 794d3b7250089..4c70468d9c49a 100644 --- a/DISTRIBUTING.md +++ b/DISTRIBUTING.md @@ -12,7 +12,7 @@ separated most of the notes by OS. Note that while the code for Julia is [MIT-licensed, with a few exceptions](https://github.com/JuliaLang/julia/blob/master/LICENSE.md), the distribution created by the techniques described herein will be -GPL licensed, as various dependent libraries such as `FFTW`, `Rmath`, +GPL licensed, as various dependent libraries such as `FFTW`, `SuiteSparse`, and `git` are GPL licensed. We do hope to have a non-GPL distribution of Julia in the future. @@ -162,19 +162,6 @@ set `USE_SYSTEM_BLAS=1` and `USE_SYSTEM_LAPACK=1`, you should also set `LIBLAPACK=-l$(YOURBLAS)` and `LIBLAPACKNAME=lib$(YOURBLAS)`. Else, the reference LAPACK will be used and performance will typically be much lower. -Notes on Rmath -============== - -Rmath is a library from R, which includes basic statistical -functions. Julia uses a patched version of Rmath, which uses DSFMT as -its underlying generator, and faster normal random number -generators. If the stock Rmath provided by various linux distributions -is used, the underlying random streams will not be the same for -different RNGs in Base and Distributions.jl. - -It is highly recommended that the patched Rmath provided by Julia is -used. - Compilation scripts =================== diff --git a/LICENSE.md b/LICENSE.md index 132ce06028af9..9dd48c0581898 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -70,11 +70,6 @@ The following components of Julia's standard library have separate licenses: - base/linalg/cholmod.jl (see [SUITESPARSE](http://faculty.cse.tamu.edu/davis/suitesparse.html)) -Julia builds the following libraries by default, but does not use them itself: - -- [RMATH](http://www.r-project.org/Licenses/) - - Julia's build process uses the following external tools: - [PATCHELF](http://hydra.nixos.org/build/1524660/download/1/README) diff --git a/Makefile b/Makefile index f9a02c6f3680d..9d7e02534915c 100644 --- a/Makefile +++ b/Makefile @@ -239,7 +239,7 @@ JL_LIBS := julia julia-debug # private libraries, that are installed in $(prefix)/lib/julia JL_PRIVATE_LIBS := ccalltest ifeq ($(USE_GPL_LIBS), 1) -JL_PRIVATE_LIBS += suitesparse_wrapper Rmath-julia +JL_PRIVATE_LIBS += suitesparse_wrapper endif ifeq ($(USE_SYSTEM_FFTW),0) ifeq ($(USE_GPL_LIBS), 1) diff --git a/README.md b/README.md index 590cd634bad7e..98ecf2852596e 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,6 @@ Julia uses the following external libraries, which are automatically downloaded - **[libgit2]** (>= 0.21) — Git linkable library, used by Julia's package manager - **[utf8proc]** (>= 1.3) — a library for processing UTF-8 encoded Unicode strings - **[libosxunwind]** — clone of [libunwind], a library that determines the call-chain of a program -- **[Rmath-julia]** — library for commonly used statistical functions from the R project. For a longer overview of Julia's dependencies, see these [slides](https://github.com/tkelman/BAJUtalk-Dec2014/blob/master/BAJUtalkDec2014.pdf?raw=true). @@ -323,7 +322,6 @@ For a longer overview of Julia's dependencies, see these [slides](https://github [utf8proc]: http://julialang.org/utf8proc/ [libosxunwind]: https://github.com/JuliaLang/libosxunwind [libunwind]: http://www.nongnu.org/libunwind -[Rmath-julia]: https://github.com/JuliaLang/Rmath-julia [openssl]: https://www.openssl.org <a name="System-Provided-Libraries"> diff --git a/contrib/mac/macports.make b/contrib/mac/macports.make index eb1c621a588e1..f99927ea90cfb 100644 --- a/contrib/mac/macports.make +++ b/contrib/mac/macports.make @@ -29,7 +29,6 @@ JMAKEFLAGS = USE_SYSTEM_LLVM=1 LLVM_CONFIG=${LLVM_CONFIG} \ USE_SYSTEM_SUITESPARSE=1 \ USE_SYSTEM_ZLIB=1 \ USE_SYSTEM_GRISU=0 \ - USE_SYSTEM_RMATH=0 \ USE_SYSTEM_LIBUV=0 \ PREFIX=${PREFIX} CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS} LDFLAGS=${LDFLAGS} \ CC=${CC} CXX=${CXX} CPP=${CPP} FC=${FC} USEGCC=${USEGCC} USECLANG=${USECLANG} diff --git a/contrib/vagrant/Vagrantfile b/contrib/vagrant/Vagrantfile index 1e91aedf9f12e..5ce593eb28655 100644 --- a/contrib/vagrant/Vagrantfile +++ b/contrib/vagrant/Vagrantfile @@ -5,7 +5,7 @@ $script = <<SCRIPT cat - > /home/vagrant/.bash_aliases <<"EOF" export JULIA_VAGRANT_BUILD=1 BUILDOPTS="LLVM_CONFIG=llvm-config-3.3 USE_BLAS64=0" -for lib in LLVM ZLIB SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND GRISU OPENLIBM RMATH; do +for lib in LLVM ZLIB SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND GRISU OPENLIBM; do export BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1" done diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index aefa0b2864868..2c635dab4cb7c 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -161,7 +161,7 @@ if [ -z "`which make 2>/dev/null`" ]; then fi for lib in SUITESPARSE ARPACK BLAS LAPACK FFTW \ - GMP MPFR PCRE LIBUNWIND RMATH OPENSPECFUN; do + GMP MPFR PCRE LIBUNWIND OPENSPECFUN; do echo "USE_SYSTEM_$lib = 1" >> Make.user done echo 'override LIBLAPACK = $(LIBBLAS)' >> Make.user diff --git a/deps/Makefile b/deps/Makefile index 0810b01d6d923..e376b07671922 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -18,7 +18,7 @@ include $(SRCDIR)/llvm-ver.make # if you are adding a new target, it can help to copy an similar, existing target # # autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv git -# custom Makefile rules: openlibm Rmath-julia dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind +# custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind # entirely custom: virtualenv # CMake libs: libgit2 # @@ -87,13 +87,11 @@ MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_depsbindir) libd # prevent installing libs into usr/lib64 on opensuse unexport CONFIG_SITE -ifeq ($(USE_GPL_LIBS), 1) STAGE1_DEPS := -STAGE2_DEPS := Rmath-julia +STAGE2_DEPS := +ifeq ($(USE_GPL_LIBS), 1) STAGE3_DEPS := suitesparse-wrapper else -STAGE1_DEPS := -STAGE2_DEPS := STAGE3_DEPS := endif @@ -215,7 +213,7 @@ install: $(addprefix install-, $(DEP_LIBS)) cleanall: $(addprefix clean-, $(DEP_LIBS)) distcleanall: $(addprefix distclean-, $(DEP_LIBS)) rm -rf $(build_prefix) -getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-Rmath-julia get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-libgit2 +getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-libgit2 ## PATHS ## # sort is used to remove potential duplicates @@ -252,7 +250,6 @@ include $(SRCDIR)/pcre.mk include $(SRCDIR)/openlibm.mk include $(SRCDIR)/openspecfun.mk include $(SRCDIR)/dsfmt.mk -include $(SRCDIR)/rmath-julia.mk include $(SRCDIR)/objconv.mk include $(SRCDIR)/blas.mk include $(SRCDIR)/arpack.mk diff --git a/deps/Versions.make b/deps/Versions.make index 7f29e1662d0a2..3cf63f3c7eaf4 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -13,4 +13,3 @@ MPFR_VER = 3.1.4 PATCHELF_VER = 0.9 GIT_VER = 1.8.5.6 VIRTUALENV_VER = 15.0.0 -RMATH_JULIA_VER = 0.1 diff --git a/deps/checksums/Rmath-julia-0.1.tar.gz/md5 b/deps/checksums/Rmath-julia-0.1.tar.gz/md5 deleted file mode 100644 index 67ba56c0b318b..0000000000000 --- a/deps/checksums/Rmath-julia-0.1.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -4e3f9e41e7b8cd3070225d1f5e8b21d9 diff --git a/deps/checksums/Rmath-julia-0.1.tar.gz/sha512 b/deps/checksums/Rmath-julia-0.1.tar.gz/sha512 deleted file mode 100644 index 6a64d44d8e1ed..0000000000000 --- a/deps/checksums/Rmath-julia-0.1.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -1dd94eca05bc06c3f33e8be5087d3b1742d01c8b38588e0fa79b8f6b9998760bca21bee1b65873825c4dde2d994a8271166bc21f670a34810fa9db598473a887 diff --git a/deps/rmath-julia.mk b/deps/rmath-julia.mk deleted file mode 100644 index 3f6aa483dc8b1..0000000000000 --- a/deps/rmath-julia.mk +++ /dev/null @@ -1,38 +0,0 @@ -## Rmath-julia ## - -RMATH_JULIA_OBJ_TARGET := $(build_shlibdir)/libRmath-julia.$(SHLIB_EXT) -RMATH_JULIA_OBJ_SOURCE := $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/src/libRmath-julia.$(SHLIB_EXT) - -ifeq ($(USE_SYSTEM_DSFMT),0) -$(RMATH_JULIA_OBJ_SOURCE): $(DSFMT_OBJ_TARGET) -endif - -RMATH_JULIA_FLAGS += CC="$(CC)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) \ - OS="$(OS)" ARCH="$(ARCH)" \ - USE_DSFMT=1 DSFMT_libdir="$(build_shlibdir)" \ - DSFMT_includedir="$(build_includedir)" - -$(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz: | $(SRCDIR)/srccache - $(JLDOWNLOAD) $@ https://api.github.com/repos/JuliaLang/Rmath-julia/tarball/v$(RMATH_JULIA_VER) -$(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile: $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz | $(SRCDIR)/srccache - $(JLCHECKSUM) $< - mkdir -p $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) - $(TAR) -C $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) --strip-components 1 -xf $< - touch -c $@ -$(RMATH_JULIA_OBJ_SOURCE): $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile - $(MAKE) -C $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/src $(RMATH_JULIA_FLAGS) $(MAKE_COMMON) - touch -c $@ -$(RMATH_JULIA_OBJ_TARGET): $(RMATH_JULIA_OBJ_SOURCE) | $(build_shlibdir) - cp $< $@ - $(INSTALL_NAME_CMD)libRmath-julia.$(SHLIB_EXT) $@ - -clean-Rmath-julia: - -$(MAKE) -C $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) clean -distclean-Rmath-julia: clean-Rmath-julia - -rm -rf $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER) $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER) - -get-Rmath-julia: $(SRCDIR)/srccache/Rmath-julia-$(RMATH_JULIA_VER).tar.gz -configure-Rmath-julia: $(BUILDDIR)/Rmath-julia-$(RMATH_JULIA_VER)/Makefile -compile-Rmath-julia: $(RMATH_JULIA_OBJ_SOURCE) -check-Rmath-julia: compile-Rmath-julia -install-Rmath-julia: $(RMATH_JULIA_OBJ_TARGET) From b2b31a2f36d0cc617ffea1fafad8c1bcca515078 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 6 Jul 2016 11:22:56 -0400 Subject: [PATCH 0253/1117] IOBuffer UB: avoid using minus 1 on a possibly unsigned number --- base/iobuffer.jl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index 60ffbde5f6db1..e644a7e3af497 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -275,10 +275,15 @@ function unsafe_write(to::AbstractIOBuffer, p::Ptr{UInt8}, nb::UInt) ensureroom(to, nb) ptr = (to.append ? to.size+1 : to.ptr) written = min(nb, length(to.data) - ptr + 1) - for i = 0:written - 1 - @inbounds to.data[ptr + i] = unsafe_load(p + i) + towrite = written + d = to.data + while towrite > 0 + @inbounds d[ptr] = unsafe_load(p) + ptr += 1 + p += 1 + towrite -= 1 end - to.size = max(to.size, ptr - 1 + written) + to.size = max(to.size, ptr - 1) if !to.append to.ptr += written end From 4db8c976f6894b27e6bed766d2434642c27e7955 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Mon, 4 Jul 2016 09:47:30 -0500 Subject: [PATCH 0254/1117] Docstrings for rounding modes --- base/docs/helpdb/Base.jl | 12 ++++----- base/floatfuncs.jl | 2 +- base/rounding.jl | 54 +++++++++++++++++++++++++++++++++++-- doc/stdlib/math.rst | 58 ++++++++++++++++++++++++---------------- doc/stdlib/numbers.rst | 10 +++---- 5 files changed, 99 insertions(+), 37 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index a45a26f0c5225..93854f9bb8884 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -929,7 +929,7 @@ julia> Float32(1/3, RoundUp) 0.33333334f0 ``` -See `rounding` for available rounding modes. +See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. """ Float32 @@ -2951,7 +2951,7 @@ julia> Float64(pi, RoundUp) 3.1415926535897936 ``` -See `rounding` for available rounding modes. +See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. """ Float64 @@ -5200,8 +5200,7 @@ Get the current floating point rounding mode for type `T`, controlling the round arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/`](:func:`/`) and [`sqrt`](:func:`sqrt`)) and type conversion. -Valid modes are `RoundNearest`, `RoundToZero`, `RoundUp`, `RoundDown`, and `RoundFromZero` -(`BigFloat` only). +See [`RoundingMode`](:obj:`RoundingMode`) for available modes. """ rounding @@ -5392,7 +5391,8 @@ arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/` and [`sqrt`](:func:`sqrt`)) and type conversion. Note that this may affect other types, for instance changing the rounding mode of `Float64` -will change the rounding mode of `Float32`. See `rounding` for available modes +will change the rounding mode of `Float32`. See [`RoundingMode`](:obj:`RoundingMode`) for +available modes. """ setrounding(T, mode) @@ -5407,7 +5407,7 @@ equivalent to: f() setrounding(T, old) -See `rounding` for available rounding modes. +See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. """ setrounding(f::Function, T, mode) diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index fa25bb6bd6058..92538a0fa2101 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -52,7 +52,7 @@ end round([T,] x, [digits, [base]], [r::RoundingMode]) `round(x)` rounds `x` to an integer value according to the default rounding mode (see -[`rounding`](:func:`rounding`)), returning a value of the same type as `x`. By default +[`RoundingMode`](:obj:`RoundingMode`)), returning a value of the same type as `x`. By default ([`RoundNearest`](:obj:`RoundNearest`)), this will round to the nearest integer, with ties (fractional values of 0.5) being rounded to the even integer. diff --git a/base/rounding.jl b/base/rounding.jl index c3a38659f3710..f6e5bb637e7dd 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -10,16 +10,66 @@ export get_zero_subnormals, set_zero_subnormals ## rounding modes ## +""" + RoundingMode + +A type which controls rounding behavior. Currently supported rounding modes are: + +- [`RoundNearest`](:obj:`RoundNearest`) (default) +- [`RoundNearestTiesAway`](:obj:`RoundNearestTiesAway`) +- [`RoundNearestTiesUp`](:obj:`RoundNearestTiesUp`) +- [`RoundToZero`](:obj:`RoundToZero`) +- [`RoundFromZero`](:obj:`RoundFromZero`) (`BigFloat` only) +- [`RoundUp`](:obj:`RoundUp`) +- [`RoundDown`](:obj:`RoundDown`) +""" immutable RoundingMode{T} end +""" + RoundNearest + +The default rounding mode. Rounds to the nearest integer, with ties (fractional values of +0.5) being rounded to the nearest even integer. +""" const RoundNearest = RoundingMode{:Nearest}() + +""" + RoundToZero + +[`round`](:func:`round`) using this rounding mode is an alias for [`trunc`](:func:`trunc`). +""" const RoundToZero = RoundingMode{:ToZero}() + +""" + RoundUp + +[`round`](:func:`round`) using this rounding mode is an alias for [`ceil`](:func:`ceil`). +""" const RoundUp = RoundingMode{:Up}() + +""" + RoundDown + +[`round`](:func:`round`) using this rounding mode is an alias for [`floor`](:func:`floor`). +""" const RoundDown = RoundingMode{:Down}() + const RoundFromZero = RoundingMode{:FromZero}() # mpfr only -# C-style round behaviour + +""" + RoundNearestTiesAway + +Rounds to nearest integer, with ties rounded away from zero (C/C++ +[`round`](:func:`round`) behaviour). +""" const RoundNearestTiesAway = RoundingMode{:NearestTiesAway}() -# Java-style round behaviour + +""" + RoundNearestTiesUp + +Rounds to nearest integer, with ties rounded toward positive infinity (Java/JavaScript +[`round`](:func:`round`) behaviour). +""" const RoundNearestTiesUp = RoundingMode{:NearestTiesUp}() to_fenv(::RoundingMode{:Nearest}) = JL_FE_TONEAREST diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 79782b1cf9e49..36a2996143388 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -966,7 +966,7 @@ Mathematical Functions .. Docstring generated from Julia source - ``round(x)`` rounds ``x`` to an integer value according to the default rounding mode (see :func:`rounding`\ ), returning a value of the same type as ``x``\ . By default (:obj:`RoundNearest`\ ), this will round to the nearest integer, with ties (fractional values of 0.5) being rounded to the even integer. + ``round(x)`` rounds ``x`` to an integer value according to the default rounding mode (see :obj:`RoundingMode`\ ), returning a value of the same type as ``x``\ . By default (:obj:`RoundNearest`\ ), this will round to the nearest integer, with ties (fractional values of 0.5) being rounded to the even integer. .. doctest:: @@ -1011,43 +1011,55 @@ Mathematical Functions 1.2 -.. data:: RoundingMode +.. type:: RoundingMode + + .. Docstring generated from Julia source A type which controls rounding behavior. Currently supported rounding modes are: - - :obj:`RoundNearest` (default) - - :obj:`RoundNearestTiesAway` - - :obj:`RoundNearestTiesUp` - - :obj:`RoundToZero` - - :obj:`RoundUp` - - :obj:`RoundDown` + * :obj:`RoundNearest` (default) + * :obj:`RoundNearestTiesAway` + * :obj:`RoundNearestTiesUp` + * :obj:`RoundToZero` + * :obj:`RoundFromZero` (``BigFloat`` only) + * :obj:`RoundUp` + * :obj:`RoundDown` + +.. variable:: RoundNearest + + .. Docstring generated from Julia source + + The default rounding mode. Rounds to the nearest integer, with ties (fractional values of 0.5) being rounded to the nearest even integer. + +.. variable:: RoundNearestTiesAway -.. data:: RoundNearest + .. Docstring generated from Julia source - The default rounding mode. Rounds to the nearest integer, with ties - (fractional values of 0.5) being rounded to the nearest even integer. + Rounds to nearest integer, with ties rounded away from zero (C/C++ :func:`round` behaviour). -.. data:: RoundNearestTiesAway +.. variable:: RoundNearestTiesUp - Rounds to nearest integer, with ties rounded away from zero (C/C++ - :func:`round` behaviour). + .. Docstring generated from Julia source -.. data:: RoundNearestTiesUp + Rounds to nearest integer, with ties rounded toward positive infinity (Java/JavaScript :func:`round` behaviour). - Rounds to nearest integer, with ties rounded toward positive infinity - (Java/JavaScript :func:`round` behaviour). +.. variable:: RoundToZero -.. data:: RoundToZero + .. Docstring generated from Julia source - :func:`round` using this rounding mode is an alias for :func:`trunc`. + :func:`round` using this rounding mode is an alias for :func:`trunc`\ . -.. data:: RoundUp +.. variable:: RoundUp - :func:`round` using this rounding mode is an alias for :func:`ceil`. + .. Docstring generated from Julia source -.. data:: RoundDown + :func:`round` using this rounding mode is an alias for :func:`ceil`\ . + +.. variable:: RoundDown + + .. Docstring generated from Julia source - :func:`round` using this rounding mode is an alias for :func:`floor`. + :func:`round` using this rounding mode is an alias for :func:`floor`\ . .. function:: round(z, RoundingModeReal, RoundingModeImaginary) diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index b367bb7692192..f9ffc157b7eef 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -287,7 +287,7 @@ General Number Functions and Constants julia> Float32(1/3, RoundUp) 0.33333334f0 - See ``rounding`` for available rounding modes. + See :obj:`RoundingMode` for available rounding modes. .. function:: Float64(x [, mode::RoundingMode]) @@ -303,7 +303,7 @@ General Number Functions and Constants julia> Float64(pi, RoundUp) 3.1415926535897936 - See ``rounding`` for available rounding modes. + See :obj:`RoundingMode` for available rounding modes. .. function:: BigInt(x) @@ -335,7 +335,7 @@ General Number Functions and Constants Get the current floating point rounding mode for type ``T``\ , controlling the rounding of basic arithmetic functions (:func:`+`\ , :func:`-`\ , :func:`*`\ , :func:`/` and :func:`sqrt`\ ) and type conversion. - Valid modes are ``RoundNearest``\ , ``RoundToZero``\ , ``RoundUp``\ , ``RoundDown``\ , and ``RoundFromZero`` (``BigFloat`` only). + See :obj:`RoundingMode` for available modes. .. function:: setrounding(T, mode) @@ -343,7 +343,7 @@ General Number Functions and Constants Set the rounding mode of floating point type ``T``\ , controlling the rounding of basic arithmetic functions (:func:`+`\ , :func:`-`\ , :func:`*`\ , :func:`/` and :func:`sqrt`\ ) and type conversion. - Note that this may affect other types, for instance changing the rounding mode of ``Float64`` will change the rounding mode of ``Float32``\ . See ``rounding`` for available modes + Note that this may affect other types, for instance changing the rounding mode of ``Float64`` will change the rounding mode of ``Float32``\ . See :obj:`RoundingMode` for available modes. .. function:: setrounding(f::Function, T, mode) @@ -358,7 +358,7 @@ General Number Functions and Constants f() setrounding(T, old) - See ``rounding`` for available rounding modes. + See :obj:`RoundingMode` for available rounding modes. .. function:: get_zero_subnormals() -> Bool From 48865068175db3f8c2e2265f3955177406e92e7e Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Tue, 5 Jul 2016 08:32:45 -0500 Subject: [PATCH 0255/1117] Revise round docstring --- base/floatfuncs.jl | 8 ++++---- doc/stdlib/math.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index 92538a0fa2101..4d7716d7e17cc 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -51,10 +51,10 @@ end """ round([T,] x, [digits, [base]], [r::RoundingMode]) -`round(x)` rounds `x` to an integer value according to the default rounding mode (see -[`RoundingMode`](:obj:`RoundingMode`)), returning a value of the same type as `x`. By default -([`RoundNearest`](:obj:`RoundNearest`)), this will round to the nearest integer, with ties -(fractional values of 0.5) being rounded to the even integer. +Rounds `x` to an integer value according to the provided +[`RoundingMode`](:obj:`RoundingMode`), returning a value of the same type as `x`. By default +uses [`RoundNearest`](:obj:`RoundNearest`), which rounds to the nearest integer, with ties +(fractional values of 0.5) being rounded to the nearest even integer. ```jldoctest julia> round(1.7) diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 36a2996143388..4491fcf85a065 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -966,7 +966,7 @@ Mathematical Functions .. Docstring generated from Julia source - ``round(x)`` rounds ``x`` to an integer value according to the default rounding mode (see :obj:`RoundingMode`\ ), returning a value of the same type as ``x``\ . By default (:obj:`RoundNearest`\ ), this will round to the nearest integer, with ties (fractional values of 0.5) being rounded to the even integer. + Rounds ``x`` to an integer value according to the provided :obj:`RoundingMode`\ , returning a value of the same type as ``x``\ . By default uses :obj:`RoundNearest`\ , which rounds to the nearest integer, with ties (fractional values of 0.5) being rounded to the nearest even integer. .. doctest:: From 1538d8c97ec9ae664596c6897d66d3d0fc7b42ed Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Tue, 5 Jul 2016 10:20:20 -0500 Subject: [PATCH 0256/1117] Reword round and RoundingMode --- base/floatfuncs.jl | 8 +++++--- base/rounding.jl | 7 ++++++- doc/stdlib/math.rst | 6 ++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index 4d7716d7e17cc..45eb2e6e3c6fa 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -52,9 +52,11 @@ end round([T,] x, [digits, [base]], [r::RoundingMode]) Rounds `x` to an integer value according to the provided -[`RoundingMode`](:obj:`RoundingMode`), returning a value of the same type as `x`. By default -uses [`RoundNearest`](:obj:`RoundNearest`), which rounds to the nearest integer, with ties -(fractional values of 0.5) being rounded to the nearest even integer. +[`RoundingMode`](:obj:`RoundingMode`), returning a value of the same type as `x`. When not +specifying a rounding mode the global mode will be used +(see [`rounding`](:func:`rounding`)), which by default is round to the nearest integer +([`RoundNearest`](:obj:`RoundNearest`) mode), with ties (fractional values of 0.5) being +rounded to the nearest even integer. ```jldoctest julia> round(1.7) diff --git a/base/rounding.jl b/base/rounding.jl index f6e5bb637e7dd..d839a0d219bc2 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -13,7 +13,12 @@ export """ RoundingMode -A type which controls rounding behavior. Currently supported rounding modes are: +A type used for controlling the rounding mode of floating point operations (via +[`rounding`](:func:`rounding`)/[`setrounding`](:func:`setrounding`) functions), or as +optional arguments for rounding to the nearest integer (via the [`round`](:func:`round`) +function). + +Currently supported rounding modes are: - [`RoundNearest`](:obj:`RoundNearest`) (default) - [`RoundNearestTiesAway`](:obj:`RoundNearestTiesAway`) diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 4491fcf85a065..c770557bce0ca 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -966,7 +966,7 @@ Mathematical Functions .. Docstring generated from Julia source - Rounds ``x`` to an integer value according to the provided :obj:`RoundingMode`\ , returning a value of the same type as ``x``\ . By default uses :obj:`RoundNearest`\ , which rounds to the nearest integer, with ties (fractional values of 0.5) being rounded to the nearest even integer. + Rounds ``x`` to an integer value according to the provided :obj:`RoundingMode`\ , returning a value of the same type as ``x``\ . When not specifying a rounding mode the global mode will be used (see :func:`rounding`\ ), which by default is round to the nearest integer (:obj:`RoundNearest` mode), with ties (fractional values of 0.5) being rounded to the nearest even integer. .. doctest:: @@ -1015,7 +1015,9 @@ Mathematical Functions .. Docstring generated from Julia source - A type which controls rounding behavior. Currently supported rounding modes are: + A type used for controlling the rounding mode of floating point operations (via :func:`rounding`\ /:func:`setrounding` functions), or as optional arguments for rounding to the nearest integer (via the :func:`round` function). + + Currently supported rounding modes are: * :obj:`RoundNearest` (default) * :obj:`RoundNearestTiesAway` From 631e026ee22c9c0864667d4c3e8f72e94dce8c9b Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 6 Jul 2016 10:08:44 -0500 Subject: [PATCH 0257/1117] Move rounding docstrings out of helpdb --- base/docs/helpdb/Base.jl | 39 --------------------------------------- base/rounding.jl | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 93854f9bb8884..857e67900a8c6 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5193,17 +5193,6 @@ Determine whether a stream is read-only. """ isreadonly -""" - rounding(T) - -Get the current floating point rounding mode for type `T`, controlling the rounding of basic -arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/`](:func:`/`) -and [`sqrt`](:func:`sqrt`)) and type conversion. - -See [`RoundingMode`](:obj:`RoundingMode`) for available modes. -""" -rounding - """ code_llvm(f, types) @@ -5383,34 +5372,6 @@ information. """ code_warntype -""" - setrounding(T, mode) - -Set the rounding mode of floating point type `T`, controlling the rounding of basic -arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/`](:func:`/`) -and [`sqrt`](:func:`sqrt`)) and type conversion. - -Note that this may affect other types, for instance changing the rounding mode of `Float64` -will change the rounding mode of `Float32`. See [`RoundingMode`](:obj:`RoundingMode`) for -available modes. -""" -setrounding(T, mode) - -""" - setrounding(f::Function, T, mode) - -Change the rounding mode of floating point type `T` for the duration of `f`. It is logically -equivalent to: - - old = rounding(T) - setrounding(T, mode) - f() - setrounding(T, old) - -See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. -""" -setrounding(f::Function, T, mode) - """ Mmap.sync!(array) diff --git a/base/rounding.jl b/base/rounding.jl index d839a0d219bc2..43fe9565440e2 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -96,13 +96,49 @@ function from_fenv(r::Integer) end end +""" + setrounding(T, mode) + +Set the rounding mode of floating point type `T`, controlling the rounding of basic +arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/`](:func:`/`) +and [`sqrt`](:func:`sqrt`)) and type conversion. + +Note that this may affect other types, for instance changing the rounding mode of `Float64` +will change the rounding mode of `Float32`. See [`RoundingMode`](:obj:`RoundingMode`) for +available modes. +""" +setrounding(T::Type, mode) + +""" + rounding(T) + +Get the current floating point rounding mode for type `T`, controlling the rounding of basic +arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/`](:func:`/`) +and [`sqrt`](:func:`sqrt`)) and type conversion. + +See [`RoundingMode`](:obj:`RoundingMode`) for available modes. +""" +:rounding + setrounding_raw{T<:Union{Float32,Float64}}(::Type{T},i::Integer) = ccall(:fesetround, Int32, (Int32,), i) rounding_raw{T<:Union{Float32,Float64}}(::Type{T}) = ccall(:fegetround, Int32, ()) setrounding{T<:Union{Float32,Float64}}(::Type{T},r::RoundingMode) = setrounding_raw(T,to_fenv(r)) rounding{T<:Union{Float32,Float64}}(::Type{T}) = from_fenv(rounding_raw(T)) +""" + setrounding(f::Function, T, mode) +Change the rounding mode of floating point type `T` for the duration of `f`. It is logically +equivalent to: + + old = rounding(T) + setrounding(T, mode) + f() + setrounding(T, old) + +See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. +""" function setrounding{T}(f::Function, ::Type{T}, rounding::RoundingMode) old_rounding_raw = rounding_raw(T) setrounding(T,rounding) From 1889edb0f9b636e67f932aed84bd1ed001a0e45c Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 6 Jul 2016 10:25:22 -0500 Subject: [PATCH 0258/1117] Move complex round docstring out of helpdb --- base/complex.jl | 8 ++++++++ base/docs/helpdb/Base.jl | 10 ---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/base/complex.jl b/base/complex.jl index afd39316cad6f..669d9db150391 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -750,6 +750,14 @@ end #Rounding complex numbers #Requires two different RoundingModes for the real and imaginary components +""" + round(z, RoundingModeReal, RoundingModeImaginary) + +Returns the nearest integral value of the same type as the complex-valued `z` to `z`, +breaking ties using the specified [`RoundingMode`](:obj:`RoundingMode`)s. The first +[`RoundingMode`](:obj:`RoundingMode`) is used for rounding the real components while the +second is used for rounding the imaginary components. +""" function round{T<:AbstractFloat, MR, MI}(z::Complex{T}, ::RoundingMode{MR}, ::RoundingMode{MI}) Complex(round(real(z), RoundingMode{MR}()), round(imag(z), RoundingMode{MI}())) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 857e67900a8c6..5e3aec1bd05c9 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2282,16 +2282,6 @@ For matrices or vectors ``A`` and ``B``, calculates ``A / Bᴴ``. """ A_rdiv_Bc -""" - round(z, RoundingModeReal, RoundingModeImaginary) - -Returns the nearest integral value of the same type as the complex-valued `z` to `z`, -breaking ties using the specified [`RoundingMode`](:obj:`RoundingMode`)s. The first -[`RoundingMode`](:obj:`RoundingMode`) is used for rounding the real components while the -second is used for rounding the imaginary components. -""" -round(z::Real, ::Type{RoundingMode}, ::Type{RoundingMode}) - """ strwidth(s) From 14d60a5984dabc6f47992c33ec319433333032d7 Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Wed, 29 Jun 2016 15:55:12 +0200 Subject: [PATCH 0259/1117] Improve tests and documentation for unsafe_load/store!. --- base/docs/helpdb/Base.jl | 4 ++-- doc/stdlib/c.rst | 4 ++-- test/core.jl | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 5e3aec1bd05c9..d1142f0775412 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -6410,7 +6410,7 @@ Convolution of two vectors. Uses FFT algorithm. conv """ - unsafe_store!(p::Ptr{T},x,i::Integer) + unsafe_store!(p::Ptr{T}, x, [i::Integer=1]) Store a value of type `T` to the address of the ith element (1-indexed) starting at `p`. This is equivalent to the C expression `p[i-1] = x`. @@ -7986,7 +7986,7 @@ throw this exception. ProcessExitedException """ - unsafe_load(p::Ptr{T},i::Integer) + unsafe_load(p::Ptr{T}, [i::Integer=1]) Load a value of type `T` from the address of the ith element (1-indexed) starting at `p`. This is equivalent to the C expression `p[i-1]`. diff --git a/doc/stdlib/c.rst b/doc/stdlib/c.rst index d7aea1c66f3d2..4c921310f23b1 100644 --- a/doc/stdlib/c.rst +++ b/doc/stdlib/c.rst @@ -60,7 +60,7 @@ Neither ``convert`` nor ``cconvert`` should take a Julia object and turn it into a ``Ptr``\ . -.. function:: unsafe_load(p::Ptr{T},i::Integer) +.. function:: unsafe_load(p::Ptr{T}, [i::Integer=1]) .. Docstring generated from Julia source @@ -68,7 +68,7 @@ The ``unsafe`` prefix on this function indicates that no validation is performed on the pointer ``p`` to ensure that it is valid. Incorrect usage may segfault your program or return garbage answers, in the same manner as C. -.. function:: unsafe_store!(p::Ptr{T},x,i::Integer) +.. function:: unsafe_store!(p::Ptr{T}, x, [i::Integer=1]) .. Docstring generated from Julia source diff --git a/test/core.jl b/test/core.jl index e4c32b60dd554..b3a06317a759e 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1031,7 +1031,10 @@ let local X, p X = FooBar[ FooBar(3,1), FooBar(4,4) ] p = pointer(X) + @test unsafe_load(p) == FooBar(3,1) @test unsafe_load(p, 2) == FooBar(4,4) + unsafe_store!(p, FooBar(8,4)) + @test X[1] == FooBar(8,4) unsafe_store!(p, FooBar(7,3), 1) @test X[1] == FooBar(7,3) end From 5a78122d4b581c27743d86a8d20c8a62080c9922 Mon Sep 17 00:00:00 2001 From: Jiahao Chen <jiahao@mit.edu> Date: Tue, 5 Jul 2016 16:39:41 -0600 Subject: [PATCH 0260/1117] Noteworthy differences: add two MATLAB/Julia idioms Briefly describes two MATLAB idioms from @dgleich, and their Julia equivalents. The first one is logical indexing of an array with a logical vector involving the array itself, like in x(x>3) and x(x>3) = []. List filter/filter! as alternatives to logical indexing. The second one is to describe extracting or "dereferencing" a cell array, an operation which in Julia uses splatting. Also, capitalize MATLAB in the preferred way. Also describes the equivalent of the first idiom in R [ci skip] --- doc/manual/noteworthy-differences.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/manual/noteworthy-differences.rst b/doc/manual/noteworthy-differences.rst index 5bb3ebd89b92e..f5c539393266d 100644 --- a/doc/manual/noteworthy-differences.rst +++ b/doc/manual/noteworthy-differences.rst @@ -99,7 +99,18 @@ some noteworthy differences that may trip up Julia users accustomed to MATLAB: - Julia's ``type``\ s do not support dynamically adding fields at runtime, unlike MATLAB's ``class``\ es. Instead, use a :obj:`Dict`. - In Julia each module has its own global scope/namespace, whereas in - Matlab there is just one global scope. + MATLAB there is just one global scope. +- In MATLAB, an idiomatic way to remove unwanted values is to use logical + indexing, like in the expression ``x(x>3)`` or in the statement + ``x(x>3) = []`` to modify ``x`` in-place. In contrast, Julia provides the + higher order functions :func:`filter` and a :func:`filter!`, allowing users + to write ``filter(z->z>3, x)`` and ``filter!(z->z>3, x)`` as alternatives to + the corresponding transliterations ``x[x.>3]`` and ``x = x[x.>3]``. Using + :func:`filter!` reduces the use of temporary arrays. +- The analogue of extracting (or "dereferencing") all elements of a cell array, + e.g. in ``vertcat(A{:})`` in MATLAB, is written using the splat operator in + Julia, e.g. as ``vcat(A...)``. + Noteworthy differences from R ----------------------------- @@ -221,6 +232,13 @@ noteworthy differences: - Julia does not support the ``NULL`` type. - Julia lacks the equivalent of R's ``assign`` or ``get``. - In Julia, ``return`` does not require parentheses. +- In R, an idiomatic way to remove unwanted values is to use logical indexing, + like in the expression ``x[x>3]`` or in the statement ``x = x[x>3]`` to + modify ``x`` in-place. In contrast, Julia provides the higher order functions + :func:`filter` and a :func:`filter!`, allowing users to write + ``filter(z->z>3, x)`` and ``filter!(z->z>3, x)`` as alternatives to the + corresponding transliterations ``x[x.>3]`` and ``x = x[x.>3]``. Using + :func:`filter!` reduces the use of temporary arrays. Noteworthy differences from Python From 011778acc1ac508008cfbfd15acc78f4b6eb2eda Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 6 Jul 2016 16:18:04 -0400 Subject: [PATCH 0261/1117] flush cache on invalidating conflicting / replaced function entries fix #17076 --- src/gf.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/gf.c b/src/gf.c index 2c33823599904..6307c92fbebf3 100644 --- a/src/gf.c +++ b/src/gf.c @@ -976,6 +976,7 @@ static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue } // invalidate cached methods that overlap this definition +static void flush_from_cache(jl_typemap_entry_t *entry); static void invalidate_conflicting(union jl_typemap_t *pml, jl_value_t *type, jl_value_t *parent, jl_array_t *shadowed) { jl_typemap_entry_t **pl; @@ -1019,6 +1020,7 @@ static void invalidate_conflicting(union jl_typemap_t *pml, jl_value_t *type, jl } } if (replaced) { + flush_from_cache(l); *pl = l->next; jl_gc_wb(parent, *pl); } @@ -1841,6 +1843,15 @@ void call_cache_stats() { } #endif +static void flush_from_cache(jl_typemap_entry_t *entry) +{ + int i; + for (i = 0; i < N_CALL_CACHE; i++) { + if (call_cache[i] == entry) + call_cache[i] = NULL; + } +} + #ifdef _COMPILER_MICROSOFT_ #define __builtin_return_address(n) _ReturnAddress() #endif From 22ff2e60b4d85a7b5a42532c16a009e434b24219 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Wed, 6 Jul 2016 16:28:21 -0400 Subject: [PATCH 0262/1117] Simple perm gen allocator Similar to the one already used by symbol and the RW allocator for codegen. Currently used by symbols and type layout. Also rename `struct _jl_datatype_layout_t` to `jl_datatype_layout_t`. --- src/alloc.c | 36 ++++++----------------------- src/dump.c | 4 ++-- src/gc.c | 55 ++++++++++++++++++++++++++++++++++++++++++++ src/jltypes.c | 6 ++--- src/julia.h | 12 +++++----- src/julia_internal.h | 3 +++ test/core.jl | 2 -- 7 files changed, 76 insertions(+), 42 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 50d42703424a4..545d968512aea 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -645,8 +645,6 @@ jl_method_t *jl_new_method(jl_lambda_info_t *definition, jl_sym_t *name, jl_tupl // symbols -------------------------------------------------------------------- -static jl_mutex_t symbol_table_lock; - static jl_sym_t *volatile symtab = NULL; static uintptr_t hash_symbol(const char *str, size_t len) @@ -654,8 +652,6 @@ static uintptr_t hash_symbol(const char *str, size_t len) return memhash(str, len) ^ ~(uintptr_t)0/3*2; } -#define SYM_POOL_SIZE 524288 - static size_t symbol_nbytes(size_t len) { return (sizeof(jl_taggedvalue_t) + sizeof(jl_sym_t) + len + 1 + 7) & -8; @@ -666,23 +662,7 @@ static jl_sym_t *mk_symbol(const char *str, size_t len) jl_sym_t *sym; size_t nb = symbol_nbytes(len); - if (nb >= SYM_POOL_SIZE) { - jl_exceptionf(jl_argumenterror_type, "Symbol length exceeds maximum length"); - } - - jl_taggedvalue_t *tag; -#ifdef MEMDEBUG - tag = (jl_taggedvalue_t*)malloc(nb); -#else - static char *sym_pool = NULL; - static char *pool_ptr = NULL; - if (sym_pool == NULL || pool_ptr+nb > sym_pool+SYM_POOL_SIZE) { - sym_pool = (char*)malloc(SYM_POOL_SIZE); - pool_ptr = sym_pool; - } - tag = (jl_taggedvalue_t*)pool_ptr; - pool_ptr += nb; -#endif + jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc_nolock(nb); sym = (jl_sym_t*)jl_valueof(tag); // set to old marked since we don't need write barrier on it. tag->header = ((uintptr_t)jl_sym_type) | GC_OLD_MARKED; @@ -726,15 +706,15 @@ static jl_sym_t *_jl_symbol(const char *str, size_t len) jl_sym_t *volatile *slot; jl_sym_t *node = symtab_lookup(&symtab, str, len, &slot); if (node == NULL) { - JL_LOCK(&symbol_table_lock); // Might GC + JL_LOCK_NOGC(&gc_perm_lock); // Someone might have updated it, check and look up again if (*slot != NULL && (node = symtab_lookup(slot, str, len, &slot))) { - JL_UNLOCK(&symbol_table_lock); // Might GC + JL_UNLOCK_NOGC(&gc_perm_lock); return node; } node = mk_symbol(str, len); jl_atomic_store_release(slot, node); - JL_UNLOCK(&symbol_table_lock); // Might GC + JL_UNLOCK_NOGC(&gc_perm_lock); } return node; } @@ -778,8 +758,6 @@ JL_DLLEXPORT jl_sym_t *jl_gensym(void) JL_DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, int32_t len) { char gs_name[14]; - if (symbol_nbytes(len) >= SYM_POOL_SIZE) - jl_exceptionf(jl_argumenterror_type, "Symbol length exceeds maximum"); if (memchr(str, 0, len)) jl_exceptionf(jl_argumenterror_type, "Symbol name may not contain \\0"); char *name = (char*) (len >= 256 ? malloc(sizeof(gs_name)+len+3) : @@ -872,7 +850,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(void) return t; } -static struct _jl_datatype_layout_t *jl_get_layout( +static jl_datatype_layout_t *jl_get_layout( uint32_t nfields, uint32_t alignment, int haspadding, @@ -903,8 +881,8 @@ static struct _jl_datatype_layout_t *jl_get_layout( // allocate a new descriptor uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); - struct _jl_datatype_layout_t *flddesc = (struct _jl_datatype_layout_t*)malloc( - sizeof(struct _jl_datatype_layout_t) + nfields * fielddesc_size); + jl_datatype_layout_t *flddesc = (jl_datatype_layout_t*)jl_gc_perm_alloc( + sizeof(jl_datatype_layout_t) + nfields * fielddesc_size); flddesc->nfields = nfields; flddesc->alignment = alignment; flddesc->haspadding = haspadding; diff --git a/src/dump.c b/src/dump.c index d4af9d6c1703d..4780a126f3b0a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1204,8 +1204,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) uint16_t nf = read_uint16(s); uint8_t fielddesc_type = read_int8(s); size_t fielddesc_size = nf > 0 ? jl_fielddesc_size(fielddesc_type) : 0; - struct _jl_datatype_layout_t *layout = (struct _jl_datatype_layout_t*)malloc( - sizeof(struct _jl_datatype_layout_t) + nf * fielddesc_size); + jl_datatype_layout_t *layout = (jl_datatype_layout_t*)jl_gc_perm_alloc( + sizeof(jl_datatype_layout_t) + nf * fielddesc_size); layout->nfields = nf; layout->fielddesc_type = fielddesc_type; layout->alignment = read_int32(s); diff --git a/src/gc.c b/src/gc.c index c42614df2adbd..953f79a08c479 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2031,6 +2031,61 @@ JL_DLLEXPORT void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, return b; } +// Perm gen allocator +// 2M pool +#define GC_PERM_POOL_SIZE (2 * 1024 * 1024) +// 20k limit for pool allocation. At most 1% fragmentation +#define GC_PERM_POOL_LIMIT (20 * 1024) +jl_mutex_t gc_perm_lock = {0, 0}; +static char *gc_perm_pool = NULL; +static size_t gc_perm_size = 0; + +// **NOT** a safepoint +void *jl_gc_perm_alloc_nolock(size_t sz) +{ + // The caller should have acquired `gc_perm_lock` +#ifndef MEMDEBUG + if (__unlikely(sz > GC_PERM_POOL_LIMIT)) +#endif + return malloc(sz); + sz = LLT_ALIGN(sz, JL_SMALL_BYTE_ALIGNMENT); + if (__unlikely(sz > gc_perm_size)) { +#ifdef _OS_WINDOWS_ + void *pool = VirtualAlloc(NULL, + GC_PERM_POOL_SIZE + JL_SMALL_BYTE_ALIGNMENT, + MEM_COMMIT, PAGE_READWRITE); + if (__unlikely(pool == NULL)) + return NULL; + pool = (void*)LLT_ALIGN((uintptr_t)pool, JL_SMALL_BYTE_ALIGNMENT); +#else + void *pool = mmap(0, GC_PERM_POOL_SIZE, PROT_READ | PROT_WRITE, + MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (__unlikely(pool == MAP_FAILED)) + return NULL; +#endif + gc_perm_pool = (char*)pool; + gc_perm_size = GC_PERM_POOL_SIZE; + } + assert(((uintptr_t)gc_perm_pool) % JL_SMALL_BYTE_ALIGNMENT == 0); + void *p = gc_perm_pool; + gc_perm_size -= sz; + gc_perm_pool += sz; + return p; +} + +// **NOT** a safepoint +void *jl_gc_perm_alloc(size_t sz) +{ +#ifndef MEMDEBUG + if (__unlikely(sz > GC_PERM_POOL_LIMIT)) +#endif + return malloc(sz); + JL_LOCK_NOGC(&gc_perm_lock); + void *p = jl_gc_perm_alloc_nolock(sz); + JL_UNLOCK_NOGC(&gc_perm_lock); + return p; +} + JL_DLLEXPORT void jl_gc_add_finalizer(jl_value_t *v, jl_function_t *f) { jl_ptls_t ptls = jl_get_ptls_states(); diff --git a/src/jltypes.c b/src/jltypes.c index 3aefaeb42ad59..c3975601a4b15 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3741,7 +3741,7 @@ void jl_init_types(void) jl_emptysvec, jl_emptysvec, 0, 1, 0); jl_array_typename = jl_array_type->name; jl_array_type->ninitialized = 0; - static const struct _jl_datatype_layout_t _jl_array_layout = { 0, sizeof(void*), 0, 0, 0 }; + static const jl_datatype_layout_t _jl_array_layout = { 0, sizeof(void*), 0, 0, 0 }; jl_array_type->layout = &_jl_array_layout; jl_array_any_type = @@ -3976,8 +3976,8 @@ void jl_init_types(void) jl_compute_field_offsets(jl_sym_type); // TODO: don't modify layout objects - ((struct _jl_datatype_layout_t*)jl_sym_type->layout)->pointerfree = 0; - ((struct _jl_datatype_layout_t*)jl_simplevector_type->layout)->pointerfree = 0; + ((jl_datatype_layout_t*)jl_sym_type->layout)->pointerfree = 0; + ((jl_datatype_layout_t*)jl_simplevector_type->layout)->pointerfree = 0; empty_sym = jl_symbol(""); call_sym = jl_symbol("call"); diff --git a/src/julia.h b/src/julia.h index d2be17938910b..772ec1cb790f4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -328,7 +328,7 @@ typedef struct { uint32_t offset; // offset relative to data start, excluding type tag } jl_fielddesc32_t; -struct _jl_datatype_layout_t { +typedef struct { uint32_t nfields; uint32_t alignment : 28; // strictest alignment over all fields uint32_t haspadding : 1; // has internal undefined bytes @@ -339,7 +339,7 @@ struct _jl_datatype_layout_t { // jl_fielddesc16_t field16[]; // jl_fielddesc32_t field32[]; // }; -}; +} jl_datatype_layout_t; typedef struct _jl_datatype_t { JL_DATA_TYPE @@ -348,7 +348,7 @@ typedef struct _jl_datatype_t { jl_svec_t *parameters; jl_svec_t *types; jl_value_t *instance; // for singletons - const struct _jl_datatype_layout_t *layout; + const jl_datatype_layout_t *layout; int32_t size; // TODO: move to _jl_datatype_layout_t int32_t ninitialized; uint32_t uid; @@ -790,12 +790,12 @@ STATIC_INLINE char *jl_symbol_name_(jl_sym_t *s) } #define jl_symbol_name(s) jl_symbol_name_(s) -#define jl_dt_layout_fields(d) ((const char*)(d) + sizeof(struct _jl_datatype_layout_t)) +#define jl_dt_layout_fields(d) ((const char*)(d) + sizeof(jl_datatype_layout_t)) #define DEFINE_FIELD_ACCESSORS(f) \ static inline uint32_t jl_field_##f(jl_datatype_t *st, int i) \ { \ - const struct _jl_datatype_layout_t *ly = st->layout; \ + const jl_datatype_layout_t *ly = st->layout; \ assert(i >= 0 && (size_t)i < ly->nfields); \ if (ly->fielddesc_type == 0) { \ return ((const jl_fielddesc8_t*)jl_dt_layout_fields(ly))[i].f; \ @@ -812,7 +812,7 @@ DEFINE_FIELD_ACCESSORS(offset) DEFINE_FIELD_ACCESSORS(size) static inline int jl_field_isptr(jl_datatype_t *st, int i) { - const struct _jl_datatype_layout_t *ly = st->layout; + const jl_datatype_layout_t *ly = st->layout; assert(i >= 0 && (size_t)i < ly->nfields); return ((const jl_fielddesc8_t*)(jl_dt_layout_fields(ly) + (i << (ly->fielddesc_type + 1))))->isptr; } diff --git a/src/julia_internal.h b/src/julia_internal.h index 105b3641fdf47..22e40bf16f464 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -44,6 +44,9 @@ JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, int osize, int end_offset); JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz); int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset); +extern jl_mutex_t gc_perm_lock; +void *jl_gc_perm_alloc_nolock(size_t sz); +void *jl_gc_perm_alloc(size_t sz); // pools are 16376 bytes large (GC_POOL_SZ - GC_PAGE_OFFSET) static const int jl_gc_sizeclasses[JL_GC_N_POOLS] = { diff --git a/test/core.jl b/test/core.jl index b3a06317a759e..2863d79efb373 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3342,8 +3342,6 @@ typealias PossiblyInvalidUnion{T} Union{T,Int} @test_throws TypeError PossiblyInvalidUnion{1} # issue #12569 -@test_throws ArgumentError Symbol("x"^10_000_000) -@test_throws ArgumentError gensym("x"^10_000_000) @test Symbol("x") === Symbol("x") @test split(string(gensym("abc")),'#')[3] == "abc" From 5e3fe36a47e656c8b8f80340facbbfedb0bd240a Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Wed, 6 Jul 2016 19:28:17 -0400 Subject: [PATCH 0263/1117] fix #17304: f.(args...) and splatting --- src/julia-syntax.scm | 10 +++++----- test/broadcast.jl | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index a1574692dcbed..168352c1e1f10 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1562,13 +1562,13 @@ '|.| (lambda (e) ; e = (|.| f x) - (let ((f (expand-forms (cadr e))) - (x (expand-forms (caddr e)))) + (let ((f (cadr e)) + (x (caddr e))) (if (or (eq? (car x) 'quote) (eq? (car x) 'inert) (eq? (car x) '$)) - `(call (core getfield) ,f ,x) + `(call (core getfield) ,(expand-forms f) ,(expand-forms x)) ; otherwise, came from f.(args...) --> broadcast(f, args...), - ; where x = (call (top tuple) args...) at this point: - `(call broadcast ,f ,@(cddr x))))) + ; where x = (tuple args...) at this point: + (expand-forms `(call broadcast ,f ,@(cdr x)))))) '|<:| syntactic-op-to-call '|>:| syntactic-op-to-call diff --git a/test/broadcast.jl b/test/broadcast.jl index 0b43510974ae7..23d0b1c0c4245 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -213,3 +213,8 @@ end @test Base.promote_op(+, Bool) === Int @test isa(broadcast(+, true), Array{Int,0}) @test Base.promote_op(Float64, Bool) === Float64 + +# issue #17304 +let foo = [[1,2,3],[4,5,6],[7,8,9]] + @test max.(foo...) == broadcast(max, foo...) == [7,8,9] +end From 0ddea145e899d3572061a1100dc4f84d63b3a3b7 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sun, 3 Jul 2016 01:09:41 -0400 Subject: [PATCH 0264/1117] Implement poor man PLT So that `ccall`s in the sysimg doesn't need any conditional branch. Also use different GV's for the same symbol name that belongs to different libraries. --- src/ccall.cpp | 263 ++++++++++++++++++++++++++++++++++++++++-------- src/cgutils.cpp | 2 + src/codegen.cpp | 2 +- 3 files changed, 225 insertions(+), 42 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index adfced1fe94fa..5b7438046eae9 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -4,52 +4,96 @@ // --- the ccall, cglobal, and llvm intrinsics --- -static StringMap<GlobalVariable*> libMapGV; -static StringMap<GlobalVariable*> symMapGV; +// Map from symbol name (in a certain library) to its GV in sysimg and the +// DL handle address in the current session. +typedef StringMap<std::pair<GlobalVariable*,void*>> SymMapGV; +static StringMap<std::pair<GlobalVariable*,SymMapGV>> libMapGV; +#ifdef _OS_WINDOWS_ +static SymMapGV symMapExe; +static SymMapGV symMapDl; +#endif +static SymMapGV symMapDefault; + +template<typename Func> +struct LazyModule { + Func func; + Module *m; + template<typename Func2> + LazyModule(Func2 &&func) + : func(std::forward<Func2>(func)), + m(nullptr) + {} + Module *get() + { + if (!m) + m = func(); + return m; + } + Module &operator*() + { + return *get(); + } +}; -static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, const char *f_name, jl_codectx_t *ctx) +template<typename Func> +static LazyModule<typename std::remove_reference<Func>::type> +lazyModule(Func &&func) +{ + return LazyModule<typename std::remove_reference<Func>::type>( + std::forward<Func>(func)); +} + +// Find or create the GVs for the library and symbol lookup. +// Return `runtime_lib` (whether the library name is a string) +// Optionally return the symbol address in the current session +// when `symaddr != nullptr`. +// The `lib` and `sym` GV returned may not be in the current module. +template<typename MT> +static bool runtime_sym_gvs(const char *f_lib, const char *f_name, MT &&M, + GlobalVariable *&lib, GlobalVariable *&sym, + void **symaddr=nullptr) { - // in pseudo-code, this function emits the following: - // global HMODULE *libptrgv - // global void **llvmgv - // if (*llvmgv == NULL) { - // *llvmgv = jl_load_and_lookup(f_lib, f_name, libptrgv); - // } - // return (*llvmgv) void *libsym = NULL; bool runtime_lib = false; GlobalVariable *libptrgv; + SymMapGV *symMap; #ifdef _OS_WINDOWS_ if ((intptr_t)f_lib == 1) { - libptrgv = prepare_global(jlexe_var); + libptrgv = jlexe_var; libsym = jl_exe_handle; + symMap = &symMapExe; } else if ((intptr_t)f_lib == 2) { - libptrgv = prepare_global(jldll_var); + libptrgv = jldll_var; libsym = jl_dl_handle; + symMap = &symMapDl; } else #endif if (f_lib == NULL) { - libptrgv = prepare_global(jlRTLD_DEFAULT_var); + libptrgv = jlRTLD_DEFAULT_var; libsym = jl_RTLD_DEFAULT_handle; + symMap = &symMapDefault; } else { std::string name = "ccalllib_"; name += f_lib; runtime_lib = true; - libptrgv = libMapGV[f_lib]; - if (libptrgv == NULL) { - libptrgv = new GlobalVariable(*jl_Module, T_pint8, - false, GlobalVariable::ExternalLinkage, - NULL, name); - libMapGV[f_lib] = global_proto(libptrgv); + auto iter = libMapGV.find(f_lib); + if (iter == libMapGV.end()) { + libptrgv = new GlobalVariable(*M, T_pint8, false, + GlobalVariable::ExternalLinkage, + NULL, name); + auto &libgv = libMapGV[f_lib]; + libgv = std::make_pair(global_proto(libptrgv), SymMapGV()); + symMap = &libgv.second; libsym = jl_get_library(f_lib); assert(libsym != NULL); *(void**)jl_emit_and_add_to_shadow(libptrgv) = libsym; } else { - libptrgv = prepare_global(libptrgv); + libptrgv = iter->second.first; + symMap = &iter->second.second; } } if (libsym == NULL) { @@ -57,29 +101,54 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, cons } assert(libsym != NULL); - GlobalVariable *llvmgv = symMapGV[f_name]; - if (llvmgv == NULL) { + GlobalVariable *llvmgv; + auto sym_iter = symMap->find(f_name); + if (sym_iter == symMap->end()) { // MCJIT forces this to have external linkage eventually, so we would clobber // the symbol of the actual function. std::string name = "ccall_"; name += f_name; - llvmgv = new GlobalVariable(*jl_Module, T_pvoidfunc, - false, GlobalVariable::ExternalLinkage, - NULL, name); - symMapGV[f_name] = global_proto(llvmgv); - *(void**)jl_emit_and_add_to_shadow(llvmgv) = jl_dlsym_e(libsym, f_name); + name += "_"; + name += std::to_string(globalUnique++); + llvmgv = new GlobalVariable(*M, T_pvoidfunc, false, + GlobalVariable::ExternalLinkage, NULL, name); + llvmgv = global_proto(llvmgv); + void *addr = jl_dlsym_e(libsym, f_name); + (*symMap)[f_name] = std::make_pair(llvmgv, addr); + if (symaddr) + *symaddr = addr; + *(void**)jl_emit_and_add_to_shadow(llvmgv) = addr; } else { - llvmgv = prepare_global(llvmgv); + if (symaddr) + *symaddr = sym_iter->second.second; + llvmgv = sym_iter->second.first; } - BasicBlock *dlsym_lookup = BasicBlock::Create(jl_LLVMContext, "dlsym"), - *ccall_bb = BasicBlock::Create(jl_LLVMContext, "ccall"); + lib = libptrgv; + sym = llvmgv; + return runtime_lib; +} + +static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, + const char *f_name, Function *f, + GlobalVariable *libptrgv, + GlobalVariable *llvmgv, bool runtime_lib) +{ + // in pseudo-code, this function emits the following: + // global HMODULE *libptrgv + // global void **llvmgv + // if (*llvmgv == NULL) { + // *llvmgv = jl_load_and_lookup(f_lib, f_name, libptrgv); + // } + // return (*llvmgv) + BasicBlock *dlsym_lookup = BasicBlock::Create(jl_LLVMContext, "dlsym"); + BasicBlock *ccall_bb = BasicBlock::Create(jl_LLVMContext, "ccall"); Constant *initnul = ConstantPointerNull::get((PointerType*)T_pvoidfunc); builder.CreateCondBr(builder.CreateICmpNE(builder.CreateLoad(llvmgv), initnul), ccall_bb, dlsym_lookup); - assert(ctx->f->getParent() != NULL); - ctx->f->getBasicBlockList().push_back(dlsym_lookup); + assert(f->getParent() != NULL); + f->getBasicBlockList().push_back(dlsym_lookup); builder.SetInsertPoint(dlsym_lookup); Value *libname; if (runtime_lib) { @@ -96,12 +165,128 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, cons builder.CreateStore(llvmf, llvmgv); builder.CreateBr(ccall_bb); - ctx->f->getBasicBlockList().push_back(ccall_bb); + f->getBasicBlockList().push_back(ccall_bb); builder.SetInsertPoint(ccall_bb); llvmf = builder.CreateLoad(llvmgv); return builder.CreatePointerCast(llvmf,funcptype); } +static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, + const char *f_name, Function *f) +{ + GlobalVariable *libptrgv; + GlobalVariable *llvmgv; + bool runtime_lib = runtime_sym_gvs(f_lib, f_name, f->getParent(), + libptrgv, llvmgv); + libptrgv = prepare_global(libptrgv); + llvmgv = prepare_global(llvmgv); + return runtime_sym_lookup(funcptype, f_lib, f_name, f, libptrgv, llvmgv, + runtime_lib); +} + +// Map from distinct callee's to it's GOT entry. +// In principle the attribute, function type and calling convention +// don't need to be part of the key but it seems impossible to forward +// all the arguments without writing assembly directly. +// This doesn't matter too much in reality since a single function is usually +// not called with multiple signatures. +static DenseMap<AttributeSet, + std::map<std::tuple<GlobalVariable*,FunctionType*, + CallingConv::ID>,GlobalVariable*>> allPltMap; + +// Emit a "PLT" entry that will be lazily initialized +// when being called the first time. +static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, + CallingConv::ID cc, const char *f_lib, const char *f_name) +{ + assert(imaging_mode); + GlobalVariable *libptrgv; + GlobalVariable *llvmgv; + void *symaddr; + auto LM = lazyModule([&] { + Module *m = new Module(f_name, jl_LLVMContext); + jl_setup_module(m); + return m; + }); + bool runtime_lib = runtime_sym_gvs(f_lib, f_name, LM, + libptrgv, llvmgv, &symaddr); + PointerType *funcptype = PointerType::get(functype, 0); + + auto &pltMap = allPltMap[attrs]; + auto key = std::make_tuple(llvmgv, functype, cc); + auto &slot = pltMap[key]; + GlobalVariable *got; + if (!slot) { + Module *M = LM.get(); + libptrgv = prepare_global(libptrgv, M); + llvmgv = prepare_global(llvmgv, M); + BasicBlock *old = builder.GetInsertBlock(); + DebugLoc olddl = builder.getCurrentDebugLocation(); + DebugLoc noDbg; + builder.SetCurrentDebugLocation(noDbg); + std::stringstream funcName; + funcName << "jlplt_" << f_name << "_" << globalUnique++; + auto fname = funcName.str(); + Function *plt = Function::Create(functype, + GlobalVariable::ExternalLinkage, + fname, M); + plt->setAttributes(attrs); + if (cc != CallingConv::C) + plt->setCallingConv(cc); + funcName << "_got"; + auto gname = funcName.str(); + got = new GlobalVariable(*M, T_pvoidfunc, false, + GlobalVariable::ExternalLinkage, + nullptr, gname); + slot = global_proto(got); + *(void**)jl_emit_and_add_to_shadow(got) = symaddr; + BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", plt); + builder.SetInsertPoint(b0); + Value *ptr = runtime_sym_lookup(funcptype, f_lib, f_name, plt, libptrgv, + llvmgv, runtime_lib); + builder.CreateStore(builder.CreateBitCast(ptr, T_pvoidfunc), got); + SmallVector<Value*, 16> args; + for (auto &arg: plt->args()) + args.push_back(&arg); + CallInst *ret = builder.CreateCall(ptr, ArrayRef<Value*>(args)); + ret->setAttributes(attrs); + if (cc != CallingConv::C) + ret->setCallingConv(cc); + // NoReturn function can trigger LLVM verifier error when declared as + // MustTail since other passes might replace the `ret` with + // `unreachable` (LLVM should probably accept `unreachable`). + if (attrs.hasAttribute(AttributeSet::FunctionIndex, + Attribute::NoReturn)) { + builder.CreateUnreachable(); + } + else { + ret->setTailCallKind(CallInst::TCK_MustTail); + if (functype->getReturnType() == T_void) { + builder.CreateRetVoid(); + } + else { + builder.CreateRet(ret); + } + } + builder.SetInsertPoint(old); + builder.SetCurrentDebugLocation(olddl); + jl_finalize_module(std::unique_ptr<Module>(M), true); + auto shadowgot = + cast<GlobalVariable>(shadow_output->getNamedValue(gname)); + auto shadowplt = cast<Function>(shadow_output->getNamedValue(fname)); + shadowgot->setInitializer(ConstantExpr::getBitCast(shadowplt, + T_pvoidfunc)); + got = prepare_global(shadowgot); + } + else { + // `runtime_sym_gvs` shouldn't have created anything in a new module + // if it returns a GV that already exists. + assert(!LM.m); + got = prepare_global(slot); + } + return builder.CreateBitCast(builder.CreateLoad(got), funcptype); +} + // --- ABI Implementations --- // Partially based on the LDC ABI implementations licensed under the BSD 3-clause license @@ -399,8 +584,6 @@ static native_sym_arg_t interpret_symbol_arg(jl_value_t *arg, jl_codectx_t *ctx, } -typedef AttributeSet attr_type; - static jl_value_t* try_eval(jl_value_t *ex, jl_codectx_t *ctx, const char *failure, bool compiletime=false) { jl_value_t *constant = NULL; @@ -457,7 +640,7 @@ static jl_cgval_t emit_cglobal(jl_value_t **args, size_t nargs, jl_codectx_t *ct } else { if (imaging_mode) { - res = runtime_sym_lookup((PointerType*)lrt, sym.f_lib, sym.f_name, ctx); + res = runtime_sym_lookup((PointerType*)lrt, sym.f_lib, sym.f_name, ctx->f); } else { void *symaddr = jl_dlsym_e(jl_get_library(sym.f_lib), sym.f_name); @@ -860,8 +1043,6 @@ static jl_cgval_t mark_or_box_ccall_result(Value *result, bool isboxed, jl_value return mark_julia_type(result, isboxed, rt, ctx); } -typedef AttributeSet attr_type; - static std::string generate_func_sig( Type **lrt, // input parameter of the llvm return type (from julia_struct_to_llvm) Type **prt, // out parameter of the llvm return type for the function signature @@ -872,7 +1053,7 @@ static std::string generate_func_sig( Type *&fargt_vasig, // ABI coercion type for vararg list std::vector<bool> &inRegList, // vector of "inreg" parameters (vararg is the last item, if applicable) std::vector<bool> &byRefList, // vector of "byref" parameters (vararg is the last item, if applicable) - attr_type &attributes, // vector of function call site attributes (vararg is the last item, if applicable) + AttributeSet &attributes, // vector of function call site attributes (vararg is the last item, if applicable) jl_value_t *rt, // julia return type jl_svec_t *tt, // tuple of julia argument types size_t nargs) // number of actual arguments (can be different from the size of tt when varargs) @@ -1396,7 +1577,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) Type *fargt_vasig = NULL; std::vector<bool> inRegList(0); std::vector<bool> byRefList(0); - attr_type attrs; + AttributeSet attrs; Type *prt = NULL; int sret = 0; std::string err_msg = generate_func_sig(&lrt, &prt, sret, fargt, fargt_isboxed, fargt_sig, fargt_vasig, @@ -1513,7 +1694,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) PointerType *funcptype = PointerType::get(functype,0); if (imaging_mode) { - llvmf = runtime_sym_lookup(funcptype, f_lib, f_name, ctx); + llvmf = emit_plt(functype, attrs, cc, f_lib, f_name); } else { void *symaddr = jl_dlsym_e(jl_get_library(f_lib), f_name); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index d4c7f6b130b51..95cb22b4c85ff 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -12,6 +12,8 @@ static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store) static GlobalVariable *prepare_global(GlobalVariable *G, Module *M) { + if (G->getParent() == M) + return G; GlobalValue *local = M->getNamedValue(G->getName()); if (!local) { local = global_proto(G, M); diff --git a/src/codegen.cpp b/src/codegen.cpp index f627251616f6a..a8ff77a17ae83 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3548,7 +3548,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t Type *fargt_vasig; std::vector<bool> inRegList(0); std::vector<bool> byRefList(0); - attr_type attrs; + AttributeSet attrs; Type *prt = NULL; int sret = 0; size_t nargs = jl_nparams(argt); From 2cacfa628a7b12243151d6d535a7e59ada88312b Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 1 Jun 2016 14:01:25 -0400 Subject: [PATCH 0265/1117] move A_mul_B etc docs to linalg section of manual, expand docs, export Ac_ldiv_B\! and At_ldiv_B! --- base/exports.jl | 2 + base/linalg/factorization.jl | 38 ++++++++ base/linalg/linalg.jl | 2 + doc/stdlib/linalg.rst | 174 +++++++++++++++++++++++++++++++++-- doc/stdlib/math.rst | 123 ------------------------- 5 files changed, 210 insertions(+), 129 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 5af15b345eb31..a64eba3203003 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -265,6 +265,7 @@ export A_rdiv_Bc, A_rdiv_Bt, Ac_ldiv_B, + Ac_ldiv_B!, Ac_ldiv_Bc, Ac_mul_B, Ac_mul_B!, @@ -273,6 +274,7 @@ export Ac_rdiv_B, Ac_rdiv_Bc, At_ldiv_B, + At_ldiv_B!, At_ldiv_Bt, At_mul_B, At_mul_B!, diff --git a/base/linalg/factorization.jl b/base/linalg/factorization.jl index 5e9980f203c93..5b6aade370330 100644 --- a/base/linalg/factorization.jl +++ b/base/linalg/factorization.jl @@ -41,3 +41,41 @@ for (f1, f2) in ((:\, :A_ldiv_B!), end end end + +# support the same 3-arg idiom as in our other in-place A_*_B functions: +for f in (:A_ldiv_B!, :Ac_ldiv_B!, :At_ldiv_B!) + @eval $f(Y::AbstractVecOrMat, A::Factorization, B::AbstractVecOrMat) = + $f(A, copy!(Y, B)) +end + +""" + A_ldiv_B!([Y,] A, B) -> Y + +Compute `A \ B` in-place and store the result in `Y`, returning the result. +If only two arguments are passed, then `A_ldiv_B!(A, B)` overwrites `B` with +the result. + +The argument `A` should *not* be a matrix. Rather, instead of matrices it should be a +factorization object (e.g. produced by [`factorize`](:func:`factorize`) or [`cholfact`](:func:`cholfact`)). +The reason for this is that factorization itself is both expensive and typically allocates memory +(although it can also be done in-place via, e.g., [`lufact`](:func:`lufact`)), +and performance-critical situations requiring `A_ldiv_B!` usually also require fine-grained +control over the factorization of `A`. +""" +A_ldiv_B! + +""" + Ac_ldiv_B!([Y,] A, B) -> Y + +Similar to [`A_ldiv_B!`](:func:`A_ldiv_B!`), but return ``Aᴴ`` \\ ``B``, +computing the result in-place in `Y` (or overwriting `B` if `Y` is not supplied). +""" +Ac_ldiv_B! + +""" + At_ldiv_B!([Y,] A, B) -> Y + +Similar to [`A_ldiv_B!`](:func:`A_ldiv_B!`), but return ``Aᵀ`` \\ ``B``, +computing the result in-place in `Y` (or overwriting `B` if `Y` is not supplied). +""" +At_ldiv_B! diff --git a/base/linalg/linalg.jl b/base/linalg/linalg.jl index 8171bd27448fd..dc36d649fae31 100644 --- a/base/linalg/linalg.jl +++ b/base/linalg/linalg.jl @@ -147,6 +147,7 @@ export A_rdiv_Bt, Ac_ldiv_B, Ac_ldiv_Bc, + Ac_ldiv_B!, Ac_mul_B, Ac_mul_B!, Ac_mul_Bc, @@ -155,6 +156,7 @@ export Ac_rdiv_Bc, At_ldiv_B, At_ldiv_Bt, + At_ldiv_B!, At_mul_B, At_mul_B!, At_mul_Bt, diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 55b7c5296403e..30d5dd320531b 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1459,17 +1459,180 @@ Linear algebra functions in Julia are largely implemented by calling functions f If the keyword argument ``parallel`` is set to ``true``\ , ``peakflops`` is run in parallel on all the worker processors. The flop rate of the entire parallel computer is returned. When running in parallel, only 1 BLAS thread is used. The argument ``n`` still refers to the size of the problem that is solved on each processor. +Low-level matrix operations +--------------------------- + +Matrix operations involving transpositions operations like ``A' \ B`` are converted +by the Julia parser into calls to specially named functions like ``Ac_ldiv_B``. +If you want to overload these operations for your own types, then it is useful +to know the names of these functions. + +Also, in many cases there are in-place versions of matrix operations that +allow you to supply a pre-allocated output vector or matrix. This is useful +when optimizing critical code in order to avoid the overhead of repeated allocations. +These in-place operations are suffixed with ``!`` below (e.g. ``A_mul_B!``) +according to the usual Julia convention. + +.. function:: A_ldiv_B!([Y,] A, B) -> Y + + .. Docstring generated from Julia source + + Compute ``A B`` in-place and store the result in ``Y``\ , returning the result. If only two arguments are passed, then ``A_ldiv_B!(A, B)`` overwrites ``B`` with the result. + + The argument ``A`` should *not* be a matrix. Rather, instead of matrices it should be a factorization object (e.g. produced by :func:`factorize` or :func:`cholfact`\ ). The reason for this is that factorization itself is both expensive and typically allocates memory (although it can also be done in-place via, e.g., :func:`lufact`\ ), and performance-critical situations requiring ``A_ldiv_B!`` usually also require fine-grained control over the factorization of ``A``\ . + +.. function:: A_ldiv_Bc(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A` \\ :math:`Bᴴ`\ . + +.. function:: A_ldiv_Bt(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A` \\ :math:`Bᵀ`\ . + +.. function:: A_mul_B!(Y, A, B) -> Y + + .. Docstring generated from Julia source + + Calculates the matrix-matrix or matrix-vector product :math:`A⋅B` and stores the result in ``Y``\ , overwriting the existing value of ``Y``\ . Note that ``Y`` must not be aliased with either ``A`` or ``B``\ . + + .. doctest:: + + julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; Y = similar(B); A_mul_B!(Y, A, B); + + julia> Y + 2×2 Array{Float64,2}: + 3.0 3.0 + 7.0 7.0 + +.. function:: A_mul_Bc(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A⋅Bᴴ`\ . + +.. function:: A_mul_Bt(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A⋅Bᵀ`\ . + +.. function:: A_rdiv_Bc(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A / Bᴴ`\ . + +.. function:: A_rdiv_Bt(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A / Bᵀ`\ . + +.. function:: Ac_ldiv_B(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ` \\ :math:`B`\ . + +.. function:: Ac_ldiv_B!([Y,] A, B) -> Y + + .. Docstring generated from Julia source + + Similar to :func:`A_ldiv_B!`\ , but return :math:`Aᴴ` \\ :math:`B`\ , computing the result in-place in ``Y`` (or overwriting ``B`` if ``Y`` is not supplied). + +.. function:: Ac_ldiv_Bc(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ` \\ :math:`Bᴴ`\ . + +.. function:: Ac_mul_B(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ⋅B`\ . + +.. function:: Ac_mul_Bc(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ Bᴴ`\ . + +.. function:: Ac_rdiv_B(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ / B`\ . + +.. function:: Ac_rdiv_Bc(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ / Bᴴ`\ . + +.. function:: At_ldiv_B(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ` \\ :math:`B`\ . + +.. function:: At_ldiv_B!([Y,] A, B) -> Y + + .. Docstring generated from Julia source + + Similar to :func:`A_ldiv_B!`\ , but return :math:`Aᵀ` \\ :math:`B`\ , computing the result in-place in ``Y`` (or overwriting ``B`` if ``Y`` is not supplied). + +.. function:: At_ldiv_Bt(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ` \\ :math:`Bᵀ`\ . + +.. function:: At_mul_B(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ⋅B`\ . + +.. function:: At_mul_Bt(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ⋅Bᵀ`\ . + +.. function:: At_rdiv_B(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ / B`\ . + +.. function:: At_rdiv_Bt(A, B) + + .. Docstring generated from Julia source + + For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ / Bᵀ`\ . + BLAS Functions -------------- .. module:: Base.LinAlg.BLAS -:mod:`Base.LinAlg.BLAS` provides wrappers for some of the BLAS functions for -linear algebra. Those BLAS functions that overwrite one of the input -arrays have names ending in ``'!'``. +In Julia (as in much of scientific computation), dense linear-algebra operations are +based on the `LAPACK library <http://www.netlib.org/lapack/>`_, which in turn +is built on top of basic linear-algebra building-blocks known as the +`BLAS <http://www.netlib.org/blas/>`_. There are highly optimized implementations +of BLAS available for every computer architecture, and sometimes in +high-performance linear algebra routines it is useful to call the BLAS +functions directly. -Usually a function has 4 methods defined, one each for ``Float64``, -``Float32``, ``Complex128`` and ``Complex64`` arrays. +:mod:`Base.LinAlg.BLAS` provides wrappers for some of the BLAS functions. +Those BLAS functions that overwrite one of the input +arrays have names ending in ``'!'``. Usually, a BLAS function has four methods defined, +for ``Float64``, ``Float32``, ``Complex128``, and ``Complex64`` arrays. .. currentmodule:: Base.LinAlg.BLAS @@ -2362,4 +2525,3 @@ set of functions in future releases. Solves the Sylvester matrix equation ``A * X +/- X * B = scale*C`` where ``A`` and ``B`` are both quasi-upper triangular. If ``transa = N``\ , ``A`` is not modified. If ``transa = T``\ , ``A`` is transposed. If ``transa = C``\ , ``A`` is conjugate transposed. Similarly for ``transb`` and ``B``\ . If ``isgn = 1``\ , the equation ``A * X + X * B = scale * C`` is solved. If ``isgn = -1``\ , the equation ``A * X - X * B = scale * C`` is solved. Returns ``X`` (overwriting ``C``\ ) and ``scale``\ . - diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index c770557bce0ca..2fa7dd7dbfcc0 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -478,129 +478,6 @@ Mathematical Operators Short-circuiting boolean OR. -.. function:: A_ldiv_Bc(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A` \\ :math:`Bᴴ`\ . - -.. function:: A_ldiv_Bt(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A` \\ :math:`Bᵀ`\ . - -.. function:: A_mul_B!(Y, A, B) -> Y - - .. Docstring generated from Julia source - - Calculates the matrix-matrix or matrix-vector product :math:`A⋅B` and stores the result in ``Y``\ , overwriting the existing value of ``Y``\ . Note that ``Y`` must not be aliased with either ``A`` or ``B``\ . - - .. doctest:: - - julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; Y = similar(B); A_mul_B!(Y, A, B); - - julia> Y - 2×2 Array{Float64,2}: - 3.0 3.0 - 7.0 7.0 - -.. function:: A_mul_Bc(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A⋅Bᴴ`\ . - -.. function:: A_mul_Bt(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A⋅Bᵀ`\ . - -.. function:: A_rdiv_Bc(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A / Bᴴ`\ . - -.. function:: A_rdiv_Bt(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`A / Bᵀ`\ . - -.. function:: Ac_ldiv_B(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ` \\ :math:`B`\ . - -.. function:: Ac_ldiv_Bc(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ` \\ :math:`Bᴴ`\ . - -.. function:: Ac_mul_B(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ⋅B`\ . - -.. function:: Ac_mul_Bc(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ Bᴴ`\ . - -.. function:: Ac_rdiv_B(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ / B`\ . - -.. function:: Ac_rdiv_Bc(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᴴ / Bᴴ`\ . - -.. function:: At_ldiv_B(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ` \\ :math:`B`\ . - -.. function:: At_ldiv_Bt(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ` \\ :math:`Bᵀ`\ . - -.. function:: At_mul_B(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ⋅B`\ . - -.. function:: At_mul_Bt(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ⋅Bᵀ`\ . - -.. function:: At_rdiv_B(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ / B`\ . - -.. function:: At_rdiv_Bt(A, B) - - .. Docstring generated from Julia source - - For matrices or vectors :math:`A` and :math:`B`\ , calculates :math:`Aᵀ / Bᵀ`\ . - Mathematical Functions ---------------------- From b3d28d3ce98d8021c2d94639d8ce8c3ce5c3ccef Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 7 Jul 2016 13:06:06 +0000 Subject: [PATCH 0266/1117] Fix AArch64 ABI segfault in early bootstrap --- src/abi_aarch64.cpp | 3 ++- src/abi_arm.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/abi_aarch64.cpp b/src/abi_aarch64.cpp index 61e1cb5d688b3..a1c6089505a5e 100644 --- a/src/abi_aarch64.cpp +++ b/src/abi_aarch64.cpp @@ -83,7 +83,8 @@ static Type *get_llvm_fptype(jl_datatype_t *dt) default: return nullptr; } - return jl_is_floattype((jl_value_t*)dt) ? lltype : nullptr; + return ((jl_floatingpoint_type && jl_is_floattype((jl_value_t*)dt)) ? + lltype : nullptr); } static Type *get_llvm_fp_or_vectype(jl_datatype_t *dt) diff --git a/src/abi_arm.cpp b/src/abi_arm.cpp index a79d1de1a6ed3..69ca14dfb8f70 100644 --- a/src/abi_arm.cpp +++ b/src/abi_arm.cpp @@ -54,7 +54,8 @@ static Type *get_llvm_fptype(jl_datatype_t *dt) default: return NULL; } - return jl_is_floattype((jl_value_t*)dt) ? lltype : NULL; + return ((jl_floatingpoint_type && jl_is_floattype((jl_value_t*)dt)) ? + lltype : NULL); } static size_t isLegalHA(jl_datatype_t *dt, Type *&base); From 8716dbc05156969da38adbbe2bf6c998cb3c9854 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 7 Jul 2016 13:30:04 +0000 Subject: [PATCH 0267/1117] Disable PLT on ARM, PPC and PPC64 for vararg functions. Workaround LLVM bug. Fix #17312 --- src/ccall.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 5b7438046eae9..f6bf9f81722a7 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1694,7 +1694,14 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) PointerType *funcptype = PointerType::get(functype,0); if (imaging_mode) { - llvmf = emit_plt(functype, attrs, cc, f_lib, f_name); +#if defined(_CPU_ARM_) || defined(_CPU_PPC_) || defined(_CPU_PPC64_) + // ARM, PPC, PPC64 (as of LLVM 3.9) doesn't support `musttail` + // for vararg functions. + if (functype) + llvmf = runtime_sym_lookup(funcptype, f_lib, f_name, ctx->f); + else +#endif + llvmf = emit_plt(functype, attrs, cc, f_lib, f_name); } else { void *symaddr = jl_dlsym_e(jl_get_library(f_lib), f_name); From 038dc0091cc6c369fe8df1ec4bb6afa4fbd3fea9 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Thu, 7 Jul 2016 10:44:50 -0400 Subject: [PATCH 0268/1117] broadcasting over scalars should produce a scalar --- base/abstractarray.jl | 1 + base/broadcast.jl | 5 +++++ test/abstractarray.jl | 4 ++++ test/broadcast.jl | 6 +++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index fec3620e2a2bb..6d9c7937ac4b5 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1504,6 +1504,7 @@ end map!{F}(f::F, dest::AbstractArray, As::AbstractArray...) = map_n!(f, dest, As) +map(f) = f() map(f, iters...) = collect(Generator(f, iters...)) # multi-item push!, unshift! (built on top of type-specific 1-item version) diff --git a/base/broadcast.jl b/base/broadcast.jl index ab8fee3c97753..30aeafe7ee19a 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -10,6 +10,11 @@ export broadcast_getindex, broadcast_setindex! ## Broadcasting utilities ## +# fallback routines for broadcasting with no arguments or with scalars +# to just produce a scalar result: +broadcast(f) = f() +broadcast(f, x::Number...) = f(x...) + ## Calculate the broadcast shape of the arguments, or error if incompatible # array inputs broadcast_shape() = () diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 438e7b70e0274..c810704a018a3 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -550,6 +550,10 @@ end # issue #15689, mapping an abstract type @test isa(map(Set, Array[[1,2],[3,4]]), Vector{Set{Int}}) +# mapping over scalars and empty arguments: +@test map(sin, 1) == sin(1) +@test map(()->1234) == 1234 + function test_UInt_indexing(::Type{TestAbstractArray}) A = [1:100...] _A = Expr(:quote, A) diff --git a/test/broadcast.jl b/test/broadcast.jl index 23d0b1c0c4245..5519994ac604e 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -197,6 +197,10 @@ let a = broadcast(Float32, [3, 4, 5]) @test eltype(a) == Float32 end +# broadcasting scalars: +@test sin.(1) == broadcast(sin, 1) == sin(1) +@test (()->1234).() == broadcast(()->1234) == 1234 + # issue #4883 @test isa(broadcast(tuple, [1 2 3], ["a", "b", "c"]), Matrix{Tuple{Int,String}}) @test isa(broadcast((x,y)->(x==1?1.0:x,y), [1 2 3], ["a", "b", "c"]), Matrix{Tuple{Real,String}}) @@ -211,7 +215,7 @@ end # PR 16988 @test Base.promote_op(+, Bool) === Int -@test isa(broadcast(+, true), Array{Int,0}) +@test isa(broadcast(+, [true]), Array{Int,1}) @test Base.promote_op(Float64, Bool) === Float64 # issue #17304 From 21b8c65eb15af6d7894f88325b243a3ab1e79de5 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 7 Jul 2016 10:24:25 -0700 Subject: [PATCH 0269/1117] Add link to LegacyStrings.jl in doc/manual/strings.rst --- doc/manual/strings.rst | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 45e556f45f99a..a5a89f554425c 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -54,9 +54,8 @@ There are a few noteworthy high-level features about Julia's strings: strings. - Julia supports the full range of `Unicode <https://en.wikipedia.org/wiki/Unicode>`_ characters: literal - strings are always `ASCII <https://en.wikipedia.org/wiki/ASCII>`_ or - `UTF-8 <https://en.wikipedia.org/wiki/UTF-8>`_ but other encodings for - strings from external sources can be supported. + strings are always `UTF-8 <https://en.wikipedia.org/wiki/UTF-8>`_ but + other encodings for strings from external sources can be supported. .. _man-characters: @@ -272,8 +271,8 @@ string literals: Whether these Unicode characters are displayed as escapes or shown as special characters depends on your terminal's locale settings and its -support for Unicode. Non-ASCII string literals are encoded using the -UTF-8 encoding. UTF-8 is a variable-width encoding, meaning that not all +support for Unicode. String literals are encoded using the UTF-8 +encoding. UTF-8 is a variable-width encoding, meaning that not all characters are encoded in the same number of bytes. In UTF-8, ASCII characters — i.e. those with code points less than 0x80 (128) — are encoded as they are in ASCII, using a single byte, while code points @@ -317,11 +316,11 @@ inefficient and verbose way to iterate through the characters of ``s``: .. doctest:: julia> for i = 1:endof(s) - try - println(s[i]) - catch - # ignore the index error - end + try + println(s[i]) + catch + # ignore the index error + end end ∀ <BLANKLINE> @@ -339,7 +338,7 @@ exception handling required: .. doctest:: julia> for c in s - println(c) + println(c) end ∀ <BLANKLINE> @@ -350,10 +349,12 @@ exception handling required: y Julia uses UTF-8 encoding by default, and support for new encodings can -be added by packages. Additional discussion of other encodings and how -to implement support for them is beyond the scope of this document for -the time being. For further discussion of UTF-8 encoding issues, see -the section below on `byte array literals <#Byte+Array+Literals>`_, +be added by packages. For example, the `LegacyStrings.jl +<https://github.com/JuliaArchive/LegacyStrings.jl>`_ package implements +``UTF16String`` and ``UTF32String`` types. Additional discussion of other +encodings and how to implement support for them is beyond the scope of this +document for the time being. For further discussion of UTF-8 encoding issues, +see the section below on `byte array literals <#Byte+Array+Literals>`_, which goes into some greater detail. .. _man-string-interpolation: @@ -903,10 +904,9 @@ encodings. If this is all extremely confusing, try reading `"The Absolute Minimum Every Software Developer Absolutely, Positively Must Know About Unicode -and Character -Sets" <http://www.joelonsoftware.com/articles/Unicode.html>`_. It's an -excellent introduction to Unicode and UTF-8, and may help alleviate some -confusion regarding the matter. +and Character Sets" <http://www.joelonsoftware.com/articles/Unicode.html>`_. +It's an excellent introduction to Unicode and UTF-8, and may help alleviate +some confusion regarding the matter. .. _man-version-number-literals: From 032ebab18552dea994cdabad5016c4d5b5e90961 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Thu, 7 Jul 2016 14:32:53 -0400 Subject: [PATCH 0270/1117] check types as well as values --- test/abstractarray.jl | 4 ++-- test/broadcast.jl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index c810704a018a3..a686be2af4175 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -551,8 +551,8 @@ end @test isa(map(Set, Array[[1,2],[3,4]]), Vector{Set{Int}}) # mapping over scalars and empty arguments: -@test map(sin, 1) == sin(1) -@test map(()->1234) == 1234 +@test map(sin, 1) === sin(1) +@test map(()->1234) === 1234 function test_UInt_indexing(::Type{TestAbstractArray}) A = [1:100...] diff --git a/test/broadcast.jl b/test/broadcast.jl index 5519994ac604e..4eaccdb0c1b28 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -198,8 +198,8 @@ let a = broadcast(Float32, [3, 4, 5]) end # broadcasting scalars: -@test sin.(1) == broadcast(sin, 1) == sin(1) -@test (()->1234).() == broadcast(()->1234) == 1234 +@test sin.(1) === broadcast(sin, 1) === sin(1) +@test (()->1234).() === broadcast(()->1234) === 1234 # issue #4883 @test isa(broadcast(tuple, [1 2 3], ["a", "b", "c"]), Matrix{Tuple{Int,String}}) From d3844348d530f13767e4a4dd7b54ac1616fefa8a Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 7 Jul 2016 17:08:42 -0400 Subject: [PATCH 0271/1117] fix a number of minor nits with inline line numbers * the inlining pass would discard the first LineNumberNode, without checking that it matched the line number of the function declaration * the CurrentDebugLocation shouldn't be set while emitting the prologue * implement inlining info for pre-MCJIT llvm * improve printing of relative symbols in disassembly (e.g. 'callq "_setindex!"') and hide the first line number if it is zero * print inlining info during jl_gdblookup also fix llvm 3.3 build fix #17317 --- base/inference.jl | 7 ++-- src/ccall.cpp | 18 +++++++---- src/codegen.cpp | 68 ++++++++++++++++++++++---------------- src/debuginfo.cpp | 81 ++++++++++++++++++++++++++-------------------- src/disasm.cpp | 73 +++++++++++++++++++++-------------------- src/intrinsics.cpp | 2 +- src/stackwalk.c | 13 ++++---- 7 files changed, 149 insertions(+), 113 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 7747386288874..ea7055217c838 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2910,8 +2910,11 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference stmts) empty!(stmts) else - isa(stmts[1], LineNumberNode) && shift!(stmts) - unshift!(stmts, Expr(:meta, :push_loc, linfo.def.file, linfo.def.name, linfo.def.line)) + local line::Int = linfo.def.line + if isa(stmts[1], LineNumberNode) + line = shift!(stmts).line + end + unshift!(stmts, Expr(:meta, :push_loc, linfo.def.file, linfo.def.name, line)) isa(stmts[end], LineNumberNode) && pop!(stmts) push!(stmts, Expr(:meta, :pop_loc)) end diff --git a/src/ccall.cpp b/src/ccall.cpp index f6bf9f81722a7..4a883fc63d134 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -194,6 +194,7 @@ static DenseMap<AttributeSet, std::map<std::tuple<GlobalVariable*,FunctionType*, CallingConv::ID>,GlobalVariable*>> allPltMap; +#ifdef LLVM37 // needed for musttail // Emit a "PLT" entry that will be lazily initialized // when being called the first time. static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, @@ -246,8 +247,8 @@ static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, llvmgv, runtime_lib); builder.CreateStore(builder.CreateBitCast(ptr, T_pvoidfunc), got); SmallVector<Value*, 16> args; - for (auto &arg: plt->args()) - args.push_back(&arg); + for (Function::arg_iterator arg = plt->arg_begin(), arg_e = plt->arg_end(); arg != arg_e; ++arg) + args.push_back(&*arg); CallInst *ret = builder.CreateCall(ptr, ArrayRef<Value*>(args)); ret->setAttributes(attrs); if (cc != CallingConv::C) @@ -286,6 +287,7 @@ static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, } return builder.CreateBitCast(builder.CreateLoad(got), funcptype); } +#endif // --- ABI Implementations --- // Partially based on the LDC ABI implementations licensed under the BSD 3-clause license @@ -1694,14 +1696,16 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) PointerType *funcptype = PointerType::get(functype,0); if (imaging_mode) { -#if defined(_CPU_ARM_) || defined(_CPU_PPC_) || defined(_CPU_PPC64_) - // ARM, PPC, PPC64 (as of LLVM 3.9) doesn't support `musttail` - // for vararg functions. - if (functype) +#ifdef LLVM37 + // ARM, PPC, PPC64 (as of LLVM 3.9) doesn't support `musttail` for vararg functions. + // And musttail can't proceed unreachable, but is required for vararg (https://llvm.org/bugs/show_bug.cgi?id=23766) + if (functype->isVarArg()) llvmf = runtime_sym_lookup(funcptype, f_lib, f_name, ctx->f); else -#endif llvmf = emit_plt(functype, attrs, cc, f_lib, f_name); +#else + llvmf = runtime_sym_lookup(funcptype, f_lib, f_name, ctx->f); +#endif } else { void *symaddr = jl_dlsym_e(jl_get_library(f_lib), f_name); diff --git a/src/codegen.cpp b/src/codegen.cpp index a8ff77a17ae83..23f7fd5542d20 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4248,14 +4248,13 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func (jl_options.malloc_log == JL_LOG_USER && in_user_code); StringRef filename = "<missing>"; StringRef dbgFuncName = ctx.name; - int lno = -1; + int toplineno = -1; if (lam->def) { - lno = lam->def->line; + toplineno = lam->def->line; if (lam->def->file != empty_sym) filename = jl_symbol_name(lam->def->file); } ctx.file = filename; - int toplineno = lno; DIBuilder dbuilder(*M); ctx.dbuilder = &dbuilder; @@ -4267,15 +4266,17 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func #else DIFile topfile; DISubprogram SP; + std::vector<DebugLoc> DI_loc_stack; + std::vector<DISubprogram> DI_sp_stack; #endif BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", f); builder.SetInsertPoint(b0); // jl_printf(JL_STDERR, "\n*** compiling %s at %s:%d\n\n", - // jl_symbol_name(lam->name), filename.c_str(), lno); + // jl_symbol_name(lam->name), filename.c_str(), toplineno); - DebugLoc noDbg; + DebugLoc noDbg, topdebugloc; ctx.debug_enabled = true; if (dbgFuncName.empty()) { // special value: if function name is empty, disable debug info @@ -4348,8 +4349,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func #else f); // Function #endif - // set initial line number - builder.SetCurrentDebugLocation(DebugLoc::get(lno, 0, (MDNode*)SP, NULL)); + topdebugloc = DebugLoc::get(toplineno, 0, SP, NULL); #ifdef LLVM38 f->setSubprogram(SP); #endif @@ -4357,9 +4357,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func assert(SP.Verify() && SP.describes(f) && SP.getFunction() == f); #endif } - else { - builder.SetCurrentDebugLocation(noDbg); - } + builder.SetCurrentDebugLocation(noDbg); if (ctx.debug_enabled) { const bool AlwaysPreserve = true; @@ -4519,7 +4517,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func assert((Metadata*)varinfo.dinfo->getType() != jl_pvalue_dillvmt); ctx.dbuilder->insertDeclare(lv, varinfo.dinfo, ctx.dbuilder->createExpression(), #ifdef LLVM37 - builder.getCurrentDebugLocation(), + topdebugloc, #endif builder.GetInsertBlock()); } @@ -4547,7 +4545,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } ctx.dbuilder->insertDeclare(av, varinfo.dinfo, expr, #ifdef LLVM37 - builder.getCurrentDebugLocation(), + topdebugloc, #endif builder.GetInsertBlock()); } @@ -4606,7 +4604,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func addr.push_back(llvm::dwarf::DW_OP_deref); ctx.dbuilder->insertDeclare(pargArray, vi.dinfo, ctx.dbuilder->createExpression(addr), #ifdef LLVM37 - builder.getCurrentDebugLocation(), + topdebugloc, #endif builder.GetInsertBlock()); } @@ -4638,7 +4636,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } ctx.dbuilder->insertDeclare(parg, vi.dinfo, ctx.dbuilder->createExpression(addr), #ifdef LLVM37 - builder.getCurrentDebugLocation(), + topdebugloc, #endif builder.GetInsertBlock()); } @@ -4718,11 +4716,14 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } // step 13. compile body statements + if (ctx.debug_enabled) + // set initial line number + builder.SetCurrentDebugLocation(topdebugloc); bool prevlabel = false; - lno = -1; + int lno = -1; int prevlno = -1; - for(i=0; i < stmtslen; i++) { - jl_value_t *stmt = jl_array_ptr_ref(stmts,i); + for (i = 0; i < stmtslen; i++) { + jl_value_t *stmt = jl_array_ptr_ref(stmts, i); if (jl_is_linenode(stmt) || (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == line_sym)) { @@ -4733,23 +4734,30 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func lno = jl_unbox_long(jl_exprarg(stmt,0)); } MDNode *inlinedAt = NULL; -#ifdef LLVM37 if (DI_loc_stack.size() > 0) { +#ifdef LLVM37 inlinedAt = DI_loc_stack.back(); - } +#else + inlinedAt = DI_loc_stack.back().getAsMDNode(jl_LLVMContext); #endif - if (ctx.debug_enabled) builder.SetCurrentDebugLocation(DebugLoc::get(lno, 0, (MDNode*)SP, inlinedAt)); + } + if (ctx.debug_enabled) + builder.SetCurrentDebugLocation(DebugLoc::get(lno, 0, SP, inlinedAt)); } else if (ctx.debug_enabled && jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == meta_sym && jl_array_len(((jl_expr_t*)stmt)->args) >= 1) { -#ifdef LLVM37 jl_expr_t *stmt_e = (jl_expr_t*)stmt; jl_value_t *meta_arg = jl_exprarg(stmt_e, 0); if (meta_arg == (jl_value_t*)jl_symbol("push_loc")) { std::string new_filename = "<missing>"; assert(jl_array_len(stmt_e->args) > 1); jl_sym_t *filesym = (jl_sym_t*)jl_exprarg(stmt_e, 1); - new_filename = jl_symbol_name(filesym); + if (filesym != empty_sym) + new_filename = jl_symbol_name(filesym); +#ifdef LLVM37 DIFile *new_file = dbuilder.createFile(new_filename, "."); +#else + DIFile new_file = dbuilder.createFile(new_filename, "."); +#endif DI_sp_stack.push_back(SP); DI_loc_stack.push_back(builder.getCurrentDebugLocation()); std::string inl_name; @@ -4760,8 +4768,10 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func jl_value_t *arg = jl_exprarg(stmt_e, ii); if (jl_is_symbol(arg)) inl_name = jl_symbol_name((jl_sym_t*)arg); - else if (jl_is_long(arg)) - inlined_func_lineno = jl_unbox_long(arg); + else if (jl_is_int32(arg)) + inlined_func_lineno = jl_unbox_int32(arg); + else if (jl_is_int64(arg)) + inlined_func_lineno = jl_unbox_int64(arg); } } else { @@ -4779,7 +4789,13 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func 0, true, nullptr); - builder.SetCurrentDebugLocation(DebugLoc::get(inlined_func_lineno, 0, (MDNode*)SP, builder.getCurrentDebugLocation())); + MDNode *inlinedAt = NULL; +#ifdef LLVM37 + inlinedAt = builder.getCurrentDebugLocation(); +#else + inlinedAt = builder.getCurrentDebugLocation().getAsMDNode(jl_LLVMContext); +#endif + builder.SetCurrentDebugLocation(DebugLoc::get(inlined_func_lineno, 0, SP, inlinedAt)); } else if (meta_arg == (jl_value_t*)jl_symbol("pop_loc")) { SP = DI_sp_stack.back(); @@ -4787,10 +4803,8 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func builder.SetCurrentDebugLocation(DI_loc_stack.back()); DI_loc_stack.pop_back(); } -#endif } - DebugLoc loc; if (do_coverage) coverageVisitLine(filename, lno); if (jl_is_labelnode(stmt)) { diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 0b265cf9d98df..e00c0b7ca8075 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -662,7 +662,7 @@ JITEventListener *CreateJuliaJITEventListener() static int lookup_pointer(DIContext *context, jl_frame_t **frames, size_t pointer, int demangle, int noInline) { - // This function is not allowed to reference any TLS variables if noInline + // This function is not allowed to reference any TLS variables // since it can be called from an unmanaged thread on OSX. if (!context) { if (demangle && (*frames)[0].func_name != NULL) { @@ -689,7 +689,7 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, n_frames = 1; if (n_frames > 1) { jl_frame_t *new_frames = (jl_frame_t*)calloc(sizeof(jl_frame_t), n_frames); - memcpy(&new_frames[n_frames-1], *frames, sizeof(jl_frame_t)); + memcpy(&new_frames[n_frames - 1], *frames, sizeof(jl_frame_t)); free(*frames); *frames = new_frames; } @@ -1277,50 +1277,61 @@ int jl_getFunctionInfo(jl_frame_t **frames_out, size_t pointer, int skipC, int n std::vector<JITEvent_EmittedFunctionDetails::LineStart>::iterator vit = (*it).second.lines.begin(); JITEvent_EmittedFunctionDetails::LineStart prev = *vit; - - if ((*it).second.func) { - DISubprogram debugscope = - DISubprogram(prev.Loc.getScope((*it).second.func->getContext())); - jl_copy_str(&frames[0].file_name, debugscope.getFilename().str().c_str()); - // the DISubprogram has the un-mangled name, so use that if - // available. However, if the scope need not be the current - // subprogram. - if (debugscope.getName().data() != NULL) { - jl_copy_str(&frames[0].func_name, debugscope.getName().str().c_str()); - } - else { - char *oldname = frames[0].func_name; - frames[0].func_name = jl_demangle(frames[0].func_name); - free(oldname); - } + LLVMContext &Ctx = (*it).second.func->getContext(); + + DISubprogram debugscope(prev.Loc.getScope(Ctx)); + jl_copy_str(&frames[0].file_name, debugscope.getFilename().str().c_str()); + // the DISubprogram has the un-mangled name, so use that if + // available. However, if the scope need not be the current + // subprogram. + if (debugscope.getName().data() != NULL) { + jl_copy_str(&frames[0].func_name, debugscope.getName().str().c_str()); + } + else { + char *oldname = frames[0].func_name; + frames[0].func_name = jl_demangle(frames[0].func_name); + free(oldname); } - vit++; - + // find nearest line info + ++vit; while (vit != (*it).second.lines.end()) { if (pointer <= (*vit).Address) { - frames[0].line = prev.Loc.getLine(); break; } prev = *vit; - vit++; - } - if (frames[0].line == -1) { - frames[0].line = prev.Loc.getLine(); + ++vit; } - DILexicalBlockFile locscope = DILexicalBlockFile(prev.Loc.getScope((*it).second.func->getContext())); - jl_copy_str(&frames[0].file_name, locscope.getFilename().str().c_str()); + // read out inlining and line number information + int n_frames = 1; + if (!noInline) { + MDNode *inlinedAt = prev.Loc.getInlinedAt(Ctx); + while (inlinedAt != NULL) { + DebugLoc inlineloc = DebugLoc::getFromDILocation(inlinedAt); + inlinedAt = inlineloc.getInlinedAt(Ctx); + n_frames++; + } + if (n_frames > 1) { + frames = (jl_frame_t*)calloc(sizeof(jl_frame_t), n_frames); + memcpy(&frames[n_frames - 1], *frames_out, sizeof(jl_frame_t)); + free(*frames_out); + *frames_out = frames; + } + } + DebugLoc inlineloc = prev.Loc; + for (int i = 0; i < n_frames; i++) { + frames[i].inlined = i != n_frames - 1; + frames[i].line = inlineloc.getLine(); + DISubprogram locscope(inlineloc.getScope(Ctx)); + jl_copy_str(&frames[i].file_name, locscope.getFilename().str().c_str()); + jl_copy_str(&frames[i].func_name, locscope.getName().str().c_str()); + MDNode *inlinedAt = inlineloc.getInlinedAt(Ctx); + inlineloc = DebugLoc::getFromDILocation(inlinedAt); + } - /*MDNode *inlinedAt = skipInline ? NULL : prev.Loc.getInlinedAt((*it).second.func->getContext()); - if ((!skipInline) && (inlinedAt != NULL)) { - DebugLoc inlineloc = DebugLoc::getFromDILocation(inlinedAt); - DILexicalBlockFile inlinescope = DILexicalBlockFile(inlineloc.getScope((*it).second.func->getContext())); - jl_copy_str(&frames, inlinescope.getFilename().str().c_str()); - *inlinedat_line = inlineloc.getLine(); - }*/ uv_rwlock_rdunlock(&threadsafe); - return 1; + return n_frames; } uv_rwlock_rdunlock(&threadsafe); #endif // USE_MCJIT diff --git a/src/disasm.cpp b/src/disasm.cpp index 063923c5f7b42..20392068d2380 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -128,10 +128,11 @@ class SymbolTable { MCContext& Ctx; const FuncMCView &MemObj; int Pass; + int64_t slide; uint64_t ip; // virtual instruction pointer of the current instruction public: - SymbolTable(MCContext &Ctx, const FuncMCView &MemObj): - Ctx(Ctx), MemObj(MemObj), ip(0) {} + SymbolTable(MCContext &Ctx, intptr_t slide, const FuncMCView &MemObj): + Ctx(Ctx), MemObj(MemObj), slide(slide), ip(0) {} const FuncMCView &getMemoryObject() const { return MemObj; } void setPass(int Pass) { this->Pass = Pass; } int getPass() const { return Pass; } @@ -142,6 +143,7 @@ class SymbolTable { MCSymbol *lookupSymbol(uint64_t addr); void setIP(uint64_t addr); uint64_t getIP() const; + int64_t getSlide() const; }; void SymbolTable::setIP(uint64_t addr) { @@ -151,6 +153,10 @@ uint64_t SymbolTable::getIP() const { return ip; } +int64_t SymbolTable::getSlide() const +{ + return slide; +} // Insert an address void SymbolTable::insertAddress(uint64_t addr) { @@ -232,9 +238,12 @@ static int OpInfoLookup(void *DisInfo, uint64_t PC, uint64_t Offset, uint64_t Si case 8: { uint64_t val; std::memcpy(&val, bytes, 8); pointer = val; break; } default: return 0; // Cannot handle input address size } - int skipC = 0; jl_frame_t *frame = NULL; - jl_getFunctionInfo(&frame, pointer, skipC, 1); + jl_getFunctionInfo(&frame, + pointer - SymTab->getSlide(), // TODO: This is wrong for PCrel addresses, now that we are getting the unrelocated binary. + // Try looking up addresses in the DIContext first? + /*skipC*/0, + /*noInline*/1/* the entry pointer shouldn't have inlining */); char *name = frame->func_name; free(frame->file_name); free(frame); @@ -249,23 +258,6 @@ static int OpInfoLookup(void *DisInfo, uint64_t PC, uint64_t Offset, uint64_t Si } } // namespace -#ifndef USE_MCJIT -static void print_source_line(raw_ostream &stream, DebugLoc Loc) -{ - MDNode *inlinedAt = Loc.getInlinedAt(jl_LLVMContext); - if (inlinedAt != NULL) { - DebugLoc inlineloc = DebugLoc::getFromDILocation(inlinedAt); - stream << "Source line: " << inlineloc.getLine() << "\n"; - - DILexicalBlockFile innerscope = DILexicalBlockFile(Loc.getScope(jl_LLVMContext)); - stream << "Source line: [inline] " << innerscope.getFilename().str().c_str() << ':' << Loc.getLine() << "\n"; - } - else { - stream << "Source line: " << Loc.getLine() << "\n"; - } -} -#endif - extern "C" { JL_DLLEXPORT LLVMDisasmContextRef jl_LLVMCreateDisasm(const char *TripleName, void *DisInfo, int TagType, LLVMOpInfoCallback GetOpInfo, LLVMSymbolLookupCallback SymbolLookUp) { @@ -431,7 +423,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #else FuncMCView memoryObject((const uint8_t*)Fptr, Fsize); #endif - SymbolTable DisInfo(Ctx, memoryObject); + SymbolTable DisInfo(Ctx, slide, memoryObject); #ifndef USE_MCJIT typedef std::vector<JITEvent_EmittedFunctionDetails::LineStart> LInfoVec; @@ -489,6 +481,12 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #else stream << "Filename: " << di_lineIter->second.getFileName() << "\n"; #endif +#ifdef LLVM35 + if (di_lineIter->second.Line <= 0) +#else + if (di_lineIter->second.getLine() <= 0) +#endif + ++di_lineIter; nextLineAddr = di_lineIter->first; } } @@ -498,19 +496,18 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, if (lineIter != lineEnd) { DebugLoc Loc = (*lineIter).Loc; MDNode *outer = Loc.getInlinedAt(jl_LLVMContext); - StringRef FileName; - if (!outer) { - DISubprogram debugscope = DISubprogram(Loc.getScope(jl_LLVMContext)); - FileName = debugscope.getFilename(); + while (outer) { + Loc = DebugLoc::getFromDILocation(outer); + outer = Loc.getInlinedAt(jl_LLVMContext); } - else { - DebugLoc inlineloc = DebugLoc::getFromDILocation(outer); - DILexicalBlockFile debugscope = DILexicalBlockFile(inlineloc.getScope(jl_LLVMContext)); - FileName = debugscope.getFilename(); - } - stream << "Filename: " << FileName << "\n"; - print_source_line(stream, (*lineIter).Loc); + StringRef FileName; + DISubprogram debugscope = DISubprogram(Loc.getScope(jl_LLVMContext)); + stream << "Filename: " << debugscope.getFilename() << "\n"; + if (Loc.getLine() > 0) + stream << "Source line: " << Loc.getLine() << "\n"; +#if defined(LLVM35) nextLineAddr = (*lineIter).Address; +#endif } } #endif @@ -528,7 +525,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, std::ostringstream buf; buf << "Source line: " << di_lineIter->second.Line << "\n"; Streamer->EmitRawText(buf.str()); -#elif defined LLVM35 +#elif defined(LLVM35) stream << "Source line: " << di_lineIter->second.Line << "\n"; #else stream << "Source line: " << di_lineIter->second.getLine() << "\n"; @@ -537,7 +534,13 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, } #ifndef USE_MCJIT else { - print_source_line(stream, (*lineIter).Loc); + DebugLoc Loc = (*lineIter).Loc; + MDNode *outer = Loc.getInlinedAt(jl_LLVMContext); + while (outer) { + Loc = DebugLoc::getFromDILocation(outer); + outer = Loc.getInlinedAt(jl_LLVMContext); + } + stream << "Source line: " << Loc.getLine() << "\n"; nextLineAddr = (*++lineIter).Address; } #endif diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 639a279893fca..211d14f5fdb24 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -803,7 +803,7 @@ static jl_cgval_t emit_runtime_pointerset(jl_value_t *e, jl_value_t *x, jl_value #ifdef LLVM37 builder.CreateCall(prepare_call(jlpset_func), { boxed(parg, ctx), xarg, iarg, alignarg }); #else - builder.CreateCall3(prepare_call(jlpset_func), boxed(parg, ctx), xarg, iarg, alignarg); + builder.CreateCall4(prepare_call(jlpset_func), boxed(parg, ctx), xarg, iarg, alignarg); #endif return parg; } diff --git a/src/stackwalk.c b/src/stackwalk.c index d6bd218e2b741..18f991efa3a4a 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -410,20 +410,21 @@ JL_DLLEXPORT void jl_gdblookup(uintptr_t ip) // it can be called from an unmanaged thread on OSX. // it means calling getFunctionInfo with noInline = 1 jl_frame_t *frames = NULL; - int n = jl_getFunctionInfo(&frames, ip, 0, 1); + int n = jl_getFunctionInfo(&frames, ip, 0, 0); int i; - for(i=0; i < n; i++) { + for (i = 0; i < n; i++) { jl_frame_t frame = frames[i]; if (!frame.func_name) { jl_safe_printf("unknown function (ip: %p)\n", (void*)ip); } else { + const char *inlined = frame.inlined ? " [inlined]" : ""; if (frame.line != -1) { - jl_safe_printf("%s at %s:%" PRIuPTR "\n", frame.func_name, frame.file_name, (uintptr_t)frame.line); + jl_safe_printf("%s at %s:%" PRIuPTR "%s\n", frame.func_name, frame.file_name, (uintptr_t)frame.line, inlined); } else { - jl_safe_printf("%s at %s (unknown line)\n", frame.func_name, frame.file_name); + jl_safe_printf("%s at %s (unknown line)%s\n", frame.func_name, frame.file_name, inlined); } free(frame.func_name); free(frame.file_name); @@ -435,8 +436,8 @@ JL_DLLEXPORT void jl_gdblookup(uintptr_t ip) JL_DLLEXPORT void jlbacktrace(void) { jl_ptls_t ptls = jl_get_ptls_states(); - size_t n = ptls->bt_size; // ptls->bt_size > 40 ? 40 : ptls->bt_size; - for (size_t i=0; i < n; i++) + size_t i, n = ptls->bt_size; // ptls->bt_size > 400 ? 400 : ptls->bt_size; + for (i = 0; i < n; i++) jl_gdblookup(ptls->bt_data[i] - 1); } From 5c4797e5245c719aa43f2af695d4cd884ddbc39a Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 26 Jun 2016 10:39:43 -0500 Subject: [PATCH 0272/1117] Add specialized Range type, OneTo, for use with indices --- base/abstractarray.jl | 48 ++++++++--------- base/multidimensional.jl | 2 +- base/range.jl | 112 ++++++++++++++++++++++++++------------- test/ranges.jl | 30 +++++++++++ 4 files changed, 129 insertions(+), 63 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 6d9c7937ac4b5..94288e1a4ee8c 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -5,9 +5,11 @@ typealias AbstractVector{T} AbstractArray{T,1} typealias AbstractMatrix{T} AbstractArray{T,2} typealias AbstractVecOrMat{T} Union{AbstractVector{T}, AbstractMatrix{T}} -typealias RangeIndex Union{Int, Range{Int}, UnitRange{Int}, Colon} -typealias UnitRangeInteger{T<:Integer} UnitRange{T} -typealias Indices{N} NTuple{N,UnitRangeInteger} +typealias RangeIndex Union{Int, Range{Int}, AbstractUnitRange{Int}, Colon} +typealias Indices{N} NTuple{N,AbstractUnitRange} +typealias IndicesOne{N} NTuple{N,OneTo} +typealias DimOrInd Union{Integer, AbstractUnitRange} +typealias DimsOrInds{N} NTuple{N,DimOrInd} ## Basic functions ## @@ -30,20 +32,20 @@ Returns the valid range of indices for array `A` along dimension `d`. """ function indices(A::AbstractArray, d) @_inline_meta - 1:size(A,d) + OneTo(size(A,d)) end """ indices(A) Returns the tuple of valid indices for array `A`. """ -indices{T,N}(A::AbstractArray{T,N}) = _indices((), A) +indices{T,N}(A::AbstractArray{T,N}) = _indices((), A) # faster than ntuple(d->indices(A,d), Val{N}) _indices{T,N}(out::NTuple{N}, A::AbstractArray{T,N}) = out function _indices(out, A::AbstractArray) @_inline_meta _indices((out..., indices(A, length(out)+1)), A) end -# This simpler implementation suffers from #16327 +# Note: a simpler implementation suffers from #16327 # function indices{T,N}(A::AbstractArray{T,N}) # @_inline_meta # ntuple(d->indices(A, d), Val{N}) @@ -203,15 +205,15 @@ Return `true` if the given `index` is within the bounds of arrays can extend this method in order to provide a specialized bounds checking implementation. """ -checkindex(::Type{Bool}, inds::UnitRange, i) = throw(ArgumentError("unable to check bounds for indices of type $(typeof(i))")) -checkindex(::Type{Bool}, inds::UnitRange, i::Real) = (first(inds) <= i) & (i <= last(inds)) -checkindex(::Type{Bool}, inds::UnitRange, ::Colon) = true -function checkindex(::Type{Bool}, inds::UnitRange, r::Range) +checkindex(::Type{Bool}, inds::AbstractUnitRange, i) = throw(ArgumentError("unable to check bounds for indices of type $(typeof(i))")) +checkindex(::Type{Bool}, inds::AbstractUnitRange, i::Real) = (first(inds) <= i) & (i <= last(inds)) +checkindex(::Type{Bool}, inds::AbstractUnitRange, ::Colon) = true +function checkindex(::Type{Bool}, inds::AbstractUnitRange, r::Range) @_propagate_inbounds_meta isempty(r) | (checkindex(Bool, inds, first(r)) & checkindex(Bool, inds, last(r))) end -checkindex{N}(::Type{Bool}, indx::UnitRange, I::AbstractArray{Bool,N}) = N == 1 && indx == indices1(I) -function checkindex(::Type{Bool}, inds::UnitRange, I::AbstractArray) +checkindex{N}(::Type{Bool}, indx::AbstractUnitRange, I::AbstractArray{Bool,N}) = N == 1 && indx == indices1(I) +function checkindex(::Type{Bool}, inds::AbstractUnitRange, I::AbstractArray) @_inline_meta b = true for i in I @@ -312,7 +314,6 @@ checkbounds(A::AbstractArray) = checkbounds(A, 1) # 0-d case ## Constructors ## # default arguments to similar() -typealias SimIdx Union{Integer,UnitRangeInteger} """ similar(array, [element_type=eltype(array)], [dims=size(array)]) @@ -351,17 +352,15 @@ different element type it will create a regular `Array` instead: See also `allocate_for`. """ similar{T}(a::AbstractArray{T}) = similar(a, T) -similar( a::AbstractArray, T::Type) = _similar(indicesbehavior(a), a, T) +similar( a::AbstractArray, T::Type) = similar(a, T, indices(a)) similar{T}(a::AbstractArray{T}, dims::Tuple) = similar(a, T, dims) -similar{T}(a::AbstractArray{T}, dims::SimIdx...) = similar(a, T, dims) -similar( a::AbstractArray, T::Type, dims::SimIdx...) = similar(a, T, dims) -similar( a::AbstractArray, T::Type, dims::DimsInteger) = similar(a, T, convert(Dims, dims)) +similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, dims) +similar( a::AbstractArray, T::Type, dims::DimOrInd...) = similar(a, T, dims) +similar( a::AbstractArray, T::Type, dims::DimsInteger) = similar(a, T, Dims(dims)) # similar creates an Array by default +similar( a::AbstractArray, T::Type, inds::IndicesOne) = similar(a, T, map(last, inds)) similar( a::AbstractArray, T::Type, dims::Dims) = Array(T, dims) -_similar(::IndicesStartAt1, a::AbstractArray, T::Type) = similar(a, T, size(a)) -_similar(::IndicesBehavior, a::AbstractArray, T::Type) = similar(a, T, indices(a)) - """ allocate_for(storagetype, referencearray, [shape]) @@ -1300,10 +1299,11 @@ function _sub2ind(inds, L, ind, i::Integer, I::Integer...) end nextL(L, l::Integer) = L*l -nextL(L, r::UnitRange) = L*unsafe_length(r) +nextL(L, r::AbstractUnitRange) = L*unsafe_length(r) offsetin(i, l::Integer) = i-1 -offsetin(i, r::UnitRange) = i-first(r) +offsetin(i, r::AbstractUnitRange) = i-first(r) unsafe_length(r::UnitRange) = r.stop-r.start+1 +unsafe_length(r::OneTo) = length(r) ind2sub(::Tuple{}, ind::Integer) = (@_inline_meta; ind == 1 ? () : throw(BoundsError())) ind2sub(dims::DimsInteger, ind::Integer) = (@_inline_meta; _ind2sub((), dims, ind-1)) @@ -1323,9 +1323,9 @@ function _ind2sub(out, inds, ind) end _lookup(ind, d::Integer) = ind+1 -_lookup(ind, r::UnitRange) = ind+first(r) +_lookup(ind, r::AbstractUnitRange) = ind+first(r) _div(ind, d::Integer) = div(ind, d), 1, d -_div(ind, r::UnitRange) = (d = unsafe_length(r); (div(ind, d), first(r), d)) +_div(ind, r::AbstractUnitRange) = (d = unsafe_length(r); (div(ind, d), first(r), d)) smart_ind2sub(shape::NTuple{1}, ind) = (ind,) smart_ind2sub(shape, ind) = ind2sub(shape, ind) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 838101efd8328..197a585ec5439 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -78,7 +78,7 @@ end CartesianRange{N}(index::CartesianIndex{N}) = CartesianRange(one(index), index) CartesianRange(::Tuple{}) = CartesianRange{CartesianIndex{0}}(CartesianIndex{0}(()),CartesianIndex{0}(())) CartesianRange{N}(sz::NTuple{N,Int}) = CartesianRange(CartesianIndex(sz)) -CartesianRange{N}(rngs::NTuple{N,Union{Int,UnitRange{Int}}}) = CartesianRange(CartesianIndex(map(r->first(r), rngs)), CartesianIndex(map(r->last(r), rngs))) +CartesianRange{N}(rngs::NTuple{N,Union{Integer,AbstractUnitRange}}) = CartesianRange(CartesianIndex(map(r->first(r), rngs)), CartesianIndex(map(r->last(r), rngs))) ndims(R::CartesianRange) = length(R.start) ndims{I<:CartesianIndex}(::Type{CartesianRange{I}}) = length(I) diff --git a/base/range.jl b/base/range.jl index 58ec756d1b078..a3badcf109535 100644 --- a/base/range.jl +++ b/base/range.jl @@ -10,6 +10,7 @@ abstract Range{T} <: AbstractArray{T,1} ## ordinal ranges abstract OrdinalRange{T,S} <: Range{T} +abstract AbstractUnitRange{T} <: OrdinalRange{T,Int} immutable StepRange{T,S} <: OrdinalRange{T,S} start::T @@ -64,7 +65,7 @@ steprem(start,stop,step) = (stop-start) % step StepRange{T,S}(start::T, step::S, stop::T) = StepRange{T,S}(start, step, stop) -immutable UnitRange{T<:Real} <: OrdinalRange{T,Int} +immutable UnitRange{T<:Real} <: AbstractUnitRange{T} start::T stop::T UnitRange(start, stop) = new(start, unitrange_last(start,stop)) @@ -78,6 +79,12 @@ unitrange_last{T}(start::T, stop::T) = ifelse(stop >= start, convert(T,start+floor(stop-start)), convert(T,start-one(stop-start))) +immutable OneTo{T<:Integer} <: AbstractUnitRange{T} + stop::T + OneTo(stop) = new(max(zero(T), stop)) +end +OneTo{T<:Integer}(stop::T) = OneTo{T}(stop) + colon(a::Real, b::Real) = colon(promote(a,b)...) colon{T<:Real}(start::T, stop::T) = UnitRange{T}(start, stop) @@ -310,12 +317,12 @@ size(r::Range) = (length(r),) isempty(r::StepRange) = (r.start != r.stop) & ((r.step > zero(r.step)) != (r.stop > r.start)) -isempty(r::UnitRange) = r.start > r.stop +isempty(r::AbstractUnitRange) = first(r) > last(r) isempty(r::FloatRange) = length(r) == 0 isempty(r::LinSpace) = length(r) == 0 step(r::StepRange) = r.step -step(r::UnitRange) = 1 +step(r::AbstractUnitRange) = 1 step(r::FloatRange) = r.step/r.divisor step{T}(r::LinSpace{T}) = ifelse(r.len <= 0, convert(T,NaN), (r.stop-r.start)/r.divisor) @@ -323,7 +330,8 @@ function length(r::StepRange) n = Integer(div(r.stop+r.step - r.start, r.step)) isempty(r) ? zero(n) : n end -length(r::UnitRange) = Integer(r.stop - r.start + 1) +length(r::AbstractUnitRange) = Integer(last(r) - first(r) + 1) +length(r::OneTo) = r.stop length(r::FloatRange) = Integer(r.len) length(r::LinSpace) = Integer(r.len + signbit(r.len - 1)) @@ -338,11 +346,12 @@ function length{T<:Union{Int,UInt,Int64,UInt64}}(r::StepRange{T}) end end -length{T<:Union{Int,Int64}}(r::UnitRange{T}) = - checked_add(checked_sub(r.stop, r.start), one(T)) +length{T<:Union{Int,Int64}}(r::AbstractUnitRange{T}) = + checked_add(checked_sub(last(r), first(r)), one(T)) +length{T<:Union{Int,Int64}}(r::OneTo{T}) = T(r.stop) -length{T<:Union{UInt,UInt64}}(r::UnitRange{T}) = - r.stop < r.start ? zero(T) : checked_add(r.stop - r.start, one(T)) +length{T<:Union{UInt,UInt64}}(r::AbstractUnitRange{T}) = + r.stop < r.start ? zero(T) : checked_add(last(r) - first(r), one(T)) # some special cases to favor default Int type let smallint = (Int === Int64 ? @@ -355,20 +364,21 @@ let smallint = (Int === Int64 ? div(Int(r.stop)+Int(r.step) - Int(r.start), Int(r.step)) end - length{T <: smallint}(r::UnitRange{T}) = Int(r.stop) - Int(r.start) + 1 + length{T <: smallint}(r::AbstractUnitRange{T}) = Int(last(r)) - Int(first(r)) + 1 + length{T <: smallint}(r::OneTo{T}) = Int(r.stop) end first{T}(r::OrdinalRange{T}) = convert(T, r.start) +first{T}(r::OneTo{T}) = one(T) first{T}(r::FloatRange{T}) = convert(T, r.start/r.divisor) first{T}(r::LinSpace{T}) = convert(T, (r.len-1)*r.start/r.divisor) -last{T}(r::StepRange{T}) = r.stop -last(r::UnitRange) = r.stop +last{T}(r::OrdinalRange{T}) = convert(T, r.stop) last{T}(r::FloatRange{T}) = convert(T, (r.start + (r.len-1)*r.step)/r.divisor) last{T}(r::LinSpace{T}) = convert(T, (r.len-1)*r.stop/r.divisor) -minimum(r::UnitRange) = isempty(r) ? throw(ArgumentError("range must be non-empty")) : first(r) -maximum(r::UnitRange) = isempty(r) ? throw(ArgumentError("range must be non-empty")) : last(r) +minimum(r::AbstractUnitRange) = isempty(r) ? throw(ArgumentError("range must be non-empty")) : first(r) +maximum(r::AbstractUnitRange) = isempty(r) ? throw(ArgumentError("range must be non-empty")) : last(r) minimum(r::Range) = isempty(r) ? throw(ArgumentError("range must be non-empty")) : min(first(r), last(r)) maximum(r::Range) = isempty(r) ? throw(ArgumentError("range must be non-empty")) : max(first(r), last(r)) @@ -398,9 +408,10 @@ done{T,S}(r::StepRange{T,S}, i::Integer) = isempty(r) | (i == oftype(i, r.stop) + r.step) start{T}(r::UnitRange{T}) = oftype(r.start + one(T), r.start) -next{T}(r::UnitRange{T}, i) = (convert(T, i), i + one(T)) -done{T}(r::UnitRange{T}, i) = i == oftype(i, r.stop) + one(T) +next{T}(r::AbstractUnitRange{T}, i) = (convert(T, i), i + one(T)) +done{T}(r::AbstractUnitRange{T}, i) = i == oftype(i, r.stop) + one(T) +start{T}(r::OneTo{T}) = one(T) # some special cases to favor default Int type to avoid overflow let smallint = (Int === Int64 ? @@ -411,7 +422,8 @@ let smallint = (Int === Int64 ? start{T<:smallint}(r::StepRange{T}) = convert(Int, r.start) next{T<:smallint}(r::StepRange{T}, i) = (i % T, i + r.step) start{T<:smallint}(r::UnitRange{T}) = convert(Int, r.start) - next{T<:smallint}(r::UnitRange{T}, i) = (i % T, i + 1) + next{T<:smallint}(r::AbstractUnitRange{T}, i) = (i % T, i + 1) + start{T<:smallint}(r::OneTo{T}) = 1 end ## indexing @@ -423,6 +435,12 @@ function getindex{T}(v::UnitRange{T}, i::Integer) ret end +function getindex{T}(v::OneTo{T}, i::Integer) + @_inline_meta + @boundscheck ((i > 0) & (i <= v.stop)) || throw_boundserror(v, i) + convert(T, i) +end + function getindex{T}(v::Range{T}, i::Integer) @_inline_meta ret = convert(T, first(v) + (i - 1)*step(v)) @@ -447,17 +465,23 @@ end getindex(r::Range, ::Colon) = copy(r) -function getindex{T<:Integer}(r::UnitRange, s::UnitRange{T}) +function getindex{T<:Integer}(r::UnitRange, s::AbstractUnitRange{T}) @_inline_meta @boundscheck checkbounds(r, s) - st = oftype(r.start, r.start + s.start-1) + st = oftype(r.start, r.start + first(s)-1) range(st, length(s)) end -function getindex{T<:Integer}(r::UnitRange, s::StepRange{T}) +function getindex{T}(r::OneTo{T}, s::OneTo) @_inline_meta @boundscheck checkbounds(r, s) - st = oftype(r.start, r.start + s.start-1) + OneTo(T(s.stop)) +end + +function getindex{T<:Integer}(r::AbstractUnitRange, s::StepRange{T}) + @_inline_meta + @boundscheck checkbounds(r, s) + st = oftype(first(r), first(r) + s.start-1) range(st, step(s), length(s)) end @@ -487,6 +511,7 @@ end show(io::IO, r::Range) = print(io, repr(first(r)), ':', repr(step(r)), ':', repr(last(r))) show(io::IO, r::UnitRange) = print(io, repr(first(r)), ':', repr(last(r))) +show(io::IO, r::OneTo) = print(io, "Base.OneTo(", r.stop, ")") =={T<:Range}(r::T, s::T) = (first(r) == first(s)) & (step(r) == step(s)) & (last(r) == last(s)) ==(r::OrdinalRange, s::OrdinalRange) = (first(r) == first(s)) & (step(r) == step(s)) & (last(r) == last(s)) @@ -508,15 +533,17 @@ function ==(r::Range, s::Range) return true end -intersect{T1<:Integer, T2<:Integer}(r::UnitRange{T1}, s::UnitRange{T2}) = max(r.start,s.start):min(last(r),last(s)) +intersect(r::OneTo, s::OneTo) = OneTo(min(r.stop,s.stop)) -intersect{T<:Integer}(i::Integer, r::UnitRange{T}) = +intersect{T1<:Integer, T2<:Integer}(r::AbstractUnitRange{T1}, s::AbstractUnitRange{T2}) = max(first(r),first(s)):min(last(r),last(s)) + +intersect{T<:Integer}(i::Integer, r::AbstractUnitRange{T}) = i < first(r) ? (first(r):i) : i > last(r) ? (i:last(r)) : (i:i) intersect{T<:Integer}(r::UnitRange{T}, i::Integer) = intersect(i, r) -function intersect{T1<:Integer, T2<:Integer}(r::UnitRange{T1}, s::StepRange{T2}) +function intersect{T1<:Integer, T2<:Integer}(r::AbstractUnitRange{T1}, s::StepRange{T2}) if isempty(s) range(first(r), 0) elseif step(s) == 0 @@ -535,7 +562,7 @@ function intersect{T1<:Integer, T2<:Integer}(r::UnitRange{T1}, s::StepRange{T2}) end end -function intersect{T1<:Integer, T2<:Integer}(r::StepRange{T1}, s::UnitRange{T2}) +function intersect{T1<:Integer, T2<:Integer}(r::StepRange{T1}, s::AbstractUnitRange{T2}) if step(r) < 0 reverse(intersect(s, reverse(r))) else @@ -597,7 +624,7 @@ function intersect(r1::Range, r2::Range, r3::Range, r::Range...) end # findin (the index of intersection) -function _findin{T1<:Integer, T2<:Integer}(r::Range{T1}, span::UnitRange{T2}) +function _findin{T1<:Integer, T2<:Integer}(r::Range{T1}, span::AbstractUnitRange{T2}) local ifirst local ilast fspan = first(span) @@ -630,11 +657,11 @@ end ## linear operations on ranges ## --(r::OrdinalRange) = range(-r.start, -step(r), length(r)) +-(r::OrdinalRange) = range(-first(r), -step(r), length(r)) -(r::FloatRange) = FloatRange(-r.start, -r.step, r.len, r.divisor) -(r::LinSpace) = LinSpace(-r.start, -r.stop, r.len, r.divisor) -.+(x::Real, r::UnitRange) = range(x + r.start, length(r)) +.+(x::Real, r::AbstractUnitRange) = range(x + first(r), length(r)) .+(x::Real, r::Range) = (x+first(r)):step(r):(x+last(r)) #.+(x::Real, r::StepRange) = range(x + r.start, r.step, length(r)) .+(x::Real, r::FloatRange) = FloatRange(r.divisor*x + r.start, r.step, r.len, r.divisor) @@ -651,7 +678,7 @@ function .-(x::Real, r::LinSpace) x2 = x * r.divisor / (r.len - 1) LinSpace(x2 - r.start, x2 - r.stop, r.len, r.divisor) end -.-(r::UnitRange, x::Real) = range(r.start-x, length(r)) +.-(r::AbstractUnitRange, x::Real) = range(first(r)-x, length(r)) .-(r::StepRange , x::Real) = range(r.start-x, r.step, length(r)) .-(r::FloatRange, x::Real) = FloatRange(r.start - r.divisor*x, r.step, r.len, r.divisor) function .-(r::LinSpace, x::Real) @@ -659,14 +686,14 @@ function .-(r::LinSpace, x::Real) LinSpace(r.start - x2, r.stop - x2, r.len, r.divisor) end -.*(x::Real, r::OrdinalRange) = range(x*r.start, x*step(r), length(r)) +.*(x::Real, r::OrdinalRange) = range(x*first(r), x*step(r), length(r)) .*(x::Real, r::FloatRange) = FloatRange(x*r.start, x*r.step, r.len, r.divisor) .*(x::Real, r::LinSpace) = LinSpace(x * r.start, x * r.stop, r.len, r.divisor) .*(r::Range, x::Real) = x .* r .*(r::FloatRange, x::Real) = x .* r .*(r::LinSpace, x::Real) = x .* r -./(r::OrdinalRange, x::Real) = range(r.start/x, step(r)/x, length(r)) +./(r::OrdinalRange, x::Real) = range(first(r)/x, step(r)/x, length(r)) ./(r::FloatRange, x::Real) = FloatRange(r.start/x, r.step/x, r.len, r.divisor) ./(r::LinSpace, x::Real) = LinSpace(r.start / x, r.stop / x, r.len, r.divisor) @@ -675,15 +702,24 @@ promote_rule{T1,T2}(::Type{UnitRange{T1}},::Type{UnitRange{T2}}) = convert{T<:Real}(::Type{UnitRange{T}}, r::UnitRange{T}) = r convert{T<:Real}(::Type{UnitRange{T}}, r::UnitRange) = UnitRange{T}(r.start, r.stop) +promote_rule{T1,T2}(::Type{OneTo{T1}},::Type{OneTo{T2}}) = + OneTo{promote_type(T1,T2)} +convert{T<:Real}(::Type{OneTo{T}}, r::OneTo{T}) = r +convert{T<:Real}(::Type{OneTo{T}}, r::OneTo) = OneTo{T}(r.stop) + +promote_rule{T1,UR<:AbstractUnitRange}(::Type{UnitRange{T1}}, ::Type{UR}) = + UnitRange{promote_type(T1,eltype(UR))} +convert{T<:Real}(::Type{UnitRange{T}}, r::AbstractUnitRange) = UnitRange{T}(first(r), last(r)) + promote_rule{T1a,T1b,T2a,T2b}(::Type{StepRange{T1a,T1b}},::Type{StepRange{T2a,T2b}}) = StepRange{promote_type(T1a,T2a),promote_type(T1b,T2b)} convert{T1,T2}(::Type{StepRange{T1,T2}}, r::StepRange{T1,T2}) = r -promote_rule{T1a,T1b,T2}(::Type{StepRange{T1a,T1b}},::Type{UnitRange{T2}}) = - StepRange{promote_type(T1a,T2),promote_type(T1b,T2)} +promote_rule{T1a,T1b,UR<:AbstractUnitRange}(::Type{StepRange{T1a,T1b}},::Type{UR}) = + StepRange{promote_type(T1a,eltype(UR)),promote_type(T1b,eltype(UR))} convert{T1,T2}(::Type{StepRange{T1,T2}}, r::Range) = StepRange{T1,T2}(convert(T1, first(r)), convert(T2, step(r)), convert(T1, last(r))) -convert{T}(::Type{StepRange}, r::UnitRange{T}) = +convert{T}(::Type{StepRange}, r::AbstractUnitRange{T}) = StepRange{T,T}(first(r), step(r), last(r)) promote_rule{T1,T2}(::Type{FloatRange{T1}},::Type{FloatRange{T2}}) = @@ -765,15 +801,15 @@ reverse(r::LinSpace) = LinSpace(r.stop, r.start, r.len, r.divisor) ## sorting ## -issorted(r::UnitRange) = true +issorted(r::AbstractUnitRange) = true issorted(r::Range) = step(r) >= zero(step(r)) -sort(r::UnitRange) = r -sort!(r::UnitRange) = r +sort(r::AbstractUnitRange) = r +sort!(r::AbstractUnitRange) = r sort(r::Range) = issorted(r) ? r : reverse(r) -sortperm(r::UnitRange) = 1:length(r) +sortperm(r::AbstractUnitRange) = 1:length(r) sortperm(r::Range) = issorted(r) ? (1:1:length(r)) : (length(r):-1:1) function sum{T<:Real}(r::Range{T}) @@ -806,6 +842,6 @@ function in(x, r::Range) n >= 1 && n <= length(r) && r[n] == x end -in{T<:Integer}(x::Integer, r::UnitRange{T}) = (first(r) <= x) & (x <= last(r)) +in{T<:Integer}(x::Integer, r::AbstractUnitRange{T}) = (first(r) <= x) & (x <= last(r)) in{T<:Integer}(x, r::Range{T}) = isinteger(x) && !isempty(r) && x>=minimum(r) && x<=maximum(r) && (mod(convert(T,x),step(r))-mod(first(r),step(r)) == 0) in(x::Char, r::Range{Char}) = !isempty(r) && x >= minimum(r) && x <= maximum(r) && (mod(Int(x) - Int(first(r)), step(r)) == 0) diff --git a/test/ranges.jl b/test/ranges.jl index bbe8a4ad931f4..d6c5a0787a95b 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -736,3 +736,33 @@ let r = 1:3, a = [1,2,3] @test convert(Array{Int,1}, r) == a @test convert(Array{Float64,1}, r) == a end + +# OneTo +r = Base.OneTo(-5) +@test isempty(r) +@test length(r) == 0 +@test size(r) == (0,) +r = Base.OneTo(3) +@test !isempty(r) +@test length(r) == 3 +@test size(r) == (3,) +@test step(r) == 1 +@test first(r) == 1 +@test last(r) == 3 +@test minimum(r) == 1 +@test maximum(r) == 3 +@test r[2] == 2 +@test_throws BoundsError r[4] +@test_throws BoundsError r[0] +@test r+1 === 2:4 +@test 2*r === 2:2:6 +k = 0 +for i in r + @test i == (k+=1) +end +@test intersect(r, Base.OneTo(2)) == Base.OneTo(2) +@test intersect(r, 0:5) == 1:3 +io = IOBuffer() +show(io, r) +str = takebuf_string(io) +@test str == "Base.OneTo(3)" From 96eacf5f213a902e62e3f174b6d311f64ca0aeb8 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 27 Jun 2016 04:22:49 -0500 Subject: [PATCH 0273/1117] Iteration improvements for arrays with non-1 indices Better to have this logic in the main functions rather than in specific packages --- base/abstractarray.jl | 1 + base/multidimensional.jl | 2 +- test/offsetarray.jl | 2 -- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 94288e1a4ee8c..8969d66c004f1 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -628,6 +628,7 @@ next(A::AbstractArray,i) = (@_propagate_inbounds_meta; (idx, s) = next(i[1], i[2 done(A::AbstractArray,i) = (@_propagate_inbounds_meta; done(i[1], i[2])) # eachindex iterates over all indices. LinearSlow definitions are later. +eachindex(A::AbstractVector) = (@_inline_meta(); indices1(A)) eachindex(A::AbstractArray) = (@_inline_meta(); eachindex(linearindexing(A), A)) function eachindex(A::AbstractArray, B::AbstractArray) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 197a585ec5439..63fb01a5c9f74 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -83,7 +83,7 @@ CartesianRange{N}(rngs::NTuple{N,Union{Integer,AbstractUnitRange}}) = CartesianR ndims(R::CartesianRange) = length(R.start) ndims{I<:CartesianIndex}(::Type{CartesianRange{I}}) = length(I) -eachindex(::LinearSlow, A::AbstractArray) = CartesianRange(size(A)) +eachindex(::LinearSlow, A::AbstractArray) = CartesianRange(indices(A)) @inline eachindex(::LinearSlow, A::AbstractArray, B::AbstractArray...) = CartesianRange(maxsize((), A, B...)) maxsize(sz) = sz diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 790edde60d350..853186ba46730 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -30,8 +30,6 @@ parenttype(A::OffsetArray) = parenttype(typeof(A)) Base.parent(A::OffsetArray) = A.parent Base.size(A::OffsetArray) = size(parent(A)) -Base.eachindex(::LinearSlow, A::OffsetArray) = CartesianRange(indices(A)) -Base.eachindex(::LinearFast, A::OffsetVector) = indices(A, 1) Base.summary(A::OffsetArray) = string(typeof(A))*" with indices "*string(indices(A)) # Implementations of indices and indices1. Since bounds-checking is From 0669e21f118ad6d3e3f89f300b67a47f92881fb6 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 27 Jun 2016 04:33:53 -0500 Subject: [PATCH 0274/1117] Replace shape with indices, and redesign similar --- base/abstractarray.jl | 57 +++++++++++++++------------------------- base/bitarray.jl | 2 ++ base/broadcast.jl | 12 ++++----- base/essentials.jl | 2 +- base/exports.jl | 2 +- base/multidimensional.jl | 23 +++++++--------- base/number.jl | 2 +- base/operators.jl | 2 +- base/reshapedarray.jl | 2 -- base/sort.jl | 10 +++---- base/subarray.jl | 19 ++++++++------ test/offsetarray.jl | 8 +++--- 12 files changed, 62 insertions(+), 79 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 8969d66c004f1..a60006002ccfc 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -162,29 +162,6 @@ immutable IndicesSlow1D <: IndicesPerformance end # indices(A) is better than i indicesperformance(A::AbstractArray) = indicesperformance(typeof(A)) indicesperformance{T<:AbstractArray}(::Type{T}) = IndicesFast1D() -""" - shape(A) - -Returns a tuple specifying the "shape" of array `A`. For arrays with -conventional indexing (indices start at 1), this is equivalent to -`size(A)`; otherwise it is equivalent to `indices(A)`. -""" -shape(a) = shape(indicesbehavior(a), a) -""" - shape(A, d) - -Specifies the "shape" of the array `A` along dimension `d`. For arrays -with conventional indexing (starting at 1), this is equivalent to -`size(A, d)`; for arrays with unconventional indexing (indexing may -start at something different from 1), it is equivalent to `indices(A, -d)`. -""" -shape(a, d) = shape(indicesbehavior(a), a, d) -shape(::IndicesStartAt1, a) = size(a) -shape(::IndicesStartAt1, a, d) = size(a, d) -shape(::IndicesBehavior, a) = indices(a) -shape(::IndicesBehavior, a, d) = indices(a, d) - ## Bounds checking ## @generated function trailingsize{T,N,n}(A::AbstractArray{T,N}, ::Type{Val{n}}) (isa(n, Int) && isa(N, Int)) || error("Must have concrete type") @@ -352,14 +329,22 @@ different element type it will create a regular `Array` instead: See also `allocate_for`. """ similar{T}(a::AbstractArray{T}) = similar(a, T) -similar( a::AbstractArray, T::Type) = similar(a, T, indices(a)) -similar{T}(a::AbstractArray{T}, dims::Tuple) = similar(a, T, dims) -similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, dims) -similar( a::AbstractArray, T::Type, dims::DimOrInd...) = similar(a, T, dims) -similar( a::AbstractArray, T::Type, dims::DimsInteger) = similar(a, T, Dims(dims)) +similar( a::AbstractArray, T::Type) = similar(a, T, to_shape(indices(a))) +similar{T}(a::AbstractArray{T}, dims::Tuple) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, to_shape(dims)) +similar( a::AbstractArray, T::Type, dims::DimOrInd...) = similar(a, T, to_shape(dims)) +similar( a::AbstractArray, T::Type, dims) = similar(a, T, to_shape(dims)) # similar creates an Array by default -similar( a::AbstractArray, T::Type, inds::IndicesOne) = similar(a, T, map(last, inds)) -similar( a::AbstractArray, T::Type, dims::Dims) = Array(T, dims) +similar( a::AbstractArray, T::Type, dims::Dims) = Array{T}(dims) + +to_shape(::Tuple{}) = () +to_shape(dims::Dims) = dims +to_shape(dims::DimsOrInds) = map(to_shape, dims) +# each dimension +to_shape(i::Int) = i +to_shape(i::Integer) = Int(i) +to_shape(r::OneTo) = Int(last(r)) +to_shape(r::UnitRange) = convert(UnitRange{Int}, r) """ allocate_for(storagetype, referencearray, [shape]) @@ -381,7 +366,7 @@ conventional indexing, this will likely just call `Array{Int}(size(A))`, but if `A` has unconventional indexing then the indices of the result will match `A`. - allocate_for(BitArray, A, (shape(A, 2),)) + allocate_for(BitArray, A, (indices(A, 2),)) would create a 1-dimensional logical array whose indices match those of the columns of `A`. @@ -392,10 +377,10 @@ possible that several different ones will be simultaneously in use). See also `similar`. """ -allocate_for(f, a, shape::Union{SimIdx,Tuple{Vararg{SimIdx}}}) = f(shape) -allocate_for(f, a) = allocate_for(f, a, shape(a)) +allocate_for(f, a, shape::Union{DimOrInd,DimsOrInds}) = f(to_shape(shape)) +allocate_for(f, a) = allocate_for(f, a, indices(a)) # allocate_for when passed multiple arrays. Necessary for broadcast, etc. -function allocate_for(f, as::Tuple, shape::Union{SimIdx,Tuple{Vararg{SimIdx}}}) +function allocate_for(f, as::Tuple, shape::Union{DimOrInd,DimsOrInds}) @_inline_meta a = promote_indices(as...) allocate_for(f, a, shape) @@ -1406,7 +1391,7 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector) return map(f,A) end - dimsA = [shape(A)...] + dimsA = [indices(A)...] ndimsA = ndims(A) alldims = [1:ndimsA;] @@ -1431,7 +1416,7 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector) if eltype(Rsize) == Int Rsize[dims] = [size(r1)..., ntuple(d->1, nextra)...] else - Rsize[dims] = [indices(r1)..., ntuple(d->1:1, nextra)...] + Rsize[dims] = [indices(r1)..., ntuple(d->OneTo(1), nextra)...] end R = similar(r1, tuple(Rsize...,)) diff --git a/base/bitarray.jl b/base/bitarray.jl index 946deb1b4fa3c..e0754de709666 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -47,6 +47,8 @@ end isassigned{N}(B::BitArray{N}, i::Int) = 1 <= i <= length(B) +linearindexing{A<:BitArray}(::Type{A}) = LinearFast() + ## aux functions ## const _msk64 = ~UInt64(0) diff --git a/base/broadcast.jl b/base/broadcast.jl index 30aeafe7ee19a..969246e4957a9 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -3,7 +3,7 @@ module Broadcast using Base.Cartesian -using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, shape, linearindices, allocate_for, tail, dimlength +using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, allocate_for, tail, dimlength import Base: .+, .-, .*, ./, .\, .//, .==, .<, .!=, .<=, .÷, .%, .<<, .>>, .^ export broadcast, broadcast!, bitbroadcast export broadcast_getindex, broadcast_setindex! @@ -18,8 +18,8 @@ broadcast(f, x::Number...) = f(x...) ## Calculate the broadcast shape of the arguments, or error if incompatible # array inputs broadcast_shape() = () -broadcast_shape(A) = shape(A) -@inline broadcast_shape(A, B...) = broadcast_shape((), shape(A), map(shape, B)...) +broadcast_shape(A) = indices(A) +@inline broadcast_shape(A, B...) = broadcast_shape((), indices(A), map(indices, B)...) # shape inputs broadcast_shape(shape::Tuple) = shape @inline broadcast_shape(shape::Tuple, shape1::Tuple, shapes::Tuple...) = broadcast_shape(_bcs((), shape, shape1), shapes...) @@ -45,7 +45,7 @@ _bcsm(a::Number, b::Number) = a == b || b == 1 ## Check that all arguments are broadcast compatible with shape # comparing one input against a shape check_broadcast_shape(shp) = nothing -check_broadcast_shape(shp, A) = check_broadcast_shape(shp, shape(A)) +check_broadcast_shape(shp, A) = check_broadcast_shape(shp, indices(A)) check_broadcast_shape(::Tuple{}, ::Tuple{}) = nothing check_broadcast_shape(shp, ::Tuple{}) = nothing check_broadcast_shape(::Tuple{}, Ashp::Tuple) = throw(DimensionMismatch("cannot broadcast array to have fewer dimensions")) @@ -138,7 +138,7 @@ end end @inline function broadcast!{nargs}(f, B::AbstractArray, As::Vararg{Any,nargs}) - check_broadcast_shape(shape(B), As...) + check_broadcast_shape(indices(B), As...) sz = size(B) mapindex = map(x->newindexer(sz, x), As) _broadcast!(f, B, mapindex, As, Val{nargs}) @@ -368,7 +368,7 @@ for (f, scalarf) in ((:.==, :(==)), :((A,ind)->A), :((B,ind)->B[ind])), (:AbstractArray, :Any, :A, :((A,ind)->A[ind]), :((B,ind)->B))) - shape = :(shape($active)) + shape = :(indices($active)) @eval begin function ($f)(A::$sigA, B::$sigB) P = allocate_for(BitArray, $active, $shape) diff --git a/base/essentials.jl b/base/essentials.jl index bdc95b133b7f4..6635d99670c0b 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -171,7 +171,7 @@ start(v::SimpleVector) = 1 next(v::SimpleVector,i) = (v[i],i+1) done(v::SimpleVector,i) = (i > v.length) isempty(v::SimpleVector) = (v.length == 0) -indices(v::SimpleVector, d) = d == 1 ? (1:length(v)) : (1:1) +indices(v::SimpleVector, d) = d == 1 ? OneTo(length(v)) : OneTo(1) linearindices(v::SimpleVector) = indices(v, 1) function ==(v1::SimpleVector, v2::SimpleVector) diff --git a/base/exports.jl b/base/exports.jl index a64eba3203003..33f11efedad4d 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -28,6 +28,7 @@ export # Types AbstractChannel, AbstractMatrix, + AbstractUnitRange, AbstractVector, AbstractVecOrMat, Array, @@ -583,7 +584,6 @@ export searchsortedlast, select!, select, - shape, shuffle, shuffle!, size, diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 63fb01a5c9f74..3346d5983dd07 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -10,9 +10,6 @@ using Base: LinearFast, LinearSlow, AbstractCartesianIndex, fill_to_length, tail export CartesianIndex, CartesianRange -# Traits for linear indexing -linearindexing{A<:BitArray}(::Type{A}) = LinearFast() - # CartesianIndex immutable CartesianIndex{N} <: AbstractCartesianIndex{N} I::NTuple{N,Int} @@ -178,18 +175,18 @@ index_lengths_dim(A, dim, ::Colon) = (trailingsize(A, dim),) # whose length is equal to the dimension we're to process next. This # allows us to dispatch, which is important for the type-stability of # the lines involving Colon as the final index. -index_shape(A::AbstractVector, I::Colon) = shape(A) +index_shape(A::AbstractVector, I::Colon) = indices(A) index_shape(A::AbstractArray, I::Colon) = (length(A),) @inline index_shape(A::AbstractArray, I...) = index_shape_dim(A, (true,), I...) @inline index_shape_dim(A, dim, ::Colon) = (trailingsize(A, length(dim)),) -@inline index_shape_dim{T,N}(A::AbstractArray{T,N}, dim::NTuple{N}, ::Colon) = (shape(A, N),) +@inline index_shape_dim{T,N}(A::AbstractArray{T,N}, dim::NTuple{N}, ::Colon) = (indices(A, N),) @inline index_shape_dim(A, dim, I::Real...) = () -@inline index_shape_dim(A, dim, ::Colon, i, I...) = (shape(A, length(dim)), index_shape_dim(A, (dim...,true), i, I...)...) +@inline index_shape_dim(A, dim, ::Colon, i, I...) = (indices(A, length(dim)), index_shape_dim(A, (dim...,true), i, I...)...) @inline index_shape_dim(A, dim, ::Real, I...) = (index_shape_dim(A, (dim...,true), I...)...) @inline index_shape_dim{N}(A, dim, ::CartesianIndex{N}, I...) = (index_shape_dim(A, (dim...,ntuple(d->true,Val{N})...), I...)...) -@inline index_shape_dim(A, dim, i::AbstractArray, I...) = (shape(i)..., index_shape_dim(A, (dim...,true), I...)...) +@inline index_shape_dim(A, dim, i::AbstractArray, I...) = (indices(i)..., index_shape_dim(A, (dim...,true), I...)...) @inline index_shape_dim(A, dim, i::AbstractArray{Bool}, I...) = (sum(i), index_shape_dim(A, (dim...,true), I...)...) -@inline index_shape_dim{N}(A, dim, i::AbstractArray{CartesianIndex{N}}, I...) = (shape(i)..., index_shape_dim(A, (dim...,ntuple(d->true,Val{N})...), I...)...) +@inline index_shape_dim{N}(A, dim, i::AbstractArray{CartesianIndex{N}}, I...) = (indices(i)..., index_shape_dim(A, (dim...,ntuple(d->true,Val{N})...), I...)...) @inline decolon(A::AbstractVector, ::Colon) = (indices(A,1),) @inline decolon(A::AbstractArray, ::Colon) = (1:length(A),) @@ -291,8 +288,6 @@ end end end -dimlength(r::Range) = length(r) -dimlength(i::Integer) = i @noinline throw_checksize_error(A, sz) = throw(DimensionMismatch("output array is the wrong size; expected $sz, got $(size(A))")) ## setindex! ## @@ -787,7 +782,7 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. @generated function unique{T,N}(A::AbstractArray{T,N}, dim::Int) quote 1 <= dim <= $N || return copy(A) - hashes = allocate_for(inds->zeros(UInt, inds), A, shape(A, dim)) + hashes = allocate_for(inds->zeros(UInt, inds), A, indices(A, dim)) # Compute hash for each row k = 0 @@ -796,7 +791,7 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. end # Collect index of first row for each hash - uniquerow = allocate_for(Array{Int}, A, shape(A, dim)) + uniquerow = allocate_for(Array{Int}, A, indices(A, dim)) firstrow = Dict{Prehashed,Int}() for k = indices(A, dim) uniquerow[k] = get!(firstrow, Prehashed(hashes[k]), k) @@ -804,7 +799,7 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. uniquerows = collect(values(firstrow)) # Check for collisions - collided = allocate_for(falses, A, shape(A, dim)) + collided = allocate_for(falses, A, indices(A, dim)) @inbounds begin @nloops $N i A d->(if d == dim k = i_d @@ -819,7 +814,7 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. end if any(collided) - nowcollided = allocate_for(BitArray, A, shape(A, dim)) + nowcollided = allocate_for(BitArray, A, indices(A, dim)) while any(collided) # Collect index of first row for each collided hash empty!(firstrow) diff --git a/base/number.jl b/base/number.jl index 3f8cbfc37c610..d2d3270f1cfc4 100644 --- a/base/number.jl +++ b/base/number.jl @@ -7,7 +7,7 @@ isinteger(x::Integer) = true size(x::Number) = () size(x::Number,d) = convert(Int,d)<1 ? throw(BoundsError()) : 1 indices(x::Number) = () -indices(x::Number,d) = convert(Int,d)<1 ? throw(BoundsError()) : (1:1) +indices(x::Number,d) = convert(Int,d)<1 ? throw(BoundsError()) : OneTo(1) eltype{T<:Number}(::Type{T}) = T ndims(x::Number) = 0 ndims{T<:Number}(::Type{T}) = 0 diff --git a/base/operators.jl b/base/operators.jl index 6570a14e3e612..dab3177c82d77 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -387,7 +387,7 @@ function promote_shape(a::AbstractArray, b::AbstractArray) throw(DimensionMismatch("dimensions must match")) end end - return shape(a) + return indices(a) end function throw_setindex_mismatch(X, I) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index a28a7fb2d4721..a8577b1d4bc84 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -87,8 +87,6 @@ size_strides(out::Tuple) = out @inline size_strides(out, s, sz...) = size_strides((out..., out[end]*s), sz...) size(A::ReshapedArray) = A.dims -size(A::ReshapedArray, d) = d <= ndims(A) ? A.dims[d] : 1 -similar(A::ReshapedArray, eltype::Type) = similar(parent(A), eltype, size(A)) similar(A::ReshapedArray, eltype::Type, dims::Dims) = similar(parent(A), eltype, dims) linearindexing{R<:ReshapedArrayLF}(::Type{R}) = LinearFast() parent(A::ReshapedArray) = A.parent diff --git a/base/sort.jl b/base/sort.jl index 9b2bd7ea4e724..e5aa1522374b1 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -2,7 +2,7 @@ module Sort -using Base: Order, copymutable, linearindices, allocate_for, shape, linearindexing, viewindexing, LinearFast +using Base: Order, copymutable, linearindices, allocate_for, linearindexing, viewindexing, LinearFast import Base.sort, @@ -447,7 +447,7 @@ function sortperm(v::AbstractVector; by=identity, rev::Bool=false, order::Ordering=Forward) - p = Base.allocate_for(Vector{Int}, v, shape(v, 1)) + p = Base.allocate_for(Vector{Int}, v, indices(v, 1)) for (i,ind) in zip(eachindex(p), indices(v, 1)) p[i] = ind end @@ -507,7 +507,7 @@ end function sortrows(A::AbstractMatrix; kws...) inds = indices(A,1) T = slicetypeof(A, inds, :) - rows = allocate_for(Vector{T}, A, shape(A, 1)) + rows = allocate_for(Vector{T}, A, indices(A, 1)) for i in inds rows[i] = view(A, i, :) end @@ -518,7 +518,7 @@ end function sortcols(A::AbstractMatrix; kws...) inds = indices(A,2) T = slicetypeof(A, :, inds) - cols = allocate_for(Vector{T}, A, shape(A, 2)) + cols = allocate_for(Vector{T}, A, indices(A, 2)) for i in inds cols[i] = view(A, :, i) end @@ -532,7 +532,7 @@ function slicetypeof{T,N}(A::AbstractArray{T,N}, i1, i2) SubArray{T,1,typeof(A),typeof(I),fast} end slice_dummy(::Colon) = Colon() -slice_dummy{T}(::UnitRange{T}) = one(T) +slice_dummy{T}(::AbstractUnitRange{T}) = one(T) ## fast clever sorting for floats ## diff --git a/base/subarray.jl b/base/subarray.jl index adf7807b321ec..81ffd7a9d0a05 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -18,7 +18,7 @@ immutable SubArray{T,N,P,I,L} <: AbstractArray{T,N} end # Compute the linear indexability of the indices, and combine it with the linear indexing of the parent function SubArray(parent::AbstractArray, indexes::Tuple, dims::Tuple) - SubArray(linearindexing(viewindexing(indexes), linearindexing(parent)), parent, indexes, convert(Dims, dims)) + SubArray(linearindexing(viewindexing(indexes), linearindexing(parent)), parent, indexes, map(dimlength, dims)) end function SubArray{P, I, N}(::LinearSlow, parent::P, indexes::I, dims::NTuple{N, Int}) SubArray{eltype(P), N, P, I, false}(parent, indexes, dims, 0, 0) @@ -47,6 +47,9 @@ viewindexing(I::Tuple{Vararg{Any}}) = LinearSlow() # Of course, all other array types are slow viewindexing(I::Tuple{AbstractArray, Vararg{Any}}) = LinearSlow() +dimlength(r::Range) = length(r) +dimlength(i::Integer) = i + # Simple utilities size(V::SubArray) = V.dims length(V::SubArray) = prod(V.dims) @@ -57,7 +60,7 @@ parent(V::SubArray) = V.parent parentindexes(V::SubArray) = V.indexes parent(a::AbstractArray) = a -parentindexes(a::AbstractArray) = ntuple(i->1:size(a,i), ndims(a)) +parentindexes(a::AbstractArray) = ntuple(i->OneTo(size(a,i)), ndims(a)) ## SubArray creation # Drops singleton dimensions (those indexed with a scalar) @@ -273,19 +276,19 @@ end # they are taken from the range/vector # Since bounds-checking is performance-critical and uses # indices, it's worth optimizing these implementations thoroughly -indices(S::SubArray, d::Integer) = 1 <= d <= ndims(S) ? indices(S)[d] : (d > ndims(S) ? (1:1) : error("dimension $d out of range")) +indices(S::SubArray, d::Integer) = 1 <= d <= ndims(S) ? indices(S)[d] : (d > ndims(S) ? OneTo(1) : error("dimension $d out of range")) indices(S::SubArray) = (@_inline_meta; _indices(indicesbehavior(parent(S)), S)) -_indices(::IndicesStartAt1, S::SubArray) = (@_inline_meta; map(s->1:s, size(S))) +_indices(::IndicesStartAt1, S::SubArray) = (@_inline_meta; map(s->OneTo(s), size(S))) _indices(::IndicesBehavior, S::SubArray) = (@_inline_meta; _indices((), 1, S, S.indexes...)) _indices(out::Tuple, dim, S::SubArray) = out -_indices(out::Tuple, dim, S::SubArray, i1, I...) = (@_inline_meta; _indices((out..., 1:length(i1)), dim+1, S, I...)) +_indices(out::Tuple, dim, S::SubArray, i1, I...) = (@_inline_meta; _indices((out..., OneTo(length(i1))), dim+1, S, I...)) _indices(out::Tuple, dim, S::SubArray, ::Real, I...) = (@_inline_meta; _indices(out, dim+1, S, I...)) _indices(out::Tuple, dim, S::SubArray, ::Colon, I...) = (@_inline_meta; _indices((out..., indices(parent(S), dim)), dim+1, S, I...)) -indices1{T}(S::SubArray{T,0}) = 1:1 +indices1{T}(S::SubArray{T,0}) = OneTo(1) indices1(S::SubArray) = (@_inline_meta; _indices1(indicesbehavior(parent(S)), S)) -_indices1(::IndicesStartAt1, S::SubArray) = 1:S.dims[1] +_indices1(::IndicesStartAt1, S::SubArray) = OneTo(S.dims[1]) _indices1(::IndicesBehavior, S::SubArray) = (@_inline_meta; _indices1(S, 1, S.indexes...)) -_indices1(S::SubArray, dim, i1, I...) = (@_inline_meta; 1:length(i1)) +_indices1(S::SubArray, dim, i1, I...) = (@_inline_meta; OneTo(length(i1))) _indices1(S::SubArray, dim, i1::Real, I...) = (@_inline_meta; _indices1(S, dim+1, I...)) _indices1(S::SubArray, dim, i1::Colon, I...) = (@_inline_meta; indices(parent(S), dim)) diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 853186ba46730..4d1c1cdb7ee4f 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -7,7 +7,7 @@ module OAs -using Base: SimIdx, Indices, LinearSlow, LinearFast +using Base: DimOrInd, Indices, LinearSlow, LinearFast export OffsetArray @@ -46,13 +46,13 @@ Base.indices1{T}(A::OffsetArray{T,0}) = 1:1 function Base.similar(A::OffsetArray, T::Type, dims::Dims) B = similar(parent(A), T, dims) end -function Base.similar(A::AbstractArray, T::Type, inds::Tuple{Vararg{SimIdx}}) +function Base.similar(A::AbstractArray, T::Type, inds::Tuple{Vararg{DimOrInd}}) B = similar(A, T, map(Base.dimlength, inds)) OffsetArray(B, map(indsoffset, inds)) end -Base.allocate_for(f, A::OffsetArray, shape::SimIdx) = OffsetArray(f(Base.dimlength(shape)), (indsoffset(shape),)) -Base.allocate_for(f, A::OffsetArray, shape::Tuple{Vararg{SimIdx}}) = OffsetArray(f(map(Base.dimlength, shape)), map(indsoffset, shape)) +Base.allocate_for(f, A::OffsetArray, shape::DimOrInd) = OffsetArray(f(Base.dimlength(shape)), (indsoffset(shape),)) +Base.allocate_for(f, A::OffsetArray, shape::Tuple{Vararg{DimOrInd}}) = OffsetArray(f(map(Base.dimlength, shape)), map(indsoffset, shape)) Base.promote_indices(a::OffsetArray, b::OffsetArray) = a Base.reshape(A::AbstractArray, inds::Indices) = OffsetArray(reshape(A, map(Base.dimlength, inds)), map(indsoffset, inds)) From 3fe28b2b12f6f384cf7d12ffa1aeda19668fae3a Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 26 Jun 2016 17:41:06 -0500 Subject: [PATCH 0275/1117] Update broadcast and its callers to the new indices behavior --- base/broadcast.jl | 44 +++++++++++++++++++------------------ base/deprecated.jl | 6 +++++ base/operators.jl | 10 ++++----- base/sparse/sparse.jl | 2 +- base/sparse/sparsematrix.jl | 26 +++++++++++----------- test/broadcast.jl | 33 ++++++++++++++-------------- 6 files changed, 64 insertions(+), 57 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 969246e4957a9..d4c5c48ae5a30 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -3,7 +3,7 @@ module Broadcast using Base.Cartesian -using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, allocate_for, tail, dimlength +using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, to_shape, allocate_for, tail, dimlength, OneTo import Base: .+, .-, .*, ./, .\, .//, .==, .<, .!=, .<=, .÷, .%, .<<, .>>, .^ export broadcast, broadcast!, bitbroadcast export broadcast_getindex, broadcast_setindex! @@ -44,6 +44,8 @@ _bcsm(a::Number, b::Number) = a == b || b == 1 ## Check that all arguments are broadcast compatible with shape ## Check that all arguments are broadcast compatible with shape # comparing one input against a shape +check_broadcast_shape(::Tuple{}) = nothing +check_broadcast_shape(::Tuple{}, A::Union{AbstractArray,Number}) = check_broadcast_shape((), indices(A)) check_broadcast_shape(shp) = nothing check_broadcast_shape(shp, A) = check_broadcast_shape(shp, indices(A)) check_broadcast_shape(::Tuple{}, ::Tuple{}) = nothing @@ -222,13 +224,13 @@ end @inline bitbroadcast(f, As...) = broadcast!(f, allocate_for(BitArray, As, broadcast_shape(As...)), As...) -broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex!(Array{eltype(src)}(broadcast_shape(I...)), src, I...) +broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex!(Array{eltype(src)}(to_shape(broadcast_shape(I...))), src, I...) @generated function broadcast_getindex!(dest::AbstractArray, src::AbstractArray, I::AbstractArray...) N = length(I) Isplat = Expr[:(I[$d]) for d = 1:N] quote @nexprs $N d->(I_d = I[d]) - check_broadcast_shape(size(dest), $(Isplat...)) # unnecessary if this function is never called directly + check_broadcast_shape(indices(dest), $(Isplat...)) # unnecessary if this function is never called directly checkbounds(src, $(Isplat...)) @nloops $N i dest d->(@nexprs $N k->(j_d_k = size(I_k, d) == 1 ? 1 : i_d)) begin @nexprs $N k->(@inbounds J_k = @nref $N I_k d->j_d_k) @@ -245,22 +247,22 @@ end @nexprs $N d->(I_d = I[d]) checkbounds(A, $(Isplat...)) shape = broadcast_shape($(Isplat...)) - @nextract $N shape d->(length(shape) < d ? 1 : shape[d]) + @nextract $N shape d->(length(shape) < d ? OneTo(1) : shape[d]) if !isa(x, AbstractArray) - @nloops $N i d->(1:shape_d) d->(@nexprs $N k->(j_d_k = size(I_k, d) == 1 ? 1 : i_d)) begin + xA = convert(eltype(A), x) + @nloops $N i d->shape_d d->(@nexprs $N k->(j_d_k = size(I_k, d) == 1 ? 1 : i_d)) begin @nexprs $N k->(@inbounds J_k = @nref $N I_k d->j_d_k) - @inbounds (@nref $N A J) = x + @inbounds (@nref $N A J) = xA end else X = x - # To call setindex_shape_check, we need to create fake 1-d indexes of the proper size - @nexprs $N d->(fakeI_d = 1:shape_d) - @ncall $N Base.setindex_shape_check X shape - k = 1 - @nloops $N i d->(1:shape_d) d->(@nexprs $N k->(j_d_k = size(I_k, d) == 1 ? 1 : i_d)) begin - @nexprs $N k->(@inbounds J_k = @nref $N I_k d->j_d_k) - @inbounds (@nref $N A J) = X[k] - k += 1 + @nexprs $N d->(shapelen_d = dimlength(shape_d)) + @ncall $N Base.setindex_shape_check X shapelen + Xstate = start(X) + @inbounds @nloops $N i d->shape_d d->(@nexprs $N k->(j_d_k = size(I_k, d) == 1 ? 1 : i_d)) begin + @nexprs $N k->(J_k = @nref $N I_k d->j_d_k) + x_el, Xstate = next(X, Xstate) + (@nref $N A J) = x_el end end A @@ -276,22 +278,22 @@ end eltype_plus(As::AbstractArray...) = promote_eltype_op(+, As...) -.+(As::AbstractArray...) = broadcast!(+, Array{eltype_plus(As...)}(broadcast_shape(As...)), As...) +.+(As::AbstractArray...) = broadcast!(+, Array{eltype_plus(As...)}(to_shape(broadcast_shape(As...))), As...) function .-(A::AbstractArray, B::AbstractArray) - broadcast!(-, Array{promote_op(-, eltype(A), eltype(B))}(broadcast_shape(A,B)), A, B) + broadcast!(-, Array{promote_op(-, eltype(A), eltype(B))}(to_shape(broadcast_shape(A,B))), A, B) end eltype_mul(As::AbstractArray...) = promote_eltype_op(*, As...) -.*(As::AbstractArray...) = broadcast!(*, Array{eltype_mul(As...)}(broadcast_shape(As...)), As...) +.*(As::AbstractArray...) = broadcast!(*, Array{eltype_mul(As...)}(to_shape(broadcast_shape(As...))), As...) function ./(A::AbstractArray, B::AbstractArray) - broadcast!(/, Array{promote_op(/, eltype(A), eltype(B))}(broadcast_shape(A, B)), A, B) + broadcast!(/, Array{promote_op(/, eltype(A), eltype(B))}(to_shape(broadcast_shape(A, B))), A, B) end function .\(A::AbstractArray, B::AbstractArray) - broadcast!(\, Array{promote_op(\, eltype(A), eltype(B))}(broadcast_shape(A, B)), A, B) + broadcast!(\, Array{promote_op(\, eltype(A), eltype(B))}(to_shape(broadcast_shape(A, B))), A, B) end typealias RatIntT{T<:Integer} Union{Type{Rational{T}},Type{T}} @@ -301,11 +303,11 @@ type_rdiv{T<:Integer,S<:Integer}(::RatIntT{T}, ::RatIntT{S}) = type_rdiv{T<:Integer,S<:Integer}(::CRatIntT{T}, ::CRatIntT{S}) = Complex{Rational{promote_type(T,S)}} function .//(A::AbstractArray, B::AbstractArray) - broadcast!(//, Array{type_rdiv(eltype(A), eltype(B))}(broadcast_shape(A, B)), A, B) + broadcast!(//, Array{type_rdiv(eltype(A), eltype(B))}(to_shape(broadcast_shape(A, B))), A, B) end function .^(A::AbstractArray, B::AbstractArray) - broadcast!(^, Array{promote_op(^, eltype(A), eltype(B))}(broadcast_shape(A, B)), A, B) + broadcast!(^, Array{promote_op(^, eltype(A), eltype(B))}(to_shape(broadcast_shape(A, B))), A, B) end # ## element-wise comparison operators returning BitArray ## diff --git a/base/deprecated.jl b/base/deprecated.jl index dc862dc46eca6..10a50952a8cf7 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -756,6 +756,12 @@ function first(::Colon) 1 end +# Not exported, but may be useful just in case +function Broadcast.check_broadcast_shape(sz::Dims, As::Union{AbstractArray,Number}...) + depwarn("check_broadcast_shape(size(A), B...) should be replaced with check_broadcast_shape(indices(A), B...)", :check_broadcast_shape) + Broadcast.check_broadcast_shape(map(OneTo, sz), As...) +end + @deprecate slice view @deprecate sub view diff --git a/base/operators.jl b/base/operators.jl index dab3177c82d77..ae9ccd639d525 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -403,7 +403,7 @@ end # for permutations that leave array elements in the same linear order. # those are the permutations that preserve the order of the non-singleton # dimensions. -function setindex_shape_check(X::AbstractArray, I...) +function setindex_shape_check(X::AbstractArray, I::Integer...) li = ndims(X) lj = length(I) i = j = 1 @@ -440,16 +440,16 @@ end setindex_shape_check(X::AbstractArray) = (length(X)==1 || throw_setindex_mismatch(X,())) -setindex_shape_check(X::AbstractArray, i) = +setindex_shape_check(X::AbstractArray, i::Integer) = (length(X)==i || throw_setindex_mismatch(X, (i,))) -setindex_shape_check{T}(X::AbstractArray{T,1}, i) = +setindex_shape_check{T}(X::AbstractArray{T,1}, i::Integer) = (length(X)==i || throw_setindex_mismatch(X, (i,))) -setindex_shape_check{T}(X::AbstractArray{T,1}, i, j) = +setindex_shape_check{T}(X::AbstractArray{T,1}, i::Integer, j::Integer) = (length(X)==i*j || throw_setindex_mismatch(X, (i,j))) -function setindex_shape_check{T}(X::AbstractArray{T,2}, i, j) +function setindex_shape_check{T}(X::AbstractArray{T,2}, i::Integer, j::Integer) if length(X) != i*j throw_setindex_mismatch(X, (i,j)) end diff --git a/base/sparse/sparse.jl b/base/sparse/sparse.jl index b09d036e6d8b1..7a7a5a82fc905 100644 --- a/base/sparse/sparse.jl +++ b/base/sparse/sparse.jl @@ -2,7 +2,7 @@ module SparseArrays -using Base: ReshapedArray, setindex_shape_check +using Base: ReshapedArray, setindex_shape_check, to_shape using Base.Sort: Forward using Base.LinAlg: AbstractTriangular, PosDefException diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index a2f2391a2d6e5..9be5e09d73950 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1413,8 +1413,8 @@ end function gen_broadcast_body_sparse(f::Function, is_first_sparse::Bool) F = Expr(:quote, f) quote - Base.Broadcast.check_broadcast_shape(size(B), A_1) - Base.Broadcast.check_broadcast_shape(size(B), A_2) + Base.Broadcast.check_broadcast_shape(indices(B), A_1) + Base.Broadcast.check_broadcast_shape(indices(B), A_2) colptrB = B.colptr; rowvalB = B.rowval; nzvalB = B.nzval colptr1 = A_1.colptr; rowval1 = A_1.rowval; nzval1 = A_1.nzval @@ -1577,8 +1577,8 @@ function gen_broadcast_body_zpreserving(f::Function, is_first_sparse::Bool) op2 = :(val1) end quote - Base.Broadcast.check_broadcast_shape(size(B), $A1) - Base.Broadcast.check_broadcast_shape(size(B), $A2) + Base.Broadcast.check_broadcast_shape(indices(B), $A1) + Base.Broadcast.check_broadcast_shape(indices(B), $A2) nnzB = isempty(B) ? 0 : nnz($A1) * div(B.n, ($A1).n) * div(B.m, ($A1).m) @@ -1647,16 +1647,16 @@ end broadcast{Tv1,Ti1,Tv2,Ti2}(f::Function, A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = - broadcast!(f, spzeros(promote_type(Tv1, Tv2), promote_type(Ti1, Ti2), broadcast_shape(A_1, A_2)...), A_1, A_2) + broadcast!(f, spzeros(promote_type(Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) broadcast_zpreserving!(args...) = broadcast!(args...) broadcast_zpreserving(args...) = broadcast(args...) broadcast_zpreserving{Tv1,Ti1,Tv2,Ti2}(f::Function, A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = - broadcast_zpreserving!(f, spzeros(promote_type(Tv1, Tv2), promote_type(Ti1, Ti2), broadcast_shape(A_1, A_2)...), A_1, A_2) + broadcast_zpreserving!(f, spzeros(promote_type(Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) broadcast_zpreserving{Tv,Ti}(f::Function, A_1::SparseMatrixCSC{Tv,Ti}, A_2::Union{Array,BitArray,Number}) = - broadcast_zpreserving!(f, spzeros(promote_eltype(A_1, A_2), Ti, broadcast_shape(A_1, A_2)...), A_1, A_2) + broadcast_zpreserving!(f, spzeros(promote_eltype(A_1, A_2), Ti, to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) broadcast_zpreserving{Tv,Ti}(f::Function, A_1::Union{Array,BitArray,Number}, A_2::SparseMatrixCSC{Tv,Ti}) = - broadcast_zpreserving!(f, spzeros(promote_eltype(A_1, A_2), Ti, broadcast_shape(A_1, A_2)...), A_1, A_2) + broadcast_zpreserving!(f, spzeros(promote_eltype(A_1, A_2), Ti, to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) ## Binary arithmetic and boolean operators @@ -1676,7 +1676,7 @@ for (op, pro) in ((+, :eltype_plus), throw(DimensionMismatch("")) end Tv = ($pro)(A_1, A_2) - B = spzeros(Tv, promote_type(Ti1, Ti2), broadcast_shape(A_1, A_2)...) + B = spzeros(Tv, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...) $body B end @@ -1718,15 +1718,15 @@ end # macro (.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, full(B)) .+{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = - broadcast!(+, spzeros(eltype_plus(A_1, A_2), promote_type(Ti1, Ti2), broadcast_shape(A_1, A_2)...), A_1, A_2) + broadcast!(+, spzeros(eltype_plus(A_1, A_2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) function .-{Tva,Tia,Tvb,Tib}(A::SparseMatrixCSC{Tva,Tia}, B::SparseMatrixCSC{Tvb,Tib}) - broadcast!(-, spzeros(eltype_plus(A, B), promote_type(Tia, Tib), broadcast_shape(A, B)...), A, B) + broadcast!(-, spzeros(eltype_plus(A, B), promote_type(Tia, Tib), to_shape(broadcast_shape(A, B))...), A, B) end ## element-wise comparison operators returning SparseMatrixCSC ## -.<{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(<, spzeros( Bool, promote_type(Ti1, Ti2), broadcast_shape(A_1, A_2)...), A_1, A_2) -.!={Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(!=, spzeros( Bool, promote_type(Ti1, Ti2), broadcast_shape(A_1, A_2)...), A_1, A_2) +.<{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(<, spzeros( Bool, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) +.!={Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(!=, spzeros( Bool, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) ## full equality function ==(A1::SparseMatrixCSC, A2::SparseMatrixCSC) diff --git a/test/broadcast.jl b/test/broadcast.jl index 4eaccdb0c1b28..6750e05061b75 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -3,7 +3,7 @@ module TestBroadcastInternals using Base.Broadcast: broadcast_shape, check_broadcast_shape, newindex, _bcs, _bcsm -using Base.Test +using Base: Test, OneTo @test @inferred(_bcs((), (3,5), (3,5))) == (3,5) @test @inferred(_bcs((), (3,1), (3,5))) == (3,5) @@ -18,21 +18,21 @@ using Base.Test @test_throws DimensionMismatch _bcs((), (-1:1, 2:6), (-1:1, 2:5)) @test_throws DimensionMismatch _bcs((), (-1:1, 2:5), (2, 2:5)) -@test @inferred(broadcast_shape(zeros(3,4), zeros(3,4))) == (3,4) -@test @inferred(broadcast_shape(zeros(3,4), zeros(3))) == (3,4) -@test @inferred(broadcast_shape(zeros(3), zeros(3,4))) == (3,4) -@test @inferred(broadcast_shape(zeros(3), zeros(1,4), zeros(1))) == (3,4) - -check_broadcast_shape((3,5), zeros(3,5)) -check_broadcast_shape((3,5), zeros(3,1)) -check_broadcast_shape((3,5), zeros(3)) -check_broadcast_shape((3,5), zeros(3,5), zeros(3)) -check_broadcast_shape((3,5), zeros(3,5), 1) -check_broadcast_shape((3,5), 5, 2) -@test_throws DimensionMismatch check_broadcast_shape((3,5), zeros(2,5)) -@test_throws DimensionMismatch check_broadcast_shape((3,5), zeros(3,4)) -@test_throws DimensionMismatch check_broadcast_shape((3,5), zeros(3,4,2)) -@test_throws DimensionMismatch check_broadcast_shape((3,5), zeros(3,5), zeros(2)) +@test @inferred(broadcast_shape(zeros(3,4), zeros(3,4))) == (OneTo(3),OneTo(4)) +@test @inferred(broadcast_shape(zeros(3,4), zeros(3))) == (OneTo(3),OneTo(4)) +@test @inferred(broadcast_shape(zeros(3), zeros(3,4))) == (OneTo(3),OneTo(4)) +@test @inferred(broadcast_shape(zeros(3), zeros(1,4), zeros(1))) == (OneTo(3),OneTo(4)) + +check_broadcast_shape((OneTo(3),OneTo(5)), zeros(3,5)) +check_broadcast_shape((OneTo(3),OneTo(5)), zeros(3,1)) +check_broadcast_shape((OneTo(3),OneTo(5)), zeros(3)) +check_broadcast_shape((OneTo(3),OneTo(5)), zeros(3,5), zeros(3)) +check_broadcast_shape((OneTo(3),OneTo(5)), zeros(3,5), 1) +check_broadcast_shape((OneTo(3),OneTo(5)), 5, 2) +@test_throws DimensionMismatch check_broadcast_shape((OneTo(3),OneTo(5)), zeros(2,5)) +@test_throws DimensionMismatch check_broadcast_shape((OneTo(3),OneTo(5)), zeros(3,4)) +@test_throws DimensionMismatch check_broadcast_shape((OneTo(3),OneTo(5)), zeros(3,4,2)) +@test_throws DimensionMismatch check_broadcast_shape((OneTo(3),OneTo(5)), zeros(3,5), zeros(2)) check_broadcast_shape((-1:1, 6:9), (-1:1, 6:9)) check_broadcast_shape((-1:1, 6:9), (-1:1, 1)) @@ -40,7 +40,6 @@ check_broadcast_shape((-1:1, 6:9), (1, 6:9)) @test_throws DimensionMismatch check_broadcast_shape((-1:1, 6:9), (-1, 6:9)) @test_throws DimensionMismatch check_broadcast_shape((-1:1, 6:9), (-1:1, 6)) check_broadcast_shape((-1:1, 6:9), 1) -check_broadcast_shape((-1:1, 6:9), zeros(1,1)) ci(x) = CartesianIndex(x) @test @inferred(newindex(ci((2,2)), (true, true))) == ci((2,2)) From 8ed5bf0e7520dffdd82521a3409158008216ea04 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 26 Jun 2016 17:41:51 -0500 Subject: [PATCH 0276/1117] Updates for subarray for new indices --- base/abstractarray.jl | 5 ----- base/serialize.jl | 2 +- base/subarray.jl | 18 ++++++++++-------- test/subarray.jl | 15 +++++++++++---- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index a60006002ccfc..ca8a4df6814b9 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1313,11 +1313,6 @@ _lookup(ind, r::AbstractUnitRange) = ind+first(r) _div(ind, d::Integer) = div(ind, d), 1, d _div(ind, r::AbstractUnitRange) = (d = unsafe_length(r); (div(ind, d), first(r), d)) -smart_ind2sub(shape::NTuple{1}, ind) = (ind,) -smart_ind2sub(shape, ind) = ind2sub(shape, ind) -smart_sub2ind(shape::NTuple{1}, i) = (i,) -smart_sub2ind(shape, I...) = (@_inline_meta; sub2ind(shape, I...)) - # Vectorized forms function sub2ind{N,T<:Integer}(inds::Union{Dims{N},Indices{N}}, I::AbstractVector{T}...) I1 = I[1] diff --git a/base/serialize.jl b/base/serialize.jl index debf09797391f..30c38312a1f2b 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -234,7 +234,7 @@ _trimmedsubarray(A, V, newindexes, index::ViewIndex, indexes...) = _trimmedsubar trimmedindex(P, d, i::Real) = oftype(i, 1) trimmedindex(P, d, i::Colon) = i -trimmedindex(P, d, i::AbstractVector) = oftype(i, 1:length(i)) +trimmedindex(P, d, i::AbstractArray) = oftype(i, reshape(linearindices(i), indices(i))) function serialize{T<:AbstractString}(s::AbstractSerializer, ss::SubString{T}) # avoid saving a copy of the parent string, keeping the type of ss diff --git a/base/subarray.jl b/base/subarray.jl index 81ffd7a9d0a05..aa527607da513 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -48,7 +48,7 @@ viewindexing(I::Tuple{Vararg{Any}}) = LinearSlow() viewindexing(I::Tuple{AbstractArray, Vararg{Any}}) = LinearSlow() dimlength(r::Range) = length(r) -dimlength(i::Integer) = i +dimlength(i::Integer) = Int(i) # Simple utilities size(V::SubArray) = V.dims @@ -79,11 +79,17 @@ function view{N}(A::AbstractArray, I::Vararg{ViewIndex,N}) # TODO: DEPRECATE FOR @boundscheck checkbounds(A, I...) unsafe_view(reshape(A, Val{N}), I...) end + function unsafe_view{T,N}(A::AbstractArray{T,N}, I::Vararg{ViewIndex,N}) @_inline_meta J = to_indexes(I...) SubArray(A, J, map(dimlength, index_shape(A, J...))) end +function unsafe_view{T,N}(V::SubArray{T,N}, I::Vararg{ViewIndex,N}) + @_inline_meta + idxs = reindex(V, V.indexes, to_indexes(I...)) + SubArray(V.parent, idxs, map(dimlength, (index_shape(V.parent, idxs...)))) +end # Re-indexing is the heart of a view, transforming A[i, j][x, y] to A[i[x], j[y]] # @@ -169,12 +175,6 @@ function setindex!(V::FastContiguousSubArray, x, i::Real) V end -function unsafe_view{T,N}(V::SubArray{T,N}, I::Vararg{ViewIndex,N}) - @_inline_meta - idxs = reindex(V, V.indexes, to_indexes(I...)) - SubArray(V.parent, idxs, index_shape(V.parent, idxs...)) -end - linearindexing{T<:FastSubArray}(::Type{T}) = LinearFast() linearindexing{T<:SubArray}(::Type{T}) = LinearSlow() @@ -261,7 +261,9 @@ unsafe_convert{T,N,P,I<:Tuple{Vararg{RangeIndex}}}(::Type{Ptr{T}}, V::SubArray{T pointer(V::FastSubArray, i::Int) = pointer(V.parent, V.offset1 + V.stride1*i) pointer(V::FastContiguousSubArray, i::Int) = pointer(V.parent, V.offset1 + i) -pointer(V::SubArray, i::Int) = pointer(V, smart_ind2sub(shape(V), i)) +pointer(V::SubArray, i::Int) = _pointer(V, i) +_pointer{T}(V::SubArray{T,1}, i::Int) = pointer(V, (i,)) +_pointer(V::SubArray, i::Int) = pointer(V, ind2sub(indices(V), i)) function pointer{T,N,P<:Array,I<:Tuple{Vararg{RangeIndex}}}(V::SubArray{T,N,P,I}, is::Tuple{Vararg{Int}}) index = first_index(V) diff --git a/test/subarray.jl b/test/subarray.jl index 771e0c963f1e0..5181bf4f6671b 100644 --- a/test/subarray.jl +++ b/test/subarray.jl @@ -1,10 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license using Base.Test -# import Base: ViewIndex, dimsizeexpr, rangetype, merge_indexes, first_index, stride1expr, tailsize -using Base.Cartesian - -print_underestimates = false ######## Utilities ########### @@ -336,6 +332,7 @@ sA = view(A, 2:2, 1:5, :) @test parentindexes(sA) == (2:2, 1:5, :) @test Base.parentdims(sA) == [1:3;] @test size(sA) == (1, 5, 8) +@test indices(sA) === (Base.OneTo(1), Base.OneTo(5), Base.OneTo(8)) @test sA[1, 2, 1:8][:] == [5:15:120;] sA[2:5:end] = -1 @test all(sA[2:5:end] .== -1) @@ -355,6 +352,7 @@ test_bounds(sA) sA = view(A, 1:3, 3:3, 2:5) @test Base.parentdims(sA) == [1:3;] @test size(sA) == (3,1,4) +@test indices(sA) === (Base.OneTo(3), Base.OneTo(1), Base.OneTo(4)) @test sA == A[1:3,3:3,2:5] @test sA[:] == A[1:3,3,2:5][:] test_bounds(sA) @@ -367,6 +365,12 @@ sA = view(A, 1:2:3, 1:3:5, 1:2:8) # Test with mixed types @test sA[:, Int16[1,2], big(2)] == [31 40; 33 42] test_bounds(sA) +sA = view(A, 1:1, 1:5, [1 3; 4 2]) +@test ndims(sA) == 4 +@test indices(sA) === (Base.OneTo(1), Base.OneTo(5), Base.OneTo(2), Base.OneTo(2)) +sA = view(A, 1:2, 3, [1 3; 4 2]) +@test ndims(sA) == 3 +@test indices(sA) === (Base.OneTo(2), Base.OneTo(2), Base.OneTo(2)) # sub logical indexing #4763 A = view([1:10;], 5:8) @@ -384,6 +388,7 @@ sA = view(A, 2, :, 1:8) @test parentindexes(sA) == (2, :, 1:8) @test Base.parentdims(sA) == [2:3;] @test size(sA) == (5, 8) +@test indices(sA) === (Base.OneTo(5), Base.OneTo(8)) @test strides(sA) == (3,15) @test sA[2, 1:8][:] == [5:15:120;] @test sA[:,1] == [2:3:14;] @@ -395,11 +400,13 @@ test_bounds(sA) sA = view(A, 1:3, 1:5, 5) @test Base.parentdims(sA) == [1:2;] @test size(sA) == (3,5) +@test indices(sA) === (Base.OneTo(3),Base.OneTo(5)) @test strides(sA) == (1,3) test_bounds(sA) sA = view(A, 1:2:3, 3, 1:2:8) @test Base.parentdims(sA) == [1,3] @test size(sA) == (2,4) +@test indices(sA) === (Base.OneTo(2), Base.OneTo(4)) @test strides(sA) == (2,30) @test sA[:] == A[sA.indexes...][:] test_bounds(sA) From 9acbe8aeb80bdbd873c4f206076dbce9539ad1d8 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 27 Jun 2016 04:12:33 -0500 Subject: [PATCH 0277/1117] Rename allocate_for -> similar. Fixes #17124. --- base/abstractarray.jl | 33 +++++++++++---------------------- base/broadcast.jl | 12 ++++++------ base/exports.jl | 1 - base/multidimensional.jl | 8 ++++---- base/sort.jl | 8 ++++---- test/offsetarray.jl | 13 ++++++------- 6 files changed, 31 insertions(+), 44 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index ca8a4df6814b9..d4bda7ad18eb1 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -326,7 +326,6 @@ different element type it will create a regular `Array` instead: 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 -See also `allocate_for`. """ similar{T}(a::AbstractArray{T}) = similar(a, T) similar( a::AbstractArray, T::Type) = similar(a, T, to_shape(indices(a))) @@ -347,44 +346,34 @@ to_shape(r::OneTo) = Int(last(r)) to_shape(r::UnitRange) = convert(UnitRange{Int}, r) """ - allocate_for(storagetype, referencearray, [shape]) + similar(storagetype, indices) Create an uninitialized mutable array analogous to that specified by -`storagetype`, but with type and shape specified by the final two -arguments. The main purpose of this function is to support allocation -of arrays that may have unconventional indexing (starting at other -than 1), as determined by `referencearray` and the optional `shape` -information. +`storagetype`, but with `indices` specified by the last +argument. `storagetype` might be a type or a function. **Examples**: - allocate_for(Array{Int}, A) + similar(Array{Int}, indices(A)) creates an array that "acts like" an `Array{Int}` (and might indeed be backed by one), but which is indexed identically to `A`. If `A` has -conventional indexing, this will likely just call +conventional indexing, this will be identical to `Array{Int}(size(A))`, but if `A` has unconventional indexing then the indices of the result will match `A`. - allocate_for(BitArray, A, (indices(A, 2),)) + similar(BitArray, (indices(A, 2),)) would create a 1-dimensional logical array whose indices match those of the columns of `A`. -The main purpose of the `referencearray` argument is to select a -particular array type supporting unconventional indexing (as it is -possible that several different ones will be simultaneously in use). + similar(dims->zeros(Int, dims), indices(A)) -See also `similar`. +would create an array of `Int`, initialized to zero, matching the +indices of `A`. """ -allocate_for(f, a, shape::Union{DimOrInd,DimsOrInds}) = f(to_shape(shape)) -allocate_for(f, a) = allocate_for(f, a, indices(a)) -# allocate_for when passed multiple arrays. Necessary for broadcast, etc. -function allocate_for(f, as::Tuple, shape::Union{DimOrInd,DimsOrInds}) - @_inline_meta - a = promote_indices(as...) - allocate_for(f, a, shape) -end +similar(f, shape::Tuple) = f(to_shape(shape)) +similar(f, dims::DimOrInd...) = similar(f, dims) promote_indices(a) = a function promote_indices(a, b, c...) diff --git a/base/broadcast.jl b/base/broadcast.jl index d4c5c48ae5a30..7eff313075d95 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -3,7 +3,7 @@ module Broadcast using Base.Cartesian -using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, to_shape, allocate_for, tail, dimlength, OneTo +using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, to_shape, tail, dimlength, OneTo import Base: .+, .-, .*, ./, .\, .//, .==, .<, .!=, .<=, .÷, .%, .<<, .>>, .^ export broadcast, broadcast!, bitbroadcast export broadcast_getindex, broadcast_setindex! @@ -186,7 +186,7 @@ function broadcast_t(f, ::Type{Any}, As...) shp = broadcast_shape(As...) iter = CartesianRange(shp) if isempty(iter) - return allocate_for(Array{Union{}}, As, shp) + return similar(Array{Union{}}, shp) end nargs = length(As) sz = size(iter) @@ -194,12 +194,12 @@ function broadcast_t(f, ::Type{Any}, As...) st = start(iter) I, st = next(iter, st) val = f([ As[i][newindex(I, indexmaps[i])] for i=1:nargs ]...) - B = allocate_for(Array{typeof(val)}, As, shp) + B = similar(Array{typeof(val)}, shp) B[I] = val return _broadcast!(f, B, indexmaps, As, Val{nargs}, iter, st, 1) end -@inline broadcast_t(f, T, As...) = broadcast!(f, allocate_for(Array{T}, As, broadcast_shape(As...)), As...) +@inline broadcast_t(f, T, As...) = broadcast!(f, similar(Array{T}, broadcast_shape(As...)), As...) @inline broadcast(f, As...) = broadcast_t(f, promote_eltype_op(f, As...), As...) @@ -222,7 +222,7 @@ function broadcast(f, As...) end =# -@inline bitbroadcast(f, As...) = broadcast!(f, allocate_for(BitArray, As, broadcast_shape(As...)), As...) +@inline bitbroadcast(f, As...) = broadcast!(f, similar(BitArray, broadcast_shape(As...)), As...) broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex!(Array{eltype(src)}(to_shape(broadcast_shape(I...))), src, I...) @generated function broadcast_getindex!(dest::AbstractArray, src::AbstractArray, I::AbstractArray...) @@ -373,7 +373,7 @@ for (f, scalarf) in ((:.==, :(==)), shape = :(indices($active)) @eval begin function ($f)(A::$sigA, B::$sigB) - P = allocate_for(BitArray, $active, $shape) + P = similar(BitArray, $shape) F = parent(P) l = length(F) l == 0 && return F diff --git a/base/exports.jl b/base/exports.jl index 33f11efedad4d..8641e82cd8fba 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -484,7 +484,6 @@ export zeta, # arrays - allocate_for, bitbroadcast, broadcast!, broadcast, diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 3346d5983dd07..3c4e3b3bef930 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -782,7 +782,7 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. @generated function unique{T,N}(A::AbstractArray{T,N}, dim::Int) quote 1 <= dim <= $N || return copy(A) - hashes = allocate_for(inds->zeros(UInt, inds), A, indices(A, dim)) + hashes = similar(inds->zeros(UInt, inds), indices(A, dim)) # Compute hash for each row k = 0 @@ -791,7 +791,7 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. end # Collect index of first row for each hash - uniquerow = allocate_for(Array{Int}, A, indices(A, dim)) + uniquerow = similar(Array{Int}, indices(A, dim)) firstrow = Dict{Prehashed,Int}() for k = indices(A, dim) uniquerow[k] = get!(firstrow, Prehashed(hashes[k]), k) @@ -799,7 +799,7 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. uniquerows = collect(values(firstrow)) # Check for collisions - collided = allocate_for(falses, A, indices(A, dim)) + collided = similar(falses, indices(A, dim)) @inbounds begin @nloops $N i A d->(if d == dim k = i_d @@ -814,7 +814,7 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. end if any(collided) - nowcollided = allocate_for(BitArray, A, indices(A, dim)) + nowcollided = similar(BitArray, indices(A, dim)) while any(collided) # Collect index of first row for each collided hash empty!(firstrow) diff --git a/base/sort.jl b/base/sort.jl index e5aa1522374b1..9cac327aa92c5 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -2,7 +2,7 @@ module Sort -using Base: Order, copymutable, linearindices, allocate_for, linearindexing, viewindexing, LinearFast +using Base: Order, copymutable, linearindices, linearindexing, viewindexing, LinearFast import Base.sort, @@ -447,7 +447,7 @@ function sortperm(v::AbstractVector; by=identity, rev::Bool=false, order::Ordering=Forward) - p = Base.allocate_for(Vector{Int}, v, indices(v, 1)) + p = similar(Vector{Int}, indices(v, 1)) for (i,ind) in zip(eachindex(p), indices(v, 1)) p[i] = ind end @@ -507,7 +507,7 @@ end function sortrows(A::AbstractMatrix; kws...) inds = indices(A,1) T = slicetypeof(A, inds, :) - rows = allocate_for(Vector{T}, A, indices(A, 1)) + rows = similar(Vector{T}, indices(A, 1)) for i in inds rows[i] = view(A, i, :) end @@ -518,7 +518,7 @@ end function sortcols(A::AbstractMatrix; kws...) inds = indices(A,2) T = slicetypeof(A, :, inds) - cols = allocate_for(Vector{T}, A, indices(A, 2)) + cols = similar(Vector{T}, indices(A, 2)) for i in inds cols[i] = view(A, :, i) end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 4d1c1cdb7ee4f..ce38c5721a8e5 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -7,7 +7,7 @@ module OAs -using Base: DimOrInd, Indices, LinearSlow, LinearFast +using Base: DimOrInd, DimsOrInds, Indices, LinearSlow, LinearFast export OffsetArray @@ -46,16 +46,15 @@ Base.indices1{T}(A::OffsetArray{T,0}) = 1:1 function Base.similar(A::OffsetArray, T::Type, dims::Dims) B = similar(parent(A), T, dims) end -function Base.similar(A::AbstractArray, T::Type, inds::Tuple{Vararg{DimOrInd}}) +function Base.similar(A::AbstractArray, T::Type, inds::Tuple{UnitRange,Vararg{UnitRange}}) B = similar(A, T, map(Base.dimlength, inds)) OffsetArray(B, map(indsoffset, inds)) end -Base.allocate_for(f, A::OffsetArray, shape::DimOrInd) = OffsetArray(f(Base.dimlength(shape)), (indsoffset(shape),)) -Base.allocate_for(f, A::OffsetArray, shape::Tuple{Vararg{DimOrInd}}) = OffsetArray(f(map(Base.dimlength, shape)), map(indsoffset, shape)) +Base.similar(f::Union{Function,Type}, shape::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(f(map(Base.dimlength, shape)), map(indsoffset, shape)) Base.promote_indices(a::OffsetArray, b::OffsetArray) = a -Base.reshape(A::AbstractArray, inds::Indices) = OffsetArray(reshape(A, map(Base.dimlength, inds)), map(indsoffset, inds)) +Base.reshape(A::AbstractArray, inds::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(reshape(A, map(Base.dimlength, inds)), map(indsoffset, inds)) @inline function Base.getindex{T,N}(A::OffsetArray{T,N}, I::Vararg{Int,N}) @boundscheck checkbounds(A, I...) @@ -207,10 +206,10 @@ B = similar(A, (3,4)) @test isa(B, Array{Int,2}) @test size(B) == (3,4) @test indices(B) == (1:3, 1:4) -B = similar(A, (-3:3,4)) +B = similar(A, (-3:3,1:4)) @test isa(B, OffsetArray{Int,2}) @test indices(B) == (-3:3, 1:4) -B = similar(parent(A), (-3:3,4)) +B = similar(parent(A), (-3:3,1:4)) @test isa(B, OffsetArray{Int,2}) @test indices(B) == (-3:3, 1:4) From 2873b6e65479cd1388251190c174e080ce888b8d Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Wed, 29 Jun 2016 11:16:47 -0500 Subject: [PATCH 0278/1117] Eliminate the indicesbehavior trait --- base/abstractarray.jl | 52 +++++---------------------------------- base/arraymath.jl | 18 ++++++-------- base/permuteddimsarray.jl | 8 +----- base/reshapedarray.jl | 26 +++++++++----------- base/sort.jl | 2 +- base/subarray.jl | 17 ++----------- doc/manual/interfaces.rst | 17 ++++++------- 7 files changed, 37 insertions(+), 103 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index d4bda7ad18eb1..90e6395403f27 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -146,15 +146,6 @@ linearindexing(A::AbstractArray, B::AbstractArray...) = linearindexing(linearind linearindexing(::LinearFast, ::LinearFast) = LinearFast() linearindexing(::LinearIndexing, ::LinearIndexing) = LinearSlow() -abstract IndicesBehavior -immutable IndicesStartAt1 <: IndicesBehavior end # indices 1:size(A,d) -immutable IndicesUnitRange <: IndicesBehavior end # arb UnitRange indices -immutable IndicesList <: IndicesBehavior end # indices like (:cat, :dog, :mouse) - -indicesbehavior(A::AbstractArray) = indicesbehavior(typeof(A)) -indicesbehavior{T<:AbstractArray}(::Type{T}) = IndicesStartAt1() -indicesbehavior(::Number) = IndicesStartAt1() - abstract IndicesPerformance immutable IndicesFast1D <: IndicesPerformance end # indices(A, d) is fast immutable IndicesSlow1D <: IndicesPerformance end # indices(A) is better than indices(A,d) @@ -375,26 +366,6 @@ indices of `A`. similar(f, shape::Tuple) = f(to_shape(shape)) similar(f, dims::DimOrInd...) = similar(f, dims) -promote_indices(a) = a -function promote_indices(a, b, c...) - @_inline_meta - promote_indices(promote_indices(a, b), c...) -end -# overload this to return true for your type, e.g., -# promote_indices(a::OffsetArray, b::OffsetArray) = a -promote_indices(a::AbstractArray, b::AbstractArray) = _promote_indices(indicesbehavior(a), indicesbehavior(b), a, b) -_promote_indices(::IndicesStartAt1, ::IndicesStartAt1, a, b) = a -_promote_indices(::IndicesBehavior, ::IndicesBehavior, a, b) = throw(ArgumentError("types $(typeof(a)) and $(typeof(b)) do not have promote_indices defined")) -promote_indices(a::Number, b::AbstractArray) = b -promote_indices(a::AbstractArray, b::Number) = a - -# Strip off the index-changing container---this assumes that `parent` -# performs such an operation. TODO: since few things in Base need this, it -# would be great to find a way to eliminate this function. -normalize_indices(A) = normalize_indices(indicesbehavior(A), A) -normalize_indices(::IndicesStartAt1, A) = A -normalize_indices(::IndicesBehavior, A) = parent(A) - ## from general iterable to any array function copy!(dest::AbstractArray, src) @@ -1229,38 +1200,26 @@ end # sub2ind and ind2sub # fallbacks function sub2ind(A::AbstractArray, I...) - @_inline_meta - sub2ind(indicesbehavior(A), A, I...) -end -function sub2ind(::IndicesStartAt1, A::AbstractArray, I...) - @_inline_meta - sub2ind(size(A), I...) -end -function sub2ind(::IndicesBehavior, A::AbstractArray, I...) @_inline_meta sub2ind(indices(A), I...) end function ind2sub(A::AbstractArray, ind) - @_inline_meta - ind2sub(indicesbehavior(A), A, ind) -end -function ind2sub(::IndicesStartAt1, A::AbstractArray, ind) - @_inline_meta - ind2sub(size(A), ind) -end -function ind2sub(::IndicesBehavior, A::AbstractArray, ind) @_inline_meta ind2sub(indices(A), ind) end +# 0-dimensional arrays and indexing with [] sub2ind(::Tuple{}) = 1 sub2ind(::DimsInteger) = 1 sub2ind(::Indices) = 1 sub2ind(::Tuple{}, I::Integer...) = (@_inline_meta; _sub2ind((), 1, 1, I...)) +# Generic cases sub2ind(dims::DimsInteger, I::Integer...) = (@_inline_meta; _sub2ind(dims, 1, 1, I...)) sub2ind(inds::Indices, I::Integer...) = (@_inline_meta; _sub2ind(inds, 1, 1, I...)) -# In 1d, there's a question of whether we're doing cartesian indexing or linear indexing. Support only the former. +# In 1d, there's a question of whether we're doing cartesian indexing +# or linear indexing. Support only the former. sub2ind(inds::Indices{1}, I::Integer...) = throw(ArgumentError("Linear indexing is not defined for one-dimensional arrays")) +sub2ind(inds::Tuple{OneTo}, I::Integer...) = (@_inline_meta; _sub2ind(inds, 1, 1, I...)) # only OneTo is safe _sub2ind(::Any, L, ind) = ind function _sub2ind(::Tuple{}, L, ind, i::Integer, I::Integer...) @@ -1284,6 +1243,7 @@ ind2sub(::Tuple{}, ind::Integer) = (@_inline_meta; ind == 1 ? () : throw(BoundsE ind2sub(dims::DimsInteger, ind::Integer) = (@_inline_meta; _ind2sub((), dims, ind-1)) ind2sub(inds::Indices, ind::Integer) = (@_inline_meta; _ind2sub((), inds, ind-1)) ind2sub(inds::Indices{1}, ind::Integer) = throw(ArgumentError("Linear indexing is not defined for one-dimensional arrays")) +ind2sub(inds::Tuple{OneTo}, ind::Integer) = (@_inline_meta; _ind2sub((), inds, ind-1)) _ind2sub(::Tuple{}, ::Tuple{}, ind) = (ind+1,) function _ind2sub(out, indslast::NTuple{1}, ind) diff --git a/base/arraymath.jl b/base/arraymath.jl index cb3cd4625e26e..c89ac3c2d6a30 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -211,8 +211,8 @@ function flipdim{T}(A::Array{T}, d::Integer) end function rotl90(A::AbstractMatrix) - B = similar_transpose(A) - ind2 = indices(A,2) + ind1, ind2 = indices(A) + B = similar(A, (ind2,ind1)) n = first(ind2)+last(ind2) for i=indices(A,1), j=ind2 B[n-j,i] = A[i,j] @@ -220,8 +220,8 @@ function rotl90(A::AbstractMatrix) return B end function rotr90(A::AbstractMatrix) - B = similar_transpose(A) - ind1 = indices(A,1) + ind1, ind2 = indices(A) + B = similar(A, (ind2,ind1)) m = first(ind1)+last(ind1) for i=ind1, j=indices(A,2) B[j,m-i] = A[i,j] @@ -246,10 +246,6 @@ end rotr90(A::AbstractMatrix, k::Integer) = rotl90(A,-k) rot180(A::AbstractMatrix, k::Integer) = mod(k, 2) == 1 ? rot180(A) : copy(A) -similar_transpose(A::AbstractMatrix) = similar_transpose(indicesbehavior(A), A) -similar_transpose(::IndicesStartAt1, A::AbstractMatrix) = similar(A, (size(A,2), size(A,1))) -similar_transpose(::IndicesBehavior, A::AbstractMatrix) = similar(A, (indices(A,2), indices(A,1))) - ## Transpose ## transpose!(B::AbstractMatrix, A::AbstractMatrix) = transpose_f!(transpose, B, A) ctranspose!(B::AbstractMatrix, A::AbstractMatrix) = transpose_f!(ctranspose, B, A) @@ -315,11 +311,13 @@ function ccopy!(B, A) end function transpose(A::AbstractMatrix) - B = similar_transpose(A) + ind1, ind2 = indices(A) + B = similar(A, (ind2, ind1)) transpose!(B, A) end function ctranspose(A::AbstractMatrix) - B = similar_transpose(A) + ind1, ind2 = indices(A) + B = similar(A, (ind2, ind1)) ctranspose!(B, A) end ctranspose{T<:Real}(A::AbstractVecOrMat{T}) = transpose(A) diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index c5a461509d494..dd64416fae86b 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -2,8 +2,6 @@ module PermutedDimsArrays -using Base: IndicesStartAt1, IndicesBehavior, indicesbehavior - export permutedims # Some day we will want storage-order-aware iteration, so put perm in the parameters @@ -47,14 +45,10 @@ _genperm(out, I) = out @inline genperm(I, perm::AbstractVector{Int}) = genperm(I, (perm...,)) function Base.permutedims{T,N}(A::AbstractArray{T,N}, perm) - dest = similar_permute(A, perm) + dest = similar(A, genperm(indices(A), perm)) permutedims!(dest, A, perm) end -similar_permute(A::AbstractArray, perm) = similar_permute(indicesbehavior(A), A, perm) -similar_permute{T,N}(::IndicesStartAt1, A::AbstractArray{T,N}, perm) = similar(A, genperm(size(A), perm)) -similar_permute{T,N}(::IndicesBehavior, A::AbstractArray{T,N}, perm) = similar(A, genperm(indices(A), perm)) - function Base.permutedims!(dest, src::AbstractArray, perm) Base.checkdims_perm(dest, src, perm) P = PermutedDimsArray(dest, invperm(perm)) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index a8577b1d4bc84..053a554eb821a 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -36,24 +36,20 @@ start(R::ReshapedArrayIterator) = start(R.iter) end length(R::ReshapedArrayIterator) = length(R.iter) -reshape(parent::AbstractArray, ref::AbstractArray) = reshape(indicesbehavior(ref), parent, ref) -reshape(::IndicesStartAt1, parent::AbstractArray, ref::AbstractArray) = reshape(parent, size(ref)) -reshape(::IndicesBehavior, parent::AbstractArray, ref::AbstractArray) = reshape(parent, indices(ref)) - -reshape(parent::AbstractArray, dims::Dims) = _reshape(parent, dims) -reshape(parent::AbstractArray, len::Integer) = reshape(parent, (Int(len),)) -reshape(parent::AbstractArray, dims::Int...) = reshape(parent, dims) +reshape(parent::AbstractArray, shp::Tuple) = _reshape(parent, to_shape(shp)) +reshape(parent::AbstractArray, dims::DimOrInd...) = reshape(parent, dims) reshape{T,N}(parent::AbstractArray{T,N}, ndims::Type{Val{N}}) = parent function reshape{T,AN,N}(parent::AbstractArray{T,AN}, ndims::Type{Val{N}}) - reshape(parent, rdims((), size(parent), Val{N})) -end -# Move elements from sz to out until out reaches the desired dimensionality N, -# either filling with 1 or collapsing the product of trailing dims into the last element -@pure rdims{N}(out::NTuple{N}, sz::Tuple{}, ::Type{Val{N}}) = out -@pure rdims{N}(out::NTuple{N}, sz::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = (front(out)..., last(out) * prod(sz)) -@pure rdims{N}(out::Tuple, sz::Tuple{}, ::Type{Val{N}}) = rdims((out..., 1), (), Val{N}) -@pure rdims{N}(out::Tuple, sz::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = rdims((out..., first(sz)), tail(sz), Val{N}) + reshape(parent, rdims((), indices(parent), Val{N})) +end +# Move elements from inds to out until out reaches the desired +# dimensionality N, either filling with OneTo(1) or collapsing the +# product of trailing dims into the last element +@pure rdims{N}(out::NTuple{N}, inds::Tuple{}, ::Type{Val{N}}) = out +@pure rdims{N}(out::NTuple{N}, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = (front(out)..., length(last(out)) * prod(map(length, inds))) +@pure rdims{N}(out::Tuple, inds::Tuple{}, ::Type{Val{N}}) = rdims((out..., OneTo(1)), (), Val{N}) +@pure rdims{N}(out::Tuple, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = rdims((out..., first(inds)), tail(inds), Val{N}) function _reshape(parent::AbstractArray, dims::Dims) prod(dims) == length(parent) || throw(DimensionMismatch("parent has $(length(parent)) elements, which is incompatible with size $dims")) diff --git a/base/sort.jl b/base/sort.jl index 9cac327aa92c5..99900be146ef4 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -492,7 +492,7 @@ function sort(A::AbstractArray, dim::Integer; else Av = A[:] sort_chunks!(Av, size(A,1), alg, order) - reshape(Av, A) + reshape(Av, indices(A)) end end diff --git a/base/subarray.jl b/base/subarray.jl index aa527607da513..02b4cb5b3d95a 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -279,17 +279,13 @@ end # Since bounds-checking is performance-critical and uses # indices, it's worth optimizing these implementations thoroughly indices(S::SubArray, d::Integer) = 1 <= d <= ndims(S) ? indices(S)[d] : (d > ndims(S) ? OneTo(1) : error("dimension $d out of range")) -indices(S::SubArray) = (@_inline_meta; _indices(indicesbehavior(parent(S)), S)) -_indices(::IndicesStartAt1, S::SubArray) = (@_inline_meta; map(s->OneTo(s), size(S))) -_indices(::IndicesBehavior, S::SubArray) = (@_inline_meta; _indices((), 1, S, S.indexes...)) +indices(S::SubArray) = (@_inline_meta; _indices((), 1, S, S.indexes...)) _indices(out::Tuple, dim, S::SubArray) = out _indices(out::Tuple, dim, S::SubArray, i1, I...) = (@_inline_meta; _indices((out..., OneTo(length(i1))), dim+1, S, I...)) _indices(out::Tuple, dim, S::SubArray, ::Real, I...) = (@_inline_meta; _indices(out, dim+1, S, I...)) _indices(out::Tuple, dim, S::SubArray, ::Colon, I...) = (@_inline_meta; _indices((out..., indices(parent(S), dim)), dim+1, S, I...)) indices1{T}(S::SubArray{T,0}) = OneTo(1) -indices1(S::SubArray) = (@_inline_meta; _indices1(indicesbehavior(parent(S)), S)) -_indices1(::IndicesStartAt1, S::SubArray) = OneTo(S.dims[1]) -_indices1(::IndicesBehavior, S::SubArray) = (@_inline_meta; _indices1(S, 1, S.indexes...)) +indices1(S::SubArray) = (@_inline_meta; _indices1(S, 1, S.indexes...)) _indices1(S::SubArray, dim, i1, I...) = (@_inline_meta; OneTo(length(i1))) _indices1(S::SubArray, dim, i1::Real, I...) = (@_inline_meta; _indices1(S, dim+1, I...)) _indices1(S::SubArray, dim, i1::Colon, I...) = (@_inline_meta; indices(parent(S), dim)) @@ -297,15 +293,6 @@ _indices1(S::SubArray, dim, i1::Colon, I...) = (@_inline_meta; indices(parent(S) # Moreover, incides(S) is fast but indices(S, d) is slower indicesperformance{T<:SubArray}(::Type{T}) = IndicesSlow1D() -indicesbehavior(S::SubArray) = _indicesbehavior(indicesbehavior(parent(S)), S) -_indicesbehavior(::IndicesStartAt1, S::SubArray) = IndicesStartAt1() -_indicesbehavior(::IndicesBehavior, S::SubArray) = (@_inline_meta; _indicesbehavior(keepcolon((), S.indexes...), S)) -keepcolon(out) = out -keepcolon(out, ::Colon, I...) = (@_inline_meta; (Colon(),)) -keepcolon(out, i1, I...) = (@_inline_meta; keepcolon(out, I...)) -_indicesbehavior(::Tuple{}, S::SubArray) = IndicesStartAt1() -_indicesbehavior(::Tuple{Colon}, S::SubArray) = indicesbehavior(parent(S)) - ## Compatability # deprecate? function parentdims(s::SubArray) diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 5ede56eb24f8c..b0f0886da6c22 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -166,9 +166,8 @@ Methods to implement :func:`similar(A, dims::NTuple{Int}) <similar>` ``similar(A, eltype(A), dims)`` Return a mutable array with the same element type and size `dims` :func:`similar(A, ::Type{S}, dims::NTuple{Int}) <similar>` ``Array{S}(dims)`` Return a mutable array with the specified element type and size **Non-traditional indices** **Default definition** **Brief description** -:func:`Base.indicesbehavior(::Type) <indicesbehavior>` ``Base.IndicesStartAt1()`` Trait with values ``IndicesStartAt1()``, ``IndicesUnitRange()``, ``IndicesList()`` -:func:`indices(A, d) <indices>` ``1:size(A, d)`` Return the range of valid indices along dimension ``d`` -:func:`Base.similar(A, ::Type{S}, inds::NTuple{Ind}) <similar>` ``similar(A, S, map(Base.dimlength, inds))`` Return a mutable array with the specified indices ``inds`` (see below for discussion of ``Ind``) +:func:`indices(A, d) <indices>` ``OneTo(size(A, d))`` Return the ``AbstractUnitRange`` of valid indices along dimension ``d`` +:func:`Base.similar(A, ::Type{S}, inds::NTuple{Ind}) <similar>` ``similar(A, S, map(Base.dimlength, inds))`` Return a mutable array with the specified indices ``inds`` (see below) ===================================================================== ============================================ ======================================================================================= If a type is defined as a subtype of ``AbstractArray``, it inherits a very large set of rich behaviors including iteration and multidimensional indexing built on top of single-element access. See the :ref:`arrays manual page <man-arrays>` and :ref:`standard library section <stdlib-arrays>` for more supported methods. @@ -290,9 +289,9 @@ In addition to all the iterable and indexable methods from above, these types ca If you are defining an array type that allows non-traditional indexing (indices that start at something other than 1), you should specialize -``indices`` and ``indicesbehavior``. You should also specialize -``similar`` so that the ``dims`` argument (ordinarily a ``Dims`` -size-tuple) can be a mixture of ``Integer`` and ``UnitRange`` objects; -the ``Integer`` entries imply that the indexing starts from 1, whereas -the dimensions encoded with ``UnitRange`` may have arbitrary starting -index. +``indices``. You should also specialize ``similar`` so that the +``dims`` argument (ordinarily a ``Dims`` size-tuple) can accept +``AbstractUnitRange`` objects, perhaps range-types ``Ind`` of your own +design. For example, if indexing always starts with 0 for your +arrays, you likely want to define a ``ZeroTo`` range type. Otherwise, +you can use standard ``UnitRange``. From 8f8d1073ee672d7518d14b58776841822bb7c128 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Wed, 6 Jul 2016 10:03:50 -0500 Subject: [PATCH 0279/1117] Simpler sub2ind/ind2sub for Tuple{OneTo} --- base/abstractarray.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 90e6395403f27..7ba848b37b058 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1220,6 +1220,7 @@ sub2ind(inds::Indices, I::Integer...) = (@_inline_meta; _sub2ind(inds, 1, 1, I.. # or linear indexing. Support only the former. sub2ind(inds::Indices{1}, I::Integer...) = throw(ArgumentError("Linear indexing is not defined for one-dimensional arrays")) sub2ind(inds::Tuple{OneTo}, I::Integer...) = (@_inline_meta; _sub2ind(inds, 1, 1, I...)) # only OneTo is safe +sub2ind(inds::Tuple{OneTo}, i::Integer) = i _sub2ind(::Any, L, ind) = ind function _sub2ind(::Tuple{}, L, ind, i::Integer, I::Integer...) @@ -1243,7 +1244,7 @@ ind2sub(::Tuple{}, ind::Integer) = (@_inline_meta; ind == 1 ? () : throw(BoundsE ind2sub(dims::DimsInteger, ind::Integer) = (@_inline_meta; _ind2sub((), dims, ind-1)) ind2sub(inds::Indices, ind::Integer) = (@_inline_meta; _ind2sub((), inds, ind-1)) ind2sub(inds::Indices{1}, ind::Integer) = throw(ArgumentError("Linear indexing is not defined for one-dimensional arrays")) -ind2sub(inds::Tuple{OneTo}, ind::Integer) = (@_inline_meta; _ind2sub((), inds, ind-1)) +ind2sub(inds::Tuple{OneTo}, ind::Integer) = (ind,) _ind2sub(::Tuple{}, ::Tuple{}, ind) = (ind+1,) function _ind2sub(out, indslast::NTuple{1}, ind) From 46432337c9e0836b3d8609a700556526037e8f8c Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Wed, 6 Jul 2016 09:49:28 -0500 Subject: [PATCH 0280/1117] Fix offsetarray tests and move them earlier The reason to move earlier is to test whether the methods corrupt other operations. --- test/choosetests.jl | 4 ++-- test/offsetarray.jl | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/test/choosetests.jl b/test/choosetests.jl index 9818d9acbeb12..9484d0e394f7c 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -17,7 +17,7 @@ function choosetests(choices = []) testnames = [ "linalg", "subarray", "core", "inference", "keywordargs", "numbers", "printf", "char", "string", "triplequote", "unicode", - "dates", "dict", "hashing", "iobuffer", "staged", + "dates", "dict", "hashing", "iobuffer", "staged", "offsetarray", "arrayops", "tuple", "reduce", "reducedim", "random", "abstractarray", "intfuncs", "simdloop", "vecelement", "blas", "sparse", "bitarray", "copy", "math", "fastmath", "functional", @@ -33,7 +33,7 @@ function choosetests(choices = []) "markdown", "base64", "serialize", "misc", "threads", "enums", "cmdlineargs", "i18n", "workspace", "libdl", "int", "checked", "intset", "floatfuncs", "compile", "parallel", "inline", - "boundscheck", "error", "ambiguous", "offsetarray", "cartesian" + "boundscheck", "error", "ambiguous", "cartesian" ] if Base.USE_GPL_LIBS diff --git a/test/offsetarray.jl b/test/offsetarray.jl index ce38c5721a8e5..4a91bf6a29f26 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -24,7 +24,6 @@ OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::Vararg{Int,N}) = OffsetArray(A, (::Type{OffsetArray{T}}){T,N}(inds::Indices{N}) = OffsetArray{T,N}(inds) Base.linearindexing{T<:OffsetArray}(::Type{T}) = Base.linearindexing(parenttype(T)) -Base.indicesbehavior{T<:OffsetArray}(::Type{T}) = Base.IndicesUnitRange() parenttype{T,N,AA}(::Type{OffsetArray{T,N,AA}}) = AA parenttype(A::OffsetArray) = parenttype(typeof(A)) @@ -52,7 +51,6 @@ function Base.similar(A::AbstractArray, T::Type, inds::Tuple{UnitRange,Vararg{Un end Base.similar(f::Union{Function,Type}, shape::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(f(map(Base.dimlength, shape)), map(indsoffset, shape)) -Base.promote_indices(a::OffsetArray, b::OffsetArray) = a Base.reshape(A::AbstractArray, inds::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(reshape(A, map(Base.dimlength, inds)), map(indsoffset, inds)) @@ -99,6 +97,7 @@ end using OAs +let # Basics A0 = [1 3; 2 4] A = OffsetArray(A0, (-1,2)) # LinearFast @@ -133,26 +132,30 @@ S = OffsetArray(view(A0, 1:2, 1:2), (-1,2)) # LinearSlow @test eachindex(A) == 1:4 @test eachindex(S) == CartesianRange((0:1,3:4)) -# slice +# view S = view(A, :, 3) @test S == OffsetArray([1,2], (A.offsets[1],)) @test S[0] == 1 @test S[1] == 2 @test_throws BoundsError S[2] +@test indices(S) === (0:1,) S = view(A, 0, :) @test S == OffsetArray([1,3], (A.offsets[2],)) @test S[3] == 1 @test S[4] == 3 @test_throws BoundsError S[1] +@test indices(S) === (3:4,) S = view(A, 0:0, 4) @test S == [3] @test S[1] == 3 @test_throws BoundsError S[0] +@test indices(S) === (Base.OneTo(1),) S = view(A, 1, 3:4) @test S == [2,4] @test S[1] == 2 @test S[2] == 4 @test_throws BoundsError S[3] +@test indices(S) === (Base.OneTo(2),) S = view(A, :, :) @test S == A @test S[0,3] == S[1] == 1 @@ -160,6 +163,7 @@ S = view(A, :, :) @test S[0,4] == S[3] == 3 @test S[1,4] == S[4] == 4 @test_throws BoundsError S[1,1] +@test indices(S) === (0:1, 3:4) # iteration for (a,d) in zip(A, A0) @@ -201,29 +205,33 @@ cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,10^3), (10,-9))) # neithe B = similar(A, Float32) @test isa(B, OffsetArray{Float32,2}) @test size(B) == size(A) -@test indices(B) == indices(A) +@test indices(B) === indices(A) B = similar(A, (3,4)) @test isa(B, Array{Int,2}) @test size(B) == (3,4) -@test indices(B) == (1:3, 1:4) +@test indices(B) === (Base.OneTo(3), Base.OneTo(4)) B = similar(A, (-3:3,1:4)) @test isa(B, OffsetArray{Int,2}) -@test indices(B) == (-3:3, 1:4) +@test indices(B) === (-3:3, 1:4) B = similar(parent(A), (-3:3,1:4)) @test isa(B, OffsetArray{Int,2}) -@test indices(B) == (-3:3, 1:4) +@test indices(B) === (-3:3, 1:4) # Indexing with OffsetArray indices i1 = OffsetArray([2,1], (-5,)) i1 = OffsetArray([2,1], -5) b = A0[i1, 1] -@test indices(b) == (-4:-3,) +@test indices(b) === (-4:-3,) @test b[-4] == 2 @test b[-3] == 1 b = A0[1,i1] -@test indices(b) == (-4:-3,) +@test indices(b) === (-4:-3,) @test b[-4] == 3 @test b[-3] == 1 +v = view(A0, i1, 1) +@test indices(v) === (-4:-3,) +v = view(A0, 1:1, i1) +@test indices(v) === (Base.OneTo(1), -4:-3) # logical indexing @test A[A .> 2] == [3,4] @@ -354,3 +362,4 @@ v = OffsetArray(rand(8), (-2,)) @test 2*A == OffsetArray(2*parent(A), A.offsets) @test A+A == OffsetArray(parent(A)+parent(A), A.offsets) @test A.*A == OffsetArray(parent(A).*parent(A), A.offsets) +end From a7af4e4b05a8fb65ca5ec5b9bfb21fbdb9a5ad34 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Wed, 6 Jul 2016 06:14:22 -0500 Subject: [PATCH 0281/1117] subarray indices: work around an inference hang (#17278) --- base/subarray.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/base/subarray.jl b/base/subarray.jl index 02b4cb5b3d95a..e41e1faf3b639 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -279,16 +279,16 @@ end # Since bounds-checking is performance-critical and uses # indices, it's worth optimizing these implementations thoroughly indices(S::SubArray, d::Integer) = 1 <= d <= ndims(S) ? indices(S)[d] : (d > ndims(S) ? OneTo(1) : error("dimension $d out of range")) -indices(S::SubArray) = (@_inline_meta; _indices((), 1, S, S.indexes...)) -_indices(out::Tuple, dim, S::SubArray) = out -_indices(out::Tuple, dim, S::SubArray, i1, I...) = (@_inline_meta; _indices((out..., OneTo(length(i1))), dim+1, S, I...)) -_indices(out::Tuple, dim, S::SubArray, ::Real, I...) = (@_inline_meta; _indices(out, dim+1, S, I...)) -_indices(out::Tuple, dim, S::SubArray, ::Colon, I...) = (@_inline_meta; _indices((out..., indices(parent(S), dim)), dim+1, S, I...)) +indices(S::SubArray) = (@_inline_meta; _indices_sub(S, 1, S.indexes...)) +_indices_sub(S::SubArray, dim::Int) = () +_indices_sub(S::SubArray, dim::Int, ::Real, I...) = (@_inline_meta; _indices_sub(S, dim+1, I...)) +_indices_sub(S::SubArray, dim::Int, ::Colon, I...) = (@_inline_meta; (indices(parent(S), dim), _indices_sub(S, dim+1, I...)...)) +_indices_sub(S::SubArray, dim::Int, i1::AbstractArray, I...) = (@_inline_meta; (indices(i1)..., _indices_sub(S, dim+1, I...)...)) indices1{T}(S::SubArray{T,0}) = OneTo(1) indices1(S::SubArray) = (@_inline_meta; _indices1(S, 1, S.indexes...)) -_indices1(S::SubArray, dim, i1, I...) = (@_inline_meta; OneTo(length(i1))) _indices1(S::SubArray, dim, i1::Real, I...) = (@_inline_meta; _indices1(S, dim+1, I...)) _indices1(S::SubArray, dim, i1::Colon, I...) = (@_inline_meta; indices(parent(S), dim)) +_indices1(S::SubArray, dim, i1::AbstractArray, I...) = (@_inline_meta; indices1(i1)) # Moreover, incides(S) is fast but indices(S, d) is slower indicesperformance{T<:SubArray}(::Type{T}) = IndicesSlow1D() From ace86a7bbcd6af11eb17832ae217b75840c693ef Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Wed, 29 Jun 2016 13:46:36 -0500 Subject: [PATCH 0282/1117] Performance improvements for indices --- base/abstractarray.jl | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 7ba848b37b058..128f6439eb8b0 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -11,6 +11,10 @@ typealias IndicesOne{N} NTuple{N,OneTo} typealias DimOrInd Union{Integer, AbstractUnitRange} typealias DimsOrInds{N} NTuple{N,DimOrInd} +macro _inline_pure_meta() + Expr(:meta, :inline, :pure) +end + ## Basic functions ## vect() = Array{Any}(0) @@ -30,27 +34,19 @@ size{N}(x, d1::Integer, d2::Integer, dx::Vararg{Integer, N}) = (size(x, d1), siz Returns the valid range of indices for array `A` along dimension `d`. """ -function indices(A::AbstractArray, d) - @_inline_meta - OneTo(size(A,d)) -end +indices{T,N}(A::AbstractArray{T,N}, d) = d <= N ? indices(A)[d] : OneTo(1) """ indices(A) Returns the tuple of valid indices for array `A`. """ -indices{T,N}(A::AbstractArray{T,N}) = _indices((), A) # faster than ntuple(d->indices(A,d), Val{N}) -_indices{T,N}(out::NTuple{N}, A::AbstractArray{T,N}) = out -function _indices(out, A::AbstractArray) - @_inline_meta - _indices((out..., indices(A, length(out)+1)), A) +function indices{T,N}(A::AbstractArray{T,N}) + @_inline_pure_meta + map(s->OneTo(s), size(A)) end -# Note: a simpler implementation suffers from #16327 -# function indices{T,N}(A::AbstractArray{T,N}) -# @_inline_meta -# ntuple(d->indices(A, d), Val{N}) -# end + indices1(A) = (@_inline_meta; indices(A, 1)) + """ linearindices(A) @@ -1241,21 +1237,21 @@ unsafe_length(r::UnitRange) = r.stop-r.start+1 unsafe_length(r::OneTo) = length(r) ind2sub(::Tuple{}, ind::Integer) = (@_inline_meta; ind == 1 ? () : throw(BoundsError())) -ind2sub(dims::DimsInteger, ind::Integer) = (@_inline_meta; _ind2sub((), dims, ind-1)) -ind2sub(inds::Indices, ind::Integer) = (@_inline_meta; _ind2sub((), inds, ind-1)) +ind2sub(dims::DimsInteger, ind::Integer) = (@_inline_meta; _ind2sub(dims, ind-1)) +ind2sub(inds::Indices, ind::Integer) = (@_inline_meta; _ind2sub(inds, ind-1)) ind2sub(inds::Indices{1}, ind::Integer) = throw(ArgumentError("Linear indexing is not defined for one-dimensional arrays")) ind2sub(inds::Tuple{OneTo}, ind::Integer) = (ind,) -_ind2sub(::Tuple{}, ::Tuple{}, ind) = (ind+1,) -function _ind2sub(out, indslast::NTuple{1}, ind) +_ind2sub(::Tuple{}, ind) = (ind+1,) +function _ind2sub(indslast::NTuple{1}, ind) @_inline_meta - (out..., _lookup(ind, indslast[1])) + (_lookup(ind, indslast[1]),) end -function _ind2sub(out, inds, ind) +function _ind2sub(inds, ind) @_inline_meta r1 = inds[1] indnext, f, l = _div(ind, r1) - _ind2sub((out..., ind-l*indnext+f), tail(inds), indnext) + (ind-l*indnext+f, _ind2sub(tail(inds), indnext)...) end _lookup(ind, d::Integer) = ind+1 From 68db3b5ecbb1978cb4e5e78017f2975de7bc073c Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 7 Jul 2016 06:16:12 -0500 Subject: [PATCH 0283/1117] Elim. `indicesperformance` and use only a single path for bounds-checking --- base/abstractarray.jl | 70 ++++++--------------------------------- base/essentials.jl | 2 +- base/permuteddimsarray.jl | 2 +- base/subarray.jl | 4 --- 4 files changed, 12 insertions(+), 66 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 128f6439eb8b0..839b89b175a5f 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -142,13 +142,6 @@ linearindexing(A::AbstractArray, B::AbstractArray...) = linearindexing(linearind linearindexing(::LinearFast, ::LinearFast) = LinearFast() linearindexing(::LinearIndexing, ::LinearIndexing) = LinearSlow() -abstract IndicesPerformance -immutable IndicesFast1D <: IndicesPerformance end # indices(A, d) is fast -immutable IndicesSlow1D <: IndicesPerformance end # indices(A) is better than indices(A,d) - -indicesperformance(A::AbstractArray) = indicesperformance(typeof(A)) -indicesperformance{T<:AbstractArray}(::Type{T}) = IndicesFast1D() - ## Bounds checking ## @generated function trailingsize{T,N,n}(A::AbstractArray{T,N}, ::Type{Val{n}}) (isa(n, Int) && isa(N, Int)) || error("Must have concrete type") @@ -187,7 +180,8 @@ function checkindex(::Type{Bool}, inds::AbstractUnitRange, I::AbstractArray) end # check all indices/dimensions -# To make extension easier, avoid specializing checkbounds on index types +# To make extension easier, avoid specializations of checkbounds on index types +# (That said, typically one does not need to specialize this function.) """ checkbounds(Bool, array, indexes...) @@ -197,15 +191,12 @@ behaviors. """ function checkbounds(::Type{Bool}, A::AbstractArray, I...) @_inline_meta - _checkbounds(Bool, indicesperformance(A), A, I...) -end -function _checkbounds(::Type{Bool}, ::IndicesSlow1D, A::AbstractArray, I...) - @_inline_meta - checkbounds_indices(indices(A), I) + _chkbounds(A, I...) end -_checkbounds(::Type{Bool}, ::IndicesSlow1D, A::AbstractArray, I::AbstractArray{Bool}) = _chkbnds(A, (true,), I) +_chkbounds(A::AbstractArray, i::Integer) = (@_inline_meta; checkindex(Bool, linearindices(A), i)) +_chkbounds(A::AbstractArray, I::AbstractArray{Bool}) = (@_inline_meta; checkbounds_logical(A, I)) +_chkbounds(A::AbstractArray, I...) = (@_inline_meta; checkbounds_indices(indices(A), I)) -# Bounds-checking for arrays for which indices(A) is faster than indices(A, d) checkbounds_indices(::Tuple{}, ::Tuple{}) = true checkbounds_indices(::Tuple{}, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, 1:1, I[1])) checkbounds_indices(::Tuple{}, I::Tuple) = (@_inline_meta; checkindex(Bool, 1:1, I[1]) & checkbounds_indices((), tail(I))) @@ -213,52 +204,11 @@ checkbounds_indices(inds::Tuple{Any}, I::Tuple{Any}) = (@_inline_meta; checkinde checkbounds_indices(inds::Tuple, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, 1:prod(map(dimlength, inds)), I[1])) checkbounds_indices(inds::Tuple, I::Tuple) = (@_inline_meta; checkindex(Bool, inds[1], I[1]) & checkbounds_indices(tail(inds), tail(I))) -# Bounds-checking for arrays for which indices(A, d) is fast -function _checkbounds(::Type{Bool}, ::IndicesFast1D, A::AbstractArray, I...) - @_inline_meta - # checked::NTuple{M} means we have checked dimensions 1:M-1, now - # need to check dimension M. checked[M] indicates whether all the - # previous ones are in-bounds. - # By growing checked, it allows us to test whether we've processed - # the same number of dimensions as the array, even while - # supporting CartesianIndex - _chkbnds(A, (true,), I...) -end -checkbounds(::Type{Bool}, A::AbstractArray) = checkbounds(Bool, A, 1) # 0-d case # Single logical array indexing: -_chkbnds(A::AbstractArray, ::NTuple{1,Bool}, I::AbstractArray{Bool}) = indices(A) == indices(I) -_chkbnds(A::AbstractArray, ::NTuple{1,Bool}, I::AbstractVector{Bool}) = length(A) == length(I) -_chkbnds(A::AbstractVector, ::NTuple{1,Bool}, I::AbstractArray{Bool}) = length(A) == length(I) -_chkbnds(A::AbstractVector, ::NTuple{1,Bool}, I::AbstractVector{Bool}) = indices(A) == indices(I) -# Linear indexing: -function _chkbnds(A::AbstractVector, ::NTuple{1,Bool}, I) - @_inline_meta - checkindex(Bool, indices1(A), I) -end -function _chkbnds(A::AbstractArray, ::NTuple{1,Bool}, I) - @_inline_meta - checkindex(Bool, 1:length(A), I) -end -# When all indices have been checked: -_chkbnds{M}(A, checked::NTuple{M,Bool}) = checked[M] -# When the number of indices matches the array dimensionality: -function _chkbnds{T,N}(A::AbstractArray{T,N}, checked::NTuple{N,Bool}, I1) - @_inline_meta - checked[N] & checkindex(Bool, indices(A, N), I1) -end -# When the last checked dimension is not equal to the array dimensionality: -# TODO: for #14770 (deprecating partial linear indexing), change to 1:trailingsize(...) to 1:1 -function _chkbnds{T,N,M}(A::AbstractArray{T,N}, checked::NTuple{M,Bool}, I1) - @_inline_meta - checked[M] & checkindex(Bool, 1:trailingsize(A, Val{M}), I1) -end -# Checking an interior dimension: -function _chkbnds{T,N,M}(A::AbstractArray{T,N}, checked::NTuple{M,Bool}, I1, I...) - @_inline_meta - # grow checked by one - newchecked = (checked..., checked[M] & checkindex(Bool, indices(A, M), I1)) - _chkbnds(A, newchecked, I...) -end +checkbounds_logical(A::AbstractArray, I::AbstractArray{Bool}) = indices(A) == indices(I) +checkbounds_logical(A::AbstractArray, I::AbstractVector{Bool}) = length(A) == length(I) +checkbounds_logical(A::AbstractVector, I::AbstractArray{Bool}) = length(A) == length(I) +checkbounds_logical(A::AbstractVector, I::AbstractVector{Bool}) = indices(A) == indices(I) throw_boundserror(A, I) = (@_noinline_meta; throw(BoundsError(A, I))) diff --git a/base/essentials.jl b/base/essentials.jl index 6635d99670c0b..2f7ecbeff7645 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -171,7 +171,7 @@ start(v::SimpleVector) = 1 next(v::SimpleVector,i) = (v[i],i+1) done(v::SimpleVector,i) = (i > v.length) isempty(v::SimpleVector) = (v.length == 0) -indices(v::SimpleVector, d) = d == 1 ? OneTo(length(v)) : OneTo(1) +indices(v::SimpleVector) = (OneTo(length(v)),) linearindices(v::SimpleVector) = indices(v, 1) function ==(v1::SimpleVector, v2::SimpleVector) diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index dd64416fae86b..8ab7d60f26301 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -25,7 +25,7 @@ end Base.parent(A::PermutedDimsArray) = A.parent Base.size(A::PermutedDimsArray) = A.dims -Base.indices{T,N,perm}(A::PermutedDimsArray{T,N,perm}, d) = indices(parent(A), perm[d]) +Base.indices{T,N,perm}(A::PermutedDimsArray{T,N,perm}) = genperm(indices(parent(A)), perm) @inline function Base.getindex{T,N,perm,iperm}(A::PermutedDimsArray{T,N,perm,iperm}, I::Vararg{Int,N}) @boundscheck checkbounds(A, I...) diff --git a/base/subarray.jl b/base/subarray.jl index e41e1faf3b639..e12e3e039db1e 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -278,7 +278,6 @@ end # they are taken from the range/vector # Since bounds-checking is performance-critical and uses # indices, it's worth optimizing these implementations thoroughly -indices(S::SubArray, d::Integer) = 1 <= d <= ndims(S) ? indices(S)[d] : (d > ndims(S) ? OneTo(1) : error("dimension $d out of range")) indices(S::SubArray) = (@_inline_meta; _indices_sub(S, 1, S.indexes...)) _indices_sub(S::SubArray, dim::Int) = () _indices_sub(S::SubArray, dim::Int, ::Real, I...) = (@_inline_meta; _indices_sub(S, dim+1, I...)) @@ -290,9 +289,6 @@ _indices1(S::SubArray, dim, i1::Real, I...) = (@_inline_meta; _indices1(S, dim+1 _indices1(S::SubArray, dim, i1::Colon, I...) = (@_inline_meta; indices(parent(S), dim)) _indices1(S::SubArray, dim, i1::AbstractArray, I...) = (@_inline_meta; indices1(i1)) -# Moreover, incides(S) is fast but indices(S, d) is slower -indicesperformance{T<:SubArray}(::Type{T}) = IndicesSlow1D() - ## Compatability # deprecate? function parentdims(s::SubArray) From c1276a96e33c61f29e19fd5ec63017f371daa5d3 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 7 Jul 2016 06:16:58 -0500 Subject: [PATCH 0284/1117] Reduce use of splatting in sparse matrix size operations Splatting forces dynamic method lookup in places where that's a major cost --- base/sparse/sparsematrix.jl | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 9be5e09d73950..6e18bf43b08ed 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -251,7 +251,7 @@ end similar(S::SparseMatrixCSC, Tv::Type=eltype(S)) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), Array{Tv}(length(S.nzval))) similar{Tv,Ti,TvNew,TiNew}(S::SparseMatrixCSC{Tv,Ti}, ::Type{TvNew}, ::Type{TiNew}) = SparseMatrixCSC(S.m, S.n, convert(Array{TiNew},S.colptr), convert(Array{TiNew}, S.rowval), Array{TvNew}(length(S.nzval))) -similar{Tv}(S::SparseMatrixCSC, ::Type{Tv}, d::Dims) = spzeros(Tv, d...) +@inline similar{Tv}(S::SparseMatrixCSC, ::Type{Tv}, d::Dims) = spzeros(Tv, d...) function convert{Tv,Ti,TvS,TiS}(::Type{SparseMatrixCSC{Tv,Ti}}, S::SparseMatrixCSC{TvS,TiS}) if Tv == TvS && Ti == TiS @@ -1239,7 +1239,8 @@ function spzeros(Tv::Type, Ti::Type, m::Integer, n::Integer) ((m < 0) || (n < 0)) && throw(ArgumentError("invalid Array dimensions")) SparseMatrixCSC(m, n, ones(Ti, n+1), Array{Ti}(0), Array{Tv}(0)) end - +# de-splatting variant +spzeros(Tv::Type, Ti::Type, sz::Tuple{Integer,Integer}) = spzeros(Tv, Ti, sz[1], sz[2]) speye(n::Integer) = speye(Float64, n) speye(T::Type, n::Integer) = speye(T, n, n) @@ -1647,16 +1648,16 @@ end broadcast{Tv1,Ti1,Tv2,Ti2}(f::Function, A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = - broadcast!(f, spzeros(promote_type(Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) + broadcast!(f, spzeros(promote_type(Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) -broadcast_zpreserving!(args...) = broadcast!(args...) -broadcast_zpreserving(args...) = broadcast(args...) +@inline broadcast_zpreserving!(args...) = broadcast!(args...) +@inline broadcast_zpreserving(args...) = broadcast(args...) broadcast_zpreserving{Tv1,Ti1,Tv2,Ti2}(f::Function, A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = - broadcast_zpreserving!(f, spzeros(promote_type(Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) + broadcast_zpreserving!(f, spzeros(promote_type(Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) broadcast_zpreserving{Tv,Ti}(f::Function, A_1::SparseMatrixCSC{Tv,Ti}, A_2::Union{Array,BitArray,Number}) = - broadcast_zpreserving!(f, spzeros(promote_eltype(A_1, A_2), Ti, to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) + broadcast_zpreserving!(f, spzeros(promote_eltype(A_1, A_2), Ti, to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) broadcast_zpreserving{Tv,Ti}(f::Function, A_1::Union{Array,BitArray,Number}, A_2::SparseMatrixCSC{Tv,Ti}) = - broadcast_zpreserving!(f, spzeros(promote_eltype(A_1, A_2), Ti, to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) + broadcast_zpreserving!(f, spzeros(promote_eltype(A_1, A_2), Ti, to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) ## Binary arithmetic and boolean operators @@ -1676,7 +1677,7 @@ for (op, pro) in ((+, :eltype_plus), throw(DimensionMismatch("")) end Tv = ($pro)(A_1, A_2) - B = spzeros(Tv, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...) + B = spzeros(Tv, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))) $body B end @@ -1718,15 +1719,15 @@ end # macro (.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, full(B)) .+{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = - broadcast!(+, spzeros(eltype_plus(A_1, A_2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) + broadcast!(+, spzeros(eltype_plus(A_1, A_2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) function .-{Tva,Tia,Tvb,Tib}(A::SparseMatrixCSC{Tva,Tia}, B::SparseMatrixCSC{Tvb,Tib}) - broadcast!(-, spzeros(eltype_plus(A, B), promote_type(Tia, Tib), to_shape(broadcast_shape(A, B))...), A, B) + broadcast!(-, spzeros(eltype_plus(A, B), promote_type(Tia, Tib), to_shape(broadcast_shape(A, B))), A, B) end ## element-wise comparison operators returning SparseMatrixCSC ## -.<{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(<, spzeros( Bool, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) -.!={Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(!=, spzeros( Bool, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))...), A_1, A_2) +.<{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(<, spzeros( Bool, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) +.!={Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(!=, spzeros( Bool, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) ## full equality function ==(A1::SparseMatrixCSC, A2::SparseMatrixCSC) From 306a175e73259ff9e74f3491877802dd98116402 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 7 Jul 2016 11:44:32 -0500 Subject: [PATCH 0285/1117] Fixes for indices1(::Subarray) and indices1 specializations This leads to noticeable performance improvements for several benchmarks --- base/abstractarray.jl | 3 ++- base/subarray.jl | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 839b89b175a5f..71b29d82f4810 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -45,7 +45,8 @@ function indices{T,N}(A::AbstractArray{T,N}) map(s->OneTo(s), size(A)) end -indices1(A) = (@_inline_meta; indices(A, 1)) +indices1{T}(A::AbstractArray{T,0}) = OneTo(1) +indices1{T}(A::AbstractArray{T}) = indices(A)[1] """ linearindices(A) diff --git a/base/subarray.jl b/base/subarray.jl index e12e3e039db1e..e9a4cf1c57765 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -283,11 +283,12 @@ _indices_sub(S::SubArray, dim::Int) = () _indices_sub(S::SubArray, dim::Int, ::Real, I...) = (@_inline_meta; _indices_sub(S, dim+1, I...)) _indices_sub(S::SubArray, dim::Int, ::Colon, I...) = (@_inline_meta; (indices(parent(S), dim), _indices_sub(S, dim+1, I...)...)) _indices_sub(S::SubArray, dim::Int, i1::AbstractArray, I...) = (@_inline_meta; (indices(i1)..., _indices_sub(S, dim+1, I...)...)) -indices1{T}(S::SubArray{T,0}) = OneTo(1) indices1(S::SubArray) = (@_inline_meta; _indices1(S, 1, S.indexes...)) +_indices1(S::SubArray, dim) = OneTo(1) _indices1(S::SubArray, dim, i1::Real, I...) = (@_inline_meta; _indices1(S, dim+1, I...)) _indices1(S::SubArray, dim, i1::Colon, I...) = (@_inline_meta; indices(parent(S), dim)) _indices1(S::SubArray, dim, i1::AbstractArray, I...) = (@_inline_meta; indices1(i1)) +_indices1{T}(S::SubArray, dim, i1::AbstractArray{T,0}, I...) = (@_inline_meta; _indices1(S, dim+1, I...)) ## Compatability # deprecate? From f3d49cec28aab594f4f5dfb3c17c5f00ac4bd458 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 7 Jul 2016 16:28:05 -0500 Subject: [PATCH 0286/1117] Update the docs --- doc/stdlib/arrays.rst | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index f2deb6bbf98be..960cca00318a7 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -43,18 +43,6 @@ Basic functions Returns the valid range of indices for array ``A`` along dimension ``d``\ . -.. function:: shape(A) - - .. Docstring generated from Julia source - - Returns a tuple specifying the "shape" of array ``A``\ . For arrays with conventional indexing (indices start at 1), this is equivalent to ``size(A)``\ ; otherwise it is equivalent to ``indices(A)``\ . - -.. function:: shape(A, d) - - .. Docstring generated from Julia source - - Specifies the "shape" of the array ``A`` along dimension ``d``\ . For arrays with conventional indexing (starting at 1), this is equivalent to ``size(A, d)``\ ; for arrays with unconventional indexing (indexing may start at something different from 1), it is equivalent to ``indices(A, d)``\ . - .. function:: length(A) -> Integer .. Docstring generated from Julia source @@ -283,31 +271,31 @@ Constructors 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 - See also ``allocate_for``\ . - -.. function:: allocate_for(storagetype, referencearray, [shape]) +.. function:: similar(storagetype, indices) .. Docstring generated from Julia source - Create an uninitialized mutable array analogous to that specified by ``storagetype``\ , but with type and shape specified by the final two arguments. The main purpose of this function is to support allocation of arrays that may have unconventional indexing (starting at other than 1), as determined by ``referencearray`` and the optional ``shape`` information. + Create an uninitialized mutable array analogous to that specified by ``storagetype``\ , but with ``indices`` specified by the last argument. ``storagetype`` might be a type or a function. .. code-block:: julia **Examples**: - allocate_for(Array{Int}, A) + similar(Array{Int}, indices(A)) - creates an array that "acts like" an ``Array{Int}`` (and might indeed be backed by one), but which is indexed identically to ``A``\ . If ``A`` has conventional indexing, this will likely just call ``Array{Int}(size(A))``\ , but if ``A`` has unconventional indexing then the indices of the result will match ``A``\ . + creates an array that "acts like" an ``Array{Int}`` (and might indeed be backed by one), but which is indexed identically to ``A``\ . If ``A`` has conventional indexing, this will be identical to ``Array{Int}(size(A))``\ , but if ``A`` has unconventional indexing then the indices of the result will match ``A``\ . .. code-block:: julia - allocate_for(BitArray, A, (shape(A, 2),)) + similar(BitArray, (indices(A, 2),)) would create a 1-dimensional logical array whose indices match those of the columns of ``A``\ . - The main purpose of the ``referencearray`` argument is to select a particular array type supporting unconventional indexing (as it is possible that several different ones will be simultaneously in use). + .. code-block:: julia + + similar(dims->zeros(Int, dims), indices(A)) - See also ``similar``\ . + would create an array of ``Int``\ , initialized to zero, matching the indices of ``A``\ . .. function:: reinterpret(type, A) From 4104a970f136b489dacd8bed238366ad23d25fae Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Thu, 7 Jul 2016 17:14:41 -0500 Subject: [PATCH 0287/1117] Append newline to the end of dump methods (#17202) * Append newline to the end of dump methods * Made dump methods return nothing --- base/show.jl | 18 ++++++++++++++++-- test/show.jl | 6 ++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/base/show.jl b/base/show.jl index 9fbe79d85a7c7..0706ed905b6b6 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1040,6 +1040,7 @@ function dump(io::IO, x::SimpleVector, n::Int, indent) end end end + isempty(indent) && println(io) nothing end @@ -1065,6 +1066,7 @@ function dump(io::IO, x::ANY, n::Int, indent) else !isa(x,Function) && print(io, " ", x) end + isempty(indent) && println(io) nothing end @@ -1102,12 +1104,21 @@ function dump(io::IO, x::Array, n::Int, indent) end end end + isempty(indent) && println(io) + nothing +end +function dump(io::IO, x::Symbol, n::Int, indent) + print(io, typeof(x), " ", x) + isempty(indent) && println(io) nothing end -dump(io::IO, x::Symbol, n::Int, indent) = print(io, typeof(x), " ", x) # Types -dump(io::IO, x::Union, n::Int, indent) = print(io, x) +function dump(io::IO, x::Union, n::Int, indent) + print(io, x) + isempty(indent) && println(io) + nothing +end function dump(io::IO, x::DataType, n::Int, indent) print(io, x) @@ -1124,6 +1135,7 @@ function dump(io::IO, x::DataType, n::Int, indent) end end end + isempty(indent) && println(io) nothing end @@ -1167,6 +1179,8 @@ function dumptype(io::IO, x::ANY, n::Int, indent) end end end + isempty(indent) && println(io) + nothing end # For abstract types, use _dumptype only if it's a form that will be called diff --git a/test/show.jl b/test/show.jl index 989e311b3a565..9956c803c527b 100644 --- a/test/show.jl +++ b/test/show.jl @@ -500,3 +500,9 @@ let s = IOBuffer(Array{UInt8}(0), true, true) Base.showarray(s, [1,2,3], false, header = false) @test String(resize!(s.data, s.size)) == " 1\n 2\n 3" end + +# The `dump` function should alway have a trailing newline +let io = IOBuffer() + dump(io, :(x = 1)) + @test takebuf_string(io)[end] == '\n' +end From 1cc383555b5dc333f51359610914cf3800b831bf Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Thu, 7 Jul 2016 18:38:00 -0400 Subject: [PATCH 0288/1117] typo Fix typo noted by @tkelman (https://github.com/JuliaLang/julia/pull/16702#discussion_r69987863) --- base/linalg/factorization.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/linalg/factorization.jl b/base/linalg/factorization.jl index 5b6aade370330..49254d2638dc1 100644 --- a/base/linalg/factorization.jl +++ b/base/linalg/factorization.jl @@ -58,7 +58,7 @@ the result. The argument `A` should *not* be a matrix. Rather, instead of matrices it should be a factorization object (e.g. produced by [`factorize`](:func:`factorize`) or [`cholfact`](:func:`cholfact`)). The reason for this is that factorization itself is both expensive and typically allocates memory -(although it can also be done in-place via, e.g., [`lufact`](:func:`lufact`)), +(although it can also be done in-place via, e.g., [`lufact!`](:func:`lufact!`)), and performance-critical situations requiring `A_ldiv_B!` usually also require fine-grained control over the factorization of `A`. """ From 3b3bdb6c71610a88cf33c2ed80476844ea9ae50a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 7 Jul 2016 22:12:09 -0400 Subject: [PATCH 0289/1117] fix a parsing bug in `try x catch 0 end` --- src/julia-parser.scm | 7 ++++++- test/parse.jl | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 3cd3896707f19..288426eebdb65 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1246,7 +1246,12 @@ (loop (require-token s) (if (or var? (not var)) catch-block - `(block ,loc ,var ,@(cdr catch-block))) + `(block ,loc ,var + ,@(if (and (length= catch-block 2) + (pair? (cadr catch-block)) + (eq? (caadr catch-block) 'line)) + '() + (cdr catch-block)))) (if var? var 'false) finalb))))) ((and (eq? nxt 'finally) diff --git a/test/parse.jl b/test/parse.jl index 734a4d94c5176..0861d00a55688 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -450,6 +450,8 @@ add_method_to_glob_fn!() @test (try error(); catch 0; end) === 0 @test (try error(); catch false; end) === false # false and true are Bool literals, not variables @test (try error(); catch true; end) === true +f16517() = try error(); catch 0; end +@test f16517() === 0 # issue #16671 @test parse("1.") === 1.0 From 6f50dd207b91648441403f36617d12806d51cd00 Mon Sep 17 00:00:00 2001 From: Simon Kornblith <simon@simonster.com> Date: Thu, 7 Jul 2016 23:01:35 -0400 Subject: [PATCH 0290/1117] Remove test for #14722 (type instability in generic matmul) This test apparently breaks with inlining disabled. --- test/linalg/matmul.jl | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 0f44b8cf8479c..3ac079b930731 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -388,15 +388,3 @@ let @test_throws DimensionMismatch A_mul_B!(full43, full43, tri44) end end - -# Ensure that matrix multiplication with a narrower output type than input type does not -# produce allocation in the inner loop (#14722), by ensuring allocation does not change -# with the size of the input. -C1 = Array(Float32, 5, 5) -A1 = rand(Float64, 5, 5) -B1 = rand(Float64, 5, 5) -C2 = Array(Float32, 6, 6) -A2 = rand(Float64, 6, 6) -B2 = rand(Float64, 6, 6) -A_mul_B!(C1, A1, B1) -@test @allocated(A_mul_B!(C1, A1, B1)) == @allocated(A_mul_B!(C2, A2, B2)) From 1ee5404cc847a2eb2bf3ad2541aa27df4021a3f9 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 23 Jun 2016 11:26:38 -0700 Subject: [PATCH 0291/1117] Remove `transpose(x) = x` fallback, instead `error`'ing. Patch up a few missing `transpose` methods and correct a test that failed for lack of a `transpose` method. --- base/char.jl | 2 ++ base/operators.jl | 2 +- base/strings/basic.jl | 2 ++ test/linalg/matmul.jl | 3 ++- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/base/char.jl b/base/char.jl index 52aa1492e2f86..e28217dc8c10a 100644 --- a/base/char.jl +++ b/base/char.jl @@ -32,6 +32,8 @@ in(x::Char, y::Char) = x == y ==(x::Char, y::Char) = UInt32(x) == UInt32(y) isless(x::Char, y::Char) = UInt32(x) < UInt32(y) +transpose(c::Char) = c + const hashchar_seed = 0xd4d64234 hash(x::Char, h::UInt) = hash_uint64(((UInt64(x)+hashchar_seed)<<32) $ UInt64(h)) diff --git a/base/operators.jl b/base/operators.jl index ae9ccd639d525..7dc3aa539ab23 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -293,7 +293,7 @@ fldmod1{T<:Real}(x::T, y::T) = (fld1(x,y), mod1(x,y)) fldmod1{T<:Integer}(x::T, y::T) = (fld1(x,y), mod1(x,y)) # transpose -transpose(x) = x +transpose(x) = throw(ArgumentError("transpose not implemented for $(typeof(x)). Consider permutedims.")) ctranspose(x) = conj(transpose(x)) conj(x) = x diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 167bef7bcbd44..caa60795b8c0c 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -101,6 +101,8 @@ function length(s::AbstractString) end end +transpose(s::AbstractString) = s + ## string comparison functions ## function cmp(a::AbstractString, b::AbstractString) diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 3ac079b930731..44bb9a6afa85b 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -320,8 +320,9 @@ end immutable RootInt i::Int end -import Base: *, promote_op +import Base: *, transpose, promote_op (*)(x::RootInt, y::RootInt) = x.i*y.i +transpose(x::RootInt) = x promote_op(::typeof(*), ::Type{RootInt}, ::Type{RootInt}) = Int a = [RootInt(3)] From 8371135be91ff970420e0757e2a297a9153d0ddd Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 7 Jul 2016 21:26:06 -0700 Subject: [PATCH 0292/1117] Remove (c)transpose no-op for Chars and remove associated test. --- base/char.jl | 2 -- test/operators.jl | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/base/char.jl b/base/char.jl index e28217dc8c10a..52aa1492e2f86 100644 --- a/base/char.jl +++ b/base/char.jl @@ -32,8 +32,6 @@ in(x::Char, y::Char) = x == y ==(x::Char, y::Char) = UInt32(x) == UInt32(y) isless(x::Char, y::Char) = UInt32(x) < UInt32(y) -transpose(c::Char) = c - const hashchar_seed = 0xd4d64234 hash(x::Char, h::UInt) = hash_uint64(((UInt64(x)+hashchar_seed)<<32) $ UInt64(h)) diff --git a/test/operators.jl b/test/operators.jl index 36f80a6033ddf..07031ed2ff81e 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -33,7 +33,7 @@ p = 1=>:foo @test (|)(2) == 2 @test ($)(2) == 2 -@test ctranspose('a') == 'a' +# @test ctranspose('a') == 'a' # (c)transpose of Chars no longer supported @test_throws ArgumentError Base.scalarmin(['a','b'],['c','d']) @test_throws ArgumentError Base.scalarmin('a',['c','d']) From dd1e922c145bc3822d23c15cd09e205f14404509 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Thu, 7 Jul 2016 20:59:25 -0400 Subject: [PATCH 0293/1117] Move option parsing to libjulia --- src/Makefile | 6 +- {ui => src}/getopt.c | 0 {ui => src}/getopt.h | 0 src/init.c | 1 + src/jloptions.c | 451 +++++++++++++++++++++++++++++++++++++++++++ src/julia.h | 9 + test/cmdlineargs.jl | 5 +- ui/Makefile | 3 - ui/repl.c | 450 +----------------------------------------- 9 files changed, 474 insertions(+), 451 deletions(-) rename {ui => src}/getopt.c (100%) rename {ui => src}/getopt.h (100%) create mode 100644 src/jloptions.c diff --git a/src/Makefile b/src/Makefile index 1617281c73eeb..ba6cf7298c465 100644 --- a/src/Makefile +++ b/src/Makefile @@ -39,7 +39,11 @@ SRCS := \ jltypes gf typemap ast builtins module interpreter \ alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \ simplevector APInt-C runtime_intrinsics runtime_ccall \ - threadgroup threading stackwalk gc gc-debug gc-pages safepoint + threadgroup threading stackwalk gc gc-debug gc-pages safepoint jloptions + +ifeq ($(USEMSVC), 1) +SRCS += getopt +endif LLVMLINK := diff --git a/ui/getopt.c b/src/getopt.c similarity index 100% rename from ui/getopt.c rename to src/getopt.c diff --git a/ui/getopt.h b/src/getopt.h similarity index 100% rename from ui/getopt.h rename to src/getopt.h diff --git a/src/init.c b/src/init.c index 7ebb1488afff9..27e5d8f8c4cea 100644 --- a/src/init.c +++ b/src/init.c @@ -80,6 +80,7 @@ jl_options_t jl_options = { 0, // quiet NULL, // outputo NULL, // outputji 0, // incremental + 0 // image_file_specified }; int jl_boot_file_loaded = 0; diff --git a/src/jloptions.c b/src/jloptions.c new file mode 100644 index 0000000000000..78297e9d98df5 --- /dev/null +++ b/src/jloptions.c @@ -0,0 +1,451 @@ +#include <limits.h> + +#include "julia.h" + +#ifndef _MSC_VER +#include <unistd.h> +#include <getopt.h> +#else +#include "getopt.h" +#endif + +static const char usage[] = "julia [switches] -- [programfile] [args...]\n"; +static const char opts[] = + " -v, --version Display version information\n" + " -h, --help Print this message\n\n" + + // startup options + " -J, --sysimage <file> Start up with the given system image file\n" + " --precompiled={yes|no} Use precompiled code from system image if available\n" + " --compilecache={yes|no} Enable/disable incremental precompilation of modules\n" + " -H, --home <dir> Set location of `julia` executable\n" + " --startup-file={yes|no} Load ~/.juliarc.jl\n" + " --handle-signals={yes|no} Enable or disable Julia's default signal handlers\n\n" + + // actions + " -e, --eval <expr> Evaluate <expr>\n" + " -E, --print <expr> Evaluate and show <expr>\n" + " -L, --load <file> Load <file> immediately on all processors\n\n" + + // parallel options + " -p, --procs {N|auto} Integer value N launches N additional local worker processes\n" + " \"auto\" launches as many workers as the number of local cores\n" + " --machinefile <file> Run processes on hosts listed in <file>\n\n" + + // interactive options + " -i Interactive mode; REPL runs and isinteractive() is true\n" + " -q, --quiet Quiet startup (no banner)\n" + " --color={yes|no} Enable or disable color text\n" + " --history-file={yes|no} Load or save history\n\n" + + // code generation options + " --compile={yes|no|all|min}Enable or disable JIT compiler, or request exhaustive compilation\n" + " -C, --cpu-target <target> Limit usage of cpu features up to <target>\n" + " -O, --optimize={0,1,2,3} Set the optimization level (default 2 if unspecified or 3 if specified as -O)\n" + " --inline={yes|no} Control whether inlining is permitted (overrides functions declared as @inline)\n" + " --check-bounds={yes|no} Emit bounds checks always or never (ignoring declarations)\n" + " --math-mode={ieee,fast} Disallow or enable unsafe floating point optimizations (overrides @fastmath declaration)\n\n" + + // error and warning options + " --depwarn={yes|no|error} Enable or disable syntax and method deprecation warnings (\"error\" turns warnings into errors)\n\n" + + // compiler output options + " --output-o name Generate an object file (including system image data)\n" + " --output-ji name Generate a system image data file (.ji)\n" + " --output-bc name Generate LLVM bitcode (.bc)\n" + " --output-incremental=no Generate an incremental output file (rather than complete)\n\n" + + // instrumentation options + " --code-coverage={none|user|all}, --code-coverage\n" + " Count executions of source lines (omitting setting is equivalent to \"user\")\n" + " --track-allocation={none|user|all}, --track-allocation\n" + " Count bytes allocated by each source line\n\n" + + "Deprecated options:\n" + " -F Load ~/.juliarc (deprecated, use --startup-file=yes)\n" + " -f, --no-startup Don't load ~/.juliarc (deprecated, use --startup-file=no)\n" + " -P, --post-boot <expr> Evaluate <expr>, but don't disable interactive mode (deprecated, use -i -e instead)\n" + " --no-history-file Don't load history file (deprecated, use --history-file=no)\n"; + +JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) +{ + enum { opt_machinefile = 300, + opt_color, + opt_history_file, + opt_no_history_file, + opt_startup_file, + opt_compile, + opt_code_coverage, + opt_track_allocation, + opt_check_bounds, + opt_output_bc, + opt_depwarn, + opt_inline, + opt_math_mode, + opt_worker, + opt_bind_to, + opt_handle_signals, + opt_output_o, + opt_output_ji, + opt_use_precompiled, + opt_use_compilecache, + opt_incremental + }; + static const char const* shortopts = "+vhqFfH:e:E:P:L:J:C:ip:O:"; + static const struct option longopts[] = { + // exposed command line options + // NOTE: This set of required arguments need to be kept in sync + // with the required arguments defined in base/client.jl `process_options()` + { "version", no_argument, 0, 'v' }, + { "help", no_argument, 0, 'h' }, + { "quiet", no_argument, 0, 'q' }, + { "home", required_argument, 0, 'H' }, + { "eval", required_argument, 0, 'e' }, + { "print", required_argument, 0, 'E' }, + { "load", required_argument, 0, 'L' }, + { "sysimage", required_argument, 0, 'J' }, + { "precompiled", required_argument, 0, opt_use_precompiled }, + { "compilecache", required_argument, 0, opt_use_compilecache }, + { "cpu-target", required_argument, 0, 'C' }, + { "procs", required_argument, 0, 'p' }, + { "machinefile", required_argument, 0, opt_machinefile }, + { "color", required_argument, 0, opt_color }, + { "history-file", required_argument, 0, opt_history_file }, + { "startup-file", required_argument, 0, opt_startup_file }, + { "compile", required_argument, 0, opt_compile }, + { "code-coverage", optional_argument, 0, opt_code_coverage }, + { "track-allocation",optional_argument, 0, opt_track_allocation }, + { "optimize", optional_argument, 0, 'O' }, + { "check-bounds", required_argument, 0, opt_check_bounds }, + { "output-bc", required_argument, 0, opt_output_bc }, + { "output-o", required_argument, 0, opt_output_o }, + { "output-ji", required_argument, 0, opt_output_ji }, + { "output-incremental",required_argument, 0, opt_incremental }, + { "depwarn", required_argument, 0, opt_depwarn }, + { "inline", required_argument, 0, opt_inline }, + { "math-mode", required_argument, 0, opt_math_mode }, + { "handle-signals", required_argument, 0, opt_handle_signals }, + // hidden command line options + { "worker", required_argument, 0, opt_worker }, + { "bind-to", required_argument, 0, opt_bind_to }, + { "lisp", no_argument, 0, 1 }, + // deprecated options + { "post-boot", required_argument, 0, 'P' }, + { "no-history-file", no_argument, 0, opt_no_history_file }, // deprecated + { "no-startup", no_argument, 0, 'f' }, // deprecated + { 0, 0, 0, 0 } + }; + + int codecov = JL_LOG_NONE; + int malloclog= JL_LOG_NONE; + // getopt handles argument parsing up to -- delineator + int argc = *argcp; + char **argv = *argvp; + if (argc > 0) { + for (int i=0; i < argc; i++) { + if (!strcmp(argv[i], "--")) { + argc = i; + break; + } + } + } + char *endptr; + opterr = 0; // suppress getopt warning messages + while (1) { + int lastind = optind; + int c = getopt_long(argc, argv, shortopts, longopts, 0); + if (c == -1) break; +restart_switch: + switch (c) { + case 0: + break; + case 1: + jl_errorf("--lisp must be specified as the first argument"); + break; + case '?': + case ':': + if (optopt) { + for (struct option *o = longopts; o->val; o++) { + if (optopt == o->val) { + if (o->has_arg == optional_argument) { + c = o->val; + goto restart_switch; + } + else if (strchr(shortopts, o->val)) { + jl_errorf("option `-%c/--%s` is missing an argument", o->val, o->name); + } + else { + jl_errorf("option `--%s` is missing an argument", o->name); + } + } + } + jl_errorf("unknown option `-%c`", optopt); + } else { + jl_errorf("unknown option `%s`", argv[lastind]); + } + break; + case 'v': // version + jl_printf(JL_STDOUT, "julia version %s\n", JULIA_VERSION_STRING); + jl_exit(0); + case 'h': // help + jl_printf(JL_STDOUT, "%s%s", usage, opts); + jl_exit(0); + case 'q': // quiet + jl_options.quiet = 1; + break; + case 'H': // home + jl_options.julia_home = strdup(optarg); + break; + case 'e': // eval + jl_options.eval = strdup(optarg); + break; + case 'E': // print + jl_options.print = strdup(optarg); + break; + case 'P': // post-boot + jl_printf(JL_STDOUT, "WARNING: julia -P/--post-boot option is deprecated, use -i -e instead.\n"); + jl_options.postboot = strdup(optarg); + break; + case 'L': // load + jl_options.load = strdup(optarg); + break; + case 'J': // sysimage + jl_options.image_file = strdup(optarg); + jl_options.image_file_specified = 1; + break; + case opt_use_precompiled: + if (!strcmp(optarg,"yes")) + jl_options.use_precompiled = JL_OPTIONS_USE_PRECOMPILED_YES; + else if (!strcmp(optarg,"no")) + jl_options.use_precompiled = JL_OPTIONS_USE_PRECOMPILED_NO; + else + jl_errorf("julia: invalid argument to --precompiled={yes|no} (%s)", optarg); + break; + case opt_use_compilecache: + if (!strcmp(optarg,"yes")) + jl_options.use_compilecache = JL_OPTIONS_USE_COMPILECACHE_YES; + else if (!strcmp(optarg,"no")) + jl_options.use_compilecache = JL_OPTIONS_USE_COMPILECACHE_NO; + else + jl_errorf("julia: invalid argument to --compilecache={yes|no} (%s)", optarg); + break; + case 'C': // cpu-target + jl_options.cpu_target = strdup(optarg); + break; + case 'p': // procs + errno = 0; + if (!strcmp(optarg,"auto")) { + jl_options.nprocs = jl_cpu_cores(); + } + else { + long nprocs = strtol(optarg, &endptr, 10); + if (errno != 0 || optarg == endptr || *endptr != 0 || nprocs < 1 || nprocs >= INT_MAX) + jl_errorf("julia: -p,--procs=<n> must be an integer >= 1"); + jl_options.nprocs = (int)nprocs; + } + break; + case opt_machinefile: + jl_options.machinefile = strdup(optarg); + break; + case opt_color: + if (!strcmp(optarg,"yes")) + jl_options.color = JL_OPTIONS_COLOR_ON; + else if (!strcmp(optarg,"no")) + jl_options.color = JL_OPTIONS_COLOR_OFF; + else + jl_errorf("julia: invalid argument to --color={yes|no} (%s)", optarg); + break; + case opt_history_file: + if (!strcmp(optarg,"yes")) + jl_options.historyfile = JL_OPTIONS_HISTORYFILE_ON; + else if (!strcmp(optarg,"no")) + jl_options.historyfile = JL_OPTIONS_HISTORYFILE_OFF; + else + jl_errorf("julia: invalid argument to --history-file={yes|no} (%s)", optarg); + break; + case opt_no_history_file: + jl_printf(JL_STDOUT, "WARNING: julia --no-history-file option is deprecated, use --history-file=no instead.\n"); + jl_options.historyfile = JL_OPTIONS_HISTORYFILE_OFF; + break; + case opt_startup_file: + if (!strcmp(optarg,"yes")) + jl_options.startupfile = JL_OPTIONS_STARTUPFILE_ON; + else if (!strcmp(optarg,"no")) + jl_options.startupfile = JL_OPTIONS_STARTUPFILE_OFF; + else + jl_errorf("julia: invalid argument to --startup-file={yes|no} (%s)", optarg); + break; + case 'f': + jl_printf(JL_STDOUT, "WARNING: julia -f/--no-startup option is deprecated, use --startup-file=no instead.\n"); + jl_options.startupfile = JL_OPTIONS_STARTUPFILE_OFF; + break; + case 'F': + jl_printf(JL_STDOUT, "WARNING: julia -F option is deprecated, use --startup-file=yes instead.\n"); + jl_options.startupfile = JL_OPTIONS_STARTUPFILE_ON; + break; + case opt_compile: + if (!strcmp(optarg,"yes")) + jl_options.compile_enabled = JL_OPTIONS_COMPILE_ON; + else if (!strcmp(optarg,"no")) + jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF; + else if (!strcmp(optarg,"all")) + jl_options.compile_enabled = JL_OPTIONS_COMPILE_ALL; + else if (!strcmp(optarg,"min")) + jl_options.compile_enabled = JL_OPTIONS_COMPILE_MIN; + else + jl_errorf("julia: invalid argument to --compile (%s)", optarg); + break; + case opt_code_coverage: + if (optarg != NULL) { + if (!strcmp(optarg,"user")) + codecov = JL_LOG_USER; + else if (!strcmp(optarg,"all")) + codecov = JL_LOG_ALL; + else if (!strcmp(optarg,"none")) + codecov = JL_LOG_NONE; + else + jl_errorf("julia: invalid argument to --code-coverage (%s)", optarg); + break; + } + else { + codecov = JL_LOG_USER; + } + break; + case opt_track_allocation: + if (optarg != NULL) { + if (!strcmp(optarg,"user")) + malloclog = JL_LOG_USER; + else if (!strcmp(optarg,"all")) + malloclog = JL_LOG_ALL; + else if (!strcmp(optarg,"none")) + malloclog = JL_LOG_NONE; + else + jl_errorf("julia: invalid argument to --track-allocation (%s)", optarg); + break; + } + else { + malloclog = JL_LOG_USER; + } + break; + case 'O': // optimize + if (optarg != NULL) { + if (!strcmp(optarg,"0")) + jl_options.opt_level = 0; + else if (!strcmp(optarg,"1")) + jl_options.opt_level = 1; + else if (!strcmp(optarg,"2")) + jl_options.opt_level = 2; + else if (!strcmp(optarg,"3")) + jl_options.opt_level = 3; + else + jl_errorf("julia: invalid argument to -O (%s)", optarg); + break; + } + else { + jl_options.opt_level = 3; + } + break; + case 'i': // isinteractive + jl_options.isinteractive = 1; + break; + case opt_check_bounds: + if (!strcmp(optarg,"yes")) + jl_options.check_bounds = JL_OPTIONS_CHECK_BOUNDS_ON; + else if (!strcmp(optarg,"no")) + jl_options.check_bounds = JL_OPTIONS_CHECK_BOUNDS_OFF; + else + jl_errorf("julia: invalid argument to --check-bounds={yes|no} (%s)", optarg); + break; + case opt_output_bc: + jl_options.outputbc = optarg; + if (!jl_options.image_file_specified) jl_options.image_file = NULL; + break; + case opt_output_o: + jl_options.outputo = optarg; + if (!jl_options.image_file_specified) jl_options.image_file = NULL; + break; + case opt_output_ji: + jl_options.outputji = optarg; + if (!jl_options.image_file_specified) jl_options.image_file = NULL; + break; + case opt_incremental: + if (!strcmp(optarg,"yes")) + jl_options.incremental = 1; + else if (!strcmp(optarg,"no")) + jl_options.incremental = 0; + else + jl_errorf("julia: invalid argument to --output-incremental={yes|no} (%s)", optarg); + break; + case opt_depwarn: + if (!strcmp(optarg,"yes")) + jl_options.depwarn = JL_OPTIONS_DEPWARN_ON; + else if (!strcmp(optarg,"no")) + jl_options.depwarn = JL_OPTIONS_DEPWARN_OFF; + else if (!strcmp(optarg,"error")) + jl_options.depwarn = JL_OPTIONS_DEPWARN_ERROR; + else + jl_errorf("julia: invalid argument to --depwarn={yes|no|error} (%s)", optarg); + break; + case opt_inline: + if (!strcmp(optarg,"yes")) + jl_options.can_inline = 1; + else if (!strcmp(optarg,"no")) + jl_options.can_inline = 0; + else { + jl_errorf("julia: invalid argument to --inline (%s)", optarg); + } + break; + case opt_math_mode: + if (!strcmp(optarg,"ieee")) + jl_options.fast_math = JL_OPTIONS_FAST_MATH_OFF; + else if (!strcmp(optarg,"fast")) + jl_options.fast_math = JL_OPTIONS_FAST_MATH_ON; + else if (!strcmp(optarg,"user")) + jl_options.fast_math = JL_OPTIONS_FAST_MATH_DEFAULT; + else + jl_errorf("julia: invalid argument to --math-mode (%s)", optarg); + break; + case opt_worker: + jl_options.worker = strdup(optarg); + break; + case opt_bind_to: + jl_options.bindto = strdup(optarg); + break; + case opt_handle_signals: + if (!strcmp(optarg,"yes")) + jl_options.handle_signals = JL_OPTIONS_HANDLE_SIGNALS_ON; + else if (!strcmp(optarg,"no")) + jl_options.handle_signals = JL_OPTIONS_HANDLE_SIGNALS_OFF; + else + jl_errorf("julia: invalid argument to --handle-signals (%s)", optarg); + break; + default: + jl_errorf("julia: unhandled option -- %c\n" + "This is a bug, please report it.", c); + } + } + jl_options.code_coverage = codecov; + jl_options.malloc_log = malloclog; + *argvp += optind; + *argcp -= optind; +} + +JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv) +{ + if (jl_core_module != NULL) { + jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); + if (args == NULL) { + args = jl_alloc_vec_any(0); + JL_GC_PUSH1(&args); + jl_set_const(jl_core_module, jl_symbol("ARGS"), (jl_value_t*)args); + JL_GC_POP(); + } + assert(jl_array_len(args) == 0); + jl_array_grow_end(args, argc); + int i; + for (i=0; i < argc; i++) { + jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); + jl_arrayset(args, s, i); + } + } +} diff --git a/src/julia.h b/src/julia.h index 772ec1cb790f4..61cfceada5f39 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1677,10 +1677,19 @@ typedef struct { const char *outputo; const char *outputji; int8_t incremental; + int8_t image_file_specified; } jl_options_t; extern JL_DLLEXPORT jl_options_t jl_options; +// Parse an argc/argv pair to extract general julia options, passing back out +// any arguments that should be passed on to the script. +JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp); + +// Set julia-level ARGS array according to the arguments provided in +// argc/argv +JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv); + JL_DLLEXPORT int jl_generating_output(void); // Settings for code_coverage and malloc_log diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 16b3c3c94117d..274f300b0b273 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -273,7 +273,10 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` end # Make sure `julia --lisp` doesn't break -run(pipeline(DevNull, `$(Base.julia_cmd()) --lisp`, DevNull)) +run(pipeline(DevNull, `$(joinpath(JULIA_HOME, Base.julia_exename())) --lisp`, DevNull)) + +# Test that `julia [some other option] --lisp` is disallowed +@test_throws ErrorException run(pipeline(DevNull, pipeline(`$(joinpath(JULIA_HOME, Base.julia_exename())) -Cnative --lisp`, stderr=DevNull), DevNull)) # --precompiled={yes|no} let exename = `$(Base.julia_cmd())` diff --git a/ui/Makefile b/ui/Makefile index c7053fcf217e1..f869b429824e0 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -10,9 +10,6 @@ override CXXFLAGS += $(JCXXFLAGS) override CPPFLAGS += $(JCPPFLAGS) SRCS := repl -ifeq ($(USEMSVC), 1) -SRCS += getopt -endif HEADERS := $(addprefix $(JULIAHOME)/src/,julia.h julia_threads.h julia_internal.h options.h) \ $(BUILDDIR)/../src/julia_version.h $(wildcard $(JULIAHOME)/src/support/*.h) $(LIBUV_INC)/uv.h diff --git a/ui/repl.c b/ui/repl.c index eb3d3184e42f0..db0cacf84fbd3 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -19,13 +19,6 @@ #include <math.h> #include <ctype.h> -#ifndef _MSC_VER -#include <unistd.h> -#include <getopt.h> -#else -#include "getopt.h" -#endif - #include "uv.h" #define WHOLE_ARCHIVE #include "../src/julia.h" @@ -48,426 +41,6 @@ __attribute__((constructor)) void jl_register_ptls_states_getter(void) } #endif -static int lisp_prompt = 0; -static int codecov = JL_LOG_NONE; -static int malloclog= JL_LOG_NONE; -static int imagepathspecified = 0; - -static const char usage[] = "julia [switches] -- [programfile] [args...]\n"; -static const char opts[] = - " -v, --version Display version information\n" - " -h, --help Print this message\n\n" - - // startup options - " -J, --sysimage <file> Start up with the given system image file\n" - " --precompiled={yes|no} Use precompiled code from system image if available\n" - " --compilecache={yes|no} Enable/disable incremental precompilation of modules\n" - " -H, --home <dir> Set location of `julia` executable\n" - " --startup-file={yes|no} Load ~/.juliarc.jl\n" - " --handle-signals={yes|no} Enable or disable Julia's default signal handlers\n\n" - - // actions - " -e, --eval <expr> Evaluate <expr>\n" - " -E, --print <expr> Evaluate and show <expr>\n" - " -L, --load <file> Load <file> immediately on all processors\n\n" - - // parallel options - " -p, --procs {N|auto} Integer value N launches N additional local worker processes\n" - " \"auto\" launches as many workers as the number of local cores\n" - " --machinefile <file> Run processes on hosts listed in <file>\n\n" - - // interactive options - " -i Interactive mode; REPL runs and isinteractive() is true\n" - " -q, --quiet Quiet startup (no banner)\n" - " --color={yes|no} Enable or disable color text\n" - " --history-file={yes|no} Load or save history\n\n" - - // code generation options - " --compile={yes|no|all|min}Enable or disable JIT compiler, or request exhaustive compilation\n" - " -C, --cpu-target <target> Limit usage of cpu features up to <target>\n" - " -O, --optimize={0,1,2,3} Set the optimization level (default 2 if unspecified or 3 if specified as -O)\n" - " --inline={yes|no} Control whether inlining is permitted (overrides functions declared as @inline)\n" - " --check-bounds={yes|no} Emit bounds checks always or never (ignoring declarations)\n" - " --math-mode={ieee,fast} Disallow or enable unsafe floating point optimizations (overrides @fastmath declaration)\n\n" - - // error and warning options - " --depwarn={yes|no|error} Enable or disable syntax and method deprecation warnings (\"error\" turns warnings into errors)\n\n" - - // compiler output options - " --output-o name Generate an object file (including system image data)\n" - " --output-ji name Generate a system image data file (.ji)\n" - " --output-bc name Generate LLVM bitcode (.bc)\n" - " --output-incremental=no Generate an incremental output file (rather than complete)\n\n" - - // instrumentation options - " --code-coverage={none|user|all}, --code-coverage\n" - " Count executions of source lines (omitting setting is equivalent to \"user\")\n" - " --track-allocation={none|user|all}, --track-allocation\n" - " Count bytes allocated by each source line\n\n" - - "Deprecated options:\n" - " -F Load ~/.juliarc (deprecated, use --startup-file=yes)\n" - " -f, --no-startup Don't load ~/.juliarc (deprecated, use --startup-file=no)\n" - " -P, --post-boot <expr> Evaluate <expr>, but don't disable interactive mode (deprecated, use -i -e instead)\n" - " --no-history-file Don't load history file (deprecated, use --history-file=no)\n"; - -void parse_opts(int *argcp, char ***argvp) -{ - enum { opt_machinefile = 300, - opt_color, - opt_history_file, - opt_no_history_file, - opt_startup_file, - opt_compile, - opt_code_coverage, - opt_track_allocation, - opt_check_bounds, - opt_output_bc, - opt_depwarn, - opt_inline, - opt_math_mode, - opt_worker, - opt_bind_to, - opt_handle_signals, - opt_output_o, - opt_output_ji, - opt_use_precompiled, - opt_use_compilecache, - opt_incremental - }; - static char* shortopts = "+vhqFfH:e:E:P:L:J:C:ip:O:"; - static struct option longopts[] = { - // exposed command line options - // NOTE: This set of required arguments need to be kept in sync - // with the required arguments defined in base/client.jl `process_options()` - { "version", no_argument, 0, 'v' }, - { "help", no_argument, 0, 'h' }, - { "quiet", no_argument, 0, 'q' }, - { "home", required_argument, 0, 'H' }, - { "eval", required_argument, 0, 'e' }, - { "print", required_argument, 0, 'E' }, - { "load", required_argument, 0, 'L' }, - { "sysimage", required_argument, 0, 'J' }, - { "precompiled", required_argument, 0, opt_use_precompiled }, - { "compilecache", required_argument, 0, opt_use_compilecache }, - { "cpu-target", required_argument, 0, 'C' }, - { "procs", required_argument, 0, 'p' }, - { "machinefile", required_argument, 0, opt_machinefile }, - { "color", required_argument, 0, opt_color }, - { "history-file", required_argument, 0, opt_history_file }, - { "startup-file", required_argument, 0, opt_startup_file }, - { "compile", required_argument, 0, opt_compile }, - { "code-coverage", optional_argument, 0, opt_code_coverage }, - { "track-allocation",optional_argument, 0, opt_track_allocation }, - { "optimize", optional_argument, 0, 'O' }, - { "check-bounds", required_argument, 0, opt_check_bounds }, - { "output-bc", required_argument, 0, opt_output_bc }, - { "output-o", required_argument, 0, opt_output_o }, - { "output-ji", required_argument, 0, opt_output_ji }, - { "output-incremental",required_argument, 0, opt_incremental }, - { "depwarn", required_argument, 0, opt_depwarn }, - { "inline", required_argument, 0, opt_inline }, - { "math-mode", required_argument, 0, opt_math_mode }, - { "handle-signals", required_argument, 0, opt_handle_signals }, - // hidden command line options - { "worker", required_argument, 0, opt_worker }, - { "bind-to", required_argument, 0, opt_bind_to }, - { "lisp", no_argument, &lisp_prompt, 1 }, - // deprecated options - { "post-boot", required_argument, 0, 'P' }, - { "no-history-file", no_argument, 0, opt_no_history_file }, // deprecated - { "no-startup", no_argument, 0, 'f' }, // deprecated - { 0, 0, 0, 0 } - }; - // getopt handles argument parsing up to -- delineator - int argc = *argcp; - char **argv = *argvp; - if (argc > 0) { - for (int i=0; i < argc; i++) { - if (!strcmp(argv[i], "--")) { - argc = i; - break; - } - } - } - char *endptr; - opterr = 0; // suppress getopt warning messages - while (1) { - int lastind = optind; - int c = getopt_long(argc, argv, shortopts, longopts, 0); - if (c == -1) break; -restart_switch: - switch (c) { - case 0: - break; - case '?': - case ':': - if (optopt) { - for (struct option *o = longopts; o->val; o++) { - if (optopt == o->val) { - if (o->has_arg == optional_argument) { - c = o->val; - goto restart_switch; - } - else if (strchr(shortopts, o->val)) { - jl_errorf("option `-%c/--%s` is missing an argument", o->val, o->name); - } - else { - jl_errorf("option `--%s` is missing an argument", o->name); - } - } - } - jl_errorf("unknown option `-%c`", optopt); - } else { - jl_errorf("unknown option `%s`", argv[lastind]); - } - break; - case 'v': // version - jl_printf(JL_STDOUT, "julia version %s\n", JULIA_VERSION_STRING); - jl_exit(0); - case 'h': // help - jl_printf(JL_STDOUT, "%s%s", usage, opts); - jl_exit(0); - case 'q': // quiet - jl_options.quiet = 1; - break; - case 'H': // home - jl_options.julia_home = strdup(optarg); - break; - case 'e': // eval - jl_options.eval = strdup(optarg); - break; - case 'E': // print - jl_options.print = strdup(optarg); - break; - case 'P': // post-boot - jl_printf(JL_STDOUT, "WARNING: julia -P/--post-boot option is deprecated, use -i -e instead.\n"); - jl_options.postboot = strdup(optarg); - break; - case 'L': // load - jl_options.load = strdup(optarg); - break; - case 'J': // sysimage - jl_options.image_file = strdup(optarg); - imagepathspecified = 1; - break; - case opt_use_precompiled: - if (!strcmp(optarg,"yes")) - jl_options.use_precompiled = JL_OPTIONS_USE_PRECOMPILED_YES; - else if (!strcmp(optarg,"no")) - jl_options.use_precompiled = JL_OPTIONS_USE_PRECOMPILED_NO; - else - jl_errorf("julia: invalid argument to --precompiled={yes|no} (%s)", optarg); - break; - case opt_use_compilecache: - if (!strcmp(optarg,"yes")) - jl_options.use_compilecache = JL_OPTIONS_USE_COMPILECACHE_YES; - else if (!strcmp(optarg,"no")) - jl_options.use_compilecache = JL_OPTIONS_USE_COMPILECACHE_NO; - else - jl_errorf("julia: invalid argument to --compilecache={yes|no} (%s)", optarg); - break; - case 'C': // cpu-target - jl_options.cpu_target = strdup(optarg); - break; - case 'p': // procs - errno = 0; - if (!strcmp(optarg,"auto")) { - jl_options.nprocs = jl_cpu_cores(); - } - else { - long nprocs = strtol(optarg, &endptr, 10); - if (errno != 0 || optarg == endptr || *endptr != 0 || nprocs < 1 || nprocs >= INT_MAX) - jl_errorf("julia: -p,--procs=<n> must be an integer >= 1"); - jl_options.nprocs = (int)nprocs; - } - break; - case opt_machinefile: - jl_options.machinefile = strdup(optarg); - break; - case opt_color: - if (!strcmp(optarg,"yes")) - jl_options.color = JL_OPTIONS_COLOR_ON; - else if (!strcmp(optarg,"no")) - jl_options.color = JL_OPTIONS_COLOR_OFF; - else - jl_errorf("julia: invalid argument to --color={yes|no} (%s)", optarg); - break; - case opt_history_file: - if (!strcmp(optarg,"yes")) - jl_options.historyfile = JL_OPTIONS_HISTORYFILE_ON; - else if (!strcmp(optarg,"no")) - jl_options.historyfile = JL_OPTIONS_HISTORYFILE_OFF; - else - jl_errorf("julia: invalid argument to --history-file={yes|no} (%s)", optarg); - break; - case opt_no_history_file: - jl_printf(JL_STDOUT, "WARNING: julia --no-history-file option is deprecated, use --history-file=no instead.\n"); - jl_options.historyfile = JL_OPTIONS_HISTORYFILE_OFF; - break; - case opt_startup_file: - if (!strcmp(optarg,"yes")) - jl_options.startupfile = JL_OPTIONS_STARTUPFILE_ON; - else if (!strcmp(optarg,"no")) - jl_options.startupfile = JL_OPTIONS_STARTUPFILE_OFF; - else - jl_errorf("julia: invalid argument to --startup-file={yes|no} (%s)", optarg); - break; - case 'f': - jl_printf(JL_STDOUT, "WARNING: julia -f/--no-startup option is deprecated, use --startup-file=no instead.\n"); - jl_options.startupfile = JL_OPTIONS_STARTUPFILE_OFF; - break; - case 'F': - jl_printf(JL_STDOUT, "WARNING: julia -F option is deprecated, use --startup-file=yes instead.\n"); - jl_options.startupfile = JL_OPTIONS_STARTUPFILE_ON; - break; - case opt_compile: - if (!strcmp(optarg,"yes")) - jl_options.compile_enabled = JL_OPTIONS_COMPILE_ON; - else if (!strcmp(optarg,"no")) - jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF; - else if (!strcmp(optarg,"all")) - jl_options.compile_enabled = JL_OPTIONS_COMPILE_ALL; - else if (!strcmp(optarg,"min")) - jl_options.compile_enabled = JL_OPTIONS_COMPILE_MIN; - else - jl_errorf("julia: invalid argument to --compile (%s)", optarg); - break; - case opt_code_coverage: - if (optarg != NULL) { - if (!strcmp(optarg,"user")) - codecov = JL_LOG_USER; - else if (!strcmp(optarg,"all")) - codecov = JL_LOG_ALL; - else if (!strcmp(optarg,"none")) - codecov = JL_LOG_NONE; - else - jl_errorf("julia: invalid argument to --code-coverage (%s)", optarg); - break; - } - else { - codecov = JL_LOG_USER; - } - break; - case opt_track_allocation: - if (optarg != NULL) { - if (!strcmp(optarg,"user")) - malloclog = JL_LOG_USER; - else if (!strcmp(optarg,"all")) - malloclog = JL_LOG_ALL; - else if (!strcmp(optarg,"none")) - malloclog = JL_LOG_NONE; - else - jl_errorf("julia: invalid argument to --track-allocation (%s)", optarg); - break; - } - else { - malloclog = JL_LOG_USER; - } - break; - case 'O': // optimize - if (optarg != NULL) { - if (!strcmp(optarg,"0")) - jl_options.opt_level = 0; - else if (!strcmp(optarg,"1")) - jl_options.opt_level = 1; - else if (!strcmp(optarg,"2")) - jl_options.opt_level = 2; - else if (!strcmp(optarg,"3")) - jl_options.opt_level = 3; - else - jl_errorf("julia: invalid argument to -O (%s)", optarg); - break; - } - else { - jl_options.opt_level = 3; - } - break; - case 'i': // isinteractive - jl_options.isinteractive = 1; - break; - case opt_check_bounds: - if (!strcmp(optarg,"yes")) - jl_options.check_bounds = JL_OPTIONS_CHECK_BOUNDS_ON; - else if (!strcmp(optarg,"no")) - jl_options.check_bounds = JL_OPTIONS_CHECK_BOUNDS_OFF; - else - jl_errorf("julia: invalid argument to --check-bounds={yes|no} (%s)", optarg); - break; - case opt_output_bc: - jl_options.outputbc = optarg; - if (!imagepathspecified) jl_options.image_file = NULL; - break; - case opt_output_o: - jl_options.outputo = optarg; - if (!imagepathspecified) jl_options.image_file = NULL; - break; - case opt_output_ji: - jl_options.outputji = optarg; - if (!imagepathspecified) jl_options.image_file = NULL; - break; - case opt_incremental: - if (!strcmp(optarg,"yes")) - jl_options.incremental = 1; - else if (!strcmp(optarg,"no")) - jl_options.incremental = 0; - else - jl_errorf("julia: invalid argument to --output-incremental={yes|no} (%s)", optarg); - break; - case opt_depwarn: - if (!strcmp(optarg,"yes")) - jl_options.depwarn = JL_OPTIONS_DEPWARN_ON; - else if (!strcmp(optarg,"no")) - jl_options.depwarn = JL_OPTIONS_DEPWARN_OFF; - else if (!strcmp(optarg,"error")) - jl_options.depwarn = JL_OPTIONS_DEPWARN_ERROR; - else - jl_errorf("julia: invalid argument to --depwarn={yes|no|error} (%s)", optarg); - break; - case opt_inline: - if (!strcmp(optarg,"yes")) - jl_options.can_inline = 1; - else if (!strcmp(optarg,"no")) - jl_options.can_inline = 0; - else { - jl_errorf("julia: invalid argument to --inline (%s)", optarg); - } - break; - case opt_math_mode: - if (!strcmp(optarg,"ieee")) - jl_options.fast_math = JL_OPTIONS_FAST_MATH_OFF; - else if (!strcmp(optarg,"fast")) - jl_options.fast_math = JL_OPTIONS_FAST_MATH_ON; - else if (!strcmp(optarg,"user")) - jl_options.fast_math = JL_OPTIONS_FAST_MATH_DEFAULT; - else - jl_errorf("julia: invalid argument to --math-mode (%s)", optarg); - break; - case opt_worker: - jl_options.worker = strdup(optarg); - break; - case opt_bind_to: - jl_options.bindto = strdup(optarg); - break; - case opt_handle_signals: - if (!strcmp(optarg,"yes")) - jl_options.handle_signals = JL_OPTIONS_HANDLE_SIGNALS_ON; - else if (!strcmp(optarg,"no")) - jl_options.handle_signals = JL_OPTIONS_HANDLE_SIGNALS_OFF; - else - jl_errorf("julia: invalid argument to --handle-signals (%s)", optarg); - break; - default: - jl_errorf("julia: unhandled option -- %c\n" - "This is a bug, please report it.", c); - } - } - jl_options.code_coverage = codecov; - jl_options.malloc_log = malloclog; - *argvp += optind; - *argcp -= optind; -} - static int exec_program(char *program) { jl_ptls_t ptls = jl_get_ptls_states(); @@ -530,22 +103,7 @@ static void print_profile(void) static NOINLINE int true_main(int argc, char *argv[]) { jl_ptls_t ptls = jl_get_ptls_states(); - if (jl_core_module != NULL) { - jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); - if (args == NULL) { - args = jl_alloc_vec_any(0); - JL_GC_PUSH1(&args); - jl_set_const(jl_core_module, jl_symbol("ARGS"), (jl_value_t*)args); - JL_GC_POP(); - } - assert(jl_array_len(args) == 0); - jl_array_grow_end(args, argc); - int i; - for (i=0; i < argc; i++) { - jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); - jl_arrayset(args, s, i); - } - } + jl_set_ARGS(argc, argv); jl_function_t *start_client = jl_base_module ? (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("_start")) : NULL; @@ -665,12 +223,12 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) } #endif libsupport_init(); - parse_opts(&argc, (char***)&argv); - if (lisp_prompt) { + if (argc >= 2 && strcmp(argv[1],"--lisp") == 0) { jl_lisp_prompt(); return 0; } - julia_init(imagepathspecified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME); + jl_parse_opts(&argc, (char***)&argv); + julia_init(jl_options.image_file_specified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME); int ret = true_main(argc, (char**)argv); jl_atexit_hook(ret); return ret; From 17f122eb26f130c7ff492247154b913a0b4dd990 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 5 May 2016 22:03:14 -0400 Subject: [PATCH 0294/1117] implement comprehensions as `collect` of a `Generator` this removes `static_typeof` and `type_goto` fixes #7258 --- base/abstractarray.jl | 4 +- base/arraymath.jl | 4 +- base/inference.jl | 111 ++------------------------ base/range.jl | 2 +- base/sparse/sparsematrix.jl | 20 ++--- src/alloc.c | 3 +- src/codegen.cpp | 14 +--- src/interpreter.c | 5 +- src/jltypes.c | 2 - src/julia-syntax.scm | 152 +++--------------------------------- src/julia.h | 4 +- src/toplevel.c | 2 - test/arrayops.jl | 2 +- test/dict.jl | 2 +- 14 files changed, 38 insertions(+), 289 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 71b29d82f4810..d9802a4c90017 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -804,8 +804,8 @@ typed_hcat{T}(::Type{T}) = Array{T}(0) ## cat: special cases vcat{T}(X::T...) = T[ X[i] for i=1:length(X) ] vcat{T<:Number}(X::T...) = T[ X[i] for i=1:length(X) ] -hcat{T}(X::T...) = T[ X[j] for i=1, j=1:length(X) ] -hcat{T<:Number}(X::T...) = T[ X[j] for i=1, j=1:length(X) ] +hcat{T}(X::T...) = T[ X[j] for i=1:1, j=1:length(X) ] +hcat{T<:Number}(X::T...) = T[ X[j] for i=1:1, j=1:length(X) ] vcat(X::Number...) = hvcat_fill(Array{promote_typeof(X...)}(length(X)), X) hcat(X::Number...) = hvcat_fill(Array{promote_typeof(X...)}(1,length(X)), X) diff --git a/base/arraymath.jl b/base/arraymath.jl index c89ac3c2d6a30..53a73c3b5ef66 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -322,8 +322,8 @@ function ctranspose(A::AbstractMatrix) end ctranspose{T<:Real}(A::AbstractVecOrMat{T}) = transpose(A) -transpose(x::AbstractVector) = [ transpose(v) for i=1, v in x ] -ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=1, v in x ] #Fixme comprehension +transpose(x::AbstractVector) = [ transpose(v) for i=1:1, v in x ] +ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=1:1, v in x ] _cumsum_type{T<:Number}(v::AbstractArray{T}) = typeof(+zero(T)) _cumsum_type(v) = typeof(v[1]+v[1]) diff --git a/base/inference.jl b/base/inference.jl index ea7055217c838..560d295d2084f 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -43,10 +43,8 @@ end type InferenceState sp::SimpleVector # static parameters label_counter::Int # index of the current highest label for this function - fedbackvars::Dict{SSAValue, Bool} mod::Module currpc::LineNum - static_typeof::Bool # info on the state of inference and the linfo linfo::LambdaInfo @@ -71,7 +69,6 @@ type InferenceState backedges::Vector{Tuple{InferenceState, Vector{LineNum}}} # iteration fixed-point detection fixedpoint::Bool - typegotoredo::Bool inworkq::Bool # optimization optimize::Bool @@ -158,13 +155,13 @@ type InferenceState inmodule = isdefined(linfo, :def) ? linfo.def.module : current_module() # toplevel thunks are inferred in the current module frame = new( - sp, nl, Dict{SSAValue, Bool}(), inmodule, 0, false, + sp, nl, inmodule, 0, linfo, la, s, Union{}, W, n, cur_hand, handler_at, n_handlers, ssavalue_uses, ssavalue_init, ObjectIdDict(), #Dict{InferenceState, Vector{LineNum}}(), Vector{Tuple{InferenceState, Vector{LineNum}}}(), - false, false, false, optimize, inlining, needtree, false) + false, false, optimize, inlining, needtree, false) push!(active, frame) nactive[] += 1 return frame @@ -1068,8 +1065,6 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState) return abstract_eval_constant(e) end e = e::Expr - # handle: - # call null new & static_typeof if is(e.head,:call) t = abstract_eval_call(e, vtypes, sv) elseif is(e.head,:null) @@ -1103,42 +1098,6 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState) t = abstract_eval_constant(val) end end - elseif is(e.head,:static_typeof) - var = e.args[1] - t = widenconst(abstract_eval(var, vtypes, sv)) - if isa(t,DataType) && typeseq(t,t.name.primary) - # remove unnecessary typevars - t = t.name.primary - end - if is(t,Bottom) - # if we haven't gotten fed-back type info yet, return Bottom. otherwise - # Bottom is the actual type of the variable, so return Type{Bottom}. - if get!(sv.fedbackvars, var, false) - t = Type{Bottom} - else - sv.static_typeof = true - end - elseif isleaftype(t) - t = Type{t} - elseif isleaftype(sv.linfo.specTypes) - if isa(t,TypeVar) - t = Type{t.ub} - else - t = Type{t} - end - else - # if there is any type uncertainty in the arguments, we are - # effectively predicting what static_typeof will say when - # the function is compiled with actual arguments. in that case - # abstract types yield Type{<:T} instead of Type{T}. - # this doesn't really model the situation perfectly, but - # "isleaftype(inference_stack.types)" should be good enough. - if isa(t,TypeVar) || isvarargtype(t) - t = Type{t} - else - t = Type{TypeVar(:_,t)} - end - end elseif is(e.head,:method) t = (length(e.args) == 1) ? Any : Void elseif is(e.head,:copyast) @@ -1666,7 +1625,6 @@ function typeinf_frame(frame) W = frame.ip s = frame.stmt_types n = frame.nstmts - @label restart_typeinf while !isempty(W) # make progress on the active ip set local pc::Int = first(W), pc´::Int @@ -1674,15 +1632,12 @@ function typeinf_frame(frame) #print(pc,": ",s[pc],"\n") delete!(W, pc) frame.currpc = pc - frame.static_typeof = false frame.cur_hand = frame.handler_at[pc] stmt = frame.linfo.code[pc] changes = abstract_interpret(stmt, s[pc]::Array{Any,1}, frame) if changes === () - # if there was a Expr(:static_typeof) on this line, - # need to continue to the next pc even though the return type was Bottom - # otherwise, this line threw an error and there is no need to continue - frame.static_typeof || break + # this line threw an error and there is no need to continue + break changes = s[pc] end if frame.cur_hand !== () @@ -1732,26 +1687,6 @@ function typeinf_frame(frame) s[l] = newstate end end - elseif is(hd, :type_goto) - for i = 2:length(stmt.args) - var = stmt.args[i]::SSAValue - # Store types that need to be fed back via type_goto - # in ssavalue_init. After finishing inference, if any - # of these types changed, start over with the fed-back - # types known from the beginning. - # See issue #3821 (using !typeseq instead of !subtype), - # and issue #7810. - id = var.id+1 - vt = frame.linfo.ssavaluetypes[id] - ot = frame.ssavalue_init[id] - if ot===NF || !(vt⊑ot && ot⊑vt) - frame.ssavalue_init[id] = vt - if get(frame.fedbackvars, var, false) - frame.typegotoredo = true - end - end - frame.fedbackvars[var] = true - end elseif is(hd, :return) pc´ = n + 1 rt = abstract_eval(stmt.args[1], s[pc], frame) @@ -1821,39 +1756,6 @@ function typeinf_frame(frame) end if finished || frame.fixedpoint - if frame.typegotoredo - # if any type_gotos changed, clear state and restart. - frame.typegotoredo = false - for ll = 2:length(s) - s[ll] = () - end - empty!(W) - push!(W, 1) - frame.cur_hand = () - frame.handler_at = Any[ () for i=1:n ] - frame.n_handlers = 0 - frame.linfo.ssavaluetypes[:] = frame.ssavalue_init - @goto restart_typeinf - else - # if a static_typeof was never reached, - # use Union{} as its real type and continue - # running type inference from its uses - # (one of which is the static_typeof) - # TODO: this restart should happen just before calling finish() - for (fbvar, seen) in frame.fedbackvars - if !seen - frame.fedbackvars[fbvar] = true - id = (fbvar::SSAValue).id + 1 - for r in frame.ssavalue_uses[id] - if !is(s[r], ()) # s[r] === () => unreached statement - push!(W, r) - end - end - @goto restart_typeinf - end - end - end - if finished finish(frame) else # fixedpoint propagation @@ -2056,7 +1958,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::InferenceState, undefs, pass) e = e::Expr head = e.head - if is(head,:static_typeof) || is(head,:line) || is(head,:const) + if is(head,:line) || is(head,:const) return e elseif is(head,:(=)) e.args[2] = eval_annotate(e.args[2], vtypes, sv, undefs, pass) @@ -2282,9 +2184,6 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) elseif isa(e,Expr) e = e::Expr head = e.head - if head === :static_typeof - return true - end if head === :static_parameter || head === :meta || head === :line || head === :inbounds || head === :boundscheck return true diff --git a/base/range.jl b/base/range.jl index a3badcf109535..335f2d89cb7d1 100644 --- a/base/range.jl +++ b/base/range.jl @@ -382,7 +382,7 @@ maximum(r::AbstractUnitRange) = isempty(r) ? throw(ArgumentError("range must be minimum(r::Range) = isempty(r) ? throw(ArgumentError("range must be non-empty")) : min(first(r), last(r)) maximum(r::Range) = isempty(r) ? throw(ArgumentError("range must be non-empty")) : max(first(r), last(r)) -ctranspose(r::Range) = [x for _=1, x=r] +ctranspose(r::Range) = [x for _=1:1, x=r] transpose(r::Range) = r' # Ranges are immutable diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 6e18bf43b08ed..4ce6e0e0a4bc4 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -3072,8 +3072,8 @@ end function vcat(X::SparseMatrixCSC...) num = length(X) - mX = [ size(x, 1) for x in X ] - nX = [ size(x, 2) for x in X ] + mX = Int[ size(x, 1) for x in X ] + nX = Int[ size(x, 2) for x in X ] m = sum(mX) n = nX[1] @@ -3090,7 +3090,7 @@ function vcat(X::SparseMatrixCSC...) Ti = promote_type(Ti, eltype(X[i].rowval)) end - nnzX = [ nnz(x) for x in X ] + nnzX = Int[ nnz(x) for x in X ] nnz_res = sum(nnzX) colptr = Array{Ti}(n + 1) rowval = Array{Ti}(nnz_res) @@ -3132,8 +3132,8 @@ end function hcat(X::SparseMatrixCSC...) num = length(X) - mX = [ size(x, 1) for x in X ] - nX = [ size(x, 2) for x in X ] + mX = Int[ size(x, 1) for x in X ] + nX = Int[ size(x, 2) for x in X ] m = mX[1] for i = 2 : num if mX[i] != m; throw(DimensionMismatch("")); end @@ -3144,7 +3144,7 @@ function hcat(X::SparseMatrixCSC...) Ti = promote_type(map(x->eltype(x.rowval), X)...) colptr = Array{Ti}(n + 1) - nnzX = [ nnz(x) for x in X ] + nnzX = Int[ nnz(x) for x in X ] nnz_res = sum(nnzX) rowval = Array{Ti}(nnz_res) nzval = Array{Tv}(nnz_res) @@ -3200,8 +3200,8 @@ Concatenate matrices block-diagonally. Currently only implemented for sparse mat """ function blkdiag(X::SparseMatrixCSC...) num = length(X) - mX = [ size(x, 1) for x in X ] - nX = [ size(x, 2) for x in X ] + mX = Int[ size(x, 1) for x in X ] + nX = Int[ size(x, 2) for x in X ] m = sum(mX) n = sum(nX) @@ -3209,7 +3209,7 @@ function blkdiag(X::SparseMatrixCSC...) Ti = promote_type(map(x->eltype(x.rowval), X)...) colptr = Array{Ti}(n + 1) - nnzX = [ nnz(x) for x in X ] + nnzX = Int[ nnz(x) for x in X ] nnz_res = sum(nnzX) rowval = Array{Ti}(nnz_res) nzval = Array{Tv}(nnz_res) @@ -3450,7 +3450,7 @@ function trace{Tv}(A::SparseMatrixCSC{Tv}) s end -diag(A::SparseMatrixCSC) = [d for d in SpDiagIterator(A)] +diag{Tv}(A::SparseMatrixCSC{Tv}) = Tv[d for d in SpDiagIterator(A)] function diagm{Tv,Ti}(v::SparseMatrixCSC{Tv,Ti}) if (size(v,1) != 1 && size(v,2) != 1) diff --git a/src/alloc.c b/src/alloc.c index 545d968512aea..cded29bb6e57f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -88,13 +88,12 @@ jl_sym_t *body_sym; jl_sym_t *method_sym; jl_sym_t *core_sym; jl_sym_t *enter_sym; jl_sym_t *leave_sym; jl_sym_t *exc_sym; jl_sym_t *error_sym; -jl_sym_t *static_typeof_sym; jl_sym_t *globalref_sym; jl_sym_t *new_sym; jl_sym_t *using_sym; jl_sym_t *const_sym; jl_sym_t *thunk_sym; jl_sym_t *anonymous_sym; jl_sym_t *underscore_sym; jl_sym_t *abstracttype_sym; jl_sym_t *bitstype_sym; -jl_sym_t *compositetype_sym; jl_sym_t *type_goto_sym; +jl_sym_t *compositetype_sym; jl_sym_t *global_sym; jl_sym_t *list_sym; jl_sym_t *dot_sym; jl_sym_t *newvar_sym; jl_sym_t *boundscheck_sym; jl_sym_t *inbounds_sym; diff --git a/src/codegen.cpp b/src/codegen.cpp index 23f7fd5542d20..c09171f0ce4c7 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3110,7 +3110,7 @@ static void emit_stmtpos(jl_value_t *expr, jl_codectx_t *ctx) if (jl_is_expr(expr)) { jl_sym_t *head = ((jl_expr_t*)expr)->head; // some expression types are metadata and can be ignored in statement position - if (head == line_sym || head == type_goto_sym || head == meta_sym) + if (head == line_sym || head == meta_sym) return; // fall-through } @@ -3284,18 +3284,6 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) literal_pointer_val(bnd)); } } - else if (head == static_typeof_sym) { - jl_value_t *extype = expr_type((jl_value_t*)ex, ctx); - if (jl_is_type_type(extype)) { - extype = jl_tparam0(extype); - if (jl_is_typevar(extype)) - extype = ((jl_tvar_t*)extype)->ub; - } - else { - extype = (jl_value_t*)jl_any_type; - } - return mark_julia_const(extype); - } else if (head == new_sym) { jl_value_t *ty = expr_type(args[0], ctx); size_t nargs = jl_array_len(ex->args); diff --git a/src/interpreter.c b/src/interpreter.c index 32900f7981f05..c348349962b9d 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -223,9 +223,6 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) else if (ex->head == copyast_sym) { return jl_copy_ast(eval(args[0], s)); } - else if (ex->head == static_typeof_sym) { - return (jl_value_t*)jl_any_type; - } else if (ex->head == exc_sym) { return ptls->exception_in_transit; } @@ -429,7 +426,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) jl_throw(args[0]); } else if (ex->head == boundscheck_sym || ex->head == inbounds_sym || ex->head == fastmath_sym || - ex->head == simdloop_sym || ex->head == meta_sym || ex->head == type_goto_sym) { + ex->head == simdloop_sym || ex->head == meta_sym) { return jl_nothing; } jl_errorf("unsupported or misplaced expression %s", jl_symbol_name(ex->head)); diff --git a/src/jltypes.c b/src/jltypes.c index c3975601a4b15..96a248db62696 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -4007,7 +4007,6 @@ void jl_init_types(void) exc_sym = jl_symbol("the_exception"); enter_sym = jl_symbol("enter"); leave_sym = jl_symbol("leave"); - static_typeof_sym = jl_symbol("static_typeof"); new_sym = jl_symbol("new"); const_sym = jl_symbol("const"); global_sym = jl_symbol("global"); @@ -4018,7 +4017,6 @@ void jl_init_types(void) abstracttype_sym = jl_symbol("abstract_type"); bitstype_sym = jl_symbol("bits_type"); compositetype_sym = jl_symbol("composite_type"); - type_goto_sym = jl_symbol("type_goto"); toplevel_sym = jl_symbol("toplevel"); dot_sym = jl_symbol("."); boundscheck_sym = jl_symbol("boundscheck"); diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 168352c1e1f10..9640baef3e3ec 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1981,78 +1981,29 @@ 'comprehension (lambda (e) - (expand-forms (lower-comprehension #f (cadr e) (cddr e)))) + (if (any (lambda (x) (eq? x ':)) (cddr e)) + (error "comprehension syntax with `:` ranges has been removed")) + (expand-forms `(call (top collect) (generator ,(cadr e) ,@(cddr e))))) 'typed_comprehension (lambda (e) + (if (any (lambda (x) (eq? x ':)) (cdddr e)) + (error "comprehension syntax with `:` ranges has been removed")) (expand-forms (lower-comprehension (cadr e) (caddr e) (cdddr e)))) 'dict_comprehension (lambda (e) (syntax-deprecation #f "[a=>b for (a,b) in c]" "Dict(a=>b for (a,b) in c)") - (expand-forms (lower-dict-comprehension (cadr e) (cddr e)))) + (expand-forms `(call (top Dict) (generator ,(cadr e) ,@(cddr e))))) 'typed_dict_comprehension - (lambda (e) - (expand-forms (lower-typed-dict-comprehension (cadr e) (caddr e) (cdddr e)))))) - -(define (lower-nd-comprehension atype expr ranges) - (let ((result (make-ssavalue)) - (ri (gensy)) - (oneresult (gensy))) - ;; evaluate one expression to figure out type and size - ;; compute just one value by inserting a break inside loops - (define (evaluate-one ranges) - (if (null? ranges) - `(= ,oneresult ,expr) - (if (eq? (car ranges) `:) - (evaluate-one (cdr ranges)) - `(for ,(car ranges) - (block ,(evaluate-one (cdr ranges)) - (break)) )))) - - ;; compute the dimensions of the result - (define (compute-dims ranges oneresult-dim) - (if (null? ranges) - (list) - (if (eq? (car ranges) `:) - (cons `(call (top size) ,oneresult ,oneresult-dim) - (compute-dims (cdr ranges) (+ oneresult-dim 1))) - (cons `(call (top length) ,(caddr (car ranges))) - (compute-dims (cdr ranges) oneresult-dim)) ))) - - ;; construct loops to cycle over all dimensions of an n-d comprehension - (define (construct-loops ranges iters oneresult-dim) - (if (null? ranges) - (if (null? iters) - `(block (call (top setindex!) ,result ,expr ,ri) - (= ,ri (call (top +) ,ri) 1)) - `(block (call (top setindex!) ,result (ref ,expr ,@(reverse iters)) ,ri) - (= ,ri (call (top +) ,ri 1))) ) - (if (eq? (car ranges) `:) - (let ((i (make-ssavalue))) - `(for (= ,i (: 1 (call (top size) ,oneresult ,oneresult-dim))) - ,(construct-loops (cdr ranges) (cons i iters) (+ oneresult-dim 1)) )) - `(for ,(car ranges) - ,(construct-loops (cdr ranges) iters oneresult-dim) )))) - - ;; Evaluate the comprehension - `(scope-block - (block - (local ,oneresult) - ,(evaluate-one ranges) - (= ,result (call (core Array) ,(if atype atype `(call (top eltype) ,oneresult)) - ,@(compute-dims ranges 1))) - (= ,ri 1) - ,(construct-loops (reverse ranges) (list) 1) - ,result )))) + (lambda (e) (expand-forms + `(call (call (core apply_type) (top Dict) ,@(cdr (cadr e))) + (generator ,(caddr e) ,@(cdddr e))))))) (define (lower-comprehension atype expr ranges) - (if (any (lambda (x) (eq? x ':)) ranges) - (lower-nd-comprehension atype expr ranges) (let ((result (make-ssavalue)) (ri (gensy)) - (initlabl (if atype #f (make-ssavalue))) (oneresult (make-ssavalue)) (lengths (map (lambda (x) (make-ssavalue)) ranges)) (states (map (lambda (x) (gensy)) ranges)) @@ -2063,7 +2014,6 @@ (define (construct-loops ranges rv is states lengths) (if (null? ranges) `(block (= ,oneresult ,expr) - ,@(if atype '() `((type_goto ,initlabl ,oneresult))) (inbounds true) (call (top setindex!) ,result ,oneresult ,ri) (inbounds pop) @@ -2089,79 +2039,10 @@ ,.(map (lambda (v r) `(= ,v (call (top length) ,r))) lengths rv) (scope-block (block - ,@(if atype '() `((label ,initlabl))) - (= ,result (call (core Array) - ,(if atype atype `(static_typeof ,oneresult)) - ,@lengths)) + (= ,result (call (core Array) ,atype ,@lengths)) (= ,ri 1) ,(construct-loops (reverse ranges) (reverse rv) is states (reverse lengths)) - ,result)))))) - -(define (lower-dict-comprehension expr ranges) - (let ((result (make-ssavalue)) - (initlabl (make-ssavalue)) - (onekey (make-ssavalue)) - (oneval (make-ssavalue)) - (rv (map (lambda (x) (make-ssavalue)) ranges))) - - ;; construct loops to cycle over all dimensions of an n-d comprehension - (define (construct-loops ranges) - (if (null? ranges) - `(block (= ,onekey ,(cadr expr)) - (= ,oneval ,(caddr expr)) - (type_goto ,initlabl ,onekey ,oneval) - (call (top setindex!) ,result ,oneval ,onekey)) - `(for ,(car ranges) - (block - ;; *** either this or force all for loop vars local - ,.(map (lambda (r) `(local ,r)) - (lhs-vars (cadr (car ranges)))) - ,(construct-loops (cdr ranges)))))) - - ;; Evaluate the comprehension - (let ((loopranges - (map (lambda (r v) `(= ,(cadr r) ,v)) ranges rv))) - `(block - ,.(map (lambda (v r) `(= ,v ,(caddr r))) rv ranges) - (scope-block - (block - #;,@(map (lambda (r) `(local ,r)) - (apply append (map (lambda (r) (lhs-vars (cadr r))) ranges))) - (label ,initlabl) - (= ,result (call (curly (top Dict) - (static_typeof ,onekey) - (static_typeof ,oneval)))) - ,(construct-loops (reverse loopranges)) - ,result)))))) - -(define (lower-typed-dict-comprehension atypes expr ranges) - (if (not (and (length= atypes 3) - (eq? (car atypes) '=>))) - (error "invalid \"typed_dict_comprehension\" syntax") - (let ( (result (make-ssavalue)) - (rs (map (lambda (x) (make-ssavalue)) ranges)) ) - - ;; construct loops to cycle over all dimensions of an n-d comprehension - (define (construct-loops ranges rs) - (if (null? ranges) - `(call (top setindex!) ,result ,(caddr expr) ,(cadr expr)) - `(for (= ,(cadr (car ranges)) ,(car rs)) - (block - ;; *** either this or force all for loop vars local - ,.(map (lambda (r) `(local ,r)) - (lhs-vars (cadr (car ranges)))) - ,(construct-loops (cdr ranges) (cdr rs)))))) - - ;; Evaluate the comprehension - `(block - ,.(map make-assignment rs (map caddr ranges)) - (= ,result (call (curly (top Dict) ,(cadr atypes) ,(caddr atypes)))) - (scope-block - (block - #;,@(map (lambda (r) `(local ,r)) - (apply append (map (lambda (r) (lhs-vars (cadr r))) ranges))) - ,(construct-loops (reverse ranges) (reverse rs)) - ,result)))))) + ,result))))) (define (lhs-vars e) (cond ((symbol? e) (list e)) @@ -3141,17 +3022,6 @@ f(x) = yt(x) (cons (list code handler-level (cadr e)) handler-goto-fixups)) '(null))) - ((type_goto) - (let ((m (get label-map (cadr e) #f))) - (if m - (emit `(type_goto ,m ,@(cddr e))) - (let ((l (make-label))) - (put! label-map (cadr e) l) - (emit `(type_goto ,l ,@(cddr e))))))) - ((static_typeof) - (assert (and value (not tail))) - e) - ;; exception handlers are lowered using ;; (enter L) - push handler with catch block at label L ;; (leave n) - pop N exception handlers diff --git a/src/julia.h b/src/julia.h index 61cfceada5f39..9c23a3eaf3ca3 100644 --- a/src/julia.h +++ b/src/julia.h @@ -574,11 +574,11 @@ extern jl_sym_t *body_sym; extern jl_sym_t *method_sym; extern jl_sym_t *slot_sym; extern jl_sym_t *enter_sym; extern jl_sym_t *leave_sym; extern jl_sym_t *exc_sym; extern jl_sym_t *new_sym; -extern jl_sym_t *static_typeof_sym; extern jl_sym_t *compiler_temp_sym; +extern jl_sym_t *compiler_temp_sym; extern jl_sym_t *const_sym; extern jl_sym_t *thunk_sym; extern jl_sym_t *anonymous_sym; extern jl_sym_t *underscore_sym; extern jl_sym_t *abstracttype_sym; extern jl_sym_t *bitstype_sym; -extern jl_sym_t *compositetype_sym; extern jl_sym_t *type_goto_sym; +extern jl_sym_t *compositetype_sym; extern jl_sym_t *global_sym; extern jl_sym_t *unused_sym; extern jl_sym_t *boundscheck_sym; extern jl_sym_t *inbounds_sym; extern jl_sym_t *copyast_sym; extern jl_sym_t *fastmath_sym; diff --git a/src/toplevel.c b/src/toplevel.c index 4ea1cb8972a8c..23db71a5e0fb6 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -269,8 +269,6 @@ int jl_has_intrinsics(jl_lambda_info_t *li, jl_value_t *v, jl_module_t *m) jl_expr_t *e = (jl_expr_t*)v; if (jl_array_len(e->args) == 0) return 0; - if (e->head == static_typeof_sym) - return 1; if (e->head == toplevel_sym || e->head == copyast_sym) return 0; jl_value_t *e0 = jl_exprarg(e, 0); diff --git a/test/arrayops.jl b/test/arrayops.jl index c7e0356e00b37..29377c403b5a2 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -983,7 +983,7 @@ X = [ i+2j for i=1:5, j=1:5 ] @test X[2,3] == 8 @test X[4,5] == 14 @test isequal(ones(2,3) * ones(2,3)', [3. 3.; 3. 3.]) -@test isequal([ [1,2] for i=1:2, : ], [1 2; 1 2]) +# @test isequal([ [1,2] for i=1:2, : ], [1 2; 1 2]) # where element type is a Union. try to confuse type inference. foo32_64(x) = (x<2) ? Int32(x) : Int64(x) boo32_64() = [ foo32_64(i) for i=1:2 ] diff --git a/test/dict.jl b/test/dict.jl index c0cde55fcee63..45dcbbe998c8d 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -98,7 +98,7 @@ let end _d = Dict("a"=>0) -@test isa([k for k in filter(x->length(x)==1, collect(keys(_d)))], Vector{Any}) +@test isa([k for k in filter(x->length(x)==1, collect(keys(_d)))], Vector{String}) let d = Dict(((1, 2), (3, 4))) From ebcee203fb8e90399059330e2745a9640f39bc36 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 13 May 2016 12:18:08 -0400 Subject: [PATCH 0295/1117] make `size` of a product iterator concatenate the sizes of its components this removes an extra layer of wrapping with `IteratorND` add some inline declarations for `Generator` iteration remove unused `IteratorND` type --- base/generator.jl | 5 +++-- base/iterator.jl | 34 ---------------------------------- src/julia-syntax.scm | 2 +- test/functional.jl | 15 ++++----------- 4 files changed, 8 insertions(+), 48 deletions(-) diff --git a/base/generator.jl b/base/generator.jl index b3ef8523a706e..732aefca93903 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -17,9 +17,10 @@ Generator(f, I1, I2, Is...) = Generator(a->f(a...), zip(I1, I2, Is...)) Generator{T,I}(::Type{T}, iter::I) = Generator{I,Type{T}}(T, iter) -start(g::Generator) = start(g.iter) -done(g::Generator, s) = done(g.iter, s) +start(g::Generator) = (@_inline_meta; start(g.iter)) +done(g::Generator, s) = (@_inline_meta; done(g.iter, s)) function next(g::Generator, s) + @_inline_meta v, s2 = next(g.iter, s) g.f(v), s2 end diff --git a/base/iterator.jl b/base/iterator.jl index 843ecc7e4dc0a..56c97b7348618 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -421,40 +421,6 @@ prod_iteratorsize(::IsInfinite, b) = IsInfinite() prod_iteratorsize(a, b) = SizeUnknown() -""" - IteratorND(iter, dims) - -Given an iterator `iter` and dimensions tuple `dims`, return an iterator that -yields the same values as `iter`, but with the specified multi-dimensional shape. -For example, this determines the shape of the array returned when `collect` is -applied to this iterator. -""" -immutable IteratorND{I,N} - iter::I - dims::NTuple{N,Int} - - function (::Type{IteratorND}){I,N}(iter::I, shape::NTuple{N,Integer}) - li = length(iter) - if li != prod(shape) - throw(DimensionMismatch("dimensions $shape must be consistent with iterator length $li")) - end - new{I,N}(iter, shape) - end - (::Type{IteratorND}){I<:AbstractProdIterator}(p::I) = IteratorND(p, size(p)) -end - -start(i::IteratorND) = start(i.iter) -done(i::IteratorND, s) = done(i.iter, s) -next(i::IteratorND, s) = next(i.iter, s) - -size(i::IteratorND) = i.dims -length(i::IteratorND) = prod(size(i)) -ndims{I,N}(::IteratorND{I,N}) = N -iteratorsize{T<:IteratorND}(::Type{T}) = HasShape() - -eltype{I}(::IteratorND{I}) = eltype(I) -iteratoreltype{I}(::Type{IteratorND{I}}) = iteratoreltype(I) - # flatten an iterator of iterators immutable Flatten{I} diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 9640baef3e3ec..0018ecfd5c2cf 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1977,7 +1977,7 @@ `(call (top Generator) (-> ,argname (block ,@splat ,expr)) ,(if (length= ranges 1) (car ranges) - `(call (top IteratorND) (call (top product) ,@ranges)))))))) + `(call (top product) ,@ranges))))))) 'comprehension (lambda (e) diff --git a/test/functional.jl b/test/functional.jl index 5042a81507bdc..e8f890f7ede7c 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -361,7 +361,10 @@ end @test Base.iteratoreltype(Base.product(take(1:2, 2))) == Base.HasEltype() @test Base.iteratoreltype(Base.product([1 2; 3 4])) == Base.HasEltype() - +@test collect(Base.product(1:2,3:4)) == [(1,3) (1,4); (2,3) (2,4)] +@test isempty(collect(Base.product(1:0,1:2))) +@test length(Base.product(1:2,1:10,4:6)) == 60 +@test Base.iteratorsize(Base.product(1:2, countfrom(1))) == Base.IsInfinite() # flatten # ------- @@ -398,14 +401,6 @@ end @test collect((i+10j for i=1:2,j=3:4)) == [31 41; 32 42] @test collect((i+10j for i=1:2,j=3:4,k=1:1)) == reshape([31 41; 32 42], (2,2,1)) -let I = Base.IteratorND(1:27,(3,3,3)) - @test collect(I) == reshape(1:27,(3,3,3)) - @test size(I) == (3,3,3) - @test length(I) == 27 - @test eltype(I) === Int - @test ndims(I) == 3 -end - let A = collect(Base.Generator(x->2x, Real[1.5,2.5])) @test A == [3,5] @test isa(A,Vector{Float64}) @@ -415,8 +410,6 @@ let f(g) = (@test size(g.iter)==(2,3)) f(i+j for i=1:2, j=3:5) end -@test_throws DimensionMismatch Base.IteratorND(1:2, (2,3)) - @test collect(Base.Generator(+, [1,2], [10,20])) == [11,22] # generator ndims #16394 From a46c065acc57363959538cfa76503efe974d45a2 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 27 May 2016 18:30:52 -0400 Subject: [PATCH 0296/1117] use inferred type for empty comprehensions if it is a leaf type this implements the "Steve Johnson compromise": - non-empty comprehensions don't depend on inference at all - in the empty case, we use either Union{} or the inferred type - therefore there is no regression in type precision in cases where we can infer a leaf type add Core.Inference.return_type This computes a return type (at compile time if possible) in a way that handles recursive dependencies in inference. --- base/array.jl | 15 +++++++++++---- base/inference.jl | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/base/array.jl b/base/array.jl index 4fc82d644a089..81e0cc2866fc5 100644 --- a/base/array.jl +++ b/base/array.jl @@ -244,16 +244,23 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown) return a end -_default_eltype(itr::ANY) = Union{} -_default_eltype{I,T}(::Generator{I,Type{T}}) = T +if isdefined(Core, :Inference) + function _default_eltype(itrt::ANY) + rt = Core.Inference.return_type(first, Tuple{itrt}) + return isleaftype(rt) ? rt : Union{} + end +else + _default_eltype(itr::ANY) = Union{} +end +_default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T _collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) = - grow_to!(_similar_for(c, _default_eltype(itr), itr, isz), itr) + grow_to!(_similar_for(c, _default_eltype(typeof(itr)), itr, isz), itr) function _collect(c, itr, ::EltypeUnknown, isz::Union{HasLength,HasShape}) st = start(itr) if done(itr,st) - return _similar_for(c, _default_eltype(itr), itr, isz) + return _similar_for(c, _default_eltype(typeof(itr)), itr, isz) end v1, st = next(itr, st) collect_to_with_first!(_similar_for(c, typeof(v1), itr, isz), v1, itr, st) diff --git a/base/inference.jl b/base/inference.jl index 560d295d2084f..cfa9ef1094b54 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -923,13 +923,32 @@ function abstract_apply(af::ANY, fargs, aargtypes::Vector{Any}, vtypes::VarTable return abstract_call(af, (), Any[type_typeof(af), Vararg{Any}], vtypes, sv) end -function pure_eval_call(f::ANY, argtypes::ANY, atype, sv) +function pure_eval_call(f::ANY, argtypes::ANY, atype, vtypes, sv) for a in drop(argtypes,1) if !(isa(a,Const) || (isType(a) && !has_typevars(a.parameters[1]))) return false end end + if f === return_type && length(argtypes) == 3 + tt = argtypes[3] + if isType(tt) + af_argtype = tt.parameters[1] + if af_argtype <: Tuple && isa(af_argtype, DataType) + af = argtypes[2] + rt = abstract_call(isa(af,Const) ? af.val : af.parameters[1], + (), Any[argtypes[2], af_argtype.parameters...], vtypes, sv) + if isa(rt,Const) + return Type{widenconst(rt)} + elseif isleaftype(rt) || isleaftype(af_argtype) || rt === Bottom + return Type{rt} + else + return Type{TypeVar(:R, rt)} + end + end + end + end + meth = _methods_by_ftype(atype, 1) if meth === false || length(meth) != 1 return false @@ -1008,7 +1027,7 @@ function abstract_call(f::ANY, fargs, argtypes::Vector{Any}, vtypes::VarTable, s end atype = argtypes_to_type(argtypes) - t = pure_eval_call(f, argtypes, atype, sv) + t = pure_eval_call(f, argtypes, atype, vtypes, sv) t !== false && return t if istopfunction(tm, f, :promote_type) || istopfunction(tm, f, :typejoin) @@ -2449,7 +2468,8 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference methsp = meth[2] method = meth[3]::Method # check whether call can be inlined to just a quoted constant value - if isa(f, widenconst(ft)) && !method.isstaged && method.lambda_template.pure && (isType(e.typ) || isa(e.typ,Const)) + if isa(f, widenconst(ft)) && !method.isstaged && (method.lambda_template.pure || f === return_type) && + (isType(e.typ) || isa(e.typ,Const)) if isType(e.typ) if !has_typevars(e.typ.parameters[1]) return inline_as_constant(e.typ.parameters[1], argexprs, sv) @@ -3567,6 +3587,16 @@ function reindex_labels!(linfo::LambdaInfo, sv::InferenceState) end end +function return_type(f::ANY, t::ANY) + rt = Union{} + for m in _methods(f, t, -1) + _, ty, inferred = typeinf(m[3], m[1], m[2], false) + !inferred && return Any + rt = tmerge(rt, ty) + rt === Any && break + end + return rt +end #### bootstrapping #### From fb724cf9614b7f5e2730469387f07f21430fa1a9 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 21 Jun 2016 18:23:32 -0400 Subject: [PATCH 0297/1117] do some eta reduction (use `f` instead of `x->f(x)`) in Generator syntax --- src/julia-syntax.scm | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 0018ecfd5c2cf..fa273a8fbc50a 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1974,7 +1974,13 @@ `(,@(map (lambda (v) `(local ,v)) (lhs-vars `(tuple ,@vars))) (= (tuple ,@vars) ,argname)))))) (expand-forms - `(call (top Generator) (-> ,argname (block ,@splat ,expr)) + `(call (top Generator) + ,(if (and (null? splat) + (length= expr 3) (eq? (car expr) 'call) + (eq? (caddr expr) argname) + (not (expr-contains-eq argname (cadr expr)))) + (cadr expr) ;; eta reduce `x->f(x)` => `f` + `(-> ,argname (block ,@splat ,expr))) ,(if (length= ranges 1) (car ranges) `(call (top product) ,@ranges))))))) From 6670241604b2d3c89dc6dc09149d6ac12753a259 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 22 Jun 2016 01:09:09 -0400 Subject: [PATCH 0298/1117] take advantage of shape-preserving comprehensions in vectorize macros --- base/operators.jl | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/base/operators.jl b/base/operators.jl index 7dc3aa539ab23..27520dcb0c01b 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -528,48 +528,36 @@ end macro vectorize_1arg(S,f) S = esc(S); f = esc(f); T = esc(:T) quote - ($f){$T<:$S}(x::AbstractArray{$T,1}) = [ ($f)(elem) for elem in x ] - ($f){$T<:$S}(x::AbstractArray{$T,2}) = - [ ($f)(x[i,j]) for i=1:size(x,1), j=1:size(x,2) ] - ($f){$T<:$S}(x::AbstractArray{$T}) = - reshape([ ($f)(y) for y in x ], size(x)) + ($f){$T<:$S}(x::AbstractArray{$T}) = [ ($f)(elem) for elem in x ] end end macro vectorize_2arg(S,f) S = esc(S); f = esc(f); T1 = esc(:T1); T2 = esc(:T2) quote - ($f){$T1<:$S, $T2<:$S}(x::($T1), y::AbstractArray{$T2}) = - reshape([ ($f)(x, z) for z in y ], size(y)) - ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::($T2)) = - reshape([ ($f)(z, y) for z in x ], size(x)) - - function ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::AbstractArray{$T2}) - shp = promote_shape(size(x),size(y)) - reshape([ ($f)(xx, yy) for (xx, yy) in zip(x, y) ], shp) - end + ($f){$T1<:$S, $T2<:$S}(x::($T1), y::AbstractArray{$T2}) = [ ($f)(x, z) for z in y ] + ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::($T2)) = [ ($f)(z, y) for z in x ] + ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::AbstractArray{$T2}) = + [ ($f)(xx, yy) for (xx, yy) in zip(x, y) ] end end # vectorized ifelse function ifelse(c::AbstractArray{Bool}, x, y) - reshape([ifelse(ci, x, y) for ci in c], size(c)) + [ifelse(ci, x, y) for ci in c] end function ifelse(c::AbstractArray{Bool}, x::AbstractArray, y::AbstractArray) - shp = promote_shape(size(c), promote_shape(size(x), size(y))) - reshape([ifelse(c_elem, x_elem, y_elem) for (c_elem, x_elem, y_elem) in zip(c, x, y)], shp) + [ifelse(c_elem, x_elem, y_elem) for (c_elem, x_elem, y_elem) in zip(c, x, y)] end function ifelse(c::AbstractArray{Bool}, x::AbstractArray, y) - shp = promote_shape(size(c), size(c)) - reshape([ifelse(c_elem, x_elem, y) for (c_elem, x_elem) in zip(c, x)], shp) + [ifelse(c_elem, x_elem, y) for (c_elem, x_elem) in zip(c, x)] end function ifelse(c::AbstractArray{Bool}, x, y::AbstractArray) - shp = promote_shape(size(c), size(y)) - reshape([ifelse(c_elem, x, y_elem) for (c_elem, y_elem) in zip(c, y)], shp) + [ifelse(c_elem, x, y_elem) for (c_elem, y_elem) in zip(c, y)] end # Pair From db4da320299aeb629c3ec7c1d6aaf07286c6a342 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 27 Jun 2016 15:46:09 -0400 Subject: [PATCH 0299/1117] add comprehension filters (fixes #550) and nested loops (fixes #4867) this also uses the new lowering for typed comprehensions, allowing all comprehensions on unknown-length iterables (fixes #1457) --- src/julia-parser.scm | 20 +++++++--- src/julia-syntax.scm | 90 +++++++++++++++++++++++++++++--------------- 2 files changed, 73 insertions(+), 37 deletions(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 288426eebdb65..bc4c96c98e7c9 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1516,22 +1516,30 @@ (else (error "missing separator in array expression"))))))) +(define (parse-generator s first) + (let ((iters (parse-comma-separated-iters s))) + (let ((iters (if (eq? (peek-token s) 'if) + (begin (take-token s) + (list `(filter ,(parse-cond s) ,@iters))) + iters))) + (if (eq? (peek-token s) 'for) + (begin (take-token s) + `(flatten (generator ,(parse-generator s first) ,@iters))) + `(generator ,first ,@iters))))) + (define (parse-comprehension s first closer) - (let ((r (parse-comma-separated-iters s))) + (let ((gen (parse-generator s first))) (if (not (eqv? (require-token s) closer)) (error (string "expected \"" closer "\"")) (take-token s)) - `(comprehension ,first ,@r))) + `(comprehension ,gen))) (define (parse-dict-comprehension s first closer) (let ((c (parse-comprehension s first closer))) - (if (dict-literal? (cadr c)) + (if (dict-literal? (cadr (cadr c))) `(dict_comprehension ,@(cdr c)) (error "invalid dict comprehension")))) -(define (parse-generator s first) - `(generator ,first ,@(parse-comma-separated-iters s))) - (define (parse-matrix s first closer gotnewline) (define (fix head v) (cons head (reverse v))) (define (update-outer v outer) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index fa273a8fbc50a..aad862acd72b7 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1526,6 +1526,26 @@ (define (syntactic-op-to-call e) `(call ,(car e) ,(expand-forms (cadr e)) ,(expand-forms (caddr e)))) +;; wrap `expr` in a function appropriate for consuming values from given ranges +(define (func-for-generator-ranges expr range-exprs) + (let* ((vars (map cadr range-exprs)) + (argname (if (and (length= vars 1) (symbol? (car vars))) + (car vars) + (gensy))) + (splat (cond ((eq? argname (car vars)) '()) + ((length= vars 1) + `(,@(map (lambda (v) `(local ,v)) (lhs-vars (car vars))) + (= ,(car vars) ,argname))) + (else + `(,@(map (lambda (v) `(local ,v)) (lhs-vars `(tuple ,@vars))) + (= (tuple ,@vars) ,argname)))))) + (if (and (null? splat) + (length= expr 3) (eq? (car expr) 'call) + (eq? (caddr expr) argname) + (not (expr-contains-eq argname (cadr expr)))) + (cadr expr) ;; eta reduce `x->f(x)` => `f` + `(-> ,argname (block ,@splat ,expr))))) + ;; table mapping expression head to a function expanding that form (define expand-table (table @@ -1960,52 +1980,60 @@ 'generator (lambda (e) - (let ((expr (cadr e)) - (vars (map cadr (cddr e))) - (ranges (map caddr (cddr e)))) - (let* ((argname (if (and (length= vars 1) (symbol? (car vars))) - (car vars) - (gensy))) - (splat (cond ((eq? argname (car vars)) '()) - ((length= vars 1) - `(,@(map (lambda (v) `(local ,v)) (lhs-vars (car vars))) - (= ,(car vars) ,argname))) - (else - `(,@(map (lambda (v) `(local ,v)) (lhs-vars `(tuple ,@vars))) - (= (tuple ,@vars) ,argname)))))) - (expand-forms - `(call (top Generator) - ,(if (and (null? splat) - (length= expr 3) (eq? (car expr) 'call) - (eq? (caddr expr) argname) - (not (expr-contains-eq argname (cadr expr)))) - (cadr expr) ;; eta reduce `x->f(x)` => `f` - `(-> ,argname (block ,@splat ,expr))) - ,(if (length= ranges 1) + (let* ((expr (cadr e)) + (filt? (eq? (car (caddr e)) 'filter)) + (range-exprs (if filt? (cddr (caddr e)) (cddr e))) + (ranges (map caddr range-exprs)) + (iter (if (length= ranges 1) (car ranges) - `(call (top product) ,@ranges))))))) + `(call (top product) ,@ranges))) + (iter (if filt? + `(call (top Filter) + ,(func-for-generator-ranges (cadr (caddr e)) range-exprs) + ,iter) + iter))) + (expand-forms + `(call (top Generator) + ,(func-for-generator-ranges expr range-exprs) + ,iter)))) + + 'flatten + (lambda (e) `(call (top Flatten) ,(expand-forms (cadr e)))) 'comprehension (lambda (e) - (if (any (lambda (x) (eq? x ':)) (cddr e)) - (error "comprehension syntax with `:` ranges has been removed")) - (expand-forms `(call (top collect) (generator ,(cadr e) ,@(cddr e))))) + (if (length> e 2) + ;; backwards compat for macros that generate :comprehension exprs + (expand-forms `(comprehension (generator ,@(cdr e)))) + (begin (if (and (eq? (caadr e) 'generator) + (any (lambda (x) (eq? x ':)) (cddr (cadr e)))) + (error "comprehension syntax with `:` ranges has been removed")) + (expand-forms `(call (top collect) ,(cadr e)))))) 'typed_comprehension (lambda (e) - (if (any (lambda (x) (eq? x ':)) (cdddr e)) - (error "comprehension syntax with `:` ranges has been removed")) - (expand-forms (lower-comprehension (cadr e) (caddr e) (cdddr e)))) + (expand-forms + (or (and (eq? (caaddr e) 'generator) + (let ((ranges (cddr (caddr e)))) + (if (any (lambda (x) (eq? x ':)) ranges) + (error "comprehension syntax with `:` ranges has been removed")) + (and (every (lambda (x) (and (pair? x) (eq? (car x) '=) + (pair? (caddr x)) (eq? (car (caddr x)) ':))) + ranges) + ;; TODO: this is a hack to lower simple comprehensions to loops very + ;; early, to greatly reduce the # of functions and load on the compiler + (lower-comprehension (cadr e) (cadr (caddr e)) ranges)))) + `(call (top collect) ,(cadr e) ,(caddr e))))) 'dict_comprehension (lambda (e) (syntax-deprecation #f "[a=>b for (a,b) in c]" "Dict(a=>b for (a,b) in c)") - (expand-forms `(call (top Dict) (generator ,(cadr e) ,@(cddr e))))) + (expand-forms `(call (top Dict) ,(cadr e)))) 'typed_dict_comprehension (lambda (e) (expand-forms `(call (call (core apply_type) (top Dict) ,@(cdr (cadr e))) - (generator ,(caddr e) ,@(cdddr e))))))) + ,(caddr e)))))) (define (lower-comprehension atype expr ranges) (let ((result (make-ssavalue)) From d088d54637ce6e0cecc8f79f942ac4f9894285b4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 23 Jun 2016 21:25:15 -0400 Subject: [PATCH 0300/1117] doc/NEWS updates for comprehension changes --- NEWS.md | 10 +++++++ doc/manual/arrays.rst | 40 +++++++++++++++++++++------- doc/manual/variables-and-scoping.rst | 6 ++--- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/NEWS.md b/NEWS.md index d332f885826bc..ed0d84c1a1d73 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,9 @@ New language features * Generator expressions, e.g. `f(i) for i in 1:n` ([#4470]). This returns an iterator that computes the specified values on demand. + * Generators and comprehensions support filtering using `if` ([#550]) and nested + iteration using multiple `for` keywords ([#4867]). + * Broadcasting syntax: ``f.(args...)`` is equivalent to ``broadcast(f, args...)`` ([#15032]). * Macro expander functions are now generic, so macros can have multiple definitions @@ -69,6 +72,13 @@ Language changes * The built-in `NTuple` type has been removed; `NTuple{N,T}` is now implemented internally as `Tuple{Vararg{T,N}}` ([#11242]). + * Array comprehensions preserve the dimensions of the input ranges. For example, + `[ 2x for x in A]` will have the same dimensions as `A`. + + * The result type of an array comprehension depends only on the types of elements + computed, instead of using type inference ([#7258]). If the result is empty, then + type inference is still used to determine the element type. + Command-line option changes --------------------------- diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 007483ae3c8db..84207da57dbb7 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -185,7 +185,7 @@ and its left and right neighbor along a 1-d grid. : .. doctest:: array-rand - julia> const x = rand(8) + julia> x = rand(8) 8-element Array{Float64,1}: 0.843025 0.869052 @@ -205,16 +205,11 @@ and its left and right neighbor along a 1-d grid. : 0.8446 0.656511 -.. note:: In the above example, ``x`` is declared as constant because type - inference in Julia does not work as well on non-constant global - variables. +The resulting array type depends on the types of the computed elements. +In order to control the type explicitly, a type can be prepended to the comprehension. +For example, we could have requested the result in single precision by writing:: -The resulting array type is inferred from the expression; in order to control -the type explicitly, the type can be prepended to the comprehension. For example, -in the above example we could have avoided declaring ``x`` as constant, and ensured -that the result is of type ``Float64`` by writing:: - - Float64[ 0.25*x[i-1] + 0.5*x[i] + 0.25*x[i+1] for i=2:length(x)-1 ] + Float32[ 0.25*x[i-1] + 0.5*x[i] + 0.25*x[i+1] for i=2:length(x)-1 ] .. _man-generator-expressions: @@ -248,6 +243,31 @@ parentheses lets us add a third argument to ``map``: (0.5,1) (0.333333,3) (0.333333,2) (0.25,4) +Ranges in generators and comprehensions can depend on previous ranges by writing +multiple ``for`` keywords: + +.. doctest:: + + julia> [(i,j) for i=1:3 for j=1:i] + 6-element Array{Tuple{Int64,Int64},1}: + (1,1) + (2,1) + (2,2) + (3,1) + (3,2) + (3,3) + +In such cases, the result is always 1-d. + +Generated values can be filtered using the ``if`` keyword: + +.. doctest:: + + julia> [(i,j) for i=1:3 for j=1:i if i+j == 4] + 2-element Array{Tuple{Int64,Int64},1}: + (2,2) + (3,1) + .. _man-array-indexing: Indexing diff --git a/doc/manual/variables-and-scoping.rst b/doc/manual/variables-and-scoping.rst index 7840eb03ae636..157110fcec7ba 100644 --- a/doc/manual/variables-and-scoping.rst +++ b/doc/manual/variables-and-scoping.rst @@ -31,7 +31,7 @@ introducing scope blocks are: +================================+==================================================================================+ | :ref:`global <man-global>` | module, baremodule, at interactive prompt (REPL) | +--------------------------------+------------------------------+---------------------------------------------------+ -| :ref:`local <man-local-scope>` | :ref:`soft <man-soft-scope>` | for, while, list-comprehensions, | +| :ref:`local <man-local-scope>` | :ref:`soft <man-soft-scope>` | for, while, comprehensions, | | | | try-catch-finally, let | | +------------------------------+---------------------------------------------------+ | | :ref:`hard <man-hard-scope>` | functions (either syntax, anonymous & do-blocks), | @@ -188,9 +188,9 @@ Soft Local Scope ``local``. Soft local scopes are introduced by for-loops, while-loops, -list-comprehensions, try-catch-finally-blocks, and let-blocks. There +comprehensions, try-catch-finally-blocks, and let-blocks. There are some extra rules for :ref:`let-blocks <man-let-blocks>` and for -:ref:`for-loops and list-comprehensions <man-for-loops-scope>`. +:ref:`for-loops and comprehensions <man-for-loops-scope>`. In the following example the ``x`` and ``y`` refer always to the same variables as the soft local scope inherits both read and write From 0d4c6a1b6eb3c7d594eba28e1fd4cd553aa9581b Mon Sep 17 00:00:00 2001 From: Jacob Quinn <quinn.jacobd@gmail.com> Date: Fri, 24 Jun 2016 21:13:24 -0400 Subject: [PATCH 0301/1117] Tests and documentation for generators with guards and nesting. --- base/generator.jl | 5 +++-- test/functional.jl | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/base/generator.jl b/base/generator.jl index 732aefca93903..75625c2c40525 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -5,8 +5,9 @@ Given a function `f` and an iterator `iter`, construct an iterator that yields the values of `f` applied to the elements of `iter`. -The syntax `f(x) for x in iter` is syntax for constructing an instance of this -type. +The syntax `f(x) [if cond(x)::Bool] for x in iter` is syntax for constructing an instance of this +type. The `[if cond(x)::Bool]` expression is optional and acts as a "guard", effectively +filtering out values where the condition is false. """ immutable Generator{I,F} f::F diff --git a/test/functional.jl b/test/functional.jl index e8f890f7ede7c..b495ddb715847 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -437,6 +437,43 @@ let i = 1 @test i == 1 end +# generators and guards + +let gen = (x for x in 1:10) + @test gen.iter == 1:10 + @test gen.f(first(1:10)) == next(gen, start(gen))[1] + for (a,b) in zip(1:10,gen) + @test a == b + end +end + +let gen = (x * y for x in 1:10, y in 1:10) + @test collect(gen) == collect(1:10) .* collect(1:10)' + @test first(gen) == 1 + @test collect(gen)[1:10] == collect(1:10) +end + +let gen = Base.Generator(+, 1:10, 1:10, 1:10) + @test first(gen) == 3 + @test collect(gen) == collect(3:3:30) +end + +let gen = (x for x in 1:10 if x % 2 == 0), gen2 = Filter(x->x % 2 == 0, x for x in 1:10) + @test collect(gen) == collect(gen2) + @test collect(gen) == collect(2:2:10) +end + +let gen = ((x,y) for x in 1:10, y in 1:10 if x % 2 == 0 && y % 2 == 0), + gen2 = Filter(x->x[1] % 2 == 0 && x[2] % 2 == 0, (x,y) for x in 1:10, y in 1:10) + @test collect(gen) == collect(gen2) +end + +# generators with nested loops (#4867) + +@test [(i,j) for i=1:3 for j=1:i] == [(1,1), (2,1), (2,2), (3,1), (3,2), (3,3)] + +@test [(i,j) for i=1:3 for j=1:i if j>1] == [(2,2), (3,2), (3,3)] + # partition(c, n) let v = collect(Base.partition([1,2,3,4,5], 1)) @test all(i->v[i][1] == i, v) From 87bf281157e11c5b945d92f928163f5ec7a6a8bd Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 6 Jul 2016 00:20:20 -0400 Subject: [PATCH 0302/1117] shorten call chain for `collect(::Generator)` --- base/array.jl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/base/array.jl b/base/array.jl index 81e0cc2866fc5..2a19d201cabda 100644 --- a/base/array.jl +++ b/base/array.jl @@ -254,6 +254,24 @@ else end _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T +_array_for(T, itr, ::HasLength) = Array(T, Int(length(itr)::Integer)) +_array_for(T, itr, ::HasShape) = Array(T, convert(Dims,size(itr))) + +function collect(itr::Generator) + isz = iteratorsize(itr.iter) + et = _default_eltype(typeof(itr)) + if isa(isz, SizeUnknown) + return grow_to!(Array(et, 0), itr) + else + st = start(itr) + if done(itr,st) + return _array_for(et, itr.iter, isz) + end + v1, st = next(itr, st) + collect_to_with_first!(_array_for(typeof(v1), itr.iter, isz), v1, itr, st) + end +end + _collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) = grow_to!(_similar_for(c, _default_eltype(typeof(itr)), itr, isz), itr) From 30a356d85fa8f86df0161e7442768702a2fa2701 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Fri, 8 Jul 2016 11:26:21 -0500 Subject: [PATCH 0303/1117] Take advantage of inline-savvy backtraces in profile printing --- base/profile.jl | 63 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/base/profile.jl b/base/profile.jl index f18aaa07962f2..66a5e0f2f2ce7 100644 --- a/base/profile.jl +++ b/base/profile.jl @@ -121,11 +121,50 @@ end function getdict(data::Vector{UInt}) uip = unique(data) - Dict{UInt, StackFrame}(map(uip) do ip - lk = lookup(ip) - # TODO handle inlined frames - ip => first(lk) - end) + Dict{UInt, Vector{StackFrame}}(ip=>lookup(ip) for ip in uip) +end + +""" + newdata, newdict = flatten(btdata, lidict) + +Produces "flattened" backtrace data. Individual instruction pointers +sometimes correspond to a multi-frame backtrace due to inlining; in +such cases, this function inserts fake instruction pointers for the +inlined calls, and returns a dictionary that is a 1-to-1 mapping +between instruction pointers and a single StackFrame. +""" +function flatten(data::Vector{UInt}, lidict::Dict{UInt,Vector{StackFrame}}) + # Makes fake instruction pointers, counting down from typemax(UInt) + newip = typemax(UInt) + taken = Set(keys(lidict)) # make sure we don't pick one that's already used + newdict = Dict{UInt,StackFrame}() + newmap = Dict{UInt,Vector{UInt}}() + for (ip, trace) in lidict + if length(trace) == 1 + newdict[ip] = trace[1] + else + newm = UInt[] + for sf in trace + while newip ∈ taken && newip > 0 + newip -= 1 + end + newip == 0 && error("all possible instruction pointers used") + push!(newm, newip) + newdict[newip] = sf + newip -= 1 + end + newmap[ip] = newm + end + end + newdata = UInt[] + for ip in data + if haskey(newmap, ip) + append!(newdata, newmap[ip]) + else + push!(newdata, ip) + end + end + newdata, newdict end """ @@ -253,7 +292,7 @@ function parse_flat(iplist, n, lidict, C::Bool) lilist, n end -function flat{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict, C::Bool, combine::Bool, cols::Integer, sortedby) +function flat{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,StackFrame}, C::Bool, combine::Bool, cols::Integer, sortedby) if !C data = purgeC(data, lidict) end @@ -266,6 +305,11 @@ function flat{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict, C::Bool, combi print_flat(io, lilist, n, combine, cols, sortedby) end +function flat{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,Vector{StackFrame}}, C::Bool, combine::Bool, cols::Integer, sortedby) + newdata, newdict = flatten(data, lidict) + flat(io, newdata, newdict, C, combine, cols, sortedby) +end + function print_flat(io::IO, lilist::Vector{StackFrame}, n::Vector{Int}, combine::Bool, cols::Integer, sortedby) p = liperm(lilist) lilist = lilist[p] @@ -476,7 +520,7 @@ function tree{T<:Unsigned}(io::IO, bt::Vector{Vector{T}}, counts::Vector{Int}, l end end -function tree{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict, C::Bool, combine::Bool, cols::Integer, maxdepth) +function tree{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,StackFrame}, C::Bool, combine::Bool, cols::Integer, maxdepth) if !C data = purgeC(data, lidict) end @@ -491,6 +535,11 @@ function tree{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict, C::Bool, combi tree(io, bt[keep], counts[keep], lidict, level, combine, cols, maxdepth) end +function tree{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,Vector{StackFrame}}, C::Bool, combine::Bool, cols::Integer, maxdepth) + newdata, newdict = flatten(data, lidict) + tree(io, newdata, newdict, C, combine, cols, maxdepth) +end + function callersf(matchfunc::Function, bt::Vector{UInt}, lidict) counts = Dict{StackFrame, Int}() lastmatched = false From 1b8d0a3175eed0f03919be4750d15f757bef5b80 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Fri, 8 Jul 2016 12:59:46 -0400 Subject: [PATCH 0304/1117] Fix compiler warnings --- src/jloptions.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jloptions.c b/src/jloptions.c index 78297e9d98df5..6265e100b3afe 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -91,7 +91,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_use_compilecache, opt_incremental }; - static const char const* shortopts = "+vhqFfH:e:E:P:L:J:C:ip:O:"; + static const char* const shortopts = "+vhqFfH:e:E:P:L:J:C:ip:O:"; static const struct option longopts[] = { // exposed command line options // NOTE: This set of required arguments need to be kept in sync @@ -165,7 +165,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) case '?': case ':': if (optopt) { - for (struct option *o = longopts; o->val; o++) { + for (const struct option *o = longopts; o->val; o++) { if (optopt == o->val) { if (o->has_arg == optional_argument) { c = o->val; From 664148295b82d33d56adf363ec9bc2e2ac949a87 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Fri, 8 Jul 2016 12:07:29 -0500 Subject: [PATCH 0305/1117] Remove redundant SubArray tests --- test/subarray.jl | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/test/subarray.jl b/test/subarray.jl index 5181bf4f6671b..0ca8b0b05c858 100644 --- a/test/subarray.jl +++ b/test/subarray.jl @@ -5,17 +5,16 @@ using Base.Test ######## Utilities ########### # Generate an array similar to A[indx1, indx2, ...], but only call -# getindex with scalar-valued indexes. This will be safe even after -# getindex starts calling sub/slice. +# getindex with scalar-valued indexes. This will be safe even if +# `getindex` someday calls `view`. -# The "nodrop" variant is similar to current getindex/sub, except it -# doesn't drop any dimensions (not even trailing ones) +# The "nodrop" variant does not drop any dimensions (not even trailing ones) function Agen_nodrop(A::AbstractArray, I...) irep = replace_colon(A, I) _Agen(A, irep...) end -# This does the same thing as slice +# This drops scalar dimensions function Agen_slice(A::AbstractArray, I...) irep = replace_colon(A, I) B = _Agen(A, irep...) @@ -179,9 +178,9 @@ function runsubarraytests(A::Array, I...) end function runsubarraytests(A::ANY, I...) - # When A was created with sub, we have to check bounds, since some + # When A was created with view, we have to check bounds, since some # of the "residual" dimensions have size 1. It's possible that we - # need dedicated tests for sub. + # need dedicated tests for view. for d = 1:length(I)-1 if !isa(I[d], Colon) && any(I[d] .> size(A,d)) return nothing @@ -203,7 +202,6 @@ function runsubarraytests(A::ANY, I...) Cdim += 1 end end - # sub local S try S = view(A, I...) @@ -216,18 +214,6 @@ function runsubarraytests(A::ANY, I...) test_linear(S, C) test_cartesian(S, C) test_mixed(S, C) - # slice - try - S = view(A, I...) - catch err - @show typeof(A) - @show A.indexes - @show I - rethrow(err) - end - test_linear(S, C) - test_cartesian(S, C) - test_mixed(S, C) end # indexN is a cartesian index, indexNN is a linear index for 2 dimensions, and indexNNN is a linear index for 3 dimensions @@ -324,7 +310,7 @@ x11289 = randn(5,5) ####### "Classical" tests ####### -# sub +# Tests where non-trailing dimensions are preserved A = copy(reshape(1:120, 3, 5, 8)) sA = view(A, 2:2, 1:5, :) @test strides(sA) == (1, 3, 15) @@ -372,7 +358,7 @@ sA = view(A, 1:2, 3, [1 3; 4 2]) @test ndims(sA) == 3 @test indices(sA) === (Base.OneTo(2), Base.OneTo(2), Base.OneTo(2)) -# sub logical indexing #4763 +# logical indexing #4763 A = view([1:10;], 5:8) @test A[A.<7] == [5, 6] @test Base.unsafe_getindex(A, A.<7) == [5, 6] @@ -381,7 +367,7 @@ sB = view(B, 2:3, 2:3) @test sB[sB.>8] == [10, 11] @test Base.unsafe_getindex(sB, sB.>8) == [10, 11] -# slice +# Tests where dimensions are dropped A = copy(reshape(1:120, 3, 5, 8)) sA = view(A, 2, :, 1:8) @test parent(sA) == A From 5af132b0c351036cddd03d2fcd29ef0bf5a5a9ef Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Fri, 8 Jul 2016 03:16:49 -0400 Subject: [PATCH 0306/1117] Symbolizer RIP-rel loads in the disassembly --- src/codegen.cpp | 2 +- src/codegen_internal.h | 1 + src/disasm.cpp | 134 +++++++++++++++++++++++++++++------------ 3 files changed, 98 insertions(+), 39 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index c09171f0ce4c7..55013ecdb224b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1395,7 +1395,7 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) #ifndef USE_MCJIT context, #endif - objcontext, + object, objcontext, #ifdef LLVM37 stream #else diff --git a/src/codegen_internal.h b/src/codegen_internal.h index 07388aa5dcb31..af172d8cc363a 100644 --- a/src/codegen_internal.h +++ b/src/codegen_internal.h @@ -12,6 +12,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #ifndef USE_MCJIT std::vector<JITEvent_EmittedFunctionDetails::LineStart> lineinfo, #endif + const object::ObjectFile *object, llvm::DIContext *context, #ifdef LLVM37 raw_ostream &rstream diff --git a/src/disasm.cpp b/src/disasm.cpp index 20392068d2380..a96710526c627 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -128,22 +128,23 @@ class SymbolTable { MCContext& Ctx; const FuncMCView &MemObj; int Pass; - int64_t slide; + const object::ObjectFile *object; uint64_t ip; // virtual instruction pointer of the current instruction + int64_t slide; public: - SymbolTable(MCContext &Ctx, intptr_t slide, const FuncMCView &MemObj): - Ctx(Ctx), MemObj(MemObj), slide(slide), ip(0) {} + SymbolTable(MCContext &Ctx, const object::ObjectFile *object, int64_t slide, const FuncMCView &MemObj): + Ctx(Ctx), MemObj(MemObj), object(object), ip(0), slide(slide) {} const FuncMCView &getMemoryObject() const { return MemObj; } void setPass(int Pass) { this->Pass = Pass; } int getPass() const { return Pass; } void insertAddress(uint64_t addr); // void createSymbol(const char *name, uint64_t addr); void createSymbols(); - const char *lookupSymbolName(uint64_t addr); + const char *lookupSymbolName(uint64_t addr, bool LocalOnly); MCSymbol *lookupSymbol(uint64_t addr); void setIP(uint64_t addr); uint64_t getIP() const; - int64_t getSlide() const; + StringRef getSymbolNameAt(uint64_t offset) const; }; void SymbolTable::setIP(uint64_t addr) { @@ -153,10 +154,56 @@ uint64_t SymbolTable::getIP() const { return ip; } -int64_t SymbolTable::getSlide() const +StringRef SymbolTable::getSymbolNameAt(uint64_t offset) const { - return slide; + if (object == NULL) return StringRef(); +#ifdef LLVM37 + object::section_iterator ESection = object->section_end(); + for (const object::SymbolRef &Sym : object->symbols()) { +#else + llvm::error_code err; + object::section_iterator ESection = object->end_sections(); + for (object::symbol_iterator I = object->begin_symbols(), E = object->end_symbols(); + !err && I != E; I.increment(err)) { + object::SymbolRef Sym = *I; +#endif + uint64_t Addr, SAddr; + object::section_iterator Sect = ESection; +#ifdef LLVM38 + auto SectOrError = Sym.getSection(); + assert(SectOrError); + Sect = SectOrError.get(); +#else + if (Sym.getSection(Sect)) continue; +#endif + if (Sect == ESection) continue; +#ifdef LLVM35 + SAddr = Sect->getAddress(); + if (SAddr == 0) continue; +#else + if (Sym.getAddress(SAddr) || SAddr == 0) continue; +#endif +#ifdef LLVM37 + auto AddrOrError = Sym.getAddress(); + assert(AddrOrError); + Addr = AddrOrError.get(); + if (Addr == offset) { + auto sNameOrError = Sym.getName(); + if (sNameOrError) + return sNameOrError.get(); + } +#else + if (Sym.getAddress(Addr)) continue; + if (Addr == offset) { + StringRef Name; + if (!Sym.getName(Name)) + return Name; + } +#endif + } + return StringRef(); } + // Insert an address void SymbolTable::insertAddress(uint64_t addr) { @@ -186,12 +233,18 @@ void SymbolTable::createSymbols() isymb->second = symb; } } -const char *SymbolTable::lookupSymbolName(uint64_t addr) +const char *SymbolTable::lookupSymbolName(uint64_t addr, bool LocalOnly) { - if (!Table.count(addr)) return NULL; - MCSymbol *symb = Table[addr]; - TempName = symb->getName().str(); - return TempName.c_str(); + TempName = std::string(); + TableType::iterator Sym = Table.find(addr); + if (Sym != Table.end()) { + MCSymbol *symb = Table[addr]; + TempName = symb->getName().str(); + } + else if (!LocalOnly) { + TempName = getSymbolNameAt(addr + slide).str(); + } + return TempName.empty() ? NULL : TempName.c_str(); } MCSymbol *SymbolTable::lookupSymbol(uint64_t addr) { @@ -204,13 +257,26 @@ static const char *SymbolLookup(void *DisInfo, uint64_t ReferenceValue, uint64_t { SymbolTable *SymTab = (SymbolTable*)DisInfo; if (SymTab->getPass() != 0) { + uint64_t addr = ReferenceValue + SymTab->getIP(); if (*ReferenceType == LLVMDisassembler_ReferenceType_In_Branch) { - uint64_t addr = ReferenceValue + SymTab->getIP(); - const char *symbolName = SymTab->lookupSymbolName(addr); + const char *symbolName = SymTab->lookupSymbolName(addr, false); + //*ReferenceType = LLVMDisassembler_ReferenceType_Out_SymbolStub; *ReferenceType = LLVMDisassembler_ReferenceType_InOut_None; *ReferenceName = NULL; return symbolName; } + else if (*ReferenceType == LLVMDisassembler_ReferenceType_In_PCrel_Load) { + const char *symbolName = SymTab->lookupSymbolName(addr, false); + if (symbolName) { +#ifdef LLVM37 + *ReferenceType = LLVMDisassembler_ReferenceType_Out_LitPool_SymAddr; +#else + *ReferenceType = LLVMDisassembler_ReferenceType_Out_LitPool_CstrAddr; +#endif + *ReferenceName = symbolName; + return NULL; + } + } } *ReferenceType = LLVMDisassembler_ReferenceType_InOut_None; *ReferenceName = NULL; @@ -240,11 +306,10 @@ static int OpInfoLookup(void *DisInfo, uint64_t PC, uint64_t Offset, uint64_t Si } jl_frame_t *frame = NULL; jl_getFunctionInfo(&frame, - pointer - SymTab->getSlide(), // TODO: This is wrong for PCrel addresses, now that we are getting the unrelocated binary. - // Try looking up addresses in the DIContext first? + pointer, /*skipC*/0, /*noInline*/1/* the entry pointer shouldn't have inlining */); - char *name = frame->func_name; + char *name = frame->func_name; // TODO: free me free(frame->file_name); free(frame); if (!name) @@ -273,6 +338,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #ifndef USE_MCJIT std::vector<JITEvent_EmittedFunctionDetails::LineStart> lineinfo, #endif + const object::ObjectFile *object, DIContext *di_ctx, #ifdef LLVM37 raw_ostream &rstream @@ -362,7 +428,6 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, unsigned OutputAsmVariant = 0; // ATT or Intel-style assembly bool ShowEncoding = false; - bool ShowInst = false; #ifdef LLVM35 std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo()); @@ -380,6 +445,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, MCInstPrinter *IP = TheTarget->createMCInstPrinter(OutputAsmVariant, *MAI, *MCII, *MRI, *STI); #endif + //IP->setPrintImmHex(true); // prefer hex or decimal immediates MCCodeEmitter *CE = 0; MCAsmBackend *MAB = 0; if (ShowEncoding) { @@ -410,7 +476,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, /*useCFI*/ true, #endif /*useDwarfDirectory*/ true, - IP, CE, MAB, ShowInst)); + IP, CE, MAB, /*ShowInst*/ false)); #ifdef LLVM36 Streamer->InitSections(true); #else @@ -423,7 +489,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #else FuncMCView memoryObject((const uint8_t*)Fptr, Fsize); #endif - SymbolTable DisInfo(Ctx, slide, memoryObject); + SymbolTable DisInfo(Ctx, object, slide, memoryObject); #ifndef USE_MCJIT typedef std::vector<JITEvent_EmittedFunctionDetails::LineStart> LInfoVec; @@ -556,7 +622,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, Streamer->EmitLabel(symbol); // emitInstructionAnnot #else - const char *symbolName = DisInfo.lookupSymbolName(Fptr+Index); + const char *symbolName = DisInfo.lookupSymbolName(Fptr + Index, true); if (symbolName) stream << symbolName << ":"; #endif @@ -566,7 +632,10 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, MCDisassembler::DecodeStatus S; FuncMCView view = memoryObject.slice(Index); S = DisAsm->getInstruction(Inst, insSize, view, 0, - /*REMOVE*/ nulls(), nulls()); + /*VStream*/ nulls(), + /*CStream*/ pass != 0 ? Streamer->GetCommentOS() : nulls()); + if (pass != 0 && Streamer->GetCommentOS().tell() > 0) + Streamer->GetCommentOS() << '\n'; switch (S) { case MCDisassembler::Fail: if (insSize == 0) // skip illegible bytes @@ -580,30 +649,19 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, if (insSize == 4) buf << "\t.long\t0x" << std::hex << std::setfill('0') << std::setw(8) - << *(uint32_t*)(Fptr+Index) << "\n"; + << *(uint32_t*)(Fptr+Index); else for (uint64_t i=0; i<insSize; ++i) buf << "\t.byte\t0x" << std::hex << std::setfill('0') << std::setw(2) - << (int)*(uint8_t*)(Fptr+Index+i) << "\n"; -#ifdef LLVM37 - Streamer->EmitRawText(buf.str()); -#else - stream << buf.str(); -#endif + << (int)*(uint8_t*)(Fptr+Index+i); + Streamer->EmitRawText(StringRef(buf.str())); } break; case MCDisassembler::SoftFail: - if (pass != 0) { -#ifdef LLVM37 - std::ostringstream buf; - buf << "potentially undefined instruction encoding:\n"; - Streamer->EmitRawText(buf.str()); -#else - stream << "potentially undefined instruction encoding:\n"; -#endif - } + if (pass != 0) + Streamer->EmitRawText(StringRef("potentially undefined instruction encoding:")); // Fall through case MCDisassembler::Success: From 2ead8ef8edb50ea98a3d0724d74e696492a11a37 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 7 Jul 2016 21:36:11 -0700 Subject: [PATCH 0307/1117] Remove (c)transpose no-op for Strings and revise associated tests. --- base/strings/basic.jl | 2 -- test/datafmt.jl | 2 +- test/strings/basic.jl | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index caa60795b8c0c..167bef7bcbd44 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -101,8 +101,6 @@ function length(s::AbstractString) end end -transpose(s::AbstractString) = s - ## string comparison functions ## function cmp(a::AbstractString, b::AbstractString) diff --git a/test/datafmt.jl b/test/datafmt.jl index 7116a2af614dc..4b97da3a32ef3 100644 --- a/test/datafmt.jl +++ b/test/datafmt.jl @@ -210,7 +210,7 @@ let i18n_data = ["Origin (English)", "Name (English)", "Origin (Native)", "Name "Yugoslavia (Cyrillic)", "Djordje Balasevic", "Југославија", "Ђорђе Балашевић", "Yugoslavia (Latin)", "Djordje Balasevic", "Jugoslavija", "Đorđe Balašević"] - i18n_arr = transpose(reshape(i18n_data, 4, Int(floor(length(i18n_data)/4)))) + i18n_arr = permutedims(reshape(i18n_data, 4, Int(floor(length(i18n_data)/4))), [2, 1]) i18n_buff = PipeBuffer() writedlm(i18n_buff, i18n_arr, ',') @test i18n_arr == readcsv(i18n_buff) diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 5d62ddb81a842..95419735d9c81 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -402,7 +402,7 @@ foobaz(ch) = reinterpret(Char, typemax(UInt32)) @test "a".*["b","c"] == ["ab","ac"] @test ["b","c"].*"a" == ["ba","ca"] -@test ["a","b"].*["c","d"]' == ["ac" "ad"; "bc" "bd"] +@test ["a","b"].*["c" "d"] == ["ac" "ad"; "bc" "bd"] # Make sure NULL pointers are handled consistently by String @test_throws ArgumentError unsafe_string(Ptr{UInt8}(0)) From c766ce435b4adfaac41a4cb98a80b0a9464db471 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Fri, 8 Jul 2016 15:29:24 -0400 Subject: [PATCH 0308/1117] make sure parseint base can be any Integer type (#17335) --- base/gmp.jl | 4 ++-- base/parse.jl | 4 ++-- test/parse.jl | 8 ++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/base/gmp.jl b/base/gmp.jl index 3b63f121ec13f..975a94814fc87 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -76,13 +76,13 @@ signed(x::BigInt) = x convert(::Type{BigInt}, x::BigInt) = x -function tryparse_internal(::Type{BigInt}, s::AbstractString, startpos::Int, endpos::Int, base::Int, raise::Bool) +function tryparse_internal(::Type{BigInt}, s::AbstractString, startpos::Int, endpos::Int, base_::Integer, raise::Bool) _n = Nullable{BigInt}() # don't make a copy in the common case where we are parsing a whole String bstr = startpos == start(s) && endpos == endof(s) ? String(s) : String(SubString(s,startpos,endpos)) - sgn, base, i = Base.parseint_preamble(true,base,bstr,start(bstr),endof(bstr)) + sgn, base, i = Base.parseint_preamble(true,Int(base_),bstr,start(bstr),endof(bstr)) if !(2 <= base <= 62) raise && throw(ArgumentError("invalid base: base must be 2 ≤ base ≤ 62, got $base")) return _n diff --git a/base/parse.jl b/base/parse.jl index 7b39f4b8e06c6..223a0ecd8f32d 100644 --- a/base/parse.jl +++ b/base/parse.jl @@ -59,9 +59,9 @@ safe_mul{T<:Integer}(n1::T, n2::T) = ((n2 > 0) ? ((n1 > div(typemax(T),n2)) || (n2 < -1) ? ((n1 > div(typemin(T),n2)) || (n1 < div(typemax(T),n2))) : ((n2 == -1) && n1 == typemin(T))) ? Nullable{T}() : Nullable{T}(n1 * n2) -function tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, startpos::Int, endpos::Int, base::Integer, raise::Bool) +function tryparse_internal{T<:Integer}(::Type{T}, s::AbstractString, startpos::Int, endpos::Int, base_::Integer, raise::Bool) _n = Nullable{T}() - sgn, base, i = parseint_preamble(T<:Signed, base, s, startpos, endpos) + sgn, base, i = parseint_preamble(T<:Signed, Int(base_), s, startpos, endpos) if !(2 <= base <= 62) raise && throw(ArgumentError("invalid base: base must be 2 ≤ base ≤ 62, got $base")) return _n diff --git a/test/parse.jl b/test/parse.jl index 0861d00a55688..e90df66d67512 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -514,6 +514,14 @@ end # to be removed post 0.5 #@test_throws MethodError eval(parse("(Any=>Any)[x=>y for (x,y) in zip([1,2,3],[4,5,6])]")) +# make sure base can be any Integer +for T in (Int, BigInt) + let n = parse(T, "123", Int8(10)) + @test n == 123 + @test isa(n, T) + end +end + # issue #16720 let err = try include_string("module A From 9acd17ff39c7aa247c3327e3e6bcaf1a19a59f19 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 8 Jul 2016 15:36:39 -0400 Subject: [PATCH 0309/1117] flatten the call chain of `checkbounds` a bit --- base/abstractarray.jl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index d9802a4c90017..75cfdd1b93c91 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -190,13 +190,22 @@ Return `true` if the specified `indexes` are in bounds for the given `array`. Su `AbstractArray` should specialize this method if they need to provide custom bounds checking behaviors. """ +function checkbounds(::Type{Bool}, A::AbstractArray, i::Integer) + @_inline_meta + checkindex(Bool, linearindices(A), i) +end +function checkbounds{T}(::Type{Bool}, A::Union{Array{T,1},Range{T}}, i::Integer) + @_inline_meta + (1 <= i) & (i <= length(A)) +end +function checkbounds(::Type{Bool}, A::AbstractArray, I::AbstractArray{Bool}) + @_inline_meta + checkbounds_logical(A, I) +end function checkbounds(::Type{Bool}, A::AbstractArray, I...) @_inline_meta - _chkbounds(A, I...) + checkbounds_indices(indices(A), I) end -_chkbounds(A::AbstractArray, i::Integer) = (@_inline_meta; checkindex(Bool, linearindices(A), i)) -_chkbounds(A::AbstractArray, I::AbstractArray{Bool}) = (@_inline_meta; checkbounds_logical(A, I)) -_chkbounds(A::AbstractArray, I...) = (@_inline_meta; checkbounds_indices(indices(A), I)) checkbounds_indices(::Tuple{}, ::Tuple{}) = true checkbounds_indices(::Tuple{}, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, 1:1, I[1])) From 52a9c26a9770ed25c43fe0e27ac2f78369be29af Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 8 Jul 2016 15:02:08 -0500 Subject: [PATCH 0310/1117] Fix comment for #17202 --- test/show.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/show.jl b/test/show.jl index 9956c803c527b..b8e9abe1a7435 100644 --- a/test/show.jl +++ b/test/show.jl @@ -501,7 +501,7 @@ let s = IOBuffer(Array{UInt8}(0), true, true) @test String(resize!(s.data, s.size)) == " 1\n 2\n 3" end -# The `dump` function should alway have a trailing newline +# The `dump` function should always have a trailing newline let io = IOBuffer() dump(io, :(x = 1)) @test takebuf_string(io)[end] == '\n' From ade5f5b05e3eb3222f3a1132858a2e6d2efabaee Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Fri, 8 Jul 2016 15:36:03 -0500 Subject: [PATCH 0311/1117] Use func(args...) -> result syntax in docstring [ci skip] --- base/profile.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/profile.jl b/base/profile.jl index 66a5e0f2f2ce7..861f2c7e2fe93 100644 --- a/base/profile.jl +++ b/base/profile.jl @@ -125,7 +125,7 @@ function getdict(data::Vector{UInt}) end """ - newdata, newdict = flatten(btdata, lidict) + flatten(btdata, lidict) -> (newdata, newdict) Produces "flattened" backtrace data. Individual instruction pointers sometimes correspond to a multi-frame backtrace due to inlining; in From 2ffaafa746ee4ab558a7b1e87fdd3df313f29f90 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Fri, 8 Jul 2016 13:02:41 -0700 Subject: [PATCH 0312/1117] Fix potential type instability in setindex! for SparseMatrixCSCs. --- base/sparse/sparsematrix.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 4ce6e0e0a4bc4..f50697a8a70e0 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -2482,11 +2482,11 @@ getindex{T<:Integer}(A::SparseMatrixCSC, I::AbstractVector{T}, J::AbstractVector getindex{T<:Integer}(A::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{T}) = A[find(I),J] ## setindex! -function setindex!{T,Ti}(A::SparseMatrixCSC{T,Ti}, v, i0::Integer, i1::Integer) - i0 = convert(Ti, i0) - i1 = convert(Ti, i1) +function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, v, i::Integer, j::Integer) + setindex!(A, convert(Tv, v), convert(Ti, i), convert(Ti, j)) +end +function setindex!{Tv,Ti<:Integer}(A::SparseMatrixCSC{Tv,Ti}, v::Tv, i0::Ti, i1::Ti) if !(1 <= i0 <= A.m && 1 <= i1 <= A.n); throw(BoundsError()); end - v = convert(T, v) r1 = Int(A.colptr[i1]) r2 = Int(A.colptr[i1+1]-1) if v == 0 #either do nothing or delete entry if it exists From 8c718ef1f0174a735d7a829d7090e795567f3d35 Mon Sep 17 00:00:00 2001 From: Gem Newman <spurll@gmail.com> Date: Fri, 8 Jul 2016 19:57:39 -0500 Subject: [PATCH 0313/1117] Date Rounding (#17037) * Added floor, ceil, round for Date and DateTime. * Refactoring and documentation for date rounding. * Added PR reference in NEWS.md. * Bugfix for date rounding test in 32-bit. Doc clarifications. * Round to nonpositive resolution throws DomainError Throw DomainError on rounding to an invalid (non-positive) resolution Clean up test cases for rounding dates that don't need rounding Add test cases for rounding to invalid (non-positive) resolutions --- NEWS.md | 4 + base/dates/Dates.jl | 1 + base/dates/rounding.jl | 178 ++++++++++++++++++++++++++++++++++++++ doc/manual/dates.rst | 91 +++++++++++++++++++ doc/stdlib/dates.rst | 97 +++++++++++++++++++++ test/dates.jl | 1 + test/dates/conversions.jl | 1 + test/dates/rounding.jl | 139 +++++++++++++++++++++++++++++ 8 files changed, 512 insertions(+) create mode 100644 base/dates/rounding.jl create mode 100644 test/dates/rounding.jl diff --git a/NEWS.md b/NEWS.md index ed0d84c1a1d73..b781026011a5e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -216,6 +216,9 @@ Library improvements * Prime number related functions have been moved from `Base` to the [Primes.jl package](https://github.com/JuliaMath/Primes.jl) ([#16481]). + * `Date` and `DateTime` values can now be rounded to a specified resolution (e.g., 1 month or + 15 minutes) with `floor`, `ceil`, and `round` ([#17037]). + Deprecated or removed --------------------- @@ -307,3 +310,4 @@ Deprecated or removed [#16731]: https://github.com/JuliaLang/julia/issues/16731 [#16972]: https://github.com/JuliaLang/julia/issues/16972 [#17266]: https://github.com/JuliaLang/julia/issues/17266 +[#17037]: https://github.com/JuliaLang/julia/issues/17037 diff --git a/base/dates/Dates.jl b/base/dates/Dates.jl index f2efebd4a00aa..ff2165de586d0 100644 --- a/base/dates/Dates.jl +++ b/base/dates/Dates.jl @@ -12,6 +12,7 @@ include("arithmetic.jl") include("conversions.jl") include("ranges.jl") include("adjusters.jl") +include("rounding.jl") include("io.jl") export Period, DatePeriod, TimePeriod, diff --git a/base/dates/rounding.jl b/base/dates/rounding.jl new file mode 100644 index 0000000000000..e9a08c6fc13c2 --- /dev/null +++ b/base/dates/rounding.jl @@ -0,0 +1,178 @@ +# The epochs used for date rounding are based ISO 8601's "year zero" notation +const DATEEPOCH = value(Date(0)) +const DATETIMEEPOCH = value(DateTime(0)) + +# According to ISO 8601, the first day of the first week of year 0000 is 0000-01-03 +const WEEKEPOCH = value(Date(0, 1, 3)) + +""" + epochdays2date(days) -> Date + +Takes the number of days since the rounding epoch (`0000-01-01T00:00:00`) and returns the +corresponding `Date`. +""" +epochdays2date(i) = Date(UTD(DATEEPOCH + Int64(i))) + +""" + epochms2datetime(milliseconds) -> DateTime + +Takes the number of milliseconds since the rounding epoch (`0000-01-01T00:00:00`) and +returns the corresponding `DateTime`. +""" +epochms2datetime(i) = DateTime(UTM(DATETIMEEPOCH + Int64(i))) + +""" + date2epochdays(dt::Date) -> Int64 + +Takes the given `Date` and returns the number of days since the rounding epoch +(`0000-01-01T00:00:00`) as an `Int64`. +""" +date2epochdays(dt::Date) = value(dt) - DATEEPOCH + +""" + datetime2epochms(dt::DateTime) -> Int64 + +Takes the given `DateTime` and returns the number of milliseconds since the rounding epoch +(`0000-01-01T00:00:00`) as an `Int64`. +""" +datetime2epochms(dt::DateTime) = value(dt) - DATETIMEEPOCH + +function Base.floor(dt::Date, p::Year) + value(p) < 1 && throw(DomainError()) + years = year(dt) + return Date(years - mod(years, value(p))) +end + +function Base.floor(dt::Date, p::Month) + value(p) < 1 && throw(DomainError()) + y, m = yearmonth(dt) + months_since_epoch = y * 12 + m - 1 + month_offset = months_since_epoch - mod(months_since_epoch, value(p)) + target_month = mod(month_offset, 12) + 1 + target_year = div(month_offset, 12) - (month_offset < 0 && target_month != 1) + return Date(target_year, target_month) +end + +function Base.floor(dt::Date, p::Week) + value(p) < 1 && throw(DomainError()) + days = value(dt) - WEEKEPOCH + days = days - mod(days, value(Day(p))) + return Date(UTD(WEEKEPOCH + Int64(days))) +end + +function Base.floor(dt::Date, p::Day) + value(p) < 1 && throw(DomainError()) + days = date2epochdays(dt) + return epochdays2date(days - mod(days, value(p))) +end + +Base.floor(dt::DateTime, p::DatePeriod) = DateTime(Base.floor(Date(dt), p)) + +function Base.floor(dt::DateTime, p::TimePeriod) + value(p) < 1 && throw(DomainError()) + milliseconds = datetime2epochms(dt) + return epochms2datetime(milliseconds - mod(milliseconds, value(Millisecond(p)))) +end + +""" + floor(dt::TimeType, p::Period) -> TimeType + +Returns the nearest `Date` or `DateTime` less than or equal to `dt` at resolution `p`. + +For convenience, `p` may be a type instead of a value: `floor(dt, Dates.Hour)` is a shortcut +for `floor(dt, Dates.Hour(1))`. + +```jldoctest +julia> floor(Date(1985, 8, 16), Dates.Month) +1985-08-01 + +julia> floor(DateTime(2013, 2, 13, 0, 31, 20), Dates.Minute(15)) +2013-02-13T00:30:00 + +julia> floor(DateTime(2016, 8, 6, 12, 0, 0), Dates.Day) +2016-08-06T00:00:00 +``` +""" +Base.floor(::Dates.TimeType, ::Dates.Period) + +""" + ceil(dt::TimeType, p::Period) -> TimeType + +Returns the nearest `Date` or `DateTime` greater than or equal to `dt` at resolution `p`. + +For convenience, `p` may be a type instead of a value: `ceil(dt, Dates.Hour)` is a shortcut +for `ceil(dt, Dates.Hour(1))`. + +```jldoctest +julia> ceil(Date(1985, 8, 16), Dates.Month) +1985-09-01 + +julia> ceil(DateTime(2013, 2, 13, 0, 31, 20), Dates.Minute(15)) +2013-02-13T00:45:00 + +julia> ceil(DateTime(2016, 8, 6, 12, 0, 0), Dates.Day) +2016-08-07T00:00:00 +``` +""" +function Base.ceil(dt::TimeType, p::Period) + f = floor(dt, p) + return (dt == f) ? f : f + p +end + +""" + floorceil(dt::TimeType, p::Period) -> (TimeType, TimeType) + +Simultaneously return the `floor` and `ceil` of a `Date` or `DateTime` at resolution `p`. +More efficient than calling both `floor` and `ceil` individually. +""" +function floorceil(dt::TimeType, p::Period) + f = floor(dt, p) + return f, (dt == f) ? f : f + p +end + +""" + round(dt::TimeType, p::Period, [r::RoundingMode]) -> TimeType + +Returns the `Date` or `DateTime` nearest to `dt` at resolution `p`. By default +(`RoundNearestTiesUp`), ties (e.g., rounding 9:30 to the nearest hour) will be rounded up. + +For convenience, `p` may be a type instead of a value: `round(dt, Dates.Hour)` is a shortcut +for `round(dt, Dates.Hour(1))`. + +```jldoctest +julia> round(Date(1985, 8, 16), Dates.Month) +1985-08-01 + +julia> round(DateTime(2013, 2, 13, 0, 31, 20), Dates.Minute(15)) +2013-02-13T00:30:00 + +julia> round(DateTime(2016, 8, 6, 12, 0, 0), Dates.Day) +2016-08-07T00:00:00 +``` + +Valid rounding modes for `round(::TimeType, ::Period, ::RoundingMode)` are +`RoundNearestTiesUp` (default), `RoundDown` (`floor`), and `RoundUp` (`ceil`). +""" +function Base.round(dt::TimeType, p::Period, r::RoundingMode{:NearestTiesUp}) + f, c = floorceil(dt, p) + return (dt - f) < (c - dt) ? f : c +end + +Base.round(dt::TimeType, p::Period, r::RoundingMode{:Down}) = Base.floor(dt, p) +Base.round(dt::TimeType, p::Period, r::RoundingMode{:Up}) = Base.ceil(dt, p) + +# No implementation of other `RoundingMode`s: rounding to nearest "even" is skipped because +# "even" is not defined for Period; rounding toward/away from zero is skipped because ISO +# 8601's year 0000 is not really "zero". +Base.round(::TimeType, ::Period, ::RoundingMode) = throw(DomainError()) + +# Default to RoundNearestTiesUp. +Base.round(dt::TimeType, p::Period) = Base.round(dt, p, RoundNearestTiesUp) + +# Make rounding functions callable using Period types in addition to values. +Base.floor{T <: Period}(dt::TimeType, p::Type{T}) = Base.floor(dt, p(1)) +Base.ceil{T <: Period}(dt::TimeType, p::Type{T}) = Base.ceil(dt, p(1)) + +function Base.round{T<:Period}(dt::TimeType, p::Type{T}, r::RoundingMode=RoundNearestTiesUp) + return Base.round(dt, p(1), r) +end diff --git a/doc/manual/dates.rst b/doc/manual/dates.rst index bf589a46fad9f..08b5ba148b228 100644 --- a/doc/manual/dates.rst +++ b/doc/manual/dates.rst @@ -367,4 +367,95 @@ Periods are a human view of discrete, sometimes irregular durations of time. Con 3 years +Rounding +-------- + +:class:`Date` and :class:`DateTime` values can be rounded to a specified resolution (e.g., +1 month or 15 minutes) with :func:`floor`, :func:`ceil`, or :func:`round`: + +.. doctest:: + + julia> floor(Date(1985, 8, 16), Dates.Month) + 1985-08-01 + + julia> ceil(DateTime(2013, 2, 13, 0, 31, 20), Dates.Minute(15)) + 2013-02-13T00:45:00 + + julia> round(DateTime(2016, 8, 6, 20, 15), Dates.Day) + 2016-08-07T00:00:00 + +Unlike the numeric :func:`round` method, which breaks ties toward the even number by +default, the :class:`TimeType` :func:`round` method uses the ``RoundNearestTiesUp`` +rounding mode. (It's difficult to guess what breaking ties to nearest "even" +:class:`TimeType` would entail.) Further details on the available ``RoundingMode`` s can +be found in the +`API reference <http://docs.julialang.org/en/latest/stdlib/dates/#rounding-functions>`_. + +Rounding should generally behave as expected, but there are a few cases in which the +expected behaviour is not obvious. + +Rounding Epoch +~~~~~~~~~~~~~~ + +In many cases, the resolution specified for rounding (e.g., ``Dates.Second(30)``) divides +evenly into the next largest period (in this case, ``Dates.Minute(1)``). But rounding +behaviour in cases in which this is not true may lead to confusion. What is the expected +result of rounding a :class:`DateTime` to the nearest 10 hours? + +.. doctest:: + + julia> round(DateTime(2016, 7, 17, 11, 55), Dates.Hour(10)) + 2016-07-17T12:00:00 + +That may seem confusing, given that the hour (12) is not divisible by 10. The reason that +``2016-07-17T12:00:00`` was chosen is that it is 17,676,660 hours after +``0000-01-01T00:00:00``, and 17,676,660 is divisible by 10. + +As Julia :class:`Date` and :class:`DateTime` values are represented according to the ISO +8601 standard, ``0000-01-01T00:00:00`` was chosen as base (or "rounding epoch") from +which to begin the count of days (and milliseconds) used in rounding calculations. (Note +that this differs slightly from Julia's internal representation of :class:`Date` s using +Rata Die notation; but since the ISO 8601 standard is most visible to the end user, +``0000-01-01T00:00:00`` was chosen as the rounding epoch instead of the +``0000-12-31T00:00:00`` used internally to minimize confusion.) + +The only exception to the use of ``0000-01-01T00:00:00`` as the rounding epoch is when +rounding to weeks. Rounding to the nearest week will always return a Monday (the first day +of the week as specified by ISO 8601). For this reason, we use ``0000-01-03T00:00:00`` +(the first day of the first week of year 0000, as defined by ISO 8601) as the base when +rounding to a number of weeks. + +Here is a related case in which the expected behaviour is not necessarily obvious: What +happens when we round to the nearest ``P(2)``, where ``P`` is a :class:`Period` type? In +some cases (specifically, when ``P <: Dates.TimePeriod``) the answer is clear: + +.. doctest:: + + julia> round(DateTime(2016, 7, 17, 8, 55, 30), Dates.Hour(2)) + 2016-07-17T08:00:00 + + julia> round(DateTime(2016, 7, 17, 8, 55, 30), Dates.Minute(2)) + 2016-07-17T08:56:00 + +This seems obvious, because two of each of these periods still divides evenly into the +next larger order period. But in the case of two months (which still divides evenly into +one year), the answer may be surprising: + +.. doctest:: + + julia> round(DateTime(2016, 7, 17, 8, 55, 30), Dates.Month(2)) + 2016-07-01T00:00:00 + +Why round to the first day in July, even though it is month 7 (an odd number)? The key is +that months are 1-indexed (the first month is assigned 1), unlike hours, minutes, seconds, +and milliseconds (the first of which are assigned 0). + +This means that rounding a :class:`DateTime` to an even multiple of seconds, minutes, +hours, or years (because the ISO 8601 specification includes a year zero) will result in +a :class:`DateTime` with an even value in that field, while rounding a :class:`DateTime` +to an even multiple of months will result in the months field having an odd value. Because +both months and years may contain an irregular number of days, whether rounding to an even +number of days will result in an even value in the days field is uncertain. + + See the `API reference <http://docs.julialang.org/en/latest/stdlib/dates/>`_ for additional information on methods exported from the :mod:`Dates` module. diff --git a/doc/stdlib/dates.rst b/doc/stdlib/dates.rst index dd9ef62218a81..1cc554b42b613 100644 --- a/doc/stdlib/dates.rst +++ b/doc/stdlib/dates.rst @@ -575,6 +575,103 @@ Periods Returns a sensible "default" value for the input Period by returning ``one(p)`` for Year, Month, and Day, and ``zero(p)`` for Hour, Minute, Second, and Millisecond. +Rounding Functions +~~~~~~~~~~~~~~~~~~ + +``Date`` and ``DateTime`` values can be rounded to a specified resolution (e.g., 1 month +or 15 minutes) with ``floor``, ``ceil``, or ``round``. + +.. function:: floor(dt::TimeType, p::Period) -> TimeType + + .. Docstring generated from Julia source + + Returns the nearest ``Date`` or ``DateTime`` less than or equal to ``dt`` at resolution ``p``\ . + + For convenience, ``p`` may be a type instead of a value: ``floor(dt, Dates.Hour)`` is a shortcut for ``floor(dt, Dates.Hour(1))``\ . + + .. doctest:: + + julia> floor(Date(1985, 8, 16), Dates.Month) + 1985-08-01 + + julia> floor(DateTime(2013, 2, 13, 0, 31, 20), Dates.Minute(15)) + 2013-02-13T00:30:00 + + julia> floor(DateTime(2016, 8, 6, 12, 0, 0), Dates.Day) + 2016-08-06T00:00:00 + +.. function:: ceil(dt::TimeType, p::Period) -> TimeType + + .. Docstring generated from Julia source + + Returns the nearest ``Date`` or ``DateTime`` greater than or equal to ``dt`` at resolution ``p``\ . + + For convenience, ``p`` may be a type instead of a value: ``ceil(dt, Dates.Hour)`` is a shortcut for ``ceil(dt, Dates.Hour(1))``\ . + + .. doctest:: + + julia> ceil(Date(1985, 8, 16), Dates.Month) + 1985-09-01 + + julia> ceil(DateTime(2013, 2, 13, 0, 31, 20), Dates.Minute(15)) + 2013-02-13T00:45:00 + + julia> ceil(DateTime(2016, 8, 6, 12, 0, 0), Dates.Day) + 2016-08-07T00:00:00 + +.. function:: round(dt::TimeType, p::Period, [r::RoundingMode]) -> TimeType + + .. Docstring generated from Julia source + + Returns the ``Date`` or ``DateTime`` nearest to ``dt`` at resolution ``p``\ . By default (``RoundNearestTiesUp``\ ), ties (e.g., rounding 9:30 to the nearest hour) will be rounded up. + + For convenience, ``p`` may be a type instead of a value: ``round(dt, Dates.Hour)`` is a shortcut for ``round(dt, Dates.Hour(1))``\ . + + .. doctest:: + + julia> round(Date(1985, 8, 16), Dates.Month) + 1985-08-01 + + julia> round(DateTime(2013, 2, 13, 0, 31, 20), Dates.Minute(15)) + 2013-02-13T00:30:00 + + julia> round(DateTime(2016, 8, 6, 12, 0, 0), Dates.Day) + 2016-08-07T00:00:00 + + Valid rounding modes for ``round(::TimeType, ::Period, ::RoundingMode)`` are ``RoundNearestTiesUp`` (default), ``RoundDown`` (``floor``\ ), and ``RoundUp`` (``ceil``\ ). + +The following functions are not exported: + +.. function:: floorceil(dt::TimeType, p::Period) -> (TimeType, TimeType) + + .. Docstring generated from Julia source + + Simultaneously return the ``floor`` and ``ceil`` of a ``Date`` or ``DateTime`` at resolution ``p``\ . More efficient than calling both ``floor`` and ``ceil`` individually. + +.. function:: epochdays2date(days) -> Date + + .. Docstring generated from Julia source + + Takes the number of days since the rounding epoch (``0000-01-01T00:00:00``\ ) and returns the corresponding ``Date``\ . + +.. function:: epochms2datetime(milliseconds) -> DateTime + + .. Docstring generated from Julia source + + Takes the number of milliseconds since the rounding epoch (``0000-01-01T00:00:00``\ ) and returns the corresponding ``DateTime``\ . + +.. function:: date2epochdays(dt::Date) -> Int64 + + .. Docstring generated from Julia source + + Takes the given ``Date`` and returns the number of days since the rounding epoch (``0000-01-01T00:00:00``\ ) as an ``Int64``\ . + +.. function:: datetime2epochms(dt::DateTime) -> Int64 + + .. Docstring generated from Julia source + + Takes the given ``DateTime`` and returns the number of milliseconds since the rounding epoch (``0000-01-01T00:00:00``\ ) as an ``Int64``\ . + Conversion Functions ~~~~~~~~~~~~~~~~~~~~ diff --git a/test/dates.jl b/test/dates.jl index 6c8467a9f1acc..ea654b7c2fe47 100644 --- a/test/dates.jl +++ b/test/dates.jl @@ -13,6 +13,7 @@ include("dates/arithmetic.jl") include("dates/conversions.jl") include("dates/ranges.jl") include("dates/adjusters.jl") +include("dates/rounding.jl") include("dates/io.jl") end diff --git a/test/dates/conversions.jl b/test/dates/conversions.jl index 3706c9ef96aa7..db3fd97948ee0 100644 --- a/test/dates/conversions.jl +++ b/test/dates/conversions.jl @@ -28,6 +28,7 @@ @test string(Dates.unix2datetime(915148801.00)) == string("1999-01-01T00:00:01") @test string(Dates.unix2datetime(915148801.25)) == string("1999-01-01T00:00:01.25") +# Test conversion to and from Rata Die @test Date(Dates.rata2datetime(734869)) == Dates.Date(2013,1,1) @test Dates.datetime2rata(Dates.rata2datetime(734869)) == 734869 diff --git a/test/dates/rounding.jl b/test/dates/rounding.jl new file mode 100644 index 0000000000000..80218fcf0db47 --- /dev/null +++ b/test/dates/rounding.jl @@ -0,0 +1,139 @@ +# Test conversion to and from the rounding epoch (ISO 8601 year 0000) +@test Dates.epochdays2date(-1) == Dates.Date(-1, 12, 31) +@test Dates.epochdays2date(0) == Dates.Date(0, 1, 1) +@test Dates.epochdays2date(1) == Dates.Date(0, 1, 2) +@test Dates.epochdays2date(736329) == Dates.Date(2016, 1, 1) +@test Dates.epochms2datetime(-86400000) == Dates.DateTime(-1, 12, 31) +@test Dates.epochms2datetime(0) == Dates.DateTime(0, 1, 1) +@test Dates.epochms2datetime(86400000) == Dates.DateTime(0, 1, 2) +@test Dates.epochms2datetime(Int64(736329) * 86400000) == Dates.DateTime(2016, 1, 1) +@test Dates.date2epochdays(Dates.Date(-1, 12, 31)) == -1 +@test Dates.date2epochdays(Dates.Date(0, 1, 1)) == 0 +@test Dates.date2epochdays(Dates.Date(2016, 1, 1)) == 736329 +@test Dates.datetime2epochms(Dates.DateTime(-1, 12, 31)) == -86400000 +@test Dates.datetime2epochms(Dates.DateTime(0, 1, 1)) == 0 +@test Dates.datetime2epochms(Dates.DateTime(2016, 1, 1)) == Int64(736329) * 86400000 + +# Basic rounding tests +dt = Dates.Date(2016, 2, 28) # Sunday +@test floor(dt, Dates.Year) == Dates.Date(2016) +@test floor(dt, Dates.Year(5)) == Dates.Date(2015) +@test floor(dt, Dates.Year(10)) == Dates.Date(2010) +@test floor(dt, Dates.Month) == Dates.Date(2016, 2) +@test floor(dt, Dates.Month(6)) == Dates.Date(2016, 1) +@test floor(dt, Dates.Week) == toprev(dt, Dates.Monday) +@test ceil(dt, Dates.Year) == Dates.Date(2017) +@test ceil(dt, Dates.Year(5)) == Dates.Date(2020) +@test ceil(dt, Dates.Month) == Dates.Date(2016, 3) +@test ceil(dt, Dates.Month(6)) == Dates.Date(2016, 7) +@test ceil(dt, Dates.Week) == tonext(dt, Dates.Monday) +@test round(dt, Dates.Year) == Dates.Date(2016) +@test round(dt, Dates.Month) == Dates.Date(2016, 3) +@test round(dt, Dates.Week) == Dates.Date(2016, 2, 29) + +dt = Dates.DateTime(2016, 2, 28, 15, 10, 50, 500) +@test floor(dt, Dates.Day) == Dates.DateTime(2016, 2, 28) +@test floor(dt, Dates.Hour) == Dates.DateTime(2016, 2, 28, 15) +@test floor(dt, Dates.Hour(2)) == Dates.DateTime(2016, 2, 28, 14) +@test floor(dt, Dates.Hour(12)) == Dates.DateTime(2016, 2, 28, 12) +@test floor(dt, Dates.Minute) == Dates.DateTime(2016, 2, 28, 15, 10) +@test floor(dt, Dates.Minute(15)) == Dates.DateTime(2016, 2, 28, 15, 0) +@test floor(dt, Dates.Second) == Dates.DateTime(2016, 2, 28, 15, 10, 50) +@test floor(dt, Dates.Second(30)) == Dates.DateTime(2016, 2, 28, 15, 10, 30) +@test ceil(dt, Dates.Day) == Dates.DateTime(2016, 2, 29) +@test ceil(dt, Dates.Hour) == Dates.DateTime(2016, 2, 28, 16) +@test ceil(dt, Dates.Hour(2)) == Dates.DateTime(2016, 2, 28, 16) +@test ceil(dt, Dates.Hour(12)) == Dates.DateTime(2016, 2, 29, 0) +@test ceil(dt, Dates.Minute) == Dates.DateTime(2016, 2, 28, 15, 11) +@test ceil(dt, Dates.Minute(15)) == Dates.DateTime(2016, 2, 28, 15, 15) +@test ceil(dt, Dates.Second) == Dates.DateTime(2016, 2, 28, 15, 10, 51) +@test ceil(dt, Dates.Second(30)) == Dates.DateTime(2016, 2, 28, 15, 11, 0) +@test round(dt, Dates.Day) == Dates.DateTime(2016, 2, 29) +@test round(dt, Dates.Hour) == Dates.DateTime(2016, 2, 28, 15) +@test round(dt, Dates.Hour(2)) == Dates.DateTime(2016, 2, 28, 16) +@test round(dt, Dates.Hour(12)) == Dates.DateTime(2016, 2, 28, 12) +@test round(dt, Dates.Minute) == Dates.DateTime(2016, 2, 28, 15, 11) +@test round(dt, Dates.Minute(15)) == Dates.DateTime(2016, 2, 28, 15, 15) +@test round(dt, Dates.Second) == Dates.DateTime(2016, 2, 28, 15, 10, 51) +@test round(dt, Dates.Second(30)) == Dates.DateTime(2016, 2, 28, 15, 11, 0) + +# Rounding for dates at the rounding epoch (year 0000) +dt = Dates.DateTime(0) +@test floor(dt, Dates.Year) == dt +@test floor(dt, Dates.Month) == dt +@test floor(dt, Dates.Week) == Dates.Date(-1, 12, 27) # Monday prior to 0000-01-01 +@test floor(Dates.Date(-1, 12, 27), Dates.Week) == Dates.Date(-1, 12, 27) +@test floor(dt, Dates.Day) == dt +@test floor(dt, Dates.Hour) == dt +@test floor(dt, Dates.Minute) == dt +@test floor(dt, Dates.Second) == dt +@test ceil(dt, Dates.Year) == dt +@test ceil(dt, Dates.Month) == dt +@test ceil(dt, Dates.Week) == Dates.Date(0, 1, 3) # Monday following 0000-01-01 +@test ceil(Dates.Date(0, 1, 3), Dates.Week) == Dates.Date(0, 1, 3) +@test ceil(dt, Dates.Day) == dt +@test ceil(dt, Dates.Hour) == dt +@test ceil(dt, Dates.Minute) == dt +@test ceil(dt, Dates.Second) == dt + +# Test rounding for multiples of a period (easiest to test close to rounding epoch) +dt = Dates.DateTime(0, 1, 19, 19, 19, 19, 19) +@test floor(dt, Dates.Year(2)) == DateTime(0) +@test floor(dt, Dates.Month(2)) == DateTime(0, 1) # Odd number; months are 1-indexed +@test floor(dt, Dates.Week(2)) == DateTime(0, 1, 17) # Third Monday of 0000 +@test floor(dt, Dates.Day(2)) == DateTime(0, 1, 19) # Odd number; days are 1-indexed +@test floor(dt, Dates.Hour(2)) == DateTime(0, 1, 19, 18) +@test floor(dt, Dates.Minute(2)) == DateTime(0, 1, 19, 19, 18) +@test floor(dt, Dates.Second(2)) == DateTime(0, 1, 19, 19, 19, 18) +@test ceil(dt, Dates.Year(2)) == DateTime(2) +@test ceil(dt, Dates.Month(2)) == DateTime(0, 3) # Odd number; months are 1-indexed +@test ceil(dt, Dates.Week(2)) == DateTime(0, 1, 31) # Fifth Monday of 0000 +@test ceil(dt, Dates.Day(2)) == DateTime(0, 1, 21) # Odd number; days are 1-indexed +@test ceil(dt, Dates.Hour(2)) == DateTime(0, 1, 19, 20) +@test ceil(dt, Dates.Minute(2)) == DateTime(0, 1, 19, 19, 20) +@test ceil(dt, Dates.Second(2)) == DateTime(0, 1, 19, 19, 19, 20) + +# Test rounding for dates with negative years +dt = Dates.DateTime(-1, 12, 29, 19, 19, 19, 19) +@test floor(dt, Dates.Year(2)) == DateTime(-2) +@test floor(dt, Dates.Month(2)) == DateTime(-1, 11) # Odd number; months are 1-indexed +@test floor(dt, Dates.Week(2)) == DateTime(-1, 12, 20) # 2 weeks prior to 0000-01-03 +@test floor(dt, Dates.Day(2)) == DateTime(-1, 12, 28) # Even; 4 days prior to 0000-01-01 +@test floor(dt, Dates.Hour(2)) == DateTime(-1, 12, 29, 18) +@test floor(dt, Dates.Minute(2)) == DateTime(-1, 12, 29, 19, 18) +@test floor(dt, Dates.Second(2)) == DateTime(-1, 12, 29, 19, 19, 18) +@test ceil(dt, Dates.Year(2)) == DateTime(0) +@test ceil(dt, Dates.Month(2)) == DateTime(0, 1) # Odd number; months are 1-indexed +@test ceil(dt, Dates.Week(2)) == DateTime(0, 1, 3) # First Monday of 0000 +@test ceil(dt, Dates.Day(2)) == DateTime(-1, 12, 30) # Even; 2 days prior to 0000-01-01 +@test ceil(dt, Dates.Hour(2)) == DateTime(-1, 12, 29, 20) +@test ceil(dt, Dates.Minute(2)) == DateTime(-1, 12, 29, 19, 20) +@test ceil(dt, Dates.Second(2)) == DateTime(-1, 12, 29, 19, 19, 20) + +# Test rounding for dates that should not need rounding +for dt in [Dates.DateTime(2016, 1, 1), Dates.DateTime(-2016, 1, 1)] + for p in [Dates.Year, Dates.Month, Dates.Day, Dates.Hour, Dates.Minute, Dates.Second] + @test floor(dt, p) == dt + @test ceil(dt, p) == dt + end +end + +# Test available RoundingModes +dt = Dates.DateTime(2016, 2, 28, 12) +@test round(dt, Dates.Day, RoundNearestTiesUp) == Dates.DateTime(2016, 2, 29) +@test round(dt, Dates.Day, RoundUp) == Dates.DateTime(2016, 2, 29) +@test round(dt, Dates.Day, RoundDown) == Dates.DateTime(2016, 2, 28) +@test_throws DomainError round(dt, Dates.Day, RoundNearest) +@test_throws DomainError round(dt, Dates.Day, RoundNearestTiesAway) +@test_throws DomainError round(dt, Dates.Day, RoundToZero) +@test round(dt, Dates.Day) == round(dt, Dates.Day, RoundNearestTiesUp) + +# Test rounding to invalid resolutions +dt = Dates.DateTime(2016, 2, 28, 12, 15) +for p in [Dates.Year, Dates.Month, Dates.Week, Dates.Day, Dates.Hour] + for v in [-1, 0] + @test_throws DomainError floor(dt, p(v)) + @test_throws DomainError ceil(dt, p(v)) + @test_throws DomainError round(dt, p(v)) + end +end From f83817a34e2c72f6adcb12767950e162d5f5e38e Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 8 Jul 2016 21:28:52 -0400 Subject: [PATCH 0314/1117] fix #17338, broken showing of SimpleVector (#17339) --- base/essentials.jl | 1 + test/show.jl | 3 +++ 2 files changed, 4 insertions(+) diff --git a/base/essentials.jl b/base/essentials.jl index 2f7ecbeff7645..5756a4571c81b 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -173,6 +173,7 @@ done(v::SimpleVector,i) = (i > v.length) isempty(v::SimpleVector) = (v.length == 0) indices(v::SimpleVector) = (OneTo(length(v)),) linearindices(v::SimpleVector) = indices(v, 1) +indices(v::SimpleVector, d) = d <= 1 ? indices(v)[d] : OneTo(1) function ==(v1::SimpleVector, v2::SimpleVector) length(v1)==length(v2) || return false diff --git a/test/show.jl b/test/show.jl index b8e9abe1a7435..ce2dab19e4d38 100644 --- a/test/show.jl +++ b/test/show.jl @@ -506,3 +506,6 @@ let io = IOBuffer() dump(io, :(x = 1)) @test takebuf_string(io)[end] == '\n' end + +# issue #17338 +@test repr(Core.svec(1,2)) == "svec(1,2)" From 6725be50cd427a6be5966f0bd34d40622761ab16 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 8 Jul 2016 19:23:23 -0700 Subject: [PATCH 0315/1117] Fix spelling, grammar in a few comments [av skip] --- src/ccall.cpp | 4 ++-- src/debuginfo.cpp | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 4a883fc63d134..76a9757050984 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1697,8 +1697,8 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) PointerType *funcptype = PointerType::get(functype,0); if (imaging_mode) { #ifdef LLVM37 - // ARM, PPC, PPC64 (as of LLVM 3.9) doesn't support `musttail` for vararg functions. - // And musttail can't proceed unreachable, but is required for vararg (https://llvm.org/bugs/show_bug.cgi?id=23766) + // ARM, PPC, PPC64 (as of LLVM 3.9) don't support `musttail` for vararg functions. + // And musttail can't precede unreachable, but is required for vararg (https://llvm.org/bugs/show_bug.cgi?id=23766) if (functype->isVarArg()) llvmf = runtime_sym_lookup(funcptype, f_lib, f_name, ctx->f); else diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index e00c0b7ca8075..4901eabb98bf5 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -1281,9 +1281,8 @@ int jl_getFunctionInfo(jl_frame_t **frames_out, size_t pointer, int skipC, int n DISubprogram debugscope(prev.Loc.getScope(Ctx)); jl_copy_str(&frames[0].file_name, debugscope.getFilename().str().c_str()); - // the DISubprogram has the un-mangled name, so use that if - // available. However, if the scope need not be the current - // subprogram. + // The DISubprogram has the un-mangled name, so use that if + // available. However, the scope need not be the current subprogram. if (debugscope.getName().data() != NULL) { jl_copy_str(&frames[0].func_name, debugscope.getName().str().c_str()); } From ceccd716dbe8d9cb3d10661eddda4d6d1e0e9849 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Fri, 8 Jul 2016 23:08:10 -0400 Subject: [PATCH 0316/1117] tryparse should still throw an exception on invalid base (#17333) --- base/parse.jl | 17 ++++++++++------- test/parse.jl | 5 +++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/base/parse.jl b/base/parse.jl index 223a0ecd8f32d..5e5cf78f91879 100644 --- a/base/parse.jl +++ b/base/parse.jl @@ -141,13 +141,16 @@ function tryparse_internal(::Type{Bool}, sbuff::Union{String,SubString}, Nullable{Bool}() end -function tryparse{T<:Integer}(::Type{T}, s::AbstractString, base::Integer=0) - return tryparse_internal(T, s, start(s), endof(s), base, false) -end - -function parse{T<:Integer}(::Type{T}, s::AbstractString, base::Integer=0) - return get(tryparse_internal(T, s, start(s), endof(s), base, true)) -end +check_valid_base(base) = 2 <= base <= 62 ? base : + throw(ArgumentError("invalid base: base must be 2 ≤ base ≤ 62, got $base")) +tryparse{T<:Integer}(::Type{T}, s::AbstractString, base::Integer) = + tryparse_internal(T, s, start(s), endof(s), check_valid_base(base), false) +tryparse{T<:Integer}(::Type{T}, s::AbstractString) = + tryparse_internal(T, s, start(s), endof(s), 0, false) +parse{T<:Integer}(::Type{T}, s::AbstractString, base::Integer) = + get(tryparse_internal(T, s, start(s), endof(s), check_valid_base(base), true)) +parse{T<:Integer}(::Type{T}, s::AbstractString) = + get(tryparse_internal(T, s, start(s), endof(s), 0, true)) ## string to float functions ## diff --git a/test/parse.jl b/test/parse.jl index e90df66d67512..b9eda6f987956 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -548,5 +548,10 @@ end @test_throws ArgumentError parse(Int, "2", 1) @test_throws ArgumentError parse(Int, "2", 63) +# issue #17333: tryparse should still throw on invalid base +for T in (Int32, BigInt), base in (0,1,100) + @test_throws ArgumentError tryparse(T, "0", base) +end + # error throwing branch from #10560 @test_throws ArgumentError Base.tryparse_internal(Bool, "foo", 1, 2, 10, true) From 9c7eb832969633fc5a3bf804704164841f003b7a Mon Sep 17 00:00:00 2001 From: Amit Murthy <amit.murthy@gmail.com> Date: Fri, 8 Jul 2016 15:41:23 +0530 Subject: [PATCH 0317/1117] Fix bug in channels with multiple for loops on the same channel --- base/channels.jl | 2 +- test/parallel_exec.jl | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/base/channels.jl b/base/channels.jl index c84e2da112f85..1461f14285b5d 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -51,7 +51,6 @@ function fetch(c::Channel) end function take!(c::Channel) - !isopen(c) && !isready(c) && throw(closed_exception()) wait(c) v = shift!(c.data) notify(c.cond_put, nothing, false, false) # notify only one, since only one slot has become available for a put!. @@ -64,6 +63,7 @@ isready(c::Channel) = n_avail(c) > 0 function wait(c::Channel) while !isready(c) + !isopen(c) && throw(closed_exception()) wait(c.cond_take) end nothing diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index e135dd1f8fa26..326b9274c99ad 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -538,6 +538,24 @@ function testcpt() end testcpt() +# Test multiple "for" loops waiting on the same channel which +# is closed after adding a few elements. +c=Channel() +results=[] +@sync begin + for i in 1:20 + @async for i in c + push!(results, i) + end + end + sleep(1.0) + for i in 1:5 + put!(c,i) + end + close(c) +end +@test sum(results) == 15 + @test_throws ArgumentError sleep(-1) @test_throws ArgumentError timedwait(()->false, 0.1, pollint=-0.5) From 84f57e2ff7b87ec23265dedb3f990f18938474be Mon Sep 17 00:00:00 2001 From: Amit Murthy <amit.murthy@gmail.com> Date: Thu, 7 Jul 2016 12:19:21 +0530 Subject: [PATCH 0318/1117] Optimize number of tasks started by AsyncCollector. Make it dynamic. AsyncCollector: Limit numer of tasks and communicate with channels --- base/asyncmap.jl | 100 ++++++++++++++++++++++++++++++------------ base/pmap.jl | 6 +-- base/workerpool.jl | 16 ++++++- test/abstractarray.jl | 2 +- test/parallel_exec.jl | 8 +++- 5 files changed, 96 insertions(+), 36 deletions(-) diff --git a/base/asyncmap.jl b/base/asyncmap.jl index 324e394b1e12f..9490f713267a5 100644 --- a/base/asyncmap.jl +++ b/base/asyncmap.jl @@ -19,44 +19,97 @@ type AsyncCollector f results enumerator::Enumerate - ntasks::Int + max_tasks::Function + task_chnl::Channel{Tuple{Int, Any}} # to communicate with the tasks + + AsyncCollector(f, r, en::Enumerate, mt::Function, c::Channel) = new(f, r, en, mt, c) end function AsyncCollector(f, results, c...; ntasks=0) if ntasks == 0 ntasks = max(nworkers(), 100) + max_tasks = ()->ntasks + elseif isa(ntasks, Integer) + max_tasks = ()->ntasks + elseif isa(ntasks, Function) + max_tasks = ntasks + else + throw(ArgumentError("ntasks must be an Integer or a zero-arg function returning the maximum number of tasks allowed.")) end - AsyncCollector(f, results, enumerate(zip(c...)), ntasks) + AsyncCollector(f, results, enumerate(zip(c...)), max_tasks, Channel{Tuple{Int, Any}}(typemax(Int))) end type AsyncCollectorState enum_state active_count::Int - task_done::Condition + item_done::Condition done::Bool in_error::Bool + nfree::Int # number of free tasks end -# Busy if the maximum number of concurrent tasks is running. -function isbusy(itr::AsyncCollector, state::AsyncCollectorState) - state.active_count == itr.ntasks -end - +isbusy(itr::AsyncCollector, state::AsyncCollectorState) = (state.nfree == 0) # Wait for @async task to end. -wait(state::AsyncCollectorState) = wait(state.task_done) +wait(state::AsyncCollectorState) = wait(state.item_done) + +function start_collector_task(itr::AsyncCollector, state::AsyncCollectorState) + t = @async begin + try + for (i, args) in itr.task_chnl + state.nfree -= 1 + + itr.results[i] = itr.f(args...) + notify(state.item_done, nothing) + + state.nfree += 1 + end + catch e + # The in_error flag causes done() to end the iteration early and call sync_end(). + # sync_end() then re-throws "e" in the main task. + state.in_error = true + clear_collector_channel(itr) + notify(state.item_done, nothing) + + rethrow(e) + end + end + state.active_count += 1 + t +end + +function clear_collector_channel(itr::AsyncCollector) + try + # empty out the channel and close it, ignore any errors in doing this + while isready(itr.task_chnl) + take!(itr.task_chnl) + end + close(itr.task_chnl) + catch + end + nothing +end # Open a @sync block and initialise iterator state. function start(itr::AsyncCollector) sync_begin() - AsyncCollectorState(start(itr.enumerator), 0, Condition(), false, false) + + state = AsyncCollectorState(start(itr.enumerator), 0, Condition(), false, false, 0) + + for _ in 1:itr.max_tasks() + start_collector_task(itr, state) + state.nfree += 1 + end + state end # Close @sync block when iterator is done. function done(itr::AsyncCollector, state::AsyncCollectorState) if state.in_error + @assert !isopen(itr.task_chnl) # Channel should have been cleared and closed in the async task. + sync_end() # state.in_error is only being set in the @async block (and an error thrown), @@ -67,13 +120,19 @@ function done(itr::AsyncCollector, state::AsyncCollectorState) if !state.done && done(itr.enumerator, state.enum_state) state.done = true + close(itr.task_chnl) sync_end() end return state.done end function next(itr::AsyncCollector, state::AsyncCollectorState) - # Wait if the maximum number of concurrent tasks are already running. + # start a task if required. + # Note: Shouldn't need to check on every iteration. Do this at a periodic interval? + if state.active_count < itr.max_tasks() + start_collector_task(itr, state) + end + while isbusy(itr, state) wait(state) if state.in_error @@ -84,28 +143,11 @@ function next(itr::AsyncCollector, state::AsyncCollectorState) # Get index and mapped function arguments from enumeration iterator. (i, args), state.enum_state = next(itr.enumerator, state.enum_state) + put!(itr.task_chnl, (i, args)) - # Execute function call and save result asynchronously. - @async begin - try - itr.results[i] = itr.f(args...) - catch e - # The in_error flag causes done() to end the iteration early and call sync_end(). - # sync_end() then re-throws "e" in the main task. - state.in_error = true - rethrow(e) - finally - state.active_count -= 1 - notify(state.task_done, nothing) - end - end - - # Count number of concurrent tasks. - state.active_count += 1 return (nothing, state) end - """ AsyncGenerator(f, c...; ntasks=0) -> iterator diff --git a/base/pmap.jl b/base/pmap.jl index a8ba5605e54af..ca48804867a42 100644 --- a/base/pmap.jl +++ b/base/pmap.jl @@ -21,7 +21,7 @@ for details. """ function pgenerate(p::WorkerPool, f, c) if length(p) == 0 - return AsyncGenerator(f, c) + return AsyncGenerator(f, c; ntasks=()->nworkers(p)) end batches = batchsplit(c, min_batch_count = length(p) * 3) return flatten(AsyncGenerator(remote(p, b -> asyncmap(f, b)), batches)) @@ -98,7 +98,7 @@ function pmap(p::AbstractWorkerPool, f, c; distributed=true, batch_size=1, on_e f = wrap_on_error(f, on_error) end - return collect(AsyncGenerator(f, c)) + return collect(AsyncGenerator(f, c; ntasks=()->nworkers(p))) else batches = batchsplit(c, min_batch_count = length(p) * 3, max_batch_size = batch_size) @@ -113,7 +113,7 @@ function pmap(p::AbstractWorkerPool, f, c; distributed=true, batch_size=1, on_e f = wrap_on_error(f, (x,e)->BatchProcessingError(x,e); capture_data=true) end f = wrap_batch(f, p, on_error) - results = collect(flatten(AsyncGenerator(f, batches))) + results = collect(flatten(AsyncGenerator(f, batches; ntasks=()->nworkers(p)))) if (on_error !== nothing) || (retry_n > 0) process_batch_errors!(p, f_orig, results, on_error, retry_on, retry_n, retry_max_delay) end diff --git a/base/workerpool.jl b/base/workerpool.jl index f82eac5fbdeea..5320d95a92c8a 100644 --- a/base/workerpool.jl +++ b/base/workerpool.jl @@ -47,7 +47,21 @@ isready(pool::AbstractWorkerPool) = isready(pool.channel) put!(pool::AbstractWorkerPool, w::Int) = (put!(pool.channel, w); pool) -workers(pool::AbstractWorkerPool) = collect(pool.workers) +function workers(pool::AbstractWorkerPool) + if length(pool) == 0 && pool === default_worker_pool() + return [1] + else + return collect(pool.workers) + end +end + +function nworkers(pool::AbstractWorkerPool) + if length(pool) == 0 && pool === default_worker_pool() + return 1 + else + return length(pool.workers) + end +end function take!(pool::AbstractWorkerPool) # Find an active worker diff --git a/test/abstractarray.jl b/test/abstractarray.jl index a686be2af4175..6847048da7770 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -489,7 +489,7 @@ Base.done{N}(::GenericIterator{N}, i) = i > N ? true : false Base.iteratorsize{N}(::Type{GenericIterator{N}}) = Base.SizeUnknown() function test_map(::Type{TestAbstractArray}) - empty_pool = WorkerPool() + empty_pool = WorkerPool([myid()]) pmap_fallback = (f, c...) -> pmap(empty_pool, f, c...) for mapf in [map, asyncmap, pmap_fallback] diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 326b9274c99ad..9f357836cfd53 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -10,8 +10,12 @@ elseif Base.JLOptions().code_coverage == 2 cov_flag = `--code-coverage=all` end -# Test a `remote` invocation when no workers are present +# Test a few "remote" invocations when no workers are present @test remote(myid)() == 1 +@test pmap(identity, 1:100) == [1:100...] +@test 100 == @parallel (+) for i in 1:100 + 1 + end addprocs(4; exeflags=`$cov_flag $inline_flag --check-bounds=yes --depwarn=error`) @@ -819,7 +823,7 @@ end @test [1:100...] == pmap(x->x, Base.Generator(x->(sleep(0.0001); x), 1:100)) # Test asyncmap -@test allunique(asyncmap(x->object_id(current_task()), 1:100)) +@test allunique(asyncmap(x->(sleep(1.0);object_id(current_task())), 1:10)) # CachingPool tests wp = CachingPool(workers()) From d9dbd33008d1a74e585b4a5301a3ae49bf6090d4 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Thu, 7 Jul 2016 22:07:38 -0700 Subject: [PATCH 0319/1117] Fix build breakage on power #17309 [ci skip] --- src/abi_ppc64le.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/abi_ppc64le.cpp b/src/abi_ppc64le.cpp index 40cd72b06b109..b2dac994d1208 100644 --- a/src/abi_ppc64le.cpp +++ b/src/abi_ppc64le.cpp @@ -115,7 +115,7 @@ Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) // Arguments are either scalar or passed by value size_t size = dt->size; // don't need to change bitstypes - if (!dt->nfields) + if (!jl_datatype_nfields(dt)) return NULL; // legalize this into [n x f32/f64] jl_datatype_t *ty0 = NULL; @@ -135,7 +135,7 @@ Type *preferred_llvm_type(jl_datatype_t *dt, bool isret) assert(jl_is_bitstype(elemty)); Type *ety = julia_type_to_llvm(elemty); - Type *vty = VectorType::get(ety, ty0->nfields); + Type *vty = VectorType::get(ety, jl_datatype_nfields(ty0)); return ArrayType::get(vty, hfa); } } From a587d2701b8d622359387e82696d40482743f3de Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Sat, 9 Jul 2016 13:09:27 +0200 Subject: [PATCH 0320/1117] Fix several typos in docs - Fix incorrect indent in an "Examples" section. - Remove some leading `.`s from two `.. function` signatures. - Use `*`, not `_`, for emphasis in `eigs` docs. - Replace `Note` section with an admonition in `eigs`. - Add closing `|`s for absolute values in `gcdx` docs. --- base/abstractarray.jl | 2 +- base/docs/helpdb/Base.jl | 9 +++++++-- base/intfuncs.jl | 4 ++-- base/linalg/arnoldi.jl | 18 +++++++++--------- doc/stdlib/arrays.rst | 16 +++++++++++----- doc/stdlib/linalg.rst | 23 ++++++++++++----------- doc/stdlib/math.rst | 2 +- 7 files changed, 43 insertions(+), 31 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 75cfdd1b93c91..f95e22c9688f0 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -299,7 +299,7 @@ Create an uninitialized mutable array analogous to that specified by `storagetype`, but with `indices` specified by the last argument. `storagetype` might be a type or a function. - **Examples**: +**Examples**: similar(Array{Int}, indices(A)) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index d1142f0775412..e5f85bbab3d5b 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -484,8 +484,13 @@ promote_type Returns a tuple of subscripts into an array with dimensions `dims`, corresponding to the linear index `index`. -**Example**: `i, j, ... = ind2sub(size(A), indmax(A))` provides the -indices of the maximum element +**Example**: + +``` +i, j, ... = ind2sub(size(A), indmax(A)) +``` + +provides the indices of the maximum element. """ ind2sub(dims::Tuple, index::Int) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index f7da37713690c..64fdabb4d01fd 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -69,8 +69,8 @@ julia> gcdx(240, 46) Bézout coefficients are *not* uniquely defined. `gcdx` returns the minimal Bézout coefficients that are computed by the extended Euclid algorithm. (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients `u` - and `v` are minimal in the sense that ``|u| < |\\frac y d`` and ``|v| < - |\\frac x d``. Furthermore, the signs of `u` and `v` are chosen so that `d` + and `v` are minimal in the sense that ``|u| < |\\frac y d|`` and ``|v| < + |\\frac x d|``. Furthermore, the signs of `u` and `v` are chosen so that `d` is positive. """ function gcdx{T<:Integer}(a::T, b::T) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index fba8d84227a46..baf05f4098126 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -52,7 +52,7 @@ final residual vector `resid`. !!! note The `sigma` and `which` keywords interact: the description of eigenvalues - searched for by `which` do _not_ necessarily refer to the eigenvalues of + searched for by `which` do *not* necessarily refer to the eigenvalues of `A`, but rather the linear operator constructed by the specification of the iteration mode implied by `sigma`. @@ -138,16 +138,16 @@ X = sprand(10, 5, 0.2) eigs(X, nsv = 2, tol = 1e-3) ``` -**Note** +!!! note -The `sigma` and `which` keywords interact: the description of eigenvalues searched for by -`which` do _not_ necessarily refer to the eigenvalue problem ``Av = Bv\\lambda``, but rather -the linear operator constructed by the specification of the iteration mode implied by `sigma`. + The `sigma` and `which` keywords interact: the description of eigenvalues searched for by + `which` do *not* necessarily refer to the eigenvalue problem ``Av = Bv\\lambda``, but rather + the linear operator constructed by the specification of the iteration mode implied by `sigma`. -| `sigma` | iteration mode | `which` refers to the problem | -|:----------------|:---------------------------------|:-----------------------------------| -| `nothing` | ordinary (forward) | ``Av = Bv\\lambda`` | -| real or complex | inverse with level shift `sigma` | ``(A - \\sigma B )^{-1}B = v\\nu`` | + | `sigma` | iteration mode | `which` refers to the problem | + |:----------------|:---------------------------------|:-----------------------------------| + | `nothing` | ordinary (forward) | ``Av = Bv\\lambda`` | + | real or complex | inverse with level shift `sigma` | ``(A - \\sigma B )^{-1}B = v\\nu`` | """ eigs(A, B; kwargs...) = _eigs(A, B; kwargs...) function _eigs(A, B; diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 960cca00318a7..e4272781be1a5 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -134,7 +134,13 @@ Basic functions Returns a tuple of subscripts into an array with dimensions ``dims``\ , corresponding to the linear index ``index``\ . - **Example**: ``i, j, ... = ind2sub(size(A), indmax(A))`` provides the indices of the maximum element + **Example**: + + .. code-block:: julia + + i, j, ... = ind2sub(size(A), indmax(A)) + + provides the indices of the maximum element. .. function:: ind2sub(a, index) -> subscripts @@ -277,9 +283,9 @@ Constructors Create an uninitialized mutable array analogous to that specified by ``storagetype``\ , but with ``indices`` specified by the last argument. ``storagetype`` might be a type or a function. - .. code-block:: julia + **Examples**: - **Examples**: + .. code-block:: julia similar(Array{Int}, indices(A)) @@ -1045,7 +1051,7 @@ dense counterparts. The following functions are specific to sparse arrays. For an in-place version and algorithmic information, see :func:`Base.SparseArrays.dropzeros!`\ . .. function:: permute{Tv,Ti,Tp<:Integer,Tq<:Integer}(A::SparseMatrixCSC{Tv,Ti}, p::AbstractVector{Tp}, -.............. q::AbstractVector{Tq}) + q::AbstractVector{Tq}) .. Docstring generated from Julia source @@ -1054,7 +1060,7 @@ dense counterparts. The following functions are specific to sparse arrays. For expert drivers and additional information, see :func:`Base.SparseArrays.permute!`\ . .. function:: permute!{Tv,Ti,Tp<:Integer,Tq<:Integer}(X::SparseMatrixCSC{Tv,Ti}, A::SparseMatrixCSC{Tv,Ti}, -.............. p::AbstractVector{Tp}, q::AbstractVector{Tq}[, C::SparseMatrixCSC{Tv,Ti}]) + p::AbstractVector{Tp}, q::AbstractVector{Tq}[, C::SparseMatrixCSC{Tv,Ti}]) .. Docstring generated from Julia source diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 30d5dd320531b..bbf6ce5893a31 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1338,7 +1338,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f ``eigs`` returns the ``nev`` requested eigenvalues in ``d``\ , the corresponding Ritz vectors ``v`` (only if ``ritzvec=true``\ ), the number of converged eigenvalues ``nconv``\ , the number of iterations ``niter`` and the number of matrix vector multiplications ``nmult``\ , as well as the final residual vector ``resid``\ . .. note:: - The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalues of ``A``\ , but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``\ . + The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do *not* necessarily refer to the eigenvalues of ``A``\ , but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``\ . +-----------------+------------------------------------+------------------------------------+ | ``sigma`` | iteration mode | ``which`` refers to eigenvalues of | @@ -1403,17 +1403,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f X = sprand(10, 5, 0.2) eigs(X, nsv = 2, tol = 1e-3) - **Note** + .. note:: + The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do *not* necessarily refer to the eigenvalue problem :math:`Av = Bv\lambda`\ , but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``\ . - The ``sigma`` and ``which`` keywords interact: the description of eigenvalues searched for by ``which`` do _not_ necessarily refer to the eigenvalue problem :math:`Av = Bv\lambda`\ , but rather the linear operator constructed by the specification of the iteration mode implied by ``sigma``\ . + +-----------------+------------------------------------+--------------------------------------+ + | ``sigma`` | iteration mode | ``which`` refers to the problem | + +=================+====================================+======================================+ + | ``nothing`` | ordinary (forward) | :math:`Av = Bv\lambda` | + +-----------------+------------------------------------+--------------------------------------+ + | real or complex | inverse with level shift ``sigma`` | :math:`(A - \sigma B )^{-1}B = v\nu` | + +-----------------+------------------------------------+--------------------------------------+ - +-----------------+------------------------------------+--------------------------------------+ - | ``sigma`` | iteration mode | ``which`` refers to the problem | - +=================+====================================+======================================+ - | ``nothing`` | ordinary (forward) | :math:`Av = Bv\lambda` | - +-----------------+------------------------------------+--------------------------------------+ - | real or complex | inverse with level shift ``sigma`` | :math:`(A - \sigma B )^{-1}B = v\nu` | - +-----------------+------------------------------------+--------------------------------------+ .. function:: svds(A; nsv=6, ritzvec=true, tol=0.0, maxiter=1000, ncv=2*nsv, u0=zeros((0,)), v0=zeros((0,))) -> (SVD([left_sv,] s, [right_sv,]), nconv, niter, nmult, resid) @@ -1479,7 +1479,7 @@ according to the usual Julia convention. Compute ``A B`` in-place and store the result in ``Y``\ , returning the result. If only two arguments are passed, then ``A_ldiv_B!(A, B)`` overwrites ``B`` with the result. - The argument ``A`` should *not* be a matrix. Rather, instead of matrices it should be a factorization object (e.g. produced by :func:`factorize` or :func:`cholfact`\ ). The reason for this is that factorization itself is both expensive and typically allocates memory (although it can also be done in-place via, e.g., :func:`lufact`\ ), and performance-critical situations requiring ``A_ldiv_B!`` usually also require fine-grained control over the factorization of ``A``\ . + The argument ``A`` should *not* be a matrix. Rather, instead of matrices it should be a factorization object (e.g. produced by :func:`factorize` or :func:`cholfact`\ ). The reason for this is that factorization itself is both expensive and typically allocates memory (although it can also be done in-place via, e.g., :func:`lufact!`\ ), and performance-critical situations requiring ``A_ldiv_B!`` usually also require fine-grained control over the factorization of ``A``\ . .. function:: A_ldiv_Bc(A, B) @@ -2525,3 +2525,4 @@ set of functions in future releases. Solves the Sylvester matrix equation ``A * X +/- X * B = scale*C`` where ``A`` and ``B`` are both quasi-upper triangular. If ``transa = N``\ , ``A`` is not modified. If ``transa = T``\ , ``A`` is transposed. If ``transa = C``\ , ``A`` is conjugate transposed. Similarly for ``transb`` and ``B``\ . If ``isgn = 1``\ , the equation ``A * X + X * B = scale * C`` is solved. If ``isgn = -1``\ , the equation ``A * X - X * B = scale * C`` is solved. Returns ``X`` (overwriting ``C``\ ) and ``scale``\ . + diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 2fa7dd7dbfcc0..de8a2b4043ae6 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1267,7 +1267,7 @@ Mathematical Functions (2,-9,47) .. note:: - Bézout coefficients are *not* uniquely defined. ``gcdx`` returns the minimal Bézout coefficients that are computed by the extended Euclid algorithm. (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients ``u`` and ``v`` are minimal in the sense that :math:`|u| < |\frac y d` and :math:`|v| < |\frac x d`\ . Furthermore, the signs of ``u`` and ``v`` are chosen so that ``d`` is positive. + Bézout coefficients are *not* uniquely defined. ``gcdx`` returns the minimal Bézout coefficients that are computed by the extended Euclid algorithm. (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients ``u`` and ``v`` are minimal in the sense that :math:`|u| < |\frac y d|` and :math:`|v| < |\frac x d|`\ . Furthermore, the signs of ``u`` and ``v`` are chosen so that ``d`` is positive. .. function:: ispow2(n) -> Bool From bfd98251798331fd2ccc38b7fa8f860d8b402499 Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Fri, 5 Feb 2016 11:46:13 +0100 Subject: [PATCH 0321/1117] Allow combining pushmeta!-based macros. Teach pushmeta! about encountering other macro calls, and make it jump across all but the last argument. --- base/expr.jl | 10 ++++++++-- test/meta.jl | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/base/expr.jl b/base/expr.jl index 1ea47248df842..8a6f5a91d39d9 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -117,11 +117,17 @@ function pushmeta!(ex::Expr, sym::Symbol, args::Any...) else tag = Expr(sym, args...) end - idx, exargs = findmeta(ex) + + inner = ex + while inner.head == :macrocall + inner = inner.args[end]::Expr + end + + idx, exargs = findmeta(inner) if idx != 0 push!(exargs[idx].args, tag) else - body::Expr = ex.args[2] + body::Expr = inner.args[2] unshift!(body.args, Expr(:meta, tag)) end ex diff --git a/test/meta.jl b/test/meta.jl index 1d7638e933cf1..c133013ea16cb 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -100,6 +100,23 @@ for m in [:foo1, :foo2, :foo3, :foo4, :foo5] end @test Base.findmeta(multi_meta.args)[1] == 0 +# Test that pushmeta! can push across other macros, +# in the case multiple pushmeta!-based macros are combined + +@attach 40 @attach 41 @attach 42 dummy_multi() = return nothing + +asts = code_lowered(dummy_multi, Tuple{}) +@test length(asts) == 1 +ast = asts[1] + +body = Expr(:block) +body.args = Base.uncompressed_ast(ast) + +@test popmeta!(body, :test) == (true, [40]) +@test popmeta!(body, :test) == (true, [41]) +@test popmeta!(body, :test) == (true, [42]) +@test popmeta!(body, :nonexistent) == (false, []) + end From d0025222f69d83c0ea13135071ad4182daa3407a Mon Sep 17 00:00:00 2001 From: Sacha <Sacha0@users.noreply.github.com> Date: Sat, 9 Jul 2016 18:41:14 -0700 Subject: [PATCH 0322/1117] Towards #12251 and #12153, reimplement all full(X) method defs. as convert(Array, X) and add convert(AbstractArray, X) methods for Factorizations (#17066) * Implement convert(Matrix, SparseMatrixCSC) <- convert(Array, SparseMatrixCSC) chain, and reimplement full(SparseMatrixCSC) as a synonymous child of convert(Array, SparseMatrixCSC). * Implement convert(Vector, AbstractSparseVector) <- convert(Array, AbstractSparseVector) chain, and reimplement full(AbstractSparseVector) as a synonymous child of convert(Array, AbstractSparseVector). * Implement convert(Matrix, Bidiagonal) <- convert(Array, Bidiagonal) chain, and reimplement full(Bidiagonal) as a synonymous child of convert(Array, Bidiagonal). * Implement convert(AbstractMatrix, X) <- convert(AbstractArray, X) <- convert(Matrix, X) <- convert(Array, X) chains for Cholesky and CholeskyPivoted types, and reimplement full(X) for both types as synonymous children of convert(Array, X). * Implement convert(Matrix, Diagonal) <- convert(Array, Diagonal) chain, and reimplement full(Diagonal) as a synonymous child of convert(Array, Diagonal). * Implement convert(AbstractMatrix, Eigen) <- convert(AbstractArray, Eigen) <- convert(Matrix, Eigen) <- convert(Array, Eigen) chain, and reimplement full(Eigen) as synonymous child of convert(Array, Eigen). * Implement convert(AbstractMatrix, Hessenberg) <- convert(AbstractArray, Hessenberg) <- convert(Matrix, Hessenberg) <- convert(Array, Hessenberg) and convert(Matrix, HessenbergQ) <- convert(Array, HessenbergQ )chains, and reimplement full(X) for both types as synonmous children of convert(Array, X). * Implement convert(SymTridiagonal, LDLt) <- convert(AbstractMatrix, LDLt) <- convert(AbstractArray, LDLt) <- convert(Matrix, LDLt) <- convert(Array, LDLt) chain, and reimplement full(LDLt) as a synonymous child of convert(Array, LDLt). * Implement convert(AbstractMatrix, LQ) <- convert(AbstractArray, LQ) <- convert(Matrix, LQ) <- convert(Array, LQ) and convert(Matrix, LQPackedQ) <- convert(Array, LQPackedQ) chains, and reimplement full(X) for both types as a synonymous child of convert(Array, X). * Implement convert(AbstractMatrix, X) < convert(AbstractArray, X) <- convert(Matrix, X) <- convert(Array, X) chains for LU and LU{T,Tridiagonal{T}} types, and reimplement full(X) for both types as synonymous children of convert(Array, X). * Implement convert(AbstractArray, Union{QR,QRCompactWY}) <- convert(AbstractMatrix, Union{QR,QRCompactWY}) <- convert(Matrix, Union{QR,QRCompactWY}) <- convert(Array, Union{QR,QRCompactWY}) and convert(Matrix, Union{QRPackedQ,QRCompactWYQ}) <- convert(Array, Union{QRPackedQ,QRCompactWYQ}) chains, and reimplement full(Union{QR,QRCompactWY}) and full(Union{QRPackedQ,QRCompactWYQ}) as synonymous children of convert(Array, Union{QR,QRCompactWY}) and convert(Array, Union{QRPackedQ,QRCompactWYQ}) respectively. * Implement convert(AbstractMatrix, Schur) <- convert(AbstractArray, Schur) <- convert(Matrix, Schur) <- convert(Array, Schur) chain, and reimplement full(Schur) as a synonymous child of convert(Array, Schur). * Implement convert(AbstractMatrix, SVD) <- convert(AbstractArray, SVD) <- convert(Matrix, SVD) <- convert(Array, SVD) chain, and reimplement full(SVD) as a synonymous child of convert(Array, SVD). * Implements convert(Matrix, Symmetric) and convert(Matrix, Hermitian). Implements convert(Array, Union{Symmetric,Hermitian}) as a short child of the two preceding methods, and reimplements full(Union{Symmetric,Hermitian}) likewise. * Implements convert(Array, AbstractTriangular) as a short child of convert(Matrix, AbstractTriangular), and reimplements full(AbstractTriangular) as a short child of convert(Array, AbstractTriangular). * Implements convert(Array, Tridiagonal) and convert(Array, SymTridiagonal) as synonymous children of convert(Matrix, Tridiagonal) and convert(Matrix, SymTridiagonal) respectively, and reimplements full(Tridiagonal) and full(SymTridiagonal) as short children of convert(Array, Tridiagonal) and convert(Array, SymTridiagonal) respectively. --- base/linalg/bidiag.jl | 3 +- base/linalg/cholesky.jl | 15 +++++-- base/linalg/diagonal.jl | 5 ++- base/linalg/eigen.jl | 8 +++- base/linalg/hessenberg.jl | 13 +++--- base/linalg/ldlt.jl | 9 +++- base/linalg/lq.jl | 39 +++++++++-------- base/linalg/lu.jl | 83 ++++++++++++++++++++----------------- base/linalg/qr.jl | 33 ++++++++++----- base/linalg/schur.jl | 7 +++- base/linalg/special.jl | 1 - base/linalg/svd.jl | 7 +++- base/linalg/symmetric.jl | 8 +++- base/linalg/triangular.jl | 3 +- base/linalg/tridiag.jl | 6 ++- base/sparse/sparsematrix.jl | 26 +++++++----- base/sparse/sparsevector.jl | 17 ++++---- 17 files changed, 179 insertions(+), 104 deletions(-) diff --git a/base/linalg/bidiag.jl b/base/linalg/bidiag.jl index dfa3aae1d44c7..15913677df72a 100644 --- a/base/linalg/bidiag.jl +++ b/base/linalg/bidiag.jl @@ -114,7 +114,6 @@ function Base.replace_in_print_matrix(A::Bidiagonal,i::Integer,j::Integer,s::Abs end #Converting from Bidiagonal to dense Matrix -full{T}(M::Bidiagonal{T}) = convert(Matrix{T}, M) function convert{T}(::Type{Matrix{T}}, A::Bidiagonal) n = size(A, 1) B = zeros(T, n, n) @@ -130,6 +129,8 @@ function convert{T}(::Type{Matrix{T}}, A::Bidiagonal) return B end convert{T}(::Type{Matrix}, A::Bidiagonal{T}) = convert(Matrix{T}, A) +convert(::Type{Array}, A::Bidiagonal) = convert(Matrix, A) +full(A::Bidiagonal) = convert(Array, A) promote_rule{T,S}(::Type{Matrix{T}}, ::Type{Bidiagonal{S}})=Matrix{promote_type(T,S)} #Converting from Bidiagonal to Tridiagonal diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index 8265d694c160a..e961ef09f5a57 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -324,11 +324,20 @@ convert{T}(::Type{CholeskyPivoted{T}},C::CholeskyPivoted) = CholeskyPivoted(AbstractMatrix{T}(C.factors),C.uplo,C.piv,C.rank,C.tol,C.info) convert{T}(::Type{Factorization{T}}, C::CholeskyPivoted) = convert(CholeskyPivoted{T}, C) -full{T,S}(C::Cholesky{T,S}) = C.uplo == 'U' ? C[:U]'C[:U] : C[:L]*C[:L]' -function full(F::CholeskyPivoted) +convert(::Type{AbstractMatrix}, C::Cholesky) = C.uplo == 'U' ? C[:U]'C[:U] : C[:L]*C[:L]' +convert(::Type{AbstractArray}, C::Cholesky) = convert(AbstractMatrix, C) +convert(::Type{Matrix}, C::Cholesky) = convert(Array, convert(AbstractArray, C)) +convert(::Type{Array}, C::Cholesky) = convert(Matrix, C) +full(C::Cholesky) = convert(Array, C) + +function convert(::Type{AbstractMatrix}, F::CholeskyPivoted) ip = invperm(F[:p]) - return (F[:L] * F[:U])[ip,ip] + (F[:L] * F[:U])[ip,ip] end +convert(::Type{AbstractArray}, F::CholeskyPivoted) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::CholeskyPivoted) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::CholeskyPivoted) = convert(Matrix, F) +full(F::CholeskyPivoted) = convert(Array, F) copy(C::Cholesky) = Cholesky(copy(C.factors), C.uplo) copy(C::CholeskyPivoted) = CholeskyPivoted(copy(C.factors), C.uplo, C.piv, C.rank, C.tol, C.info) diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 91c73aff73aa7..0d377c4b9308f 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -23,6 +23,9 @@ convert{T}(::Type{Diagonal{T}}, D::Diagonal) = Diagonal{T}(convert(Vector{T}, D. convert{T}(::Type{AbstractMatrix{T}}, D::Diagonal) = convert(Diagonal{T}, D) convert{T}(::Type{UpperTriangular}, A::Diagonal{T}) = UpperTriangular(A) convert{T}(::Type{LowerTriangular}, A::Diagonal{T}) = LowerTriangular(A) +convert(::Type{Matrix}, D::Diagonal) = diagm(D.diag) +convert(::Type{Array}, D::Diagonal) = convert(Matrix, D) +full(D::Diagonal) = convert(Array, D) function similar{T}(D::Diagonal, ::Type{T}) return Diagonal{T}(similar(D.diag, T)) @@ -39,8 +42,6 @@ function size(D::Diagonal,d::Integer) return d<=2 ? length(D.diag) : 1 end -full(D::Diagonal) = diagm(D.diag) - @inline function getindex(D::Diagonal, i::Int, j::Int) @boundscheck checkbounds(D, i, j) if i == j diff --git a/base/linalg/eigen.jl b/base/linalg/eigen.jl index e1edaabe3ffa8..6e68c72d4b30f 100644 --- a/base/linalg/eigen.jl +++ b/base/linalg/eigen.jl @@ -182,5 +182,11 @@ end eigvecs(A::AbstractMatrix, B::AbstractMatrix) = eigvecs(eigfact(A, B)) +# Conversion methods + ## Can we determine the source/result is Real? This is not stored in the type Eigen -full(F::Eigen) = F.vectors * Diagonal(F.values) / F.vectors +convert(::Type{AbstractMatrix}, F::Eigen) = F.vectors * Diagonal(F.values) / F.vectors +convert(::Type{AbstractArray}, F::Eigen) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::Eigen) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::Eigen) = convert(Matrix, F) +full(F::Eigen) = convert(Array, F) \ No newline at end of file diff --git a/base/linalg/hessenberg.jl b/base/linalg/hessenberg.jl index 6e3ac19a7fc00..83088cda465eb 100644 --- a/base/linalg/hessenberg.jl +++ b/base/linalg/hessenberg.jl @@ -53,11 +53,14 @@ function getindex(A::HessenbergQ, i::Integer, j::Integer) end ## reconstruct the original matrix -full{T<:BlasFloat}(A::HessenbergQ{T}) = LAPACK.orghr!(1, size(A.factors, 1), copy(A.factors), A.τ) -function full(F::Hessenberg) - fq = full(F[:Q]) - return (fq * F[:H]) * fq' -end +convert{T<:BlasFloat}(::Type{Matrix}, A::HessenbergQ{T}) = LAPACK.orghr!(1, size(A.factors, 1), copy(A.factors), A.τ) +convert(::Type{Array}, A::HessenbergQ) = convert(Matrix, A) +full(A::HessenbergQ) = convert(Array, A) +convert(::Type{AbstractMatrix}, F::Hessenberg) = (fq = full(F[:Q]); (fq * F[:H]) * fq') +convert(::Type{AbstractArray}, F::Hessenberg) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::Hessenberg) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::Hessenberg) = convert(Matrix, F) +full(F::Hessenberg) = convert(Array, F) A_mul_B!{T<:BlasFloat}(Q::HessenbergQ{T}, X::StridedVecOrMat{T}) = LAPACK.ormhr!('L', 'N', 1, size(Q.factors, 1), Q.factors, Q.τ, X) diff --git a/base/linalg/ldlt.jl b/base/linalg/ldlt.jl index f3631cc56750e..15ea8de561301 100644 --- a/base/linalg/ldlt.jl +++ b/base/linalg/ldlt.jl @@ -76,11 +76,16 @@ function A_ldiv_B!{T}(S::LDLt{T,SymTridiagonal{T}}, B::AbstractVecOrMat{T}) return B end -## reconstruct the original matrix, which is tridiagonal -function full(F::LDLt) +# Conversion methods +function convert(::Type{SymTridiagonal}, F::LDLt) e = copy(F.data.ev) d = copy(F.data.dv) e .*= d[1:end-1] d[2:end] += e .* F.data.ev SymTridiagonal(d, e) end +convert(::Type{AbstractMatrix}, F::LDLt) = convert(SymTridiagonal, F) +convert(::Type{AbstractArray}, F::LDLt) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::LDLt) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::LDLt) = convert(Matrix, F) +full(F::LDLt) = convert(Array, F) \ No newline at end of file diff --git a/base/linalg/lq.jl b/base/linalg/lq.jl index b49ece2abe93a..cb10bb81bac5d 100644 --- a/base/linalg/lq.jl +++ b/base/linalg/lq.jl @@ -46,8 +46,15 @@ function lq(A::Union{Number, AbstractMatrix}; thin::Bool=true) end copy(A::LQ) = LQ(copy(A.factors), copy(A.τ)) + convert{T}(::Type{LQ{T}},A::LQ) = LQ(convert(AbstractMatrix{T}, A.factors), convert(Vector{T}, A.τ)) convert{T}(::Type{Factorization{T}}, A::LQ) = convert(LQ{T}, A) +convert(::Type{AbstractMatrix}, A::LQ) = A[:L]*A[:Q] +convert(::Type{AbstractArray}, A::LQ) = convert(AbstractMatrix, A) +convert(::Type{Matrix}, A::LQ) = convert(Array, convert(AbstractArray, A)) +convert(::Type{Array}, A::LQ) = convert(Matrix, A) +full(A::LQ) = convert(Array, A) + ctranspose{T}(A::LQ{T}) = QR{T,typeof(A.factors)}(A.factors', A.τ) function getindex(A::LQ, d::Symbol) @@ -73,6 +80,21 @@ getq(A::LQ) = LQPackedQ(A.factors, A.τ) convert{T}(::Type{LQPackedQ{T}}, Q::LQPackedQ) = LQPackedQ(convert(AbstractMatrix{T}, Q.factors), convert(Vector{T}, Q.τ)) convert{T}(::Type{AbstractMatrix{T}}, Q::LQPackedQ) = convert(LQPackedQ{T}, Q) +convert(::Type{Matrix}, A::LQPackedQ) = LAPACK.orglq!(copy(A.factors),A.τ) +convert(::Type{Array}, A::LQPackedQ) = convert(Matrix, A) +function full{T}(A::LQPackedQ{T}; thin::Bool = true) + #= We construct the full eye here, even though it seems inefficient, because + every element in the output matrix is a function of all the elements of + the input matrix. The eye is modified by the elementary reflectors held + in A, so this is not just an indexing operation. Note that in general + explicitly constructing Q, rather than using the ldiv or mult methods, + may be a wasteful allocation. =# + if thin + convert(Array, A) + else + A_mul_B!(A, eye(T, size(A.factors,2), size(A.factors,1))) + end +end size(A::LQ, dim::Integer) = size(A.factors, dim) size(A::LQ) = size(A.factors) @@ -88,23 +110,6 @@ end size(A::LQPackedQ) = size(A.factors) -full(A::LQ) = A[:L]*A[:Q] -#= -We construct the full eye here, even though it seems ineffecient, because -every element in the output matrix is a function of all the elements of -the input matrix. The eye is modified by the elementary reflectors held -in A, so this is not just an indexing operation. Note that in general -explicitly constructing Q, rather than using the ldiv or mult methods, -may be a wasteful allocation. -=# -function full{T}(A::LQPackedQ{T}; thin::Bool=true) - if thin - LAPACK.orglq!(copy(A.factors),A.τ) - else - A_mul_B!(A, eye(T, size(A.factors,2), size(A.factors,1))) - end -end - ## Multiplication by LQ A_mul_B!{T<:BlasFloat}(A::LQ{T}, B::StridedVecOrMat{T}) = A[:L]*LAPACK.ormlq!('L','N',A.factors,A.τ,B) A_mul_B!{T<:BlasFloat}(A::LQ{T}, B::QR{T}) = A[:L]*LAPACK.ormlq!('L','N',A.factors,A.τ,full(B)) diff --git a/base/linalg/lu.jl b/base/linalg/lu.jl index 4113b106f425b..5201decd46fc6 100644 --- a/base/linalg/lu.jl +++ b/base/linalg/lu.jl @@ -329,41 +329,6 @@ function getindex{T}(F::Base.LinAlg.LU{T,Tridiagonal{T}}, d::Symbol) throw(KeyError(d)) end -function full{T}(F::Base.LinAlg.LU{T,Tridiagonal{T}}) - n = size(F, 1) - - dl = copy(F.factors.dl) - d = copy(F.factors.d) - du = copy(F.factors.du) - du2 = copy(F.factors.du2) - - for i = n - 1:-1:1 - li = dl[i] - dl[i] = li*d[i] - d[i + 1] += li*du[i] - if i < n - 1 - du[i + 1] += li*du2[i] - end - - if F.ipiv[i] != i - tmp = dl[i] - dl[i] = d[i] - d[i] = tmp - - tmp = d[i + 1] - d[i + 1] = du[i] - du[i] = tmp - - if i < n - 1 - tmp = du[i + 1] - du[i + 1] = du2[i] - du2[i] = tmp - end - end - end - return Tridiagonal(dl, d, du) -end - # See dgtts2.f function A_ldiv_B!{T}(A::LU{T,Tridiagonal{T}}, B::AbstractVecOrMat) n = size(A,1) @@ -467,5 +432,49 @@ end /(B::AbstractMatrix,A::LU) = At_ldiv_Bt(A,B).' -## reconstruct the original matrix -full(F::LU) = (F[:L] * F[:U])[invperm(F[:p]),:] +# Conversions +convert(::Type{AbstractMatrix}, F::LU) = (F[:L] * F[:U])[invperm(F[:p]),:] +convert(::Type{AbstractArray}, F::LU) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::LU) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::LU) = convert(Matrix, F) +full(F::LU) = convert(Array, F) + +function convert{T}(::Type{Tridiagonal}, F::Base.LinAlg.LU{T,Tridiagonal{T}}) + n = size(F, 1) + + dl = copy(F.factors.dl) + d = copy(F.factors.d) + du = copy(F.factors.du) + du2 = copy(F.factors.du2) + + for i = n - 1:-1:1 + li = dl[i] + dl[i] = li*d[i] + d[i + 1] += li*du[i] + if i < n - 1 + du[i + 1] += li*du2[i] + end + + if F.ipiv[i] != i + tmp = dl[i] + dl[i] = d[i] + d[i] = tmp + + tmp = d[i + 1] + d[i + 1] = du[i] + du[i] = tmp + + if i < n - 1 + tmp = du[i + 1] + du[i + 1] = du2[i] + du2[i] = tmp + end + end + end + return Tridiagonal(dl, d, du) +end +convert{T}(::Type{AbstractMatrix}, F::Base.LinAlg.LU{T,Tridiagonal{T}}) = convert(Tridiagonal, F) +convert{T}(::Type{AbstractArray}, F::Base.LinAlg.LU{T,Tridiagonal{T}}) = convert(AbstractMatrix, F) +convert{T}(::Type{Matrix}, F::Base.LinAlg.LU{T,Tridiagonal{T}}) = convert(Array, convert(AbstractArray, F)) +convert{T}(::Type{Array}, F::Base.LinAlg.LU{T,Tridiagonal{T}}) = convert(Matrix, F) +full{T}(F::Base.LinAlg.LU{T,Tridiagonal{T}}) = convert(Array, F) \ No newline at end of file diff --git a/base/linalg/qr.jl b/base/linalg/qr.jl index 51f37e39946d8..ba1797bc6105e 100644 --- a/base/linalg/qr.jl +++ b/base/linalg/qr.jl @@ -234,13 +234,23 @@ function qr!(v::AbstractVector) __normalize!(v, nrm), nrm end - -convert{T}(::Type{QR{T}},A::QR) = QR(convert(AbstractMatrix{T}, A.factors), convert(Vector{T}, A.τ)) +# Conversions +convert{T}(::Type{QR{T}}, A::QR) = QR(convert(AbstractMatrix{T}, A.factors), convert(Vector{T}, A.τ)) convert{T}(::Type{Factorization{T}}, A::QR) = convert(QR{T}, A) -convert{T}(::Type{QRCompactWY{T}},A::QRCompactWY) = QRCompactWY(convert(AbstractMatrix{T}, A.factors), convert(AbstractMatrix{T}, A.T)) +convert{T}(::Type{QRCompactWY{T}}, A::QRCompactWY) = QRCompactWY(convert(AbstractMatrix{T}, A.factors), convert(AbstractMatrix{T}, A.T)) convert{T}(::Type{Factorization{T}}, A::QRCompactWY) = convert(QRCompactWY{T}, A) -convert{T}(::Type{QRPivoted{T}},A::QRPivoted) = QRPivoted(convert(AbstractMatrix{T}, A.factors), convert(Vector{T}, A.τ), A.jpvt) +convert(::Type{AbstractMatrix}, F::Union{QR,QRCompactWY}) = F[:Q] * F[:R] +convert(::Type{AbstractArray}, F::Union{QR,QRCompactWY}) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::Union{QR,QRCompactWY}) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::Union{QR,QRCompactWY}) = convert(Matrix, F) +full(F::Union{QR,QRCompactWY}) = convert(Array, F) +convert{T}(::Type{QRPivoted{T}}, A::QRPivoted) = QRPivoted(convert(AbstractMatrix{T}, A.factors), convert(Vector{T}, A.τ), A.jpvt) convert{T}(::Type{Factorization{T}}, A::QRPivoted) = convert(QRPivoted{T}, A) +convert(::Type{AbstractMatrix}, F::QRPivoted) = (F[:Q] * F[:R])[:,invperm(F[:p])] +convert(::Type{AbstractArray}, F::QRPivoted) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::QRPivoted) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::QRPivoted) = convert(Matrix, F) +full(F::QRPivoted) = convert(Array, F) function getindex(A::QR, d::Symbol) m, n = size(A) @@ -283,11 +293,6 @@ function getindex{T}(A::QRPivoted{T}, d::Symbol) end end -## reconstruct the original matrix -full(F::QR) = F[:Q] * F[:R] -full(F::QRCompactWY) = F[:Q] * F[:R] -full(F::QRPivoted) = (F[:Q] * F[:R])[:,invperm(F[:p])] - # Type-stable interface to get Q getq(A::QRCompactWY) = QRCompactWYQ(A.factors,A.T) getq(A::Union{QR, QRPivoted}) = QRPackedQ(A.factors,A.τ) @@ -310,13 +315,21 @@ convert{T}(::Type{QRPackedQ{T}}, Q::QRPackedQ) = QRPackedQ(convert(AbstractMatri convert{T}(::Type{AbstractMatrix{T}}, Q::QRPackedQ) = convert(QRPackedQ{T}, Q) convert{S}(::Type{QRCompactWYQ{S}}, Q::QRCompactWYQ) = QRCompactWYQ(convert(AbstractMatrix{S}, Q.factors), convert(AbstractMatrix{S}, Q.T)) convert{S}(::Type{AbstractMatrix{S}}, Q::QRCompactWYQ) = convert(QRCompactWYQ{S}, Q) +convert{T}(::Type{Matrix}, A::Union{QRPackedQ{T},QRCompactWYQ{T}}) = A_mul_B!(A, eye(T, size(A.factors, 1), minimum(size(A.factors)))) +convert(::Type{Array}, A::Union{QRPackedQ,QRCompactWYQ}) = convert(Matrix, A) +function full{T}(A::Union{QRPackedQ{T},QRCompactWYQ{T}}; thin::Bool = true) + if thin + convert(Array, A) + else + A_mul_B!(A, eye(T, size(A.factors, 1))) + end +end size(A::Union{QR,QRCompactWY,QRPivoted}, dim::Integer) = size(A.factors, dim) size(A::Union{QR,QRCompactWY,QRPivoted}) = size(A.factors) size(A::Union{QRPackedQ,QRCompactWYQ}, dim::Integer) = 0 < dim ? (dim <= 2 ? size(A.factors, 1) : 1) : throw(BoundsError()) size(A::Union{QRPackedQ,QRCompactWYQ}) = size(A, 1), size(A, 2) -full{T}(A::Union{QRPackedQ{T},QRCompactWYQ{T}}; thin::Bool=true) = A_mul_B!(A, thin ? eye(T, size(A.factors,1), minimum(size(A.factors))) : eye(T, size(A.factors,1))) function getindex(A::Union{QRPackedQ,QRCompactWYQ}, i::Integer, j::Integer) x = zeros(eltype(A), size(A, 1)) diff --git a/base/linalg/schur.jl b/base/linalg/schur.jl index 45d679011102b..a13e839dd11e9 100644 --- a/base/linalg/schur.jl +++ b/base/linalg/schur.jl @@ -206,7 +206,12 @@ function schur(A::StridedMatrix, B::StridedMatrix) SchurF[:S], SchurF[:T], SchurF[:Q], SchurF[:Z], SchurF[:alpha], SchurF[:beta] end -full(F::Schur) = (F.Z * F.T) * F.Z' +# Conversion +convert(::Type{AbstractMatrix}, F::Schur) = (F.Z * F.T) * F.Z' +convert(::Type{AbstractArray}, F::Schur) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::Schur) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::Schur) = convert(Matrix, F) +full(F::Schur) = convert(Array, F) copy(F::Schur) = Schur(copy(F.T), copy(F.Z), copy(F.values)) copy(F::GeneralizedSchur) = GeneralizedSchur(copy(F.S), copy(F.T), copy(F.alpha), copy(F.beta), copy(F.Q), copy(F.Z)) diff --git a/base/linalg/special.jl b/base/linalg/special.jl index df77189053139..f70d67130807b 100644 --- a/base/linalg/special.jl +++ b/base/linalg/special.jl @@ -8,7 +8,6 @@ convert{T}(::Type{SymTridiagonal}, A::Diagonal{T})=SymTridiagonal(A.diag, zeros( convert{T}(::Type{Tridiagonal}, A::Diagonal{T})=Tridiagonal(zeros(T, size(A.diag,1)-1), A.diag, zeros(T, size(A.diag,1)-1)) convert(::Type{LowerTriangular}, A::Bidiagonal) = !A.isupper ? LowerTriangular(full(A)) : throw(ArgumentError("Bidiagonal matrix must have lower off diagonal to be converted to LowerTriangular")) convert(::Type{UpperTriangular}, A::Bidiagonal) = A.isupper ? UpperTriangular(full(A)) : throw(ArgumentError("Bidiagonal matrix must have upper off diagonal to be converted to UpperTriangular")) -convert(::Type{Matrix}, D::Diagonal) = diagm(D.diag) function convert(::Type{UnitUpperTriangular}, A::Diagonal) if !all(A.diag .== one(eltype(A))) diff --git a/base/linalg/svd.jl b/base/linalg/svd.jl index 896364c69d242..f5d7d1fa77726 100644 --- a/base/linalg/svd.jl +++ b/base/linalg/svd.jl @@ -236,4 +236,9 @@ function svdvals{TA,TB}(A::StridedMatrix{TA}, B::StridedMatrix{TB}) return svdvals!(copy_oftype(A, S), copy_oftype(B, S)) end -full(F::SVD) = (F.U * Diagonal(F.S)) * F.Vt +# Conversion +convert(::Type{AbstractMatrix}, F::SVD) = (F.U * Diagonal(F.S)) * F.Vt +convert(::Type{AbstractArray}, F::SVD) = convert(AbstractMatrix, F) +convert(::Type{Matrix}, F::SVD) = convert(Array, convert(AbstractArray, F)) +convert(::Type{Array}, F::SVD) = convert(Matrix, F) +full(F::SVD) = convert(Array, F) \ No newline at end of file diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index 4e5bc4de2864b..40898a9aa2789 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -80,8 +80,11 @@ function similar{T}(A::Hermitian, ::Type{T}) return Hermitian(B) end -full(A::Symmetric) = copytri!(copy(A.data), A.uplo) -full(A::Hermitian) = copytri!(copy(A.data), A.uplo, true) +# Conversion +convert(::Type{Matrix}, A::Symmetric) = copytri!(convert(Matrix, copy(A.data)), A.uplo) +convert(::Type{Matrix}, A::Hermitian) = copytri!(convert(Matrix, copy(A.data)), A.uplo, true) +convert(::Type{Array}, A::Union{Symmetric,Hermitian}) = convert(Matrix, A) +full(A::Union{Symmetric,Hermitian}) = convert(Array, A) parent(A::HermOrSym) = A.data convert{T,S<:AbstractMatrix}(::Type{Symmetric{T,S}},A::Symmetric{T,S}) = A convert{T,S<:AbstractMatrix}(::Type{Symmetric{T,S}},A::Symmetric) = Symmetric{T,S}(convert(S,A.data),A.uplo) @@ -89,6 +92,7 @@ convert{T}(::Type{AbstractMatrix{T}}, A::Symmetric) = Symmetric(convert(Abstract convert{T,S<:AbstractMatrix}(::Type{Hermitian{T,S}},A::Hermitian{T,S}) = A convert{T,S<:AbstractMatrix}(::Type{Hermitian{T,S}},A::Hermitian) = Hermitian{T,S}(convert(S,A.data),A.uplo) convert{T}(::Type{AbstractMatrix{T}}, A::Hermitian) = Hermitian(convert(AbstractMatrix{T}, A.data), Symbol(A.uplo)) + copy{T,S}(A::Symmetric{T,S}) = (B = copy(A.data); Symmetric{T,typeof(B)}(B,A.uplo)) copy{T,S}(A::Hermitian{T,S}) = (B = copy(A.data); Hermitian{T,typeof(B)}(B,A.uplo)) ishermitian(A::Hermitian) = true diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 18c0f551a7594..5dd80ac8f8cd5 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -48,7 +48,8 @@ imag(A::LowerTriangular) = LowerTriangular(imag(A.data)) imag(A::UnitLowerTriangular) = LowerTriangular(tril!(imag(A.data),-1)) imag(A::UnitUpperTriangular) = UpperTriangular(triu!(imag(A.data),1)) -full(A::AbstractTriangular) = convert(Matrix, A) +convert(::Type{Array}, A::AbstractTriangular) = convert(Matrix, A) +full(A::AbstractTriangular) = convert(Array, A) parent(A::AbstractTriangular) = A.data # then handle all methods that requires specific handling of upper/lower and unit diagonal diff --git a/base/linalg/tridiag.jl b/base/linalg/tridiag.jl index 6727d144a0c47..aee5e05483456 100644 --- a/base/linalg/tridiag.jl +++ b/base/linalg/tridiag.jl @@ -36,7 +36,6 @@ function SymTridiagonal(A::AbstractMatrix) end end -full{T}(M::SymTridiagonal{T}) = convert(Matrix{T}, M) convert{T}(::Type{SymTridiagonal{T}}, S::SymTridiagonal) = SymTridiagonal(convert(Vector{T}, S.dv), convert(Vector{T}, S.ev)) convert{T}(::Type{AbstractMatrix{T}}, S::SymTridiagonal) = @@ -55,6 +54,8 @@ function convert{T}(::Type{Matrix{T}}, M::SymTridiagonal{T}) return Mf end convert{T}(::Type{Matrix}, M::SymTridiagonal{T}) = convert(Matrix{T}, M) +convert(::Type{Array}, M::SymTridiagonal) = convert(Matrix, M) +full(M::SymTridiagonal) = convert(Array, M) size(A::SymTridiagonal) = (length(A.dv), length(A.dv)) function size(A::SymTridiagonal, d::Integer) @@ -365,7 +366,6 @@ function size(M::Tridiagonal, d::Integer) end end -full{T}(M::Tridiagonal{T}) = convert(Matrix{T}, M) function convert{T}(::Type{Matrix{T}}, M::Tridiagonal{T}) A = zeros(T, size(M)) for i = 1:length(M.d) @@ -378,6 +378,8 @@ function convert{T}(::Type{Matrix{T}}, M::Tridiagonal{T}) A end convert{T}(::Type{Matrix}, M::Tridiagonal{T}) = convert(Matrix{T}, M) +convert(::Type{Array}, M::Tridiagonal) = convert(Matrix, M) +full(M::Tridiagonal) = convert(Array, M) function similar{T}(M::Tridiagonal, ::Type{T}) Tridiagonal{T}(similar(M.dl, T), similar(M.d, T), similar(M.du, T), similar(M.du2, T)) end diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index f50697a8a70e0..62d50096a880e 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -284,9 +284,23 @@ function convert{Tv,Ti}(::Type{SparseMatrixCSC{Tv,Ti}}, M::AbstractMatrix) m, n) end convert{T}(::Type{AbstractMatrix{T}}, A::SparseMatrixCSC) = convert(SparseMatrixCSC{T}, A) -convert(::Type{Matrix}, S::SparseMatrixCSC) = full(S) convert(::Type{SparseMatrixCSC}, M::Matrix) = sparse(M) +function convert{Tv}(::Type{Matrix}, S::SparseMatrixCSC{Tv}) + # Handle cases where zero(Tv) is not defined but the array is dense. + A = length(S) == nnz(S) ? Array{Tv}(S.m, S.n) : zeros(Tv, S.m, S.n) + for Sj in 1:S.n + for Sk in nzrange(S, Sj) + Si = S.rowval[Sk] + Sv = S.nzval[Sk] + A[Si, Sj] = Sv + end + end + return A +end +convert(::Type{Array}, S::SparseMatrixCSC) = convert(Matrix, S) +full(S::SparseMatrixCSC) = convert(Array, S) + """ full(S) @@ -294,16 +308,6 @@ Convert a sparse matrix or vector `S` into a dense matrix or vector. """ full -function full{Tv}(S::SparseMatrixCSC{Tv}) - # Handle cases where zero(Tv) is not defined but the array is dense. - # (Should we really worry about this?) - A = length(S) == nnz(S) ? Array{Tv}(S.m, S.n) : zeros(Tv, S.m, S.n) - for col = 1 : S.n, k = S.colptr[col] : (S.colptr[col+1]-1) - A[S.rowval[k], col] = S.nzval[k] - end - return A -end - float(S::SparseMatrixCSC) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), float(copy(S.nzval))) complex(S::SparseMatrixCSC) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), complex(copy(S.nzval))) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index 13a35fa33f494..4c303ff911cfa 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -664,20 +664,23 @@ convert{TvD,Tv,Ti}(::Type{SparseMatrixCSC{TvD}}, x::AbstractSparseVector{Tv,Ti}) convert{Tv,Ti}(::Type{SparseMatrixCSC}, x::AbstractSparseVector{Tv,Ti}) = convert(SparseMatrixCSC{Tv,Ti}, x) - -### Array manipulation - -function full{Tv}(x::AbstractSparseVector{Tv}) +function convert{Tv}(::Type{Vector}, x::AbstractSparseVector{Tv}) n = length(x) - n == 0 && return Array{Tv}(0) + n == 0 && return Vector{Tv}(0) nzind = nonzeroinds(x) nzval = nonzeros(x) r = zeros(Tv, n) - for i = 1:length(nzind) - r[nzind[i]] = nzval[i] + for k in 1:nnz(x) + i = nzind[k] + v = nzval[k] + r[i] = v end return r end +convert(::Type{Array}, x::AbstractSparseVector) = convert(Vector, x) +full(x::AbstractSparseVector) = convert(Array, x) + +### Array manipulation vec(x::AbstractSparseVector) = x copy(x::AbstractSparseVector) = From 2e2f6dc7bc503ca77eef6cb0ab43cda819396fd4 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Sat, 9 Jul 2016 17:54:35 +0200 Subject: [PATCH 0323/1117] Update multiline signature matching explanation Adds a more explicit example of the leading whitespace requirement for docstring signatures in the RST docs. --- CONTRIBUTING.md | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index acca272e32723..adf70e7efbfee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,7 +104,8 @@ If you want to edit an existing docstring signature, you **first** have to chang edit the helpdb.jl or inline method docstrings. The existing signatures in the `doc/stdlib/*.rst` files are pattern matched to base docstrings and the new content overwrites the content in `doc/stdlib/`. The signature definitions **must** be in sync or else the pattern match will fail and documentation will be lost in the result. To add entirely new methods to the `stdlib` documentation, first add the signature in the appropriate `doc/stdlib/*.rst` file before writing the docstring, rebuilding Julia, and re-running `doc/genstdlib.jl`. -Pattern matching requires that multiline method signatures' inter-line character alignment in `doc/stdlib/*.rst` match that in the corresponding docstring. For example, docstring +Pattern matching requires that multiline method signatures' inter-line character alignment in `doc/stdlib/*.rst` match that in the corresponding docstring. In the following example, + ```julia """ foo(bar, baz, @@ -113,12 +114,25 @@ Pattern matching requires that multiline method signatures' inter-line character Foo `bar`, `baz`, `qux`, and `quux`. """ ``` -will only match entries in `doc/stdlib/*.rst` of the form -```rst + +will only match entries in `doc/stdlib/*.rst` beginning with + +``` +.. function:: foo(bar, baz, + qux, quux) +``` + +Note that the second line of the signature is indented by four spaces relative to `foo(bar, baz,` +in the first line of the signature. This leading indent matches the indent used in the +docstring exactly. If it did not match, such as in the following example, + +``` .. function:: foo(bar, baz, -.............. qux, quux) + qux, quux) ``` -where `.` in the preceding snippet's second line is wildcard, and their number preceding indent-` qux, quux)` matches the character count of `.. function:: `. + +where three spaces instead of four are used then running `genstdlib.jl` will print a warning +and not update the docstring. It is encouraged to write all new docstrings in Markdown markup. If you need to write a more complicated docstring that contains cross-references or citations it can be written in a restructured text codeblock. Many of the existing docstrings are currently restructured text codeblocks and these will be transitioned to Markdown over time. RST codeblocks are delineated with the triple-quote (\`\`\`rst \`\`\`) Makdown codeblock syntax. From cca76bd2ece6e75fc56f0b17a54c34c819635cfc Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Sun, 10 Jul 2016 16:09:49 -0400 Subject: [PATCH 0324/1117] fix #17350, regression in ambig. detection caused by c779caf0ee823128d8410f8aaf3d2255231a6acb --- src/jltypes.c | 2 -- test/ambiguous.jl | 9 +++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index 96a248db62696..5dfeeff67ef69 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2395,8 +2395,6 @@ static jl_value_t *inst_tuple_w_(jl_value_t *t, jl_value_t **env, size_t n, for(i=0; i < ntp; i++) { jl_value_t *elt = jl_svecref(tp, i); jl_value_t *pi = (jl_value_t*)inst_type_w_(elt, env, n, stack, 0); - if (jl_is_typevar(pi) && !((jl_tvar_t*)pi)->bound) - pi = ((jl_tvar_t*)pi)->ub; iparams[i] = pi; if (ip_heap) jl_gc_wb(ip_heap, pi); diff --git a/test/ambiguous.jl b/test/ambiguous.jl index fa2bc54c269e8..abf0cd6e6d8a3 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -169,4 +169,13 @@ let ms = methods(g16493, (Complex, Any)) @test first(ms).sig == Tuple{typeof(g16493), Complex{TypeVar(:T, Any, true)}, Any} end +# issue #17350 +module Ambig6 +immutable ScaleMinMax{To,From} end +map1{To<:Union{Float32,Float64},From<:Real}(mapi::ScaleMinMax{To,From}, val::From) = 1 +map1{To<:Union{Float32,Float64},From<:Real}(mapi::ScaleMinMax{To,From}, val::Union{Real,Complex}) = 2 +end + +@test isempty(detect_ambiguities(Ambig6)) + nothing From f19b2e4d03f84f738097554fb115578a13e7338d Mon Sep 17 00:00:00 2001 From: Simon Byrne <simonbyrne@gmail.com> Date: Mon, 11 Jul 2016 06:49:27 -0400 Subject: [PATCH 0325/1117] add docstrings to cwstring and transcode (#17218) --- base/c.jl | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/base/c.jl b/base/c.jl index 81aecc8c2c64b..97a1815b7d23c 100644 --- a/base/c.jl +++ b/base/c.jl @@ -110,8 +110,17 @@ end # symbols are guaranteed not to contain embedded NUL convert(::Type{Cstring}, s::Symbol) = Cstring(unsafe_convert(Ptr{Cchar}, s)) -# FIXME: this should be handled by implicit conversion to Cwstring, but good luck with that if is_windows() +""" + Base.cwstring(s) + +Converts a string `s` to a NUL-terminated `Vector{Cwchar_t}`, suitable for passing to C +functions expecting a `Ptr{Cwchar_t}`. The main advantage of using this over the implicit +conversion provided by `Cwstring` is if the function is called multiple times with the +same argument. + +This is only available on Windows. +""" function cwstring(s::AbstractString) bytes = String(s).data 0 in bytes && throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))")) @@ -120,7 +129,17 @@ end end # transcoding between data in UTF-8 and UTF-16 for Windows APIs +""" + Base.transcode(T,src::Vector{U}) + +Transcodes unicode data `src` to a different encoding, where `U` and `T` are the integers +denoting the input and output code units. Currently supported are UTF-8 and UTF-16, which +are denoted by integers `UInt8` and `UInt16`, respectively. +NULs are handled like any other character (i.e. the output will be NUL-terminated if and +only if the `src` is). +""" +function transcode end transcode{T<:Union{UInt8,UInt16}}(::Type{T}, src::Vector{T}) = src transcode(::Type{Int32}, src::Vector{UInt32}) = reinterpret(Int32, src) From b27e874d464763935fda51f3626c21c3cde80691 Mon Sep 17 00:00:00 2001 From: Carlo Baldassi <carlobaldassi@gmail.com> Date: Mon, 11 Jul 2016 13:21:38 +0200 Subject: [PATCH 0326/1117] Fix #17366 (cmdline parsing of repeated `--`) --- base/client.jl | 6 +----- test/cmdlineargs.jl | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/base/client.jl b/base/client.jl index d3b163e9ff251..84adfff03ec13 100644 --- a/base/client.jl +++ b/base/client.jl @@ -202,11 +202,7 @@ try_include(path::AbstractString) = isfile(path) && include(path) function process_options(opts::JLOptions) if !isempty(ARGS) idxs = find(x -> x == "--", ARGS) - if length(idxs) > 1 - println(STDERR, "julia: redundant option terminator `--`") - exit(1) - end - deleteat!(ARGS, idxs) + length(idxs) > 0 && deleteat!(ARGS, idxs[1]) end repl = true startup = (opts.startupfile != 2) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 274f300b0b273..1dfb0aa9dd1f9 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -213,7 +213,7 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` @test readchomp(`$exename -L $testfile -e 'exit(0)' -- foo -bar --baz`) == "String[\"foo\",\"-bar\",\"--baz\"]" @test split(readchomp(`$exename -L $testfile $testfile`), '\n') == ["String[\"$(escape(testfile))\"]", "String[]"] @test !success(`$exename --foo $testfile`) - @test !success(`$exename -L $testfile -e 'exit(0)' -- foo -bar -- baz`) + @test readchomp(`$exename -L $testfile -e 'exit(0)' -- foo -bar -- baz`) == "String[\"foo\",\"-bar\",\"--\",\"baz\"]" finally rm(testfile) end From 104da08df30049fd84494c7d596a46972e7837a4 Mon Sep 17 00:00:00 2001 From: Valentin Churavy <v.churavy@gmail.com> Date: Thu, 23 Jun 2016 17:21:26 -0400 Subject: [PATCH 0327/1117] Fix doctest code-blocks in documentation cb8aa0b - fix most doctests in the manual that changed due to backtraces. cd3797e - fix stdlib doctests 1892f72 - fix doctests in devdocs d9495c7 - resolve Warning mismatch by renaming functions 83ef854 - convert superflous doctest to code-block 78f6d70 - Fix example about typemax and typemin Due to the changes in 4706184 tuples are now printed compact. Thus to demonstrate the point in this example we have to execute the examples independently. c03e6e0 - fixup linenumbers 35a31bb - start->first Ref #15753 --- base/docs/helpdb/Base.jl | 6 +- doc/devdocs/reflection.rst | 9 ++- doc/devdocs/types.rst | 74 +++++++++--------- doc/manual/arrays.rst | 8 +- doc/manual/complex-and-rational-numbers.rst | 4 +- doc/manual/constructors.rst | 29 +++++-- doc/manual/control-flow.rst | 67 +++++++++------- doc/manual/conversion-and-promotion.rst | 17 ++-- doc/manual/documentation.rst | 2 +- doc/manual/functions.rst | 4 +- .../integers-and-floating-point-numbers.rst | 21 +++-- doc/manual/mathematical-operations.rst | 20 ++--- doc/manual/metaprogramming.rst | 9 ++- doc/manual/methods.rst | 36 +++++---- doc/manual/performance-tips.rst | 2 +- doc/manual/stacktraces.rst | 78 ++++++++++--------- doc/manual/strings.rst | 14 ++-- doc/manual/types.rst | 27 +++++-- doc/manual/variables-and-scoping.rst | 5 +- doc/manual/variables.rst | 2 + doc/stdlib/base.rst | 3 +- doc/stdlib/collections.rst | 13 +--- 22 files changed, 270 insertions(+), 180 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index e5f85bbab3d5b..f34ef98c24efc 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5450,7 +5450,8 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat! at array.jl:543 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:565 + in eval(::Module, ::Any) at ./boot.jl:237 ``` """ deleteat!(collection, itr) @@ -8109,7 +8110,8 @@ julia> convert(Int, 3.0) julia> convert(Int, 3.5) ERROR: InexactError() - in convert at int.jl:209 + in convert(::Type{Int64}, ::Float64) at ./int.jl:229 + in eval(::Module, ::Any) at ./boot.jl:237 ``` If `T` is a [`AbstractFloat`](:obj:`AbstractFloat`) or [`Rational`](:obj:`Rational`) type, diff --git a/doc/devdocs/reflection.rst b/doc/devdocs/reflection.rst index 2f3ace38417db..a9ec23394e1ff 100644 --- a/doc/devdocs/reflection.rst +++ b/doc/devdocs/reflection.rst @@ -104,9 +104,14 @@ variable assignments: .. doctest:: julia> expand( :(f() = 1) ) - :($(Expr(:method, :f, :((top(svec))((top(apply_type))(Tuple),(top(svec))())), AST(:($(Expr(:lambda, Any[], Any[Any[],Any[],0,Any[]], :(begin # none, line 1: + :(begin + $(Expr(:method, :f)) + $(Expr(:method, :f, :((top(svec))((top(apply_type))(Tuple,((top(getfield))(Core,:Typeof))(f)),(top(svec))())), LambdaInfo for anonymous + :(begin # none, line 1: return 1 - end))))), false))) + end), false)) + return f + end) .. rubric:: Intermediate and compiled representations diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index a75f1830f1fbd..1f691a1cf471a 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -130,9 +130,9 @@ one can extract the underlying :obj:`TypeVar`: .. testcode:: s g{S<:Integer}(x::S) = 0 - m = start(methods(g)) + m = first(methods(g)) p = m.sig.parameters - tv = p[1] + tv = p[2] dump(tv) .. testoutput:: s @@ -140,7 +140,7 @@ one can extract the underlying :obj:`TypeVar`: TypeVar name: Symbol S lb: Union{} - ub: Integer::DataType <: Real + ub: Integer <: Real bound: Bool true Here ``ub`` is ``Integer``, as specified in the function definition. @@ -160,27 +160,27 @@ parameters. For example: julia> h3{T<:Real}(A::Array{T}, b::T) = 1 h3 (generic function with 1 method) - julia> p1 = start(methods(h1)).sig.parameters - svec(Array{T,N},Real) + julia> p1 = first(methods(h1)).sig.parameters + svec(#h1,Array{T,N},Real) - julia> p2 = start(methods(h2)).sig.parameters - svec(Array{T,N},T<:Real) + julia> p2 = first(methods(h2)).sig.parameters + svec(#h2,Array{T,N},T<:Real) - julia> p3 = start(methods(h3)).sig.parameters - svec(Array{T<:Real,N},T<:Real) + julia> p3 = first(methods(h3)).sig.parameters + svec(#h3,Array{T<:Real,N},T<:Real) - julia> dump(p1[1].parameters[1]) + julia> dump(p1[2].parameters[1]) TypeVar name: Symbol T lb: Union{} - ub: Any::DataType <: Any + ub: Any <: Any bound: Bool false - julia> dump(p3[1].parameters[1]) + julia> dump(p3[2].parameters[1]) TypeVar name: Symbol T lb: Union{} - ub: Real::DataType <: Number + ub: Real <: Number bound: Bool true Note that ``p2`` shows two objects called ``T``, but only one of them @@ -194,7 +194,7 @@ One can construct :obj:`TypeVar`\s manually: .. doctest:: julia> TypeVar(:V, Signed, Real, false) - Signed<:V<:Real + V<:Real There are convenience versions that allow you to omit any of these arguments except the ``name`` symbol. @@ -215,20 +215,21 @@ a lot about how Julia does dispatch: julia> methods(candid) # 1 method for generic function "candid": - candid{T}(A::Array{T,N}, x::T) at none:1 + candid{T}(A::Array{T,N<:Any}, x::T) at none:1 julia> methods(sneaky) # 1 method for generic function "sneaky": - sneaky{T}(A::Array{T,N}, x::T) at none:1 + sneaky{T}(A::Array{T,N<:Any}, x::T<:Any) at none:1 These therefore print identically, but they have very different behavior: .. doctest:: julia> candid([1],3.2) - ERROR: MethodError: `candid` has no method matching candid(::Array{Int64,1}, ::Float64) + ERROR: MethodError: no method matching candid(::Array{Int64,1}, ::Float64) Closest candidates are: candid{T}(::Array{T,N}, !Matched::T) + in eval(::Module, ::Any) at ./boot.jl:237 julia> sneaky([1],3.2) 1 @@ -244,11 +245,11 @@ bound :obj:`TypeVar` objects with a hash (``#T`` instead of ``T``): .. doctest:: - julia> jl_(start(methods(candid))) - Method(sig=Tuple{Array{#T<:Any, N<:Any}, #T<:Any}, va=false, isstaged=false, tvars=#T<:Any, func=#<function>, invokes=nothing, next=nothing) + julia> jl_(first(methods(candid))) + Method(sig=Tuple{Main.#candid, Array{#T<:Any, N<:Any}, #T<:Any}, va=false, isstaged=false, tvars=#T<:Any, func=Main.candid(?), invokes=nothing, next=nothing) - julia> jl_(start(methods(sneaky))) - Method(sig=Tuple{Array{#T<:Any, N<:Any}, T<:Any}, va=false, isstaged=false, tvars=#T<:Any, func=#<function>, invokes=nothing, next=nothing) + julia> jl_(first(methods(sneaky))) + Method(sig=Tuple{Main.#sneaky, Array{#T<:Any, N<:Any}, T<:Any}, va=false, isstaged=false, tvars=#T<:Any, func=Main.sneaky(?), invokes=nothing, next=nothing) Even though both print as ``T``, in ``sneaky`` the second ``T`` is not bound, and hence it isn't constrained to be the same type as the @@ -320,8 +321,17 @@ the type, which is an object of type :obj:`TypeName`: cache: SimpleVector length: Int64 135 linearcache: SimpleVector - length: Int64 18 - uid: Int64 37 + length: Int64 60 + uid: Int64 43 + mt: MethodTable + name: Symbol Array + defs: Void nothing + cache: Void nothing + cache_arg1: Void nothing + cache_targ: Void nothing + max_args: Int64 0 + kwsorter: #undef + module: Module Core In this case, the relevant field is ``primary``, which holds a reference to the "primary" instance of the type:: @@ -357,20 +367,10 @@ type: MyType{Float32,5} julia> MyType.name.cache - svec(MyType{Float32,5},MyType{Int64,2},Evaluation succeeded, but an error occurred while showing value of type SimpleVector: - ERROR: UndefRefError: access to undefined reference - in getindex at ./essentials.jl:211 - in show_delim_array at show.jl:229 - in show at show.jl:257 - in anonymous at show.jl:1278 - in with_output_limit at ./show.jl:1255 - in showlimited at show.jl:1277 - in display at multimedia.jl:120 - [inlined code] from multimedia.jl:151 - in display at multimedia.jl:162 - -(The error is triggered because the cache is pre-allocated to have -length 8, but only the first two entries are populated.) + svec(MyType{Float32,5},MyType{Int64,2},#undef,#undef,#undef,#undef,#undef,#undef) + +(The cache is pre-allocated to have length 8, but only the first two entries +are populated.) Consequently, when you instantiate a parametric type, each concrete type gets saved in a type-cache. However, instances with :obj:`TypeVar` parameters are not cached. diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 84207da57dbb7..b68bf6220490b 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -517,9 +517,9 @@ the name of the function to vectorize. Here is a simple example: julia> methods(square) # 4 methods for generic function "square": - square{T<:Number}(::AbstractArray{T<:Number,1}) at operators.jl:374 - square{T<:Number}(::AbstractArray{T<:Number,2}) at operators.jl:375 - square{T<:Number}(::AbstractArray{T<:Number,N}) at operators.jl:377 + square{T<:Number}(x::AbstractArray{T,1}) at operators.jl:... + square{T<:Number}(x::AbstractArray{T,2}) at operators.jl:... + square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:... square(x) at none:1 julia> square([1 2 4; 5 6 7]) @@ -651,7 +651,7 @@ stride parameters. 0.507762 0.573567 0.220124 0.165816 0.211049 0.433277 0.539476 julia> b = sub(a, 2:2:8,2:2:4) - 4×2 SubArray{Float64,2,Array{Float64,2},Tuple{StepRange{Int64,Int64},StepRange{Int64,Int64}},1}: + 4×2 SubArray{Float64,2,Array{Float64,2},Tuple{StepRange{Int64,Int64},StepRange{Int64,Int64}},false}: 0.537192 0.996234 0.736979 0.228787 0.991511 0.74485 diff --git a/doc/manual/complex-and-rational-numbers.rst b/doc/manual/complex-and-rational-numbers.rst index 87712d917bb92..dcf49c8a8f0ee 100644 --- a/doc/manual/complex-and-rational-numbers.rst +++ b/doc/manual/complex-and-rational-numbers.rst @@ -159,7 +159,8 @@ versus ``-1 + 0im`` even though ``-1 == -1 + 0im``: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt at math.jl:146 + in sqrt(::Int64) at ./math.jl:146 + in eval(::Module, ::Any) at ./boot.jl:237 julia> sqrt(-1 + 0im) 0.0 + 1.0im @@ -305,6 +306,7 @@ Trying to construct a :const:`NaN` rational value, however, is not: ERROR: ArgumentError: invalid rational: zero(Int64)//zero(Int64) in Rational{Int64}(::Int64, ::Int64) at ./rational.jl:8 in //(::Int64, ::Int64) at ./rational.jl:22 + in eval(::Module, ::Any) at ./boot.jl:237 As usual, the promotion system makes interactions with other numeric types effortless: diff --git a/doc/manual/constructors.rst b/doc/manual/constructors.rst index 9207a19c42a7b..81661e656b18b 100644 --- a/doc/manual/constructors.rst +++ b/doc/manual/constructors.rst @@ -121,7 +121,8 @@ Now ``OrderedPair`` objects can only be constructed such that julia> OrderedPair(2,1) ERROR: out of order - in call at none:5 + in OrderedPair(::Int64, ::Int64) at ./none:5 + in eval(::Module, ::Any) at ./boot.jl:237 You can still reach in and directly change the field values to violate this invariant, but messing around with an object's internals uninvited is @@ -266,6 +267,7 @@ access to an uninitialized reference is an immediate error: julia> z.xx ERROR: UndefRefError: access to undefined reference + in eval(::Module, ::Any) at ./boot.jl:237 This avoids the need to continually check for ``null`` values. However, not all object fields are references. Julia considers some @@ -323,9 +325,14 @@ types of the arguments given to the constructor. Here are some examples: Point{Float64}(1.0,2.5) julia> Point(1,2.5) - ERROR: MethodError: `convert` has no method matching convert(::Type{Point{T<:Real}}, ::Int64, ::Float64) - This may have arisen from a call to the constructor Point{T<:Real}(...), - since type constructors fall back to convert methods. + ERROR: MethodError: no method matching Point{T<:Real}(::Int64, ::Float64) + Closest candidates are: + (!Matched::Type{BoundsError})(::ANY, ::ANY) + (!Matched::Type{TypeError})(::Any, ::Any, !Matched::Any, !Matched::Any) + (!Matched::Type{TypeConstructor})(::ANY, ::ANY) + ... + in eval(::Module, ::Any) at ./boot.jl:237 + ## explicit T ## @@ -334,7 +341,9 @@ types of the arguments given to the constructor. Here are some examples: julia> Point{Int64}(1.0,2.5) ERROR: InexactError() - in call at none:2 + [inlined code] from ./int.jl:229 + in Point{Int64}(::Float64, ::Float64) at ./none:2 + in eval(::Module, ::Any) at ./boot.jl:237 julia> Point{Float64}(1.0,2.5) Point{Float64}(1.0,2.5) @@ -419,9 +428,13 @@ However, other similar calls still don't work: .. doctest:: julia> Point(1.5,2) - ERROR: MethodError: `convert` has no method matching convert(::Type{Point{T<:Real}}, ::Float64, ::Int64) - This may have arisen from a call to the constructor Point{T<:Real}(...), - since type constructors fall back to convert methods. + ERROR: MethodError: no method matching Point{T<:Real}(::Float64, ::Int64) + Closest candidates are: + (!Matched::Type{BoundsError})(::ANY, ::ANY) + (!Matched::Type{TypeError})(::Any, ::Any, !Matched::Any, !Matched::Any) + (!Matched::Type{TypeConstructor})(::ANY, ::ANY) + ... + in eval(::Module, ::Any) at ./boot.jl:237 For a much more general way of making all such calls work sensibly, see :ref:`man-conversion-and-promotion`. At the risk diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 98ff5413e13dd..87ecd823829b0 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -128,7 +128,7 @@ So, we could have defined the ``test`` function above as .. doctest:: - julia> function test(x,y) + julia> function test1(x,y) if x < y relation = "less than" elseif x == y @@ -138,7 +138,7 @@ So, we could have defined the ``test`` function above as end println("x is ", relation, " y.") end - test (generic function with 1 method) + test1 (generic function with 1 method) The variable ``relation`` is declared inside the ``if`` block, but used outside. However, when depending on this behavior, make sure all possible @@ -147,7 +147,7 @@ the above function results in a runtime error .. doctest:: - julia> function test(x,y) + julia> function test2(x,y) if x < y relation = "less than" elseif x == y @@ -155,14 +155,15 @@ the above function results in a runtime error end println("x is ", relation, " y.") end - test (generic function with 1 method) + test2 (generic function with 1 method) - julia> test(1,2) + julia> test2(1,2) x is less than y. - julia> test(2,1) + julia> test2(2,1) ERROR: UndefVarError: relation not defined - in test at none:7 + in test2(::Int64, ::Int64) at ./none:7 + in eval(::Module, ::Any) at ./boot.jl:237 ``if`` blocks also return a value, which may seem unintuitive to users coming from many other languages. This value is simply the return value @@ -193,6 +194,7 @@ conditional expression is anything but ``true`` or ``false``: println("true") end ERROR: TypeError: non-boolean (Int64) used in boolean context + in eval(::Module, ::Any) at ./boot.jl:237 This error indicates that the conditional was of the wrong type: :obj:`Int64` rather than the required :obj:`Bool`. @@ -236,17 +238,17 @@ together: .. doctest:: - julia> test(x, y) = println(x < y ? "x is less than y" : + julia> test4(x, y) = println(x < y ? "x is less than y" : x > y ? "x is greater than y" : "x is equal to y") - test (generic function with 1 method) + test4 (generic function with 1 method) - julia> test(1, 2) + julia> test4(1, 2) x is less than y - julia> test(2, 1) + julia> test4(2, 1) x is greater than y - julia> test(1, 1) + julia> test4(1, 1) x is equal to y To facilitate chaining, the operator associates from right to left. @@ -365,7 +367,8 @@ For example, a recursive factorial routine could be defined like this: julia> fact(-1) ERROR: n must be non-negative - in fact at none:2 + in fact(::Int64) at ./none:2 + in eval(::Module, ::Any) at ./boot.jl:237 Boolean operations *without* short-circuit evaluation can be done with the @@ -394,6 +397,7 @@ except for the last entry in a conditional chain is an error: julia> 1 && true ERROR: TypeError: non-boolean (Int64) used in boolean context + in eval(::Module, ::Any) at ./boot.jl:237 On the other hand, any type of expression can be used at the end of a conditional chain. It will be evaluated and returned depending on the preceding conditionals: @@ -478,6 +482,7 @@ different variable name to test this: julia> j ERROR: UndefVarError: j not defined + in eval(::Module, ::Any) at ./boot.jl:237 See :ref:`man-variables-and-scoping` for a detailed explanation of variable scope and how it works in Julia. @@ -661,7 +666,8 @@ negative real value: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt at math.jl:146 + in sqrt(::Int64) at ./math.jl:146 + in eval(::Module, ::Any) at ./boot.jl:237 You may define your own exceptions in the following way: @@ -678,15 +684,16 @@ if the argument is negative: .. doctest:: - julia> f(x) = x>=0 ? exp(-x) : throw(DomainError()) - f (generic function with 1 method) + julia> g(x) = x>=0 ? exp(-x) : throw(DomainError()) + g (generic function with 1 method) - julia> f(1) + julia> g(1) 0.36787944117144233 - julia> f(-1) + julia> g(-1) ERROR: DomainError: - in f at none:1 + in g(::Int64) at ./none:1 + in eval(::Module, ::Any) at ./boot.jl:237 Note that :exc:`DomainError` without parentheses is not an exception, but a type of exception. It needs to be called to obtain an :exc:`Exception` object: @@ -706,6 +713,7 @@ error reporting: julia> throw(UndefVarError(:x)) ERROR: UndefVarError: x not defined + in eval(::Module, ::Any) at ./boot.jl:237 This mechanism can be implemented easily by custom exception types following the way :exc:`UndefVarError` is written: @@ -737,7 +745,8 @@ the :func:`sqrt` function that raises an error if its argument is negative: julia> fussy_sqrt(-1) ERROR: negative x not allowed - in fussy_sqrt at none:1 + in fussy_sqrt(::Int64) at ./none:1 + in eval(::Module, ::Any) at ./boot.jl:237 If ``fussy_sqrt`` is called with a negative value from another function, instead of trying to continue execution of the calling function, it @@ -762,7 +771,9 @@ session: julia> verbose_fussy_sqrt(-1) before fussy_sqrt ERROR: negative x not allowed - in verbose_fussy_sqrt at none:3 + [inlined code] from ./none:1 + in verbose_fussy_sqrt(::Int64) at ./none:3 + in eval(::Module, ::Any) at ./boot.jl:237 Warnings and informational messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -783,7 +794,8 @@ execution: julia> error("Hi"); 1+1 ERROR: Hi - in error at ./error.jl:21 + in error(::ASCIIString) at ./error.jl:21 + in eval(::Module, ::Any) at ./boot.jl:237 The ``try/catch`` statement @@ -796,17 +808,17 @@ call either the real or complex square root method on demand using .. doctest:: - julia> f(x) = try + julia> h(x) = try sqrt(x) catch sqrt(complex(x, 0)) end - f (generic function with 1 method) + h (generic function with 1 method) - julia> f(1) + julia> h(1) 1.0 - julia> f(-1) + julia> h(-1) 0.0 + 1.0im It is important to note that in real code computing this function, one would @@ -842,7 +854,8 @@ assumes ``x`` is a real number and returns its square root: julia> sqrt_second(-9) ERROR: DomainError: - in sqrt_second at none:7 + in sqrt_second(::Int64) at ./none:7 + in eval(::Module, ::Any) at ./boot.jl:237 Note that the symbol following ``catch`` will always be interpreted as a name for the exception, so care is needed when writing ``try/catch`` expressions diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 1166c92866677..251c7280eee95 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -101,10 +101,11 @@ requested conversion: This may have arisen from a call to the constructor AbstractFloat(...), since type constructors fall back to convert methods. Closest candidates are: - convert(::Type{AbstractFloat}, ::Bool) - convert(::Type{AbstractFloat}, ::Int8) - convert(::Type{AbstractFloat}, ::Int16) + convert(::Type{AbstractFloat}, !Matched::Bool) + convert(::Type{AbstractFloat}, !Matched::Int8) + convert(::Type{AbstractFloat}, !Matched::Int16) ... + in eval(::Module, ::Any) at ./boot.jl:237 Some languages consider parsing strings as numbers or formatting numbers as strings to be conversions (many dynamic languages will even @@ -146,7 +147,8 @@ to one and zero: julia> convert(Bool, 1im) ERROR: InexactError() - in convert at complex.jl:18 + in convert(::Type{Bool}, ::Complex{Int64}) at ./complex.jl:18 + in eval(::Module, ::Any) at ./boot.jl:237 julia> convert(Bool, 0im) false @@ -161,7 +163,8 @@ This is the actual implementation in Julia:: julia> convert(Bool, 1im) ERROR: InexactError() - in convert at complex.jl:18 + in convert(::Type{Bool}, ::Complex{Int64}) at ./complex.jl:18 + in eval(::Module, ::Any) at ./boot.jl:237 Case Study: Rational Conversions @@ -250,10 +253,10 @@ promotion is to convert numeric arguments to a common type: (1.0,2.5,3.0,0.75) julia> promote(1.5, im) - (1.5 + 0.0im,0.0 + 1.0im) + (1.5+0.0im,0.0+1.0im) julia> promote(1 + 2im, 3//4) - (1//1 + 2//1*im,3//4 + 0//1*im) + (1//1+2//1*im,3//4+0//1*im) Floating-point values are promoted to the largest of the floating-point argument types. Integer values are promoted to the larger of either the diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index 305137d358431..7b6d8945e097c 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -30,7 +30,7 @@ macros and pass them to the ``@doc`` macro just as well. Here is a more complex example, still using Markdown: -.. doctest:: +.. code-block:: julia """ bar(x[, y]) diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 7bd58491e71e3..ea6eff24eabf9 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -204,12 +204,12 @@ without being given a name, using either of these syntaxes: .. doctest:: julia> x -> x^2 + 2x - 1 - #1 (generic function with 1 method) + (::#1) (generic function with 1 method) julia> function (x) x^2 + 2x - 1 end - #2 (generic function with 1 method) + (::#3) (generic function with 1 method) This creates a function taking one argument *x* and returning the value of the polynomial *x*\ ^2 + 2\ *x* - 1 at that value. diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index c5e60a71e738c..eb4a3b3af5b19 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -412,14 +412,23 @@ types: .. doctest:: - julia> (typemin(Float16),typemax(Float16)) - (-Inf16,Inf16) + julia> typemin(Float16) + -Inf16 - julia> (typemin(Float32),typemax(Float32)) - (-Inf32,Inf32) + julia> typemax(Float16) + Inf16 - julia> (typemin(Float64),typemax(Float64)) - (-Inf,Inf) + julia> typemin(Float32) + -Inf32 + + julia> typemax(Float32) + Inf32 + + julia> typemin(Float64) + -Inf + + julia> typemax(Float64) + Inf Machine epsilon diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index d6f27627df9bb..ee9f0d5ee4c28 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -400,21 +400,23 @@ The following examples show the different forms. julia> Int8(128) ERROR: InexactError() - in call at ./essentials.jl:58 - in eval at ./boot.jl:263 + in Int8(::Int64) at ./sysimg.jl:50 + in eval(::Module, ::Any) at ./boot.jl:237 julia> Int8(127.0) 127 julia> Int8(3.14) ERROR: InexactError() - in call at ./essentials.jl:58 - in eval at ./boot.jl:263 + [inlined code] from ./int.jl:229 + in Int8(::Float64) at ./sysimg.jl:50 + in eval(::Module, ::Any) at ./boot.jl:237 julia> Int8(128.0) ERROR: InexactError() - in call at ./essentials.jl:58 - in eval at ./boot.jl:263 + [inlined code] from ./int.jl:229 + in Int8(::Float64) at ./sysimg.jl:50 + in eval(::Module, ::Any) at ./boot.jl:237 julia> 127 % Int8 127 @@ -427,9 +429,9 @@ The following examples show the different forms. julia> round(Int8,127.6) ERROR: InexactError() - in trunc at ./float.jl:357 - in round at ./float.jl:177 - in eval at ./boot.jl:263 + in trunc(::Type{Int8}, ::Float64) at ./float.jl:373 + in round(::Type{Int8}, ::Float64) at ./float.jl:180 + in eval(::Module, ::Any) at ./boot.jl:237 See :ref:`man-conversion-and-promotion` for how to define your own conversions and promotions. diff --git a/doc/manual/metaprogramming.rst b/doc/manual/metaprogramming.rst index bc32d8c0a9019..c4c3ac8444153 100644 --- a/doc/manual/metaprogramming.rst +++ b/doc/manual/metaprogramming.rst @@ -257,6 +257,7 @@ cause a compile-time error: julia> $a + b ERROR: unsupported or misplaced expression $ + in eval(::Module, ::Any) at ./boot.jl:237 In this example, the tuple ``(1,2,3)`` is interpolated as an expression into a conditional test: @@ -264,7 +265,7 @@ expression into a conditional test: .. doctest:: julia> ex = :(a in $:((1,2,3)) ) - :($(Expr(:in, :a, :((1,2,3))))) + :(a in (1,2,3)) Interpolating symbols into a nested expression requires enclosing each symbol in an enclosing quote block:: @@ -297,6 +298,9 @@ at global scope using :func:`eval`: julia> eval(ex) ERROR: UndefVarError: b not defined + in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Any) at ./boot.jl:236 + in eval(::Module, ::Any) at ./boot.jl:237 julia> a = 1; b = 2; @@ -316,6 +320,7 @@ module's environment: julia> x ERROR: UndefVarError: x not defined + in eval(::Module, ::Any) at ./boot.jl:237 julia> eval(ex) 1 @@ -420,6 +425,7 @@ Here is an extraordinarily simple macro: julia> macro sayhello() return :( println("Hello, world!") ) end + @sayhello (macro with 1 method) Macros have a dedicated character in Julia's syntax: the ``@`` (at-sign), followed by the unique name declared in a ``macro NAME ... end`` block. @@ -545,6 +551,7 @@ This macro can be used like this: julia> @assert 1==0 ERROR: AssertionError: 1 == 0 + in eval(::Module, ::Any) at ./boot.jl:237 In place of the written syntax, the macro call is expanded at parse time to its returned result. This is equivalent to writing:: diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index 2a21398f824e3..e7fb935b2891e 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -93,22 +93,26 @@ Applying it to any other types of arguments will result in a :exc:`MethodError`: .. doctest:: julia> f(2.0, 3) - ERROR: MethodError: `f` has no method matching f(::Float64, ::Int64) + ERROR: MethodError: no method matching f(::Float64, ::Int64) Closest candidates are: f(::Float64, !Matched::Float64) + in eval(::Module, ::Any) at ./boot.jl:237 julia> f(Float32(2.0), 3.0) - ERROR: MethodError: `f` has no method matching f(::Float32, ::Float64) + ERROR: MethodError: no method matching f(::Float32, ::Float64) Closest candidates are: f(!Matched::Float64, ::Float64) + in eval(::Module, ::Any) at ./boot.jl:237 julia> f(2.0, "3.0") - ERROR: MethodError: `f` has no method matching f(::Float64, ::String) + ERROR: MethodError: no method matching f(::Float64, ::String) Closest candidates are: f(::Float64, !Matched::Float64) + in eval(::Module, ::Any) at ./boot.jl:237 julia> f("2.0", "3.0") - ERROR: MethodError: `f` has no method matching f(::String, ::String) + ERROR: MethodError: no method matching f(::String, ::String) + in eval(::Module, ::Any) at ./boot.jl:237 As you can see, the arguments must be precisely of type :obj:`Float64`. Other numeric types, such as integers or 32-bit floating-point values, @@ -173,12 +177,14 @@ function ``f`` remains undefined, and applying it will still result in a .. doctest:: julia> f("foo", 3) - ERROR: MethodError: `f` has no method matching f(::String, ::Int64) + ERROR: MethodError: no method matching f(::String, ::Int64) Closest candidates are: f(!Matched::Number, ::Number) + in eval(::Module, ::Any) at ./boot.jl:237 julia> f() - ERROR: MethodError: `f` has no method matching f() + ERROR: MethodError: no method matching f() + in eval(::Module, ::Any) at ./boot.jl:237 You can easily see which methods exist for a function by entering the function object itself in an interactive session: @@ -412,19 +418,19 @@ the intersection case: .. doctest:: - julia> g(x::Float64, y::Float64) = 2x + 2y; + julia> h(x::Float64, y::Float64) = 2x + 2y; - julia> g(x::Float64, y) = 2x + y; + julia> h(x::Float64, y) = 2x + y; - julia> g(x, y::Float64) = x + 2y; + julia> h(x, y::Float64) = x + 2y; - julia> g(2.0, 3) + julia> h(2.0, 3) 7.0 - julia> g(2, 3.0) + julia> h(2, 3.0) 8.0 - julia> g(2.0, 3.0) + julia> h(2.0, 3.0) 10.0 To suppress Julia's warning, the disambiguating method must be defined @@ -492,9 +498,10 @@ signature: 4 julia> myappend([1,2,3],2.5) - ERROR: MethodError: `myappend` has no method matching myappend(::Array{Int64,1}, ::Float64) + ERROR: MethodError: no method matching myappend(::Array{Int64,1}, ::Float64) Closest candidates are: myappend{T}(::Array{T,1}, !Matched::T) + in eval(::Module, ::Any) at ./boot.jl:237 julia> myappend([1.0,2.0,3.0],4.0) 4-element Array{Float64,1}: @@ -504,9 +511,10 @@ signature: 4.0 julia> myappend([1.0,2.0,3.0],4) - ERROR: MethodError: `myappend` has no method matching myappend(::Array{Float64,1}, ::Int64) + ERROR: MethodError: no method matching myappend(::Array{Float64,1}, ::Int64) Closest candidates are: myappend{T}(::Array{T,1}, !Matched::T) + in eval(::Module, ::Any) at ./boot.jl:237 As you can see, the type of the appended element must match the element type of the vector it is appended to, or else a :exc:`MethodError` is raised. diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index c44dd59a47a30..2c88ae20a0703 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -279,7 +279,7 @@ change: .. doctest:: julia> m.a = 4.5f0 - 4.5 + 4.5f0 julia> typeof(m.a) Float64 diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index 570737a43fec0..fb56df43e72c5 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -31,8 +31,9 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example julia> example() ...-element Array{StackFrame,1}: - example at none:1 - eval at boot.jl:265 + in example() at none:1 + in eval(::Module, ::Any) at boot.jl:237 + [inlined code] from sysimg.jl:11 ... julia> @noinline child() = stacktrace() @@ -46,10 +47,11 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example julia> grandparent() ...-element Array{StackFrame,1}: - child at none:1 - parent at none:1 - grandparent at none:1 - eval at boot.jl:265 + [inlined code] from stacktraces.jl:135 + in child() at none:1 + in parent() at none:1 + in grandparent() at none:1 + in eval(::Module, ::Any) at boot.jl:237 ... Note that when calling :func:`stacktrace` you'll typically see a frame with @@ -77,7 +79,7 @@ returned by :func:`backtrace`: .. doctest:: julia> top_frame = stacktrace()[1] - eval at boot.jl:265 + in eval(::Module, ::Any) at boot.jl:237 julia> top_frame.func :eval @@ -86,7 +88,7 @@ returned by :func:`backtrace`: Symbol("./boot.jl") julia> top_frame.line - 265 + 237 julia> top_frame.inlined_file Symbol("") @@ -116,17 +118,19 @@ helpful in many places, the most obvious application is in error handling and de julia> @noinline bad_function() = undeclared_variable bad_function (generic function with 1 method) - julia> @noinline example() = try + julia> @noinline example1() = try bad_function() catch stacktrace() end - example (generic function with 1 method) + example1 (generic function with 1 method) - julia> example() + julia> example1() ...-element Array{StackFrame,1}: - example at none:4 - eval at boot.jl:265 + [inlined code] from stacktraces.jl:135 + in example1() at none:2 + in eval(::Module, ::Any) at boot.jl:237 + [inlined code] from sysimg.jl:11 ... You may notice that in the example above the first stack frame points points at line 4, @@ -142,50 +146,52 @@ returns stack information for the context of the most recent exception: .. doctest:: - julia> @noinline bad_function() = undeclared_variable - bad_function (generic function with 1 method) + julia> @noinline bad_function1() = undeclared_variable + bad_function1 (generic function with 1 method) - julia> @noinline example() = try - bad_function() + julia> @noinline example2() = try + bad_function1() catch catch_stacktrace() end - example (generic function with 1 method) + example2 (generic function with 1 method) - julia> example() + julia> example2() ...-element Array{StackFrame,1}: - bad_function at none:1 - example at none:2 - eval at boot.jl:265 + in bad_function1() at none:1 + in example2() at none:2 + in eval(::Module, ::Any) at boot.jl:237 + [inlined code] from sysimg.jl:11 ... Notice that the stack trace now indicates the appropriate line number and the missing frame. .. doctest:: - julia> @noinline child() = error("Whoops!") - child (generic function with 1 method) + julia> @noinline child1() = error("Whoops!") + child1 (generic function with 1 method) - julia> @noinline parent() = child() - parent (generic function with 1 method) + julia> @noinline parent1() = child1() + parent1 (generic function with 1 method) - julia> @noinline function grandparent() + julia> @noinline function grandparent1() try - parent() + parent1() catch err println("ERROR: ", err.msg) catch_stacktrace() end end - grandparent (generic function with 1 method) + grandparent1 (generic function with 1 method) - julia> grandparent() + julia> grandparent1() ERROR: Whoops! ...-element Array{StackFrame,1}: - child at none:1 - parent at none:1 - grandparent at none:3 - eval at boot.jl:265 + in child1() at none:1 + in parent1() at none:1 + in grandparent1() at none:3 + in eval(::Module, ::Any) at boot.jl:237 + [inlined code] from sysimg.jl:11 ... Comparison with :func:`backtrace` @@ -248,7 +254,7 @@ by passing them into :func:`StackTraces.lookup`: Ptr{Void} @0x... julia> frame = StackTraces.lookup(pointer) - [inlined code from task.c:663] rec_backtrace at task.c:723 + in jl_backtrace_from_here at stackwalk.c:103 julia> println("The top frame is from $(frame.func)!") - The top frame is from rec_backtrace! + The top frame is from jl_backtrace_from_here! diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index a5a89f554425c..15f299b73a35c 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -288,13 +288,15 @@ such an invalid byte index, an error is thrown: julia> s[2] ERROR: UnicodeError: invalid character index - in next at ./unicode/utf8.jl:65 - in getindex at strings/basic.jl:37 + in next(::UTF8String, ::Int64) at ./unicode/utf8.jl:65 + in getindex(::UTF8String, ::Int64) at ./strings/basic.jl:38 + in eval(::Module, ::Any) at ./boot.jl:237 julia> s[3] ERROR: UnicodeError: invalid character index - in next at ./unicode/utf8.jl:65 - in getindex at strings/basic.jl:37 + in next(::UTF8String, ::Int64) at ./unicode/utf8.jl:65 + in getindex(::UTF8String, ::Int64) at ./strings/basic.jl:38 + in eval(::Module, ::Any) at ./boot.jl:237 julia> s[4] ' ' @@ -545,10 +547,11 @@ contained in a string: false julia> contains("Xylophon", 'o') - ERROR: MethodError: `contains` has no method matching contains(::String, ::Char) + ERROR: MethodError: no method matching contains(::String, ::Char) Closest candidates are: contains(!Matched::Function, ::Any, !Matched::Any) contains(::AbstractString, !Matched::AbstractString) + in eval(::Module, ::Any) at ./boot.jl:237 The last error is because ``'o'`` is a character literal, and :func:`contains` is a generic function that looks for subsequences. To look for an element in a @@ -870,6 +873,7 @@ error: julia> "DATA\xff\u2200" ERROR: syntax: invalid UTF-8 sequence + in eval(::Module, ::Any) at ./boot.jl:237 Also observe the significant distinction between ``\xff`` and ``\uff``: the former escape sequence encodes the *byte 255*, whereas the latter diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 9215c075537f6..b79cbc2c3880b 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -102,6 +102,7 @@ exception is thrown, otherwise, the left-hand value is returned: julia> (1+2)::AbstractFloat ERROR: TypeError: typeassert: expected AbstractFloat, got Int64 + in eval(::Module, ::Any) at ./boot.jl:237 julia> (1+2)::Int 3 @@ -411,7 +412,8 @@ However, the value for ``baz`` must be convertible to :class:`Int`: julia> Foo((), 23.5, 1) ERROR: InexactError() - in call at none:2 + in Foo(::Tuple{}, ::Float64, ::Int64) at ./none:2 + in eval(::Module, ::Any) at ./boot.jl:237 You may find a list of field names using the ``fieldnames`` function. @@ -442,7 +444,7 @@ You can also change the values as one would expect: .. doctest:: julia> foo.qux = 2 - 2.0 + 2 julia> foo.bar = 1//2 1//2 @@ -765,9 +767,22 @@ each field: ERROR: MethodError: Cannot `convert` an object of type Float64 to an object of type Point{Float64} This may have arisen from a call to the constructor Point{Float64}(...), since type constructors fall back to convert methods. + Closest candidates are: + convert{T}(::Type{T}, !Matched::T) + (!Matched::Type{BoundsError})(::ANY) + (!Matched::Type{BoundsError})(::ANY, !Matched::ANY) + ... + in Point{Float64}(::Float64) at ./sysimg.jl:50 + in eval(::Module, ::Any) at ./boot.jl:237 julia> Point{Float64}(1.0,2.0,3.0) ERROR: MethodError: no method matching Point{Float64}(::Float64, ::Float64, ::Float64) + Closest candidates are: + (!Matched::Type{TypeError})(::Any, ::Any, ::Any, !Matched::Any) + (!Matched::Type{Expr})(::ANY...) + (!Matched::Type{Core.Inference.Generator{I,F}})(::Any, ::Any, ::Any...) + ... + in eval(::Module, ::Any) at ./boot.jl:237 Only one default constructor is generated for parametric types, since overriding it is not possible. This constructor accepts any arguments @@ -909,9 +924,11 @@ subtypes of :obj:`Real`: julia> Pointy{AbstractString} ERROR: TypeError: Pointy: in T, expected T<:Real, got Type{AbstractString} + in eval(::Module, ::Any) at ./boot.jl:237 julia> Pointy{1} ERROR: TypeError: Pointy: in T, expected T<:Real, got Int64 + in eval(::Module, ::Any) at ./boot.jl:237 Type parameters for parametric composite types can be restricted in the same manner:: @@ -1342,13 +1359,13 @@ To construct an object representing a non-missing value of type ``T``, use the .. doctest:: julia> x1 = Nullable(1) - Nullable(1) + Nullable{Int64}(1) julia> x2 = Nullable(1.0) - Nullable(1.0) + Nullable{Float64}(1.0) julia> x3 = Nullable([1, 2, 3]) - Nullable([1,2,3]) + Nullable{Array{Int64,1}}([1,2,3]) Note the core distinction between these two ways of constructing a :obj:`Nullable` object: in one style, you provide a type, ``T``, as a function parameter; in diff --git a/doc/manual/variables-and-scoping.rst b/doc/manual/variables-and-scoping.rst index 157110fcec7ba..06fb6152c39d7 100644 --- a/doc/manual/variables-and-scoping.rst +++ b/doc/manual/variables-and-scoping.rst @@ -306,11 +306,12 @@ macro definition need not come before its inner usage: .. doctest:: julia> f = y -> x + y - (anonymous function) + (::#1) (generic function with 1 method) julia> f(3) ERROR: UndefVarError: x not defined - in anonymous at none:1 + in (::##1#2)(::Int64) at ./none:1 + in eval(::Module, ::Any) at ./boot.jl:237 julia> x = 1 1 diff --git a/doc/manual/variables.rst b/doc/manual/variables.rst index c4a251fc76a9f..0f9f3f3f272b1 100644 --- a/doc/manual/variables.rst +++ b/doc/manual/variables.rst @@ -128,9 +128,11 @@ statements: julia> else = false ERROR: syntax: unexpected "else" + in eval(::Module, ::Any) at ./boot.jl:237 julia> try = "No" ERROR: syntax: unexpected "=" + in eval(::Module, ::Any) at ./boot.jl:237 Stylistic Conventions diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index daa5b1ec2ce8d..99b1972b3add2 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -367,7 +367,8 @@ All Objects julia> convert(Int, 3.5) ERROR: InexactError() - in convert at int.jl:209 + in convert(::Type{Int64}, ::Float64) at ./int.jl:229 + in eval(::Module, ::Any) at ./boot.jl:237 If ``T`` is a :obj:`AbstractFloat` or :obj:`Rational` type, then it will return the closest value to ``x`` representable by ``T``\ . diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 095277f615441..116163dc7e961 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1134,7 +1134,8 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat! at array.jl:543 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:565 + in eval(::Module, ::Any) at ./boot.jl:237 .. function:: splice!(collection, index, [replacement]) -> item @@ -1324,17 +1325,11 @@ inserted and priorities accessed or changed using indexing notation. julia> # Insert keys with associated priorities pq["a"] = 10; pq["b"] = 5; pq["c"] = 15; pq - Base.Collections.PriorityQueue{Any,Any,Base.Order.ForwardOrdering} with 3 entries: - "c" => 15 - "b" => 5 - "a" => 10 + Base.Collections.PriorityQueue{Any,Any,Base.Order.ForwardOrdering}("c"=>15,"b"=>5,"a"=>10) julia> # Change the priority of an existing key pq["a"] = 0; pq - Base.Collections.PriorityQueue{Any,Any,Base.Order.ForwardOrdering} with 3 entries: - "c" => 15 - "b" => 5 - "a" => 0 + Base.Collections.PriorityQueue{Any,Any,Base.Order.ForwardOrdering}("c"=>15,"b"=>5,"a"=>0) Heap Functions -------------- From f42d2cfc8cfa6cccf3b0a3854d96671e02844399 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Sat, 25 Jun 2016 01:04:54 -0400 Subject: [PATCH 0328/1117] Update Stack Traces docs to match implementation The Stack Traces section of the manual was very outdated. --- doc/manual/stacktraces.rst | 187 +++++++++++++++++++++---------------- 1 file changed, 106 insertions(+), 81 deletions(-) diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index fb56df43e72c5..84531327a60be 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -30,11 +30,13 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example example (generic function with 1 method) julia> example() - ...-element Array{StackFrame,1}: + 6-element Array{StackFrame,1}: in example() at none:1 - in eval(::Module, ::Any) at boot.jl:237 - [inlined code] from sysimg.jl:11 - ... + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in _start() at client.jl:359 julia> @noinline child() = stacktrace() child (generic function with 1 method) @@ -46,40 +48,40 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example grandparent (generic function with 1 method) julia> grandparent() - ...-element Array{StackFrame,1}: - [inlined code] from stacktraces.jl:135 - in child() at none:1 + 8-element Array{StackFrame,1}: + in child() at none:1 in parent() at none:1 in grandparent() at none:1 - in eval(::Module, ::Any) at boot.jl:237 - ... + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in _start() at client.jl:359 Note that when calling :func:`stacktrace` you'll typically see a frame with -``eval at boot.jl``. When calling :func:`stacktrace` from the REPL you'll also have a few +``eval(...) at boot.jl``. When calling :func:`stacktrace` from the REPL you'll also have a few extra frames in the stack from ``REPL.jl``, usually looking something like this:: julia> example() = stacktrace() example (generic function with 1 method) julia> example() - 4-element Array{StackFrame,1}: - example at none:1 - eval at boot.jl:265 - [inlined code from REPL.jl:3] eval_user_input at REPL.jl:62 - [inlined code from REPL.jl:92] anonymous at task.jl:63 + 5-element Array{StackFrame,1}: + in example() at REPL[1]:1 + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Base.REPL.REPLBackend) at REPL.jl:62 + in macro expansion at REPL.jl:92 [inlined] + in (::Base.REPL.##1#2{Base.REPL.REPLBackend})() at event.jl:46 Extracting useful information ----------------------------- -Each :obj:`StackFrame` contains the function name, file name, line number, file and line -information for inlined functions, a flag indicating whether it is a C function (by default -C functions do not appear in the stack trace), and an integer representation of the pointer -returned by :func:`backtrace`: +Each :obj:`StackFrame` contains the function name, file name, line number, lambda info, a flag indicating whether the frame has been inlined, a flag indicating whether it is a C function (by default C functions do not appear in the stack trace), and an integer representation of the pointer returned by :func:`backtrace`: .. doctest:: julia> top_frame = stacktrace()[1] - in eval(::Module, ::Any) at boot.jl:237 + in eval(::Module, ::Any) at boot.jl:231 julia> top_frame.func :eval @@ -88,13 +90,14 @@ returned by :func:`backtrace`: Symbol("./boot.jl") julia> top_frame.line - 237 + 231 - julia> top_frame.inlined_file - Symbol("") + julia> top_frame.linfo + Nullable{LambdaInfo}(LambdaInfo for eval(::Module, ::Any) + ... - julia> top_frame.inlined_line - -1 + julia> top_frame.inlined + false julia> top_frame.from_c false @@ -126,12 +129,13 @@ helpful in many places, the most obvious application is in error handling and de example1 (generic function with 1 method) julia> example1() - ...-element Array{StackFrame,1}: - [inlined code] from stacktraces.jl:135 - in example1() at none:2 - in eval(::Module, ::Any) at boot.jl:237 - [inlined code] from sysimg.jl:11 - ... + 6-element Array{StackFrame,1}: + in example1() at none:4 + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in _start() at client.jl:359 You may notice that in the example above the first stack frame points points at line 4, where :func:`stacktrace` is called, rather than line 2, where `bad_function` is called, and @@ -157,12 +161,14 @@ returns stack information for the context of the most recent exception: example2 (generic function with 1 method) julia> example2() - ...-element Array{StackFrame,1}: + 7-element Array{StackFrame,1}: in bad_function1() at none:1 in example2() at none:2 - in eval(::Module, ::Any) at boot.jl:237 - [inlined code] from sysimg.jl:11 - ... + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in _start() at client.jl:359 Notice that the stack trace now indicates the appropriate line number and the missing frame. @@ -186,13 +192,15 @@ Notice that the stack trace now indicates the appropriate line number and the mi julia> grandparent1() ERROR: Whoops! - ...-element Array{StackFrame,1}: + 8-element Array{StackFrame,1}: in child1() at none:1 in parent1() at none:1 in grandparent1() at none:3 - in eval(::Module, ::Any) at boot.jl:237 - [inlined code] from sysimg.jl:11 - ... + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Bool) at client.jl:117 + in _start() at client.jl:359 Comparison with :func:`backtrace` --------------------------------- @@ -201,60 +209,77 @@ A call to :func:`backtrace` returns a vector of ``Ptr{Void}``, which may then be :func:`stacktrace` for translation:: julia> trace = backtrace() - 15-element Array{Ptr{Void},1}: - Ptr{Void} @0x000000010527895e - Ptr{Void} @0x0000000309bf6220 - Ptr{Void} @0x0000000309bf61a0 - Ptr{Void} @0x00000001052733b4 - Ptr{Void} @0x0000000105271a0e - Ptr{Void} @0x000000010527189d - Ptr{Void} @0x0000000105272e6d - Ptr{Void} @0x0000000105272cef - Ptr{Void} @0x0000000105285b88 - Ptr{Void} @0x000000010526b50e - Ptr{Void} @0x000000010663cc28 - Ptr{Void} @0x0000000309bbc20f - Ptr{Void} @0x0000000309bbbde7 - Ptr{Void} @0x0000000309bb0262 - Ptr{Void} @0x000000010527980e + 20-element Array{Ptr{Void},1}: + Ptr{Void} @0x0000000100a26fc2 + Ptr{Void} @0x00000001029435df + Ptr{Void} @0x0000000102943635 + Ptr{Void} @0x00000001009e9620 + Ptr{Void} @0x00000001009fe1e8 + Ptr{Void} @0x00000001009fc7b6 + Ptr{Void} @0x00000001009fdae3 + Ptr{Void} @0x00000001009fe0d2 + Ptr{Void} @0x0000000100a1321b + Ptr{Void} @0x00000001009f64e7 + Ptr{Void} @0x000000010265ac5d + Ptr{Void} @0x000000010265acc1 + Ptr{Void} @0x00000001009e9620 + Ptr{Void} @0x000000031007744b + Ptr{Void} @0x0000000310077537 + Ptr{Void} @0x00000001009e9620 + Ptr{Void} @0x000000031006feec + Ptr{Void} @0x00000003100701b0 + Ptr{Void} @0x00000001009e9635 + Ptr{Void} @0x0000000100a06418 julia> stacktrace(trace) - 4-element Array{StackFrame,1}: - backtrace at error.jl:26 - eval at boot.jl:265 - [inlined code from REPL.jl:3] eval_user_input at REPL.jl:62 - [inlined code from REPL.jl:92] anonymous at task.jl:63 + 5-element Array{StackFrame,1}: + in backtrace() at error.jl:26 + in eval(::Module, ::Any) at boot.jl:231 + in eval_user_input(::Any, ::Base.REPL.REPLBackend) at REPL.jl:62 + in macro expansion at REPL.jl:92 [inlined] + in (::Base.REPL.##1#2{Base.REPL.REPLBackend})() at event.jl:46 Notice that the vector returned by :func:`backtrace` had 15 pointers, while the vector returned by :func:`stacktrace` only has 4. This is because, by default, :func:`stacktrace` removes any lower-level C functions from the stack. If you want to include stack frames from C calls, you can do it like this:: - julia> stacktrace(stack, true) - 15-element Array{StackFrame,1}: - [inlined code from task.c:651] rec_backtrace at task.c:711 - backtrace at error.jl:26 - jlcall_backtrace_23146 at :-1 - [inlined code from interpreter.c:55] jl_apply at interpreter.c:65 - eval at interpreter.c:214 - eval at interpreter.c:220 - eval_body at interpreter.c:601 - jl_toplevel_eval_body at interpreter.c:534 - jl_toplevel_eval_flex at toplevel.c:525 - jl_toplevel_eval_in_warn at builtins.c:590 - eval at boot.jl:265 - [inlined code from REPL.jl:3] eval_user_input at REPL.jl:62 - jlcall_eval_user_input_22658 at :-1 - [inlined code from REPL.jl:92] anonymous at task.jl:63 - [inlined code from julia.h:1352] jl_apply at task.c:246 + julia> stacktrace(trace, true) + 26-element Array{StackFrame,1}: + in jl_backtrace_from_here at stackwalk.c:104 + in backtrace() at error.jl:26 + in ip:0x102943635 + in jl_call_method_internal at julia_internal.h:86 [inlined] + in jl_apply_generic at gf.c:1805 + in do_call at interpreter.c:65 + in eval at interpreter.c:188 + in eval_body at interpreter.c:469 + in jl_interpret_call at interpreter.c:573 + in jl_toplevel_eval_flex at toplevel.c:543 + in jl_toplevel_eval_in_warn at builtins.c:571 + in eval(::Module, ::Any) at boot.jl:231 + in ip:0x10265acc1 + in jl_call_method_internal at julia_internal.h:86 [inlined] + in jl_apply_generic at gf.c:1805 + in eval_user_input(::Any, ::Base.REPL.REPLBackend) at REPL.jl:62 + in ip:0x310077537 + in jl_call_method_internal at julia_internal.h:86 [inlined] + in jl_apply_generic at gf.c:1805 + in macro expansion at REPL.jl:92 [inlined] + in (::Base.REPL.##1#2{Base.REPL.REPLBackend})() at event.jl:46 + in ip:0x3100701b0 + in jl_call_method_internal at julia_internal.h:86 [inlined] + in jl_apply_generic at gf.c:1795 + in jl_apply at julia.h:1388 [inlined] + in start_task at task.c:247 Individual pointers returned by :func:`backtrace` can be translated into :obj:`StackFrame` s by passing them into :func:`StackTraces.lookup`: .. doctest:: - julia> pointer = backtrace()[1] - Ptr{Void} @0x... + julia> pointer = backtrace()[1]; julia> frame = StackTraces.lookup(pointer) - in jl_backtrace_from_here at stackwalk.c:103 + 1-element Array{StackFrame,1}: + in jl_backtrace_from_here at stackwalk.c:104 - julia> println("The top frame is from $(frame.func)!") + julia> println("The top frame is from $(frame[1].func)!") The top frame is from jl_backtrace_from_here! From cc54a5f21f163c63865adeeb44641c7744bf9a64 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 13:29:39 -0400 Subject: [PATCH 0329/1117] Corrected doctest exceptions --- base/docs/helpdb/Base.jl | 8 ++--- base/test.jl | 2 +- doc/devdocs/types.rst | 2 +- doc/manual/complex-and-rational-numbers.rst | 6 ++-- doc/manual/constructors.rst | 24 ++++++--------- doc/manual/control-flow.rst | 30 +++++++++---------- doc/manual/conversion-and-promotion.rst | 11 ++----- .../integers-and-floating-point-numbers.rst | 2 ++ doc/manual/mathematical-operations.rst | 18 +++++------ doc/manual/metaprogramming.rst | 10 +++---- doc/manual/methods.rst | 24 ++++++++------- doc/manual/stacktraces.rst | 20 +++---------- doc/manual/strings.rst | 18 ++++++----- doc/manual/types.rst | 28 +++++++---------- doc/manual/variables-and-scoping.rst | 2 +- doc/manual/variables.rst | 4 +-- doc/stdlib/base.rst | 4 +-- doc/stdlib/collections.rst | 4 +-- doc/stdlib/test.rst | 2 +- 19 files changed, 96 insertions(+), 123 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index f34ef98c24efc..8d7c73adb49e0 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5450,8 +5450,8 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:565 - in eval(::Module, ::Any) at ./boot.jl:237 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:571 + in eval(::Module, ::Any) at ./boot.jl:231... ``` """ deleteat!(collection, itr) @@ -8110,8 +8110,8 @@ julia> convert(Int, 3.0) julia> convert(Int, 3.5) ERROR: InexactError() - in convert(::Type{Int64}, ::Float64) at ./int.jl:229 - in eval(::Module, ::Any) at ./boot.jl:237 + in convert(::Type{Int64}, ::Float64) at ./int.jl:239 + in eval(::Module, ::Any) at ./boot.jl:231... ``` If `T` is a [`AbstractFloat`](:obj:`AbstractFloat`) or [`Rational`](:obj:`Rational`) type, diff --git a/base/test.jl b/base/test.jl index b0382f3284f5c..c51548bfe186b 100644 --- a/base/test.jl +++ b/base/test.jl @@ -910,7 +910,7 @@ Body: julia> @inferred f(1,2,3) ERROR: return type Int64 does not match inferred return type Union{Float64,Int64} in error(::String) at ./error.jl:21 - in eval(::Module, ::Any) at ./boot.jl:226 + in eval(::Module, ::Any) at ./boot.jl:231... julia> @inferred max(1,2) 2 diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index 1f691a1cf471a..b05897ca22157 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -229,7 +229,7 @@ These therefore print identically, but they have very different behavior: ERROR: MethodError: no method matching candid(::Array{Int64,1}, ::Float64) Closest candidates are: candid{T}(::Array{T,N}, !Matched::T) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> sneaky([1],3.2) 1 diff --git a/doc/manual/complex-and-rational-numbers.rst b/doc/manual/complex-and-rational-numbers.rst index dcf49c8a8f0ee..d3a24159a76b7 100644 --- a/doc/manual/complex-and-rational-numbers.rst +++ b/doc/manual/complex-and-rational-numbers.rst @@ -159,8 +159,8 @@ versus ``-1 + 0im`` even though ``-1 == -1 + 0im``: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:146 - in eval(::Module, ::Any) at ./boot.jl:237 + in sqrt(::Int64) at ./math.jl:149 + in eval(::Module, ::Any) at ./boot.jl:231... julia> sqrt(-1 + 0im) 0.0 + 1.0im @@ -306,7 +306,7 @@ Trying to construct a :const:`NaN` rational value, however, is not: ERROR: ArgumentError: invalid rational: zero(Int64)//zero(Int64) in Rational{Int64}(::Int64, ::Int64) at ./rational.jl:8 in //(::Int64, ::Int64) at ./rational.jl:22 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... As usual, the promotion system makes interactions with other numeric types effortless: diff --git a/doc/manual/constructors.rst b/doc/manual/constructors.rst index 81661e656b18b..564bf298cda71 100644 --- a/doc/manual/constructors.rst +++ b/doc/manual/constructors.rst @@ -122,7 +122,7 @@ Now ``OrderedPair`` objects can only be constructed such that julia> OrderedPair(2,1) ERROR: out of order in OrderedPair(::Int64, ::Int64) at ./none:5 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... You can still reach in and directly change the field values to violate this invariant, but messing around with an object's internals uninvited is @@ -267,7 +267,7 @@ access to an uninitialized reference is an immediate error: julia> z.xx ERROR: UndefRefError: access to undefined reference - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... This avoids the need to continually check for ``null`` values. However, not all object fields are references. Julia considers some @@ -327,12 +327,9 @@ types of the arguments given to the constructor. Here are some examples: julia> Point(1,2.5) ERROR: MethodError: no method matching Point{T<:Real}(::Int64, ::Float64) Closest candidates are: - (!Matched::Type{BoundsError})(::ANY, ::ANY) - (!Matched::Type{TypeError})(::Any, ::Any, !Matched::Any, !Matched::Any) - (!Matched::Type{TypeConstructor})(::ANY, ::ANY) - ... - in eval(::Module, ::Any) at ./boot.jl:237 - + Point{T<:Real}{T<:Real}(::T<:Real, !Matched::T<:Real) + Point{T<:Real}{T}(::Any) + in eval(::Module, ::Any) at ./boot.jl:231... ## explicit T ## @@ -341,9 +338,8 @@ types of the arguments given to the constructor. Here are some examples: julia> Point{Int64}(1.0,2.5) ERROR: InexactError() - [inlined code] from ./int.jl:229 in Point{Int64}(::Float64, ::Float64) at ./none:2 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> Point{Float64}(1.0,2.5) Point{Float64}(1.0,2.5) @@ -430,11 +426,9 @@ However, other similar calls still don't work: julia> Point(1.5,2) ERROR: MethodError: no method matching Point{T<:Real}(::Float64, ::Int64) Closest candidates are: - (!Matched::Type{BoundsError})(::ANY, ::ANY) - (!Matched::Type{TypeError})(::Any, ::Any, !Matched::Any, !Matched::Any) - (!Matched::Type{TypeConstructor})(::ANY, ::ANY) - ... - in eval(::Module, ::Any) at ./boot.jl:237 + Point{T<:Real}{T<:Real}(::T<:Real, !Matched::T<:Real) + Point{T<:Real}{T}(::Any) + in eval(::Module, ::Any) at ./boot.jl:231... For a much more general way of making all such calls work sensibly, see :ref:`man-conversion-and-promotion`. At the risk diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 87ecd823829b0..09e7948e0308a 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -163,7 +163,7 @@ the above function results in a runtime error julia> test2(2,1) ERROR: UndefVarError: relation not defined in test2(::Int64, ::Int64) at ./none:7 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... ``if`` blocks also return a value, which may seem unintuitive to users coming from many other languages. This value is simply the return value @@ -194,7 +194,7 @@ conditional expression is anything but ``true`` or ``false``: println("true") end ERROR: TypeError: non-boolean (Int64) used in boolean context - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... This error indicates that the conditional was of the wrong type: :obj:`Int64` rather than the required :obj:`Bool`. @@ -368,7 +368,7 @@ For example, a recursive factorial routine could be defined like this: julia> fact(-1) ERROR: n must be non-negative in fact(::Int64) at ./none:2 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Boolean operations *without* short-circuit evaluation can be done with the @@ -397,7 +397,7 @@ except for the last entry in a conditional chain is an error: julia> 1 && true ERROR: TypeError: non-boolean (Int64) used in boolean context - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... On the other hand, any type of expression can be used at the end of a conditional chain. It will be evaluated and returned depending on the preceding conditionals: @@ -482,7 +482,7 @@ different variable name to test this: julia> j ERROR: UndefVarError: j not defined - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... See :ref:`man-variables-and-scoping` for a detailed explanation of variable scope and how it works in Julia. @@ -666,8 +666,8 @@ negative real value: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:146 - in eval(::Module, ::Any) at ./boot.jl:237 + in sqrt(::Int64) at ./math.jl:149 + in eval(::Module, ::Any) at ./boot.jl:231... You may define your own exceptions in the following way: @@ -693,7 +693,7 @@ if the argument is negative: julia> g(-1) ERROR: DomainError: in g(::Int64) at ./none:1 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Note that :exc:`DomainError` without parentheses is not an exception, but a type of exception. It needs to be called to obtain an :exc:`Exception` object: @@ -713,7 +713,7 @@ error reporting: julia> throw(UndefVarError(:x)) ERROR: UndefVarError: x not defined - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... This mechanism can be implemented easily by custom exception types following the way :exc:`UndefVarError` is written: @@ -746,7 +746,7 @@ the :func:`sqrt` function that raises an error if its argument is negative: julia> fussy_sqrt(-1) ERROR: negative x not allowed in fussy_sqrt(::Int64) at ./none:1 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... If ``fussy_sqrt`` is called with a negative value from another function, instead of trying to continue execution of the calling function, it @@ -771,9 +771,9 @@ session: julia> verbose_fussy_sqrt(-1) before fussy_sqrt ERROR: negative x not allowed - [inlined code] from ./none:1 + in fussy_sqrt at ./none:1 [inlined] in verbose_fussy_sqrt(::Int64) at ./none:3 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Warnings and informational messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -794,8 +794,8 @@ execution: julia> error("Hi"); 1+1 ERROR: Hi - in error(::ASCIIString) at ./error.jl:21 - in eval(::Module, ::Any) at ./boot.jl:237 + in error(::String) at ./error.jl:21 + in eval(::Module, ::Any) at ./boot.jl:231... The ``try/catch`` statement @@ -855,7 +855,7 @@ assumes ``x`` is a real number and returns its square root: julia> sqrt_second(-9) ERROR: DomainError: in sqrt_second(::Int64) at ./none:7 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Note that the symbol following ``catch`` will always be interpreted as a name for the exception, so care is needed when writing ``try/catch`` expressions diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 251c7280eee95..7eb4c9c60ee06 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -100,12 +100,7 @@ requested conversion: ERROR: MethodError: Cannot `convert` an object of type String to an object of type AbstractFloat This may have arisen from a call to the constructor AbstractFloat(...), since type constructors fall back to convert methods. - Closest candidates are: - convert(::Type{AbstractFloat}, !Matched::Bool) - convert(::Type{AbstractFloat}, !Matched::Int8) - convert(::Type{AbstractFloat}, !Matched::Int16) - ... - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Some languages consider parsing strings as numbers or formatting numbers as strings to be conversions (many dynamic languages will even @@ -148,7 +143,7 @@ to one and zero: julia> convert(Bool, 1im) ERROR: InexactError() in convert(::Type{Bool}, ::Complex{Int64}) at ./complex.jl:18 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> convert(Bool, 0im) false @@ -164,7 +159,7 @@ This is the actual implementation in Julia:: julia> convert(Bool, 1im) ERROR: InexactError() in convert(::Type{Bool}, ::Complex{Int64}) at ./complex.jl:18 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Case Study: Rational Conversions diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index eb4a3b3af5b19..c1af976478452 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -707,9 +707,11 @@ imply multiplication: julia> (x-1)(x+1) ERROR: MethodError: objects of type Int64 are not callable + ... julia> x(x+1) ERROR: MethodError: objects of type Int64 are not callable + ... Both expressions are interpreted as function application: any expression that is not a numeric literal, when immediately followed by a diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index ee9f0d5ee4c28..570d1b9be42b2 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -400,23 +400,21 @@ The following examples show the different forms. julia> Int8(128) ERROR: InexactError() - in Int8(::Int64) at ./sysimg.jl:50 - in eval(::Module, ::Any) at ./boot.jl:237 + in Int8(::Int64) at ./sysimg.jl:60 + in eval(::Module, ::Any) at ./boot.jl:231... julia> Int8(127.0) 127 julia> Int8(3.14) ERROR: InexactError() - [inlined code] from ./int.jl:229 - in Int8(::Float64) at ./sysimg.jl:50 - in eval(::Module, ::Any) at ./boot.jl:237 + in Int8(::Float64) at ./sysimg.jl:60 + in eval(::Module, ::Any) at ./boot.jl:231... julia> Int8(128.0) ERROR: InexactError() - [inlined code] from ./int.jl:229 - in Int8(::Float64) at ./sysimg.jl:50 - in eval(::Module, ::Any) at ./boot.jl:237 + in Int8(::Float64) at ./sysimg.jl:60 + in eval(::Module, ::Any) at ./boot.jl:231... julia> 127 % Int8 127 @@ -429,9 +427,9 @@ The following examples show the different forms. julia> round(Int8,127.6) ERROR: InexactError() - in trunc(::Type{Int8}, ::Float64) at ./float.jl:373 + in trunc(::Type{Int8}, ::Float64) at ./float.jl:425 in round(::Type{Int8}, ::Float64) at ./float.jl:180 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... See :ref:`man-conversion-and-promotion` for how to define your own conversions and promotions. diff --git a/doc/manual/metaprogramming.rst b/doc/manual/metaprogramming.rst index c4c3ac8444153..edcd27774b3dc 100644 --- a/doc/manual/metaprogramming.rst +++ b/doc/manual/metaprogramming.rst @@ -257,7 +257,7 @@ cause a compile-time error: julia> $a + b ERROR: unsupported or misplaced expression $ - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... In this example, the tuple ``(1,2,3)`` is interpolated as an expression into a conditional test: @@ -298,9 +298,7 @@ at global scope using :func:`eval`: julia> eval(ex) ERROR: UndefVarError: b not defined - in eval(::Module, ::Any) at ./boot.jl:237 - in eval(::Any) at ./boot.jl:236 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> a = 1; b = 2; @@ -320,7 +318,7 @@ module's environment: julia> x ERROR: UndefVarError: x not defined - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> eval(ex) 1 @@ -551,7 +549,7 @@ This macro can be used like this: julia> @assert 1==0 ERROR: AssertionError: 1 == 0 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... In place of the written syntax, the macro call is expanded at parse time to its returned result. This is equivalent to writing:: diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index e7fb935b2891e..6d6bacedff3a7 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -96,23 +96,23 @@ Applying it to any other types of arguments will result in a :exc:`MethodError`: ERROR: MethodError: no method matching f(::Float64, ::Int64) Closest candidates are: f(::Float64, !Matched::Float64) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> f(Float32(2.0), 3.0) ERROR: MethodError: no method matching f(::Float32, ::Float64) Closest candidates are: f(!Matched::Float64, ::Float64) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> f(2.0, "3.0") ERROR: MethodError: no method matching f(::Float64, ::String) Closest candidates are: f(::Float64, !Matched::Float64) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> f("2.0", "3.0") ERROR: MethodError: no method matching f(::String, ::String) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... As you can see, the arguments must be precisely of type :obj:`Float64`. Other numeric types, such as integers or 32-bit floating-point values, @@ -180,11 +180,11 @@ function ``f`` remains undefined, and applying it will still result in a ERROR: MethodError: no method matching f(::String, ::Int64) Closest candidates are: f(!Matched::Number, ::Number) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> f() ERROR: MethodError: no method matching f() - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... You can easily see which methods exist for a function by entering the function object itself in an interactive session: @@ -501,7 +501,7 @@ signature: ERROR: MethodError: no method matching myappend(::Array{Int64,1}, ::Float64) Closest candidates are: myappend{T}(::Array{T,1}, !Matched::T) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> myappend([1.0,2.0,3.0],4.0) 4-element Array{Float64,1}: @@ -514,7 +514,7 @@ signature: ERROR: MethodError: no method matching myappend(::Array{Float64,1}, ::Int64) Closest candidates are: myappend{T}(::Array{T,1}, !Matched::T) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... As you can see, the type of the appended element must match the element type of the vector it is appended to, or else a :exc:`MethodError` is raised. @@ -569,16 +569,18 @@ Function parameters can also be used to constrain the number of arguments that m .. doctest:: - julia> bar(a,b,x::Vararg{Any,2}) = (a,b,x) + julia> bar(a,b,x::Vararg{Any,2}) = (a,b,x); julia> bar(1,2,3) - ERROR: MethodError: `bar` has no matching method bar(::Int, ::Int, ::Int) + ERROR: MethodError: no method matching bar(::Int64, ::Int64, ::Int64) + ... julia> bar(1,2,3,4) (1,2,(3,4)) julia> bar(1,2,3,4,5) - ERROR: MethodError: `bar` has no method matching bar(::Int, ::Int, ::Int, ::Int, ::Int) + ERROR: MethodError: no method matching bar(::Int64, ::Int64, ::Int64, ::Int64, ::Int64) + ... More usefully, it is possible to constrain varargs methods by a parameter. For example:: diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index 84531327a60be..37d86f0e38379 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -53,10 +53,7 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example in parent() at none:1 in grandparent() at none:1 in eval(::Module, ::Any) at boot.jl:231 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in eval(::Module, ::Any) at boot.jl:231 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in _start() at client.jl:359 + ... Note that when calling :func:`stacktrace` you'll typically see a frame with ``eval(...) at boot.jl``. When calling :func:`stacktrace` from the REPL you'll also have a few @@ -132,10 +129,7 @@ helpful in many places, the most obvious application is in error handling and de 6-element Array{StackFrame,1}: in example1() at none:4 in eval(::Module, ::Any) at boot.jl:231 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in eval(::Module, ::Any) at boot.jl:231 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in _start() at client.jl:359 + ... You may notice that in the example above the first stack frame points points at line 4, where :func:`stacktrace` is called, rather than line 2, where `bad_function` is called, and @@ -165,10 +159,7 @@ returns stack information for the context of the most recent exception: in bad_function1() at none:1 in example2() at none:2 in eval(::Module, ::Any) at boot.jl:231 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in eval(::Module, ::Any) at boot.jl:231 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in _start() at client.jl:359 + ... Notice that the stack trace now indicates the appropriate line number and the missing frame. @@ -197,10 +188,7 @@ Notice that the stack trace now indicates the appropriate line number and the mi in parent1() at none:1 in grandparent1() at none:3 in eval(::Module, ::Any) at boot.jl:231 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in eval(::Module, ::Any) at boot.jl:231 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in _start() at client.jl:359 + ... Comparison with :func:`backtrace` --------------------------------- diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 15f299b73a35c..41b1f3a2a1b69 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -288,15 +288,17 @@ such an invalid byte index, an error is thrown: julia> s[2] ERROR: UnicodeError: invalid character index - in next(::UTF8String, ::Int64) at ./unicode/utf8.jl:65 - in getindex(::UTF8String, ::Int64) at ./strings/basic.jl:38 - in eval(::Module, ::Any) at ./boot.jl:237 + in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:68 + in next at ./strings/string.jl:93 [inlined] + in getindex(::String, ::Int64) at ./strings/basic.jl:70 + in eval(::Module, ::Any) at ./boot.jl:231... julia> s[3] ERROR: UnicodeError: invalid character index - in next(::UTF8String, ::Int64) at ./unicode/utf8.jl:65 - in getindex(::UTF8String, ::Int64) at ./strings/basic.jl:38 - in eval(::Module, ::Any) at ./boot.jl:237 + in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:68 + in next at ./strings/string.jl:93 [inlined] + in getindex(::String, ::Int64) at ./strings/basic.jl:70 + in eval(::Module, ::Any) at ./boot.jl:231... julia> s[4] ' ' @@ -551,7 +553,7 @@ contained in a string: Closest candidates are: contains(!Matched::Function, ::Any, !Matched::Any) contains(::AbstractString, !Matched::AbstractString) - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... The last error is because ``'o'`` is a character literal, and :func:`contains` is a generic function that looks for subsequences. To look for an element in a @@ -873,7 +875,7 @@ error: julia> "DATA\xff\u2200" ERROR: syntax: invalid UTF-8 sequence - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Also observe the significant distinction between ``\xff`` and ``\uff``: the former escape sequence encodes the *byte 255*, whereas the latter diff --git a/doc/manual/types.rst b/doc/manual/types.rst index b79cbc2c3880b..4364dc39ec7ea 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -102,7 +102,7 @@ exception is thrown, otherwise, the left-hand value is returned: julia> (1+2)::AbstractFloat ERROR: TypeError: typeassert: expected AbstractFloat, got Int64 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> (1+2)::Int 3 @@ -413,7 +413,7 @@ However, the value for ``baz`` must be convertible to :class:`Int`: julia> Foo((), 23.5, 1) ERROR: InexactError() in Foo(::Tuple{}, ::Float64, ::Int64) at ./none:2 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... You may find a list of field names using the ``fieldnames`` function. @@ -767,22 +767,15 @@ each field: ERROR: MethodError: Cannot `convert` an object of type Float64 to an object of type Point{Float64} This may have arisen from a call to the constructor Point{Float64}(...), since type constructors fall back to convert methods. - Closest candidates are: - convert{T}(::Type{T}, !Matched::T) - (!Matched::Type{BoundsError})(::ANY) - (!Matched::Type{BoundsError})(::ANY, !Matched::ANY) - ... - in Point{Float64}(::Float64) at ./sysimg.jl:50 - in eval(::Module, ::Any) at ./boot.jl:237 + in Point{Float64}(::Float64) at ./sysimg.jl:60 + in eval(::Module, ::Any) at ./boot.jl:231... julia> Point{Float64}(1.0,2.0,3.0) ERROR: MethodError: no method matching Point{Float64}(::Float64, ::Float64, ::Float64) Closest candidates are: - (!Matched::Type{TypeError})(::Any, ::Any, ::Any, !Matched::Any) - (!Matched::Type{Expr})(::ANY...) - (!Matched::Type{Core.Inference.Generator{I,F}})(::Any, ::Any, ::Any...) - ... - in eval(::Module, ::Any) at ./boot.jl:237 + Point{Float64}{T}(::Any, ::Any) + Point{Float64}{T}(::Any) + in eval(::Module, ::Any) at ./boot.jl:231... Only one default constructor is generated for parametric types, since overriding it is not possible. This constructor accepts any arguments @@ -816,6 +809,7 @@ isn't the case, the constructor will fail with a :exc:`MethodError`: julia> Point(1,2.5) ERROR: MethodError: no method matching Point{T}(::Int64, ::Float64) + ... Constructor methods to appropriately handle such mixed cases can be defined, but that will not be discussed until later on in @@ -924,11 +918,11 @@ subtypes of :obj:`Real`: julia> Pointy{AbstractString} ERROR: TypeError: Pointy: in T, expected T<:Real, got Type{AbstractString} - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> Pointy{1} ERROR: TypeError: Pointy: in T, expected T<:Real, got Int64 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Type parameters for parametric composite types can be restricted in the same manner:: @@ -1393,7 +1387,7 @@ You can safely access the value of a :obj:`Nullable` object using :func:`get`: julia> get(Nullable{Float64}()) ERROR: NullException() - in get(::Nullable{Float64}) at ./nullable.jl:45 + in get(::Nullable{Float64}) at ./nullable.jl:62... julia> get(Nullable(1.0)) 1.0 diff --git a/doc/manual/variables-and-scoping.rst b/doc/manual/variables-and-scoping.rst index 06fb6152c39d7..62370f7a8a02b 100644 --- a/doc/manual/variables-and-scoping.rst +++ b/doc/manual/variables-and-scoping.rst @@ -311,7 +311,7 @@ macro definition need not come before its inner usage: julia> f(3) ERROR: UndefVarError: x not defined in (::##1#2)(::Int64) at ./none:1 - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> x = 1 1 diff --git a/doc/manual/variables.rst b/doc/manual/variables.rst index 0f9f3f3f272b1..b85584f88ea80 100644 --- a/doc/manual/variables.rst +++ b/doc/manual/variables.rst @@ -128,11 +128,11 @@ statements: julia> else = false ERROR: syntax: unexpected "else" - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... julia> try = "No" ERROR: syntax: unexpected "=" - in eval(::Module, ::Any) at ./boot.jl:237 + in eval(::Module, ::Any) at ./boot.jl:231... Stylistic Conventions diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 99b1972b3add2..ea8c7dd21773a 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -367,8 +367,8 @@ All Objects julia> convert(Int, 3.5) ERROR: InexactError() - in convert(::Type{Int64}, ::Float64) at ./int.jl:229 - in eval(::Module, ::Any) at ./boot.jl:237 + in convert(::Type{Int64}, ::Float64) at ./int.jl:239 + in eval(::Module, ::Any) at ./boot.jl:231... If ``T`` is a :obj:`AbstractFloat` or :obj:`Rational` type, then it will return the closest value to ``x`` representable by ``T``\ . diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 116163dc7e961..8c0d45f2f7093 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1134,8 +1134,8 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:565 - in eval(::Module, ::Any) at ./boot.jl:237 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:571 + in eval(::Module, ::Any) at ./boot.jl:231... .. function:: splice!(collection, index, [replacement]) -> item diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index 05ea608d300ac..a1a6e7ea388b2 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -280,7 +280,7 @@ writing new tests. julia> @inferred f(1,2,3) ERROR: return type Int64 does not match inferred return type Union{Float64,Int64} in error(::String) at ./error.jl:21 - in eval(::Module, ::Any) at ./boot.jl:226 + in eval(::Module, ::Any) at ./boot.jl:231... julia> @inferred max(1,2) 2 From 55131802ae2ff9dbb441ba65ee40defe5dd320e1 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 13:32:55 -0400 Subject: [PATCH 0330/1117] Doctest corrections related to changes to display --- base/reflection.jl | 2 +- doc/devdocs/types.rst | 6 +++--- doc/manual/arrays.rst | 8 ++++---- doc/manual/conversion-and-promotion.rst | 4 ++-- doc/stdlib/base.rst | 2 +- doc/stdlib/collections.rst | 10 ++++++++-- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index 9e1e27a517d96..36b8ad34a2314 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -126,7 +126,7 @@ use it in the following manner to summarize information about a struct type: julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:nfields(T)]; julia> structinfo(Base.Filesystem.StatStruct) -12-element Array{Tuple{UInt64,Symbol,Type{_}},1}: +12-element Array{Tuple{UInt64,Union{Int64,Symbol},Type{_}},1}: (0x0000000000000000,:device,UInt64) (0x0000000000000008,:inode,UInt64) (0x0000000000000010,:mode,UInt64) diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index b05897ca22157..5dac7a02661e2 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -194,7 +194,7 @@ One can construct :obj:`TypeVar`\s manually: .. doctest:: julia> TypeVar(:V, Signed, Real, false) - V<:Real + Signed<:V<:Real There are convenience versions that allow you to omit any of these arguments except the ``name`` symbol. @@ -388,7 +388,7 @@ able to accommodate any tuple. Let's check the parameters: Tuple julia> Tuple.parameters - svec(Vararg{Any}) + svec(Vararg{Any,N}) It's worth noting that the parameter is a type, ``Any``, rather than a ``TypeVar T<:Any``: compare @@ -396,7 +396,7 @@ It's worth noting that the parameter is a type, ``Any``, rather than a .. doctest:: julia> jl_(Tuple.parameters) - svec(Vararg{Any}) + svec(Vararg{Any, N<:Any}) julia> jl_(Array.parameters) svec(T<:Any, N<:Any) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index b68bf6220490b..ad847f64eb20a 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -327,7 +327,7 @@ Example: .. doctest:: julia> x = reshape(1:16, 4, 4) - 4×4 Array{Int64,2}: + 4×4 Base.ReshapedArray{Int64,2,UnitRange{Int64},Tuple{}}: 1 5 9 13 2 6 10 14 3 7 11 15 @@ -347,7 +347,7 @@ Example: 16 julia> x[1, [2 3; 4 1]] - 2x2 Array{Int64,2}: + 2×2 Array{Int64,2}: 5 9 13 1 @@ -396,7 +396,7 @@ Example: .. doctest:: - julia> x = reshape(1:9, 3, 3) + julia> x = collect(reshape(1:9, 3, 3)) 3×3 Array{Int64,2}: 1 4 7 2 5 8 @@ -736,7 +736,7 @@ you can use the same names with an ``sp`` prefix: .. doctest:: julia> spzeros(3,5) - 3×5 sparse matrix with 0 Float64 nonzero entries: + 3×5 sparse matrix with 0 Float64 nonzero entries julia> speye(3,5) 3×5 sparse matrix with 3 Float64 nonzero entries: diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 7eb4c9c60ee06..78703f49c9251 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -248,10 +248,10 @@ promotion is to convert numeric arguments to a common type: (1.0,2.5,3.0,0.75) julia> promote(1.5, im) - (1.5+0.0im,0.0+1.0im) + (1.5 + 0.0im,0.0 + 1.0im) julia> promote(1 + 2im, 3//4) - (1//1+2//1*im,3//4+0//1*im) + (1//1 + 2//1*im,3//4 + 0//1*im) Floating-point values are promoted to the largest of the floating-point argument types. Integer values are promoted to the larger of either the diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index ea8c7dd21773a..897902fb076d6 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -534,7 +534,7 @@ Types julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:nfields(T)]; julia> structinfo(Base.Filesystem.StatStruct) - 12-element Array{Tuple{UInt64,Symbol,Type{_}},1}: + 12-element Array{Tuple{UInt64,Union{Int64,Symbol},Type{_}},1}: (0x0000000000000000,:device,UInt64) (0x0000000000000008,:inode,UInt64) (0x0000000000000010,:mode,UInt64) diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 8c0d45f2f7093..c745e6585fc38 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1325,11 +1325,17 @@ inserted and priorities accessed or changed using indexing notation. julia> # Insert keys with associated priorities pq["a"] = 10; pq["b"] = 5; pq["c"] = 15; pq - Base.Collections.PriorityQueue{Any,Any,Base.Order.ForwardOrdering}("c"=>15,"b"=>5,"a"=>10) + Base.Collections.PriorityQueue{Any,Any,Base.Order.ForwardOrdering} with 3 entries: + "c" => 15 + "b" => 5 + "a" => 10 julia> # Change the priority of an existing key pq["a"] = 0; pq - Base.Collections.PriorityQueue{Any,Any,Base.Order.ForwardOrdering}("c"=>15,"b"=>5,"a"=>0) + Base.Collections.PriorityQueue{Any,Any,Base.Order.ForwardOrdering} with 3 entries: + "c" => 15 + "b" => 5 + "a" => 0 Heap Functions -------------- From e4b6998fd0a55ee08d8f4b2fcdcbc9324b24ac3a Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 13:34:11 -0400 Subject: [PATCH 0331/1117] Limited number of + methods in doctest --- doc/manual/methods.rst | 154 ++++------------------------------------- 1 file changed, 14 insertions(+), 140 deletions(-) diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index 6d6bacedff3a7..adf6930a33f46 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -234,146 +234,20 @@ Julia language. Core operations typically have dozens of methods: .. doctest:: julia> methods(+) - # 139 methods for generic function "+": - +(x::Bool) at bool.jl:33 - +(x::Bool,y::Bool) at bool.jl:36 - +(y::AbstractFloat,x::Bool) at bool.jl:46 - +(x::Int64,y::Int64) at int.jl:14 - +(x::Int8,y::Int8) at int.jl:14 - +(x::UInt8,y::UInt8) at int.jl:14 - +(x::Int16,y::Int16) at int.jl:14 - +(x::UInt16,y::UInt16) at int.jl:14 - +(x::Int32,y::Int32) at int.jl:14 - +(x::UInt32,y::UInt32) at int.jl:14 - +(x::UInt64,y::UInt64) at int.jl:14 - +(x::Int128,y::Int128) at int.jl:14 - +(x::UInt128,y::UInt128) at int.jl:14 - +(x::Float32,y::Float32) at float.jl:192 - +(x::Float64,y::Float64) at float.jl:193 - +(z::Complex{T<:Real},w::Complex{T<:Real}) at complex.jl:96 - +(x::Real,z::Complex{T<:Real}) at complex.jl:106 - +(z::Complex{T<:Real},x::Real) at complex.jl:107 - +(x::Rational{T<:Integer},y::Rational{T<:Integer}) at rational.jl:167 - +(a::Float16,b::Float16) at float16.jl:136 - +(x::Base.GMP.BigInt,y::Base.GMP.BigInt) at gmp.jl:243 - +(a::Base.GMP.BigInt,b::Base.GMP.BigInt,c::Base.GMP.BigInt) at gmp.jl:266 - +(a::Base.GMP.BigInt,b::Base.GMP.BigInt,c::Base.GMP.BigInt,d::Base.GMP.BigInt) at gmp.jl:272 - +(a::Base.GMP.BigInt,b::Base.GMP.BigInt,c::Base.GMP.BigInt,d::Base.GMP.BigInt,e::Base.GMP.BigInt) at gmp.jl:279 - +(x::Base.GMP.BigInt,c::Union{UInt32,UInt16,UInt8,UInt64}) at gmp.jl:291 - +(c::Union{UInt32,UInt16,UInt8,UInt64},x::Base.GMP.BigInt) at gmp.jl:295 - +(x::Base.GMP.BigInt,c::Union{Int16,Int32,Int8,Int64}) at gmp.jl:307 - +(c::Union{Int16,Int32,Int8,Int64},x::Base.GMP.BigInt) at gmp.jl:308 - +(x::Base.MPFR.BigFloat,y::Base.MPFR.BigFloat) at mpfr.jl:206 - +(x::Base.MPFR.BigFloat,c::Union{UInt32,UInt16,UInt8,UInt64}) at mpfr.jl:213 - +(c::Union{UInt32,UInt16,UInt8,UInt64},x::Base.MPFR.BigFloat) at mpfr.jl:217 - +(x::Base.MPFR.BigFloat,c::Union{Int16,Int32,Int8,Int64}) at mpfr.jl:221 - +(c::Union{Int16,Int32,Int8,Int64},x::Base.MPFR.BigFloat) at mpfr.jl:225 - +(x::Base.MPFR.BigFloat,c::Union{Float16,Float64,Float32}) at mpfr.jl:229 - +(c::Union{Float16,Float64,Float32},x::Base.MPFR.BigFloat) at mpfr.jl:233 - +(x::Base.MPFR.BigFloat,c::Base.GMP.BigInt) at mpfr.jl:237 - +(c::Base.GMP.BigInt,x::Base.MPFR.BigFloat) at mpfr.jl:241 - +(a::Base.MPFR.BigFloat,b::Base.MPFR.BigFloat,c::Base.MPFR.BigFloat) at mpfr.jl:318 - +(a::Base.MPFR.BigFloat,b::Base.MPFR.BigFloat,c::Base.MPFR.BigFloat,d::Base.MPFR.BigFloat) at mpfr.jl:324 - +(a::Base.MPFR.BigFloat,b::Base.MPFR.BigFloat,c::Base.MPFR.BigFloat,d::Base.MPFR.BigFloat,e::Base.MPFR.BigFloat) at mpfr.jl:331 - +(x::Irrational{sym},y::Irrational{sym}) at constants.jl:71 - +{T<:Number}(x::T<:Number,y::T<:Number) at promotion.jl:205 - +{T<:AbstractFloat}(x::Bool,y::T<:AbstractFloat) at bool.jl:43 - +(x::Number,y::Number) at promotion.jl:167 - +(x::Integer,y::Ptr{T}) at pointer.jl:70 - +(x::Bool,A::AbstractArray{Bool,N}) at array.jl:829 - +(x::Integer,y::Char) at char.jl:41 - +(x::Number) at operators.jl:72 - +(r1::OrdinalRange{T,S},r2::OrdinalRange{T,S}) at operators.jl:325 - +{T<:AbstractFloat}(r1::FloatRange{T<:AbstractFloat},r2::FloatRange{T<:AbstractFloat}) at operators.jl:331 - +(r1::FloatRange{T<:AbstractFloat},r2::FloatRange{T<:AbstractFloat}) at operators.jl:348 - +(r1::FloatRange{T<:AbstractFloat},r2::OrdinalRange{T,S}) at operators.jl:349 - +(r1::OrdinalRange{T,S},r2::FloatRange{T<:AbstractFloat}) at operators.jl:350 - +(x::Ptr{T},y::Integer) at pointer.jl:68 - +{S,T}(A::Range{S},B::Range{T}) at array.jl:773 - +{S,T}(A::Range{S},B::AbstractArray{T,N}) at array.jl:791 - +(A::AbstractArray{Bool,N},x::Bool) at array.jl:828 - +(A::BitArray{N},B::BitArray{N}) at bitarray.jl:926 - +(A::Union{DenseArray{Bool,N},SubArray{Bool,N,A<:DenseArray{T,N},I<:Tuple{Vararg{Union{Colon,Range{Int64},Int64}}},LD}},B::Union{DenseArray{Bool,N},SubArray{Bool,N,A<:DenseArray{T,N},I<:Tuple{Vararg{Union{Colon,Range{Int64},Int64}}},LD}}) at array.jl:859 - +(A::Base.LinAlg.SymTridiagonal{T},B::Base.LinAlg.SymTridiagonal{T}) at linalg/tridiag.jl:59 - +(A::Base.LinAlg.Tridiagonal{T},B::Base.LinAlg.Tridiagonal{T}) at linalg/tridiag.jl:254 - +(A::Base.LinAlg.Tridiagonal{T},B::Base.LinAlg.SymTridiagonal{T}) at linalg/special.jl:113 - +(A::Base.LinAlg.SymTridiagonal{T},B::Base.LinAlg.Tridiagonal{T}) at linalg/special.jl:112 - +(A::Base.LinAlg.UpperTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.UpperTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:164 - +(A::Base.LinAlg.LowerTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.LowerTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:165 - +(A::Base.LinAlg.UpperTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.UnitUpperTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:166 - +(A::Base.LinAlg.LowerTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.UnitLowerTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:167 - +(A::Base.LinAlg.UnitUpperTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.UpperTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:168 - +(A::Base.LinAlg.UnitLowerTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.LowerTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:169 - +(A::Base.LinAlg.UnitUpperTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.UnitUpperTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:170 - +(A::Base.LinAlg.UnitLowerTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.UnitLowerTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:171 - +(A::Base.LinAlg.AbstractTriangular{T,S<:AbstractArray{T,2}},B::Base.LinAlg.AbstractTriangular{T,S<:AbstractArray{T,2}}) at linalg/triangular.jl:172 - +(Da::Base.LinAlg.Diagonal{T},Db::Base.LinAlg.Diagonal{T}) at linalg/diagonal.jl:50 - +(A::Base.LinAlg.Bidiagonal{T},B::Base.LinAlg.Bidiagonal{T}) at linalg/bidiag.jl:111 - +{T}(B::BitArray{2},J::Base.LinAlg.UniformScaling{T}) at linalg/uniformscaling.jl:28 - +(A::Base.LinAlg.Diagonal{T},B::Base.LinAlg.Bidiagonal{T}) at linalg/special.jl:103 - +(A::Base.LinAlg.Bidiagonal{T},B::Base.LinAlg.Diagonal{T}) at linalg/special.jl:104 - +(A::Base.LinAlg.Diagonal{T},B::Base.LinAlg.Tridiagonal{T}) at linalg/special.jl:103 - +(A::Base.LinAlg.Tridiagonal{T},B::Base.LinAlg.Diagonal{T}) at linalg/special.jl:104 - +(A::Base.LinAlg.Diagonal{T},B::Array{T,2}) at linalg/special.jl:103 - +(A::Array{T,2},B::Base.LinAlg.Diagonal{T}) at linalg/special.jl:104 - +(A::Base.LinAlg.Bidiagonal{T},B::Base.LinAlg.Tridiagonal{T}) at linalg/special.jl:103 - +(A::Base.LinAlg.Tridiagonal{T},B::Base.LinAlg.Bidiagonal{T}) at linalg/special.jl:104 - +(A::Base.LinAlg.Bidiagonal{T},B::Array{T,2}) at linalg/special.jl:103 - +(A::Array{T,2},B::Base.LinAlg.Bidiagonal{T}) at linalg/special.jl:104 - +(A::Base.LinAlg.Tridiagonal{T},B::Array{T,2}) at linalg/special.jl:103 - +(A::Array{T,2},B::Base.LinAlg.Tridiagonal{T}) at linalg/special.jl:104 - +(A::Base.LinAlg.SymTridiagonal{T},B::Array{T,2}) at linalg/special.jl:112 - +(A::Array{T,2},B::Base.LinAlg.SymTridiagonal{T}) at linalg/special.jl:113 - +(A::Base.LinAlg.Diagonal{T},B::Base.LinAlg.SymTridiagonal{T}) at linalg/special.jl:121 - +(A::Base.LinAlg.SymTridiagonal{T},B::Base.LinAlg.Diagonal{T}) at linalg/special.jl:122 - +(A::Base.LinAlg.Bidiagonal{T},B::Base.LinAlg.SymTridiagonal{T}) at linalg/special.jl:121 - +(A::Base.LinAlg.SymTridiagonal{T},B::Base.LinAlg.Bidiagonal{T}) at linalg/special.jl:122 - +{Tv1,Ti1,Tv2,Ti2}(A_1::Base.SparseArrays.SparseMatrixCSC{Tv1,Ti1},A_2::Base.SparseMatrix.SparseMatrixCSC{Tv2,Ti2}) at sparse/sparsematrix.jl:873 - +(A::Base.SparseArrays.SparseMatrixCSC{Tv,Ti<:Integer},B::Array{T,N}) at sparse/sparsematrix.jl:885 - +(A::Array{T,N},B::Base.SparseArrays.SparseMatrixCSC{Tv,Ti<:Integer}) at sparse/sparsematrix.jl:887 - +{P<:Base.Dates.Period}(Y::Union{SubArray{P<:Base.Dates.Period,N,A<:DenseArray{T,N},I<:Tuple{Vararg{Union{Colon,Range{Int64},Int64}}},LD},DenseArray{P<:Base.Dates.Period,N}},x::P<:Base.Dates.Period) at dates/periods.jl:50 - +{T<:Base.Dates.TimeType}(r::Range{T<:Base.Dates.TimeType},x::Base.Dates.Period) at dates/ranges.jl:39 - +{T<:Number}(x::AbstractArray{T<:Number,N}) at abstractarray.jl:442 - +{S,T}(A::AbstractArray{S,N},B::Range{T}) at array.jl:782 - +{S,T}(A::AbstractArray{S,N},B::AbstractArray{T,N}) at array.jl:800 - +(A::AbstractArray{T,N},x::Number) at array.jl:832 - +(x::Number,A::AbstractArray{T,N}) at array.jl:833 - +(x::Char,y::Integer) at char.jl:40 - +{N}(index1::Base.IteratorsMD.CartesianIndex{N},index2::Base.IteratorsMD.CartesianIndex{N}) at multidimensional.jl:121 - +(J1::Base.LinAlg.UniformScaling{T<:Number},J2::Base.LinAlg.UniformScaling{T<:Number}) at linalg/uniformscaling.jl:27 - +(J::Base.LinAlg.UniformScaling{T<:Number},B::BitArray{2}) at linalg/uniformscaling.jl:29 - +(J::Base.LinAlg.UniformScaling{T<:Number},A::AbstractArray{T,2}) at linalg/uniformscaling.jl:30 - +(J::Base.LinAlg.UniformScaling{T<:Number},x::Number) at linalg/uniformscaling.jl:31 - +(x::Number,J::Base.LinAlg.UniformScaling{T<:Number}) at linalg/uniformscaling.jl:32 - +{TA,TJ}(A::AbstractArray{TA,2},J::Base.LinAlg.UniformScaling{TJ}) at linalg/uniformscaling.jl:35 - +{T}(a::Base.Pkg.Resolve.VersionWeights.HierarchicalValue{T},b::Base.Pkg.Resolve.VersionWeights.HierarchicalValue{T}) at pkg/resolve/versionweight.jl:21 - +(a::Base.Pkg.Resolve.VersionWeights.VWPreBuildItem,b::Base.Pkg.Resolve.VersionWeights.VWPreBuildItem) at pkg/resolve/versionweight.jl:83 - +(a::Base.Pkg.Resolve.VersionWeights.VWPreBuild,b::Base.Pkg.Resolve.VersionWeights.VWPreBuild) at pkg/resolve/versionweight.jl:129 - +(a::Base.Pkg.Resolve.VersionWeights.VersionWeight,b::Base.Pkg.Resolve.VersionWeights.VersionWeight) at pkg/resolve/versionweight.jl:183 - +(a::Base.Pkg.Resolve.MaxSum.FieldValues.FieldValue,b::Base.Pkg.Resolve.MaxSum.FieldValues.FieldValue) at pkg/resolve/fieldvalue.jl:43 - +{P<:Base.Dates.Period}(x::P<:Base.Dates.Period,y::P<:Base.Dates.Period) at dates/periods.jl:43 - +{P<:Base.Dates.Period}(x::P<:Base.Dates.Period,Y::Union{SubArray{P<:Base.Dates.Period,N,A<:DenseArray{T,N},I<:Tuple{Vararg{Union{Colon,Range{Int64},Int64}}},LD},DenseArray{P<:Base.Dates.Period,N}}) at dates/periods.jl:49 - +(x::Base.Dates.Period,y::Base.Dates.Period) at dates/periods.jl:196 - +(x::Base.Dates.CompoundPeriod,y::Base.Dates.Period) at dates/periods.jl:197 - +(y::Base.Dates.Period,x::Base.Dates.CompoundPeriod) at dates/periods.jl:198 - +(x::Base.Dates.CompoundPeriod,y::Base.Dates.CompoundPeriod) at dates/periods.jl:199 - +(dt::Base.Dates.DateTime,y::Base.Dates.Year) at dates/arithmetic.jl:13 - +(dt::Base.Dates.Date,y::Base.Dates.Year) at dates/arithmetic.jl:17 - +(dt::Base.Dates.DateTime,z::Base.Dates.Month) at dates/arithmetic.jl:37 - +(dt::Base.Dates.Date,z::Base.Dates.Month) at dates/arithmetic.jl:43 - +(x::Base.Dates.Date,y::Base.Dates.Week) at dates/arithmetic.jl:60 - +(x::Base.Dates.Date,y::Base.Dates.Day) at dates/arithmetic.jl:62 - +(x::Base.Dates.DateTime,y::Base.Dates.Period) at dates/arithmetic.jl:64 - +(a::Base.Dates.TimeType,b::Base.Dates.Period,c::Base.Dates.Period) at dates/periods.jl:210 - +(a::Base.Dates.TimeType,b::Base.Dates.Period,c::Base.Dates.Period,d::Base.Dates.Period...) at dates/periods.jl:212 - +(x::Base.Dates.TimeType,y::Base.Dates.CompoundPeriod) at dates/periods.jl:216 - +(x::Base.Dates.CompoundPeriod,y::Base.Dates.TimeType) at dates/periods.jl:221 - +(x::Base.Dates.Instant) at dates/arithmetic.jl:4 - +(x::Base.Dates.TimeType) at dates/arithmetic.jl:8 - +(y::Base.Dates.Period,x::Base.Dates.TimeType) at dates/arithmetic.jl:66 - +{T<:Base.Dates.TimeType}(x::Base.Dates.Period,r::Range{T<:Base.Dates.TimeType}) at dates/ranges.jl:40 - +(a,b,c) at operators.jl:83 - +(a,b,c,xs...) at operators.jl:84 + # 166 methods for generic function "+": + ... + +(a::Float16, b::Float16) at float16.jl:136... + +(x::Float32, y::Float32) at float.jl:206... + +(x::Float64, y::Float64) at float.jl:207... + +(x::Bool, z::Complex{Bool}) at complex.jl:137... + +(x::Bool, y::Bool) at bool.jl:36... + +(x::Bool) at bool.jl:33... + +{T<:AbstractFloat}(x::Bool, y::T) at bool.jl:43... + +(x::Bool, z::Complex) at complex.jl:144... + +(x::Bool, A::AbstractArray{Bool,N<:Any}) at arraymath.jl:105... + +(x::Char, y::Integer) at char.jl:40 + ... + +(a, b, c, xs...) at operators.jl:119 Multiple dispatch together with the flexible parametric type system give Julia its ability to abstractly express high-level algorithms decoupled From 4e569b772b255b4bc296fddcfbb0e57786cb1d05 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 13:57:02 -0400 Subject: [PATCH 0332/1117] Update documentation to use `view` instead `sub` --- doc/manual/arrays.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index ad847f64eb20a..44100b50b9094 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -431,7 +431,7 @@ The first construct is used when you need the value, but not index, of each elem type with fast linear indexing; otherwise, it will be a ``CartesianIndex``:: A = rand(4,3) - B = sub(A, 1:3, 2:3) + B = view(A, 1:3, 2:3) julia> for i in eachindex(B) @show i end @@ -619,10 +619,10 @@ library can be implemented in a generic manner. :obj:`SubArray` is a specialization of :obj:`AbstractArray` that performs indexing by reference rather than by copying. A :obj:`SubArray` is created -with the :func:`sub` function, which is called the same way as :func:`getindex` (with -an array and a series of index arguments). The result of :func:`sub` looks +with the :func:`view` function, which is called the same way as :func:`getindex` +(with an array and a series of index arguments). The result of :func:`view` looks the same as the result of :func:`getindex`, except the data is left in place. -:func:`sub` stores the input index vectors in a :obj:`SubArray` object, which +:func:`view` stores the input index vectors in a :obj:`SubArray` object, which can later be used to index the original array indirectly. :obj:`StridedVector` and :obj:`StridedMatrix` are convenient aliases defined @@ -650,7 +650,7 @@ stride parameters. 0.890947 0.168877 0.32002 0.486136 0.096078 0.172048 0.77672 0.507762 0.573567 0.220124 0.165816 0.211049 0.433277 0.539476 - julia> b = sub(a, 2:2:8,2:2:4) + julia> b = view(a, 2:2:8,2:2:4) 4×2 SubArray{Float64,2,Array{Float64,2},Tuple{StepRange{Int64,Int64},StepRange{Int64,Int64}},false}: 0.537192 0.996234 0.736979 0.228787 From e8affbd4669dbfc35c26f89271ff9efdfb983fae Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 13:58:54 -0400 Subject: [PATCH 0333/1117] Update doctest for `expand` --- doc/devdocs/reflection.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/devdocs/reflection.rst b/doc/devdocs/reflection.rst index a9ec23394e1ff..a88c34d1236f7 100644 --- a/doc/devdocs/reflection.rst +++ b/doc/devdocs/reflection.rst @@ -106,7 +106,7 @@ variable assignments: julia> expand( :(f() = 1) ) :(begin $(Expr(:method, :f)) - $(Expr(:method, :f, :((top(svec))((top(apply_type))(Tuple,((top(getfield))(Core,:Typeof))(f)),(top(svec))())), LambdaInfo for anonymous + $(Expr(:method, :f, :((Core.svec)((Core.apply_type)(Tuple,(Core.Typeof)(f)),(Core.svec)())), Toplevel LambdaInfo thunk :(begin # none, line 1: return 1 end), false)) From dd05f14b242115fc653cb7dc6cb5f48ae47f7168 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 13:59:59 -0400 Subject: [PATCH 0334/1117] Update doctest for `macroexpand` --- doc/manual/metaprogramming.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/metaprogramming.rst b/doc/manual/metaprogramming.rst index edcd27774b3dc..108d48fe495a9 100644 --- a/doc/manual/metaprogramming.rst +++ b/doc/manual/metaprogramming.rst @@ -594,14 +594,14 @@ function: :(if a == b nothing else - Base.throw(Base.Main.Base.AssertionError("a == b")) + (Base.throw)(Base.Main.Base.AssertionError("a == b")) end) julia> macroexpand(:(@assert a==b "a should equal b!")) :(if a == b nothing else - Base.throw(Base.Main.Base.AssertionError("a should equal b!")) + (Base.throw)(Base.Main.Base.AssertionError("a should equal b!")) end) There is yet another case that the actual :obj:`@assert` macro handles: what From a139c4fb1c1fee057c772f9806ce893287e73fce Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 14:01:07 -0400 Subject: [PATCH 0335/1117] Update doctest to work Nullable get See JuliaLang/julia#16901 --- doc/manual/types.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 4364dc39ec7ea..85a62e57516a1 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -1403,10 +1403,10 @@ default value as a second argument to :func:`get`: .. doctest:: - julia> get(Nullable{Float64}(), 0) + julia> get(Nullable{Float64}(), 0.0) 0.0 - julia> get(Nullable(1.0), 0) + julia> get(Nullable(1.0), 0.0) 1.0 Note that this default value will automatically be converted to the type of From 115d97f1c6c17690259f8479638b913177205407 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 14:08:40 -0400 Subject: [PATCH 0336/1117] Update manual on method ambiguity on usage When ambiguous methods now report an error on usage rather than definition: see JuliaLang/julia#16125 --- doc/manual/methods.rst | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index adf6930a33f46..ac08fe17057dc 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -266,13 +266,6 @@ arguments: julia> g(x::Float64, y) = 2x + y; julia> g(x, y::Float64) = x + 2y; - WARNING: New definition - g(Any, Float64) at none:1 - is ambiguous with: - g(Float64, Any) at none:1. - To fix, define - g(Float64, Float64) - before the new definition. julia> g(2.0, 3) 7.0 @@ -281,14 +274,16 @@ arguments: 8.0 julia> g(2.0, 3.0) - 7.0 + ERROR: MethodError: g(::Float64, ::Float64) is ambiguous. Candidates: + g(x, y::Float64) at none:1 + g(x::Float64, y) at none:1 + in eval(::Module, ::Any) at ./boot.jl:231... Here the call ``g(2.0, 3.0)`` could be handled by either the ``g(Float64, Any)`` or the ``g(Any, Float64)`` method, and neither is -more specific than the other. In such cases, Julia warns you about this -ambiguity, but allows you to proceed, arbitrarily picking a method. You -should avoid method ambiguities by specifying an appropriate method for -the intersection case: +more specific than the other. In such cases, Julia raises a ``MethodError`` +rather than arbitrarily picking a method. You can avoid method ambiguities +by specifying an appropriate method for the intersection case: .. doctest:: @@ -307,9 +302,9 @@ the intersection case: julia> h(2.0, 3.0) 10.0 -To suppress Julia's warning, the disambiguating method must be defined -first, since otherwise the ambiguity exists, if transiently, until the -more specific method is defined. +It is recommended that the disambiguating method be defined first, +since otherwise the ambiguity exists, if transiently, until the more +specific method is defined. .. _man-parametric-methods: From 2f01184d225f19ea9e654ba6eec6714d62eba2e4 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 24 Jun 2016 23:19:05 -0400 Subject: [PATCH 0337/1117] Doctest fixes for dump display --- doc/devdocs/types.rst | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index 5dac7a02661e2..7f882fd0de585 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -140,7 +140,7 @@ one can extract the underlying :obj:`TypeVar`: TypeVar name: Symbol S lb: Union{} - ub: Integer <: Real + ub: Integer <: Real bound: Bool true Here ``ub`` is ``Integer``, as specified in the function definition. @@ -315,20 +315,17 @@ the type, which is an object of type :obj:`TypeName`: TypeName name: Symbol Array module: Module Core - names: SimpleVector - length: Int64 0 - primary: Array{T,N}::DataType <: DenseArray{T,N} + names: empty SimpleVector + primary: Array{T,N} <: DenseArray{T,N} cache: SimpleVector - length: Int64 135 + ... linearcache: SimpleVector - length: Int64 60 - uid: Int64 43 + ... + uid: Int64 47 mt: MethodTable name: Symbol Array defs: Void nothing cache: Void nothing - cache_arg1: Void nothing - cache_targ: Void nothing max_args: Int64 0 kwsorter: #undef module: Module Core From 08e45bb38e9a6bc8842a70d6255dee9e7028fe2e Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Sat, 25 Jun 2016 00:06:37 -0400 Subject: [PATCH 0338/1117] Fix doctest for value types --- doc/manual/types.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 85a62e57516a1..c5919884d21a4 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -1288,16 +1288,21 @@ functions in Julia's standard library accept ``Val`` types as arguments, and you can also use it to write your own functions. For example: -.. doctest:: +.. testsetup:: value-types + + firstlast(::Type{Val{true}}) = "First"; + firstlast(::Type{Val{false}}) = "Last"; + +.. doctest:: value-types firstlast(::Type{Val{true}}) = "First" firstlast(::Type{Val{false}}) = "Last" - julia> println(firstlast(Val{true})) - First + julia> firstlast(Val{true}) + "First" - julia> println(firstlast(Val{false})) - Last + julia> firstlast(Val{false}) + "Last" For consistency across Julia, the call site should always pass a ``Val`` *type* rather than creating an *instance*, i.e., use From db8a905015453dfcb457b8faa47e282af2931242 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Sat, 25 Jun 2016 00:27:39 -0400 Subject: [PATCH 0339/1117] Work around for doctest issue with empty line Sphinx does not work well when docstring output contains an empty line. Since `@code_warntype` has a empty line in the printed output this was causing Sphinx to have incorrect expected data. As it turns out the docstring example is only interested in the return type of the method which meant we could truncate part of the output using ... and avoid the empty line issue. --- base/test.jl | 11 +++-------- doc/stdlib/test.rst | 11 +++-------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/base/test.jl b/base/test.jl index c51548bfe186b..c0b2e4d7efe4a 100644 --- a/base/test.jl +++ b/base/test.jl @@ -893,19 +893,14 @@ julia> typeof(f(1,2,3)) Int64 julia> @code_warntype f(1,2,3) -Variables: - #self#::#f - a::Int64 - b::Int64 - c::Int64 - +... Body: - begin # REPL[2], line 1: + begin # none, line 1: unless (Base.slt_int)(1,b::Int64)::Bool goto 4 return 1 4: return 1.0 - end::Union{Float64,Int64} + end::UNION{FLOAT64,INT64} julia> @inferred f(1,2,3) ERROR: return type Int64 does not match inferred return type Union{Float64,Int64} diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index a1a6e7ea388b2..4a4ef04fdeaa2 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -263,19 +263,14 @@ writing new tests. Int64 julia> @code_warntype f(1,2,3) - Variables: - #self#::#f - a::Int64 - b::Int64 - c::Int64 - + ... Body: - begin # REPL[2], line 1: + begin # none, line 1: unless (Base.slt_int)(1,b::Int64)::Bool goto 4 return 1 4: return 1.0 - end::Union{Float64,Int64} + end::UNION{FLOAT64,INT64} julia> @inferred f(1,2,3) ERROR: return type Int64 does not match inferred return type Union{Float64,Int64} From 97bd55e461eda1c388adfa65ba13faea95cac0a1 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Sat, 25 Jun 2016 16:11:34 -0400 Subject: [PATCH 0340/1117] Remove eval from most stack traces --- base/docs/helpdb/Base.jl | 4 ++-- base/test.jl | 2 +- doc/devdocs/types.rst | 2 +- doc/manual/complex-and-rational-numbers.rst | 4 ++-- doc/manual/constructors.rst | 10 ++++----- doc/manual/control-flow.rst | 24 ++++++++++----------- doc/manual/conversion-and-promotion.rst | 6 +++--- doc/manual/mathematical-operations.rst | 8 +++---- doc/manual/metaprogramming.rst | 8 +++---- doc/manual/methods.rst | 18 ++++++++-------- doc/manual/strings.rst | 8 +++---- doc/manual/types.rst | 12 +++++------ doc/manual/variables-and-scoping.rst | 2 +- doc/manual/variables.rst | 4 ++-- doc/stdlib/base.rst | 2 +- doc/stdlib/collections.rst | 2 +- doc/stdlib/test.rst | 2 +- 17 files changed, 59 insertions(+), 59 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 8d7c73adb49e0..db23f5bc621c7 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5451,7 +5451,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:571 - in eval(::Module, ::Any) at ./boot.jl:231... + ... ``` """ deleteat!(collection, itr) @@ -8111,7 +8111,7 @@ julia> convert(Int, 3.0) julia> convert(Int, 3.5) ERROR: InexactError() in convert(::Type{Int64}, ::Float64) at ./int.jl:239 - in eval(::Module, ::Any) at ./boot.jl:231... + ... ``` If `T` is a [`AbstractFloat`](:obj:`AbstractFloat`) or [`Rational`](:obj:`Rational`) type, diff --git a/base/test.jl b/base/test.jl index c0b2e4d7efe4a..8eb6c1187ae1a 100644 --- a/base/test.jl +++ b/base/test.jl @@ -905,7 +905,7 @@ Body: julia> @inferred f(1,2,3) ERROR: return type Int64 does not match inferred return type Union{Float64,Int64} in error(::String) at ./error.jl:21 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> @inferred max(1,2) 2 diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index 7f882fd0de585..5a754bdf28153 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -229,7 +229,7 @@ These therefore print identically, but they have very different behavior: ERROR: MethodError: no method matching candid(::Array{Int64,1}, ::Float64) Closest candidates are: candid{T}(::Array{T,N}, !Matched::T) - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> sneaky([1],3.2) 1 diff --git a/doc/manual/complex-and-rational-numbers.rst b/doc/manual/complex-and-rational-numbers.rst index d3a24159a76b7..4a102e31407f8 100644 --- a/doc/manual/complex-and-rational-numbers.rst +++ b/doc/manual/complex-and-rational-numbers.rst @@ -160,7 +160,7 @@ versus ``-1 + 0im`` even though ``-1 == -1 + 0im``: ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). in sqrt(::Int64) at ./math.jl:149 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> sqrt(-1 + 0im) 0.0 + 1.0im @@ -306,7 +306,7 @@ Trying to construct a :const:`NaN` rational value, however, is not: ERROR: ArgumentError: invalid rational: zero(Int64)//zero(Int64) in Rational{Int64}(::Int64, ::Int64) at ./rational.jl:8 in //(::Int64, ::Int64) at ./rational.jl:22 - in eval(::Module, ::Any) at ./boot.jl:231... + ... As usual, the promotion system makes interactions with other numeric types effortless: diff --git a/doc/manual/constructors.rst b/doc/manual/constructors.rst index 564bf298cda71..aef8a8f437a76 100644 --- a/doc/manual/constructors.rst +++ b/doc/manual/constructors.rst @@ -122,7 +122,7 @@ Now ``OrderedPair`` objects can only be constructed such that julia> OrderedPair(2,1) ERROR: out of order in OrderedPair(::Int64, ::Int64) at ./none:5 - in eval(::Module, ::Any) at ./boot.jl:231... + ... You can still reach in and directly change the field values to violate this invariant, but messing around with an object's internals uninvited is @@ -267,7 +267,7 @@ access to an uninitialized reference is an immediate error: julia> z.xx ERROR: UndefRefError: access to undefined reference - in eval(::Module, ::Any) at ./boot.jl:231... + ... This avoids the need to continually check for ``null`` values. However, not all object fields are references. Julia considers some @@ -329,7 +329,7 @@ types of the arguments given to the constructor. Here are some examples: Closest candidates are: Point{T<:Real}{T<:Real}(::T<:Real, !Matched::T<:Real) Point{T<:Real}{T}(::Any) - in eval(::Module, ::Any) at ./boot.jl:231... + ... ## explicit T ## @@ -339,7 +339,7 @@ types of the arguments given to the constructor. Here are some examples: julia> Point{Int64}(1.0,2.5) ERROR: InexactError() in Point{Int64}(::Float64, ::Float64) at ./none:2 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> Point{Float64}(1.0,2.5) Point{Float64}(1.0,2.5) @@ -428,7 +428,7 @@ However, other similar calls still don't work: Closest candidates are: Point{T<:Real}{T<:Real}(::T<:Real, !Matched::T<:Real) Point{T<:Real}{T}(::Any) - in eval(::Module, ::Any) at ./boot.jl:231... + ... For a much more general way of making all such calls work sensibly, see :ref:`man-conversion-and-promotion`. At the risk diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 09e7948e0308a..221f5cd0488d6 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -163,7 +163,7 @@ the above function results in a runtime error julia> test2(2,1) ERROR: UndefVarError: relation not defined in test2(::Int64, ::Int64) at ./none:7 - in eval(::Module, ::Any) at ./boot.jl:231... + ... ``if`` blocks also return a value, which may seem unintuitive to users coming from many other languages. This value is simply the return value @@ -194,7 +194,7 @@ conditional expression is anything but ``true`` or ``false``: println("true") end ERROR: TypeError: non-boolean (Int64) used in boolean context - in eval(::Module, ::Any) at ./boot.jl:231... + ... This error indicates that the conditional was of the wrong type: :obj:`Int64` rather than the required :obj:`Bool`. @@ -368,7 +368,7 @@ For example, a recursive factorial routine could be defined like this: julia> fact(-1) ERROR: n must be non-negative in fact(::Int64) at ./none:2 - in eval(::Module, ::Any) at ./boot.jl:231... + ... Boolean operations *without* short-circuit evaluation can be done with the @@ -397,7 +397,7 @@ except for the last entry in a conditional chain is an error: julia> 1 && true ERROR: TypeError: non-boolean (Int64) used in boolean context - in eval(::Module, ::Any) at ./boot.jl:231... + ... On the other hand, any type of expression can be used at the end of a conditional chain. It will be evaluated and returned depending on the preceding conditionals: @@ -482,7 +482,7 @@ different variable name to test this: julia> j ERROR: UndefVarError: j not defined - in eval(::Module, ::Any) at ./boot.jl:231... + ... See :ref:`man-variables-and-scoping` for a detailed explanation of variable scope and how it works in Julia. @@ -667,7 +667,7 @@ negative real value: ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). in sqrt(::Int64) at ./math.jl:149 - in eval(::Module, ::Any) at ./boot.jl:231... + ... You may define your own exceptions in the following way: @@ -693,7 +693,7 @@ if the argument is negative: julia> g(-1) ERROR: DomainError: in g(::Int64) at ./none:1 - in eval(::Module, ::Any) at ./boot.jl:231... + ... Note that :exc:`DomainError` without parentheses is not an exception, but a type of exception. It needs to be called to obtain an :exc:`Exception` object: @@ -713,7 +713,7 @@ error reporting: julia> throw(UndefVarError(:x)) ERROR: UndefVarError: x not defined - in eval(::Module, ::Any) at ./boot.jl:231... + ... This mechanism can be implemented easily by custom exception types following the way :exc:`UndefVarError` is written: @@ -746,7 +746,7 @@ the :func:`sqrt` function that raises an error if its argument is negative: julia> fussy_sqrt(-1) ERROR: negative x not allowed in fussy_sqrt(::Int64) at ./none:1 - in eval(::Module, ::Any) at ./boot.jl:231... + ... If ``fussy_sqrt`` is called with a negative value from another function, instead of trying to continue execution of the calling function, it @@ -773,7 +773,7 @@ session: ERROR: negative x not allowed in fussy_sqrt at ./none:1 [inlined] in verbose_fussy_sqrt(::Int64) at ./none:3 - in eval(::Module, ::Any) at ./boot.jl:231... + ... Warnings and informational messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -795,7 +795,7 @@ execution: julia> error("Hi"); 1+1 ERROR: Hi in error(::String) at ./error.jl:21 - in eval(::Module, ::Any) at ./boot.jl:231... + ... The ``try/catch`` statement @@ -855,7 +855,7 @@ assumes ``x`` is a real number and returns its square root: julia> sqrt_second(-9) ERROR: DomainError: in sqrt_second(::Int64) at ./none:7 - in eval(::Module, ::Any) at ./boot.jl:231... + ... Note that the symbol following ``catch`` will always be interpreted as a name for the exception, so care is needed when writing ``try/catch`` expressions diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 78703f49c9251..6c64950d0c3ee 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -100,7 +100,7 @@ requested conversion: ERROR: MethodError: Cannot `convert` an object of type String to an object of type AbstractFloat This may have arisen from a call to the constructor AbstractFloat(...), since type constructors fall back to convert methods. - in eval(::Module, ::Any) at ./boot.jl:231... + ... Some languages consider parsing strings as numbers or formatting numbers as strings to be conversions (many dynamic languages will even @@ -143,7 +143,7 @@ to one and zero: julia> convert(Bool, 1im) ERROR: InexactError() in convert(::Type{Bool}, ::Complex{Int64}) at ./complex.jl:18 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> convert(Bool, 0im) false @@ -159,7 +159,7 @@ This is the actual implementation in Julia:: julia> convert(Bool, 1im) ERROR: InexactError() in convert(::Type{Bool}, ::Complex{Int64}) at ./complex.jl:18 - in eval(::Module, ::Any) at ./boot.jl:231... + ... Case Study: Rational Conversions diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index 570d1b9be42b2..0fb48f9b577f7 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -401,7 +401,7 @@ The following examples show the different forms. julia> Int8(128) ERROR: InexactError() in Int8(::Int64) at ./sysimg.jl:60 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> Int8(127.0) 127 @@ -409,12 +409,12 @@ The following examples show the different forms. julia> Int8(3.14) ERROR: InexactError() in Int8(::Float64) at ./sysimg.jl:60 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> Int8(128.0) ERROR: InexactError() in Int8(::Float64) at ./sysimg.jl:60 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> 127 % Int8 127 @@ -429,7 +429,7 @@ The following examples show the different forms. ERROR: InexactError() in trunc(::Type{Int8}, ::Float64) at ./float.jl:425 in round(::Type{Int8}, ::Float64) at ./float.jl:180 - in eval(::Module, ::Any) at ./boot.jl:231... + ... See :ref:`man-conversion-and-promotion` for how to define your own conversions and promotions. diff --git a/doc/manual/metaprogramming.rst b/doc/manual/metaprogramming.rst index 108d48fe495a9..411bfaf46f9f0 100644 --- a/doc/manual/metaprogramming.rst +++ b/doc/manual/metaprogramming.rst @@ -257,7 +257,7 @@ cause a compile-time error: julia> $a + b ERROR: unsupported or misplaced expression $ - in eval(::Module, ::Any) at ./boot.jl:231... + ... In this example, the tuple ``(1,2,3)`` is interpolated as an expression into a conditional test: @@ -298,7 +298,7 @@ at global scope using :func:`eval`: julia> eval(ex) ERROR: UndefVarError: b not defined - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> a = 1; b = 2; @@ -318,7 +318,7 @@ module's environment: julia> x ERROR: UndefVarError: x not defined - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> eval(ex) 1 @@ -549,7 +549,7 @@ This macro can be used like this: julia> @assert 1==0 ERROR: AssertionError: 1 == 0 - in eval(::Module, ::Any) at ./boot.jl:231... + ... In place of the written syntax, the macro call is expanded at parse time to its returned result. This is equivalent to writing:: diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index ac08fe17057dc..32399693aaeda 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -96,23 +96,23 @@ Applying it to any other types of arguments will result in a :exc:`MethodError`: ERROR: MethodError: no method matching f(::Float64, ::Int64) Closest candidates are: f(::Float64, !Matched::Float64) - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> f(Float32(2.0), 3.0) ERROR: MethodError: no method matching f(::Float32, ::Float64) Closest candidates are: f(!Matched::Float64, ::Float64) - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> f(2.0, "3.0") ERROR: MethodError: no method matching f(::Float64, ::String) Closest candidates are: f(::Float64, !Matched::Float64) - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> f("2.0", "3.0") ERROR: MethodError: no method matching f(::String, ::String) - in eval(::Module, ::Any) at ./boot.jl:231... + ... As you can see, the arguments must be precisely of type :obj:`Float64`. Other numeric types, such as integers or 32-bit floating-point values, @@ -180,11 +180,11 @@ function ``f`` remains undefined, and applying it will still result in a ERROR: MethodError: no method matching f(::String, ::Int64) Closest candidates are: f(!Matched::Number, ::Number) - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> f() ERROR: MethodError: no method matching f() - in eval(::Module, ::Any) at ./boot.jl:231... + ... You can easily see which methods exist for a function by entering the function object itself in an interactive session: @@ -277,7 +277,7 @@ arguments: ERROR: MethodError: g(::Float64, ::Float64) is ambiguous. Candidates: g(x, y::Float64) at none:1 g(x::Float64, y) at none:1 - in eval(::Module, ::Any) at ./boot.jl:231... + ... Here the call ``g(2.0, 3.0)`` could be handled by either the ``g(Float64, Any)`` or the ``g(Any, Float64)`` method, and neither is @@ -370,7 +370,7 @@ signature: ERROR: MethodError: no method matching myappend(::Array{Int64,1}, ::Float64) Closest candidates are: myappend{T}(::Array{T,1}, !Matched::T) - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> myappend([1.0,2.0,3.0],4.0) 4-element Array{Float64,1}: @@ -383,7 +383,7 @@ signature: ERROR: MethodError: no method matching myappend(::Array{Float64,1}, ::Int64) Closest candidates are: myappend{T}(::Array{T,1}, !Matched::T) - in eval(::Module, ::Any) at ./boot.jl:231... + ... As you can see, the type of the appended element must match the element type of the vector it is appended to, or else a :exc:`MethodError` is raised. diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 41b1f3a2a1b69..34508d1ee85fb 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -291,14 +291,14 @@ such an invalid byte index, an error is thrown: in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:68 in next at ./strings/string.jl:93 [inlined] in getindex(::String, ::Int64) at ./strings/basic.jl:70 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> s[3] ERROR: UnicodeError: invalid character index in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:68 in next at ./strings/string.jl:93 [inlined] in getindex(::String, ::Int64) at ./strings/basic.jl:70 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> s[4] ' ' @@ -553,7 +553,7 @@ contained in a string: Closest candidates are: contains(!Matched::Function, ::Any, !Matched::Any) contains(::AbstractString, !Matched::AbstractString) - in eval(::Module, ::Any) at ./boot.jl:231... + ... The last error is because ``'o'`` is a character literal, and :func:`contains` is a generic function that looks for subsequences. To look for an element in a @@ -875,7 +875,7 @@ error: julia> "DATA\xff\u2200" ERROR: syntax: invalid UTF-8 sequence - in eval(::Module, ::Any) at ./boot.jl:231... + ... Also observe the significant distinction between ``\xff`` and ``\uff``: the former escape sequence encodes the *byte 255*, whereas the latter diff --git a/doc/manual/types.rst b/doc/manual/types.rst index c5919884d21a4..35a2f1b1ea7ac 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -102,7 +102,7 @@ exception is thrown, otherwise, the left-hand value is returned: julia> (1+2)::AbstractFloat ERROR: TypeError: typeassert: expected AbstractFloat, got Int64 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> (1+2)::Int 3 @@ -413,7 +413,7 @@ However, the value for ``baz`` must be convertible to :class:`Int`: julia> Foo((), 23.5, 1) ERROR: InexactError() in Foo(::Tuple{}, ::Float64, ::Int64) at ./none:2 - in eval(::Module, ::Any) at ./boot.jl:231... + ... You may find a list of field names using the ``fieldnames`` function. @@ -768,14 +768,14 @@ each field: This may have arisen from a call to the constructor Point{Float64}(...), since type constructors fall back to convert methods. in Point{Float64}(::Float64) at ./sysimg.jl:60 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> Point{Float64}(1.0,2.0,3.0) ERROR: MethodError: no method matching Point{Float64}(::Float64, ::Float64, ::Float64) Closest candidates are: Point{Float64}{T}(::Any, ::Any) Point{Float64}{T}(::Any) - in eval(::Module, ::Any) at ./boot.jl:231... + ... Only one default constructor is generated for parametric types, since overriding it is not possible. This constructor accepts any arguments @@ -918,11 +918,11 @@ subtypes of :obj:`Real`: julia> Pointy{AbstractString} ERROR: TypeError: Pointy: in T, expected T<:Real, got Type{AbstractString} - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> Pointy{1} ERROR: TypeError: Pointy: in T, expected T<:Real, got Int64 - in eval(::Module, ::Any) at ./boot.jl:231... + ... Type parameters for parametric composite types can be restricted in the same manner:: diff --git a/doc/manual/variables-and-scoping.rst b/doc/manual/variables-and-scoping.rst index 62370f7a8a02b..e53cadb2dbbfb 100644 --- a/doc/manual/variables-and-scoping.rst +++ b/doc/manual/variables-and-scoping.rst @@ -311,7 +311,7 @@ macro definition need not come before its inner usage: julia> f(3) ERROR: UndefVarError: x not defined in (::##1#2)(::Int64) at ./none:1 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> x = 1 1 diff --git a/doc/manual/variables.rst b/doc/manual/variables.rst index b85584f88ea80..039972004a4db 100644 --- a/doc/manual/variables.rst +++ b/doc/manual/variables.rst @@ -128,11 +128,11 @@ statements: julia> else = false ERROR: syntax: unexpected "else" - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> try = "No" ERROR: syntax: unexpected "=" - in eval(::Module, ::Any) at ./boot.jl:231... + ... Stylistic Conventions diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 897902fb076d6..0492ef81c268c 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -368,7 +368,7 @@ All Objects julia> convert(Int, 3.5) ERROR: InexactError() in convert(::Type{Int64}, ::Float64) at ./int.jl:239 - in eval(::Module, ::Any) at ./boot.jl:231... + ... If ``T`` is a :obj:`AbstractFloat` or :obj:`Rational` type, then it will return the closest value to ``x`` representable by ``T``\ . diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index c745e6585fc38..4be226fc9e9ef 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1135,7 +1135,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:571 - in eval(::Module, ::Any) at ./boot.jl:231... + ... .. function:: splice!(collection, index, [replacement]) -> item diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index 4a4ef04fdeaa2..fde0dea154d5d 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -275,7 +275,7 @@ writing new tests. julia> @inferred f(1,2,3) ERROR: return type Int64 does not match inferred return type Union{Float64,Int64} in error(::String) at ./error.jl:21 - in eval(::Module, ::Any) at ./boot.jl:231... + ... julia> @inferred max(1,2) 2 From 46afbfdb0a3959711e18dbd8b4d895aaf1f5c2fe Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Sat, 25 Jun 2016 18:37:40 -0400 Subject: [PATCH 0341/1117] Correct some documentation changes Corrections include: - Remove addition of `eval(::Module, ::Any)` from most stacktraces - Use doctest groups rather than appending numbers to function names --- doc/manual/control-flow.rst | 54 +++++++++--------- .../integers-and-floating-point-numbers.rst | 22 ++------ doc/manual/metaprogramming.rst | 12 ++-- doc/manual/methods.rst | 14 ++--- doc/manual/stacktraces.rst | 55 +++++++++---------- doc/manual/types.rst | 3 +- 6 files changed, 76 insertions(+), 84 deletions(-) diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 221f5cd0488d6..eb430778eb2ed 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -126,9 +126,9 @@ This means that new variables defined inside the ``ìf`` clauses can be used after the ``if`` block, even if they weren't defined before. So, we could have defined the ``test`` function above as -.. doctest:: +.. doctest:: leaky - julia> function test1(x,y) + julia> function test(x,y) if x < y relation = "less than" elseif x == y @@ -138,16 +138,16 @@ So, we could have defined the ``test`` function above as end println("x is ", relation, " y.") end - test1 (generic function with 1 method) + test (generic function with 1 method) The variable ``relation`` is declared inside the ``if`` block, but used outside. However, when depending on this behavior, make sure all possible code paths define a value for the variable. The following change to the above function results in a runtime error -.. doctest:: +.. doctest:: bad-leaky - julia> function test2(x,y) + julia> function test(x,y) if x < y relation = "less than" elseif x == y @@ -155,14 +155,14 @@ the above function results in a runtime error end println("x is ", relation, " y.") end - test2 (generic function with 1 method) + test (generic function with 1 method) - julia> test2(1,2) + julia> test(1,2) x is less than y. - julia> test2(2,1) + julia> test(2,1) ERROR: UndefVarError: relation not defined - in test2(::Int64, ::Int64) at ./none:7 + in test(::Int64, ::Int64) at ./none:7 ... ``if`` blocks also return a value, which may seem unintuitive to users @@ -236,19 +236,19 @@ evaluates to the string ``"not less than"``. The original three-way example requires chaining multiple uses of the ternary operator together: -.. doctest:: +.. doctest:: ternary-operator - julia> test4(x, y) = println(x < y ? "x is less than y" : + julia> test(x, y) = println(x < y ? "x is less than y" : x > y ? "x is greater than y" : "x is equal to y") - test4 (generic function with 1 method) + test (generic function with 1 method) - julia> test4(1, 2) + julia> test(1, 2) x is less than y - julia> test4(2, 1) + julia> test(2, 1) x is greater than y - julia> test4(1, 1) + julia> test(1, 1) x is equal to y To facilitate chaining, the operator associates from right to left. @@ -682,23 +682,23 @@ Exceptions can be created explicitly with :func:`throw`. For example, a function defined only for nonnegative numbers could be written to :func:`throw` a :exc:`DomainError` if the argument is negative: -.. doctest:: +.. doctest:: domain-error - julia> g(x) = x>=0 ? exp(-x) : throw(DomainError()) - g (generic function with 1 method) + julia> f(x) = x>=0 ? exp(-x) : throw(DomainError()) + f (generic function with 1 method) - julia> g(1) + julia> f(1) 0.36787944117144233 - julia> g(-1) + julia> f(-1) ERROR: DomainError: - in g(::Int64) at ./none:1 + in f(::Int64) at ./none:1 ... Note that :exc:`DomainError` without parentheses is not an exception, but a type of exception. It needs to be called to obtain an :exc:`Exception` object: -.. doctest:: +.. doctest:: throw-function julia> typeof(DomainError()) <: Exception true @@ -806,19 +806,19 @@ example, a customized square root function can be written to automatically call either the real or complex square root method on demand using :exc:`Exception`\ s : -.. doctest:: +.. doctest:: try-catch - julia> h(x) = try + julia> f(x) = try sqrt(x) catch sqrt(complex(x, 0)) end - h (generic function with 1 method) + f (generic function with 1 method) - julia> h(1) + julia> f(1) 1.0 - julia> h(-1) + julia> f(-1) 0.0 + 1.0im It is important to note that in real code computing this function, one would diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index c1af976478452..9b37bff26eb08 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -412,24 +412,14 @@ types: .. doctest:: - julia> typemin(Float16) - -Inf16 + julia> (typemin(Float16),typemax(Float16)) + (-Inf16,Inf16) - julia> typemax(Float16) - Inf16 - - julia> typemin(Float32) - -Inf32 - - julia> typemax(Float32) - Inf32 - - julia> typemin(Float64) - -Inf - - julia> typemax(Float64) - Inf + julia> (typemin(Float32),typemax(Float32)) + (-Inf32,Inf32) + julia> (typemin(Float64),typemax(Float64)) + (-Inf,Inf) Machine epsilon ~~~~~~~~~~~~~~~ diff --git a/doc/manual/metaprogramming.rst b/doc/manual/metaprogramming.rst index 411bfaf46f9f0..52e4551dea937 100644 --- a/doc/manual/metaprogramming.rst +++ b/doc/manual/metaprogramming.rst @@ -418,11 +418,15 @@ Basics Here is an extraordinarily simple macro: -.. doctest:: +.. testcode:: + + macro sayhello() + return :( println("Hello, world!") ) + end + +.. testoutput:: + :hide: - julia> macro sayhello() - return :( println("Hello, world!") ) - end @sayhello (macro with 1 method) Macros have a dedicated character in Julia's syntax: the ``@`` (at-sign), diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index 32399693aaeda..9d2dff09958d2 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -285,21 +285,21 @@ more specific than the other. In such cases, Julia raises a ``MethodError`` rather than arbitrarily picking a method. You can avoid method ambiguities by specifying an appropriate method for the intersection case: -.. doctest:: +.. doctest:: unambiguous - julia> h(x::Float64, y::Float64) = 2x + 2y; + julia> g(x::Float64, y::Float64) = 2x + 2y; - julia> h(x::Float64, y) = 2x + y; + julia> g(x::Float64, y) = 2x + y; - julia> h(x, y::Float64) = x + 2y; + julia> g(x, y::Float64) = x + 2y; - julia> h(2.0, 3) + julia> g(2.0, 3) 7.0 - julia> h(2, 3.0) + julia> g(2, 3.0) 8.0 - julia> h(2.0, 3.0) + julia> g(2.0, 3.0) 10.0 It is recommended that the disambiguating method be defined first, diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index 37d86f0e38379..989b5560fd35b 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -52,7 +52,6 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example in child() at none:1 in parent() at none:1 in grandparent() at none:1 - in eval(::Module, ::Any) at boot.jl:231 ... Note that when calling :func:`stacktrace` you'll typically see a frame with @@ -113,21 +112,21 @@ Error handling While having easy access to information about the current state of the callstack can be helpful in many places, the most obvious application is in error handling and debugging. -.. doctest:: +.. doctest:: error-handling julia> @noinline bad_function() = undeclared_variable bad_function (generic function with 1 method) - julia> @noinline example1() = try + julia> @noinline example() = try bad_function() catch stacktrace() end - example1 (generic function with 1 method) + example (generic function with 1 method) - julia> example1() + julia> example() 6-element Array{StackFrame,1}: - in example1() at none:4 + in example() at none:4 in eval(::Module, ::Any) at boot.jl:231 ... @@ -142,52 +141,50 @@ This can be remedied by calling :func:`catch_stacktrace` instead of :func:`stack Instead of returning callstack information for the current context, :func:`catch_stacktrace` returns stack information for the context of the most recent exception: -.. doctest:: +.. doctest:: catch-stacktrace - julia> @noinline bad_function1() = undeclared_variable - bad_function1 (generic function with 1 method) + julia> @noinline bad_function() = undeclared_variable + bad_function (generic function with 1 method) - julia> @noinline example2() = try - bad_function1() + julia> @noinline example() = try + bad_function() catch catch_stacktrace() end - example2 (generic function with 1 method) + example (generic function with 1 method) - julia> example2() + julia> example() 7-element Array{StackFrame,1}: - in bad_function1() at none:1 - in example2() at none:2 - in eval(::Module, ::Any) at boot.jl:231 + in bad_function() at none:1 + in example() at none:2 ... Notice that the stack trace now indicates the appropriate line number and the missing frame. -.. doctest:: +.. doctest:: catch-stacktrace-demo - julia> @noinline child1() = error("Whoops!") - child1 (generic function with 1 method) + julia> @noinline child() = error("Whoops!") + child (generic function with 1 method) - julia> @noinline parent1() = child1() - parent1 (generic function with 1 method) + julia> @noinline parent() = child() + parent (generic function with 1 method) - julia> @noinline function grandparent1() + julia> @noinline function grandparent() try - parent1() + parent() catch err println("ERROR: ", err.msg) catch_stacktrace() end end - grandparent1 (generic function with 1 method) + grandparent (generic function with 1 method) - julia> grandparent1() + julia> grandparent() ERROR: Whoops! 8-element Array{StackFrame,1}: - in child1() at none:1 - in parent1() at none:1 - in grandparent1() at none:3 - in eval(::Module, ::Any) at boot.jl:231 + in child() at none:1 + in parent() at none:1 + in grandparent() at none:3 ... Comparison with :func:`backtrace` diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 35a2f1b1ea7ac..e272848841b14 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -1392,7 +1392,8 @@ You can safely access the value of a :obj:`Nullable` object using :func:`get`: julia> get(Nullable{Float64}()) ERROR: NullException() - in get(::Nullable{Float64}) at ./nullable.jl:62... + in get(::Nullable{Float64}) at ./nullable.jl:62 + ... julia> get(Nullable(1.0)) 1.0 From 592e4d69db99fe02b876c27acad089566d42768e Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Sat, 25 Jun 2016 19:04:38 -0400 Subject: [PATCH 0342/1117] Improving methods(+) doctest The `methods(+)` doctest seems to fail periodically due to the order of the methods changing. It appears that blocks of these methods typically stick together so I picked a nice reading block and attempted to format with the least amount of ellipsis. Here is an interesting failure which caused me to update the layout of the doctest. Before this commit we used the following as a doctest: ```jldoctest julia> methods(+) # 166 methods for generic function "+": ... +(a::Float16, b::Float16) at float16.jl:136... +(x::Float32, y::Float32) at float.jl:206... +(x::Float64, y::Float64) at float.jl:207... +(x::Bool, z::Complex{Bool}) at complex.jl:137... +(x::Bool, y::Bool) at bool.jl:36... +(x::Bool) at bool.jl:33... +{T<:AbstractFloat}(x::Bool, y::T) at bool.jl:43... +(x::Bool, z::Complex) at complex.jl:144... +(x::Bool, A::AbstractArray{Bool,N<:Any}) at arraymath.jl:105... +(x::Char, y::Integer) at char.jl:40 ... +(a, b, c, xs...) at operators.jl:119 ``` The output was as follows: ``` # 166 methods for generic function "+": +(a::Float16, b::Float16) at float16.jl:136 +(x::Float32, y::Float32) at float.jl:206 +(x::Float64, y::Float64) at float.jl:207 +(x::Bool, z::Complex{Bool}) at complex.jl:137 +(x::Bool, y::Bool) at bool.jl:36 +(x::Bool) at bool.jl:33 +{T<:AbstractFloat}(x::Bool, y::T) at bool.jl:43 +(x::Bool, z::Complex) at complex.jl:144 +(x::Bool, A::AbstractArray{Bool,N<:Any}) at arraymath.jl:105 +(x::Char, y::Integer) at char.jl:40 ... +(a, b, c, xs...) at operators.jl:119 ``` --- doc/manual/methods.rst | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index 9d2dff09958d2..cb96b4dd731ad 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -234,17 +234,16 @@ Julia language. Core operations typically have dozens of methods: .. doctest:: julia> methods(+) - # 166 methods for generic function "+": - ... - +(a::Float16, b::Float16) at float16.jl:136... - +(x::Float32, y::Float32) at float.jl:206... - +(x::Float64, y::Float64) at float.jl:207... - +(x::Bool, z::Complex{Bool}) at complex.jl:137... - +(x::Bool, y::Bool) at bool.jl:36... - +(x::Bool) at bool.jl:33... - +{T<:AbstractFloat}(x::Bool, y::T) at bool.jl:43... - +(x::Bool, z::Complex) at complex.jl:144... - +(x::Bool, A::AbstractArray{Bool,N<:Any}) at arraymath.jl:105... + # 166 methods for generic function "+":... + +(a::Float16, b::Float16) at float16.jl:136 + +(x::Float32, y::Float32) at float.jl:206 + +(x::Float64, y::Float64) at float.jl:207 + +(x::Bool, z::Complex{Bool}) at complex.jl:137 + +(x::Bool, y::Bool) at bool.jl:36 + +(x::Bool) at bool.jl:33 + +{T<:AbstractFloat}(x::Bool, y::T) at bool.jl:43 + +(x::Bool, z::Complex) at complex.jl:144 + +(x::Bool, A::AbstractArray{Bool,N<:Any}) at arraymath.jl:105 +(x::Char, y::Integer) at char.jl:40 ... +(a, b, c, xs...) at operators.jl:119 From 811fc003ba492504862ba3efd8783af4f24939e8 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 29 Jun 2016 13:55:04 -0500 Subject: [PATCH 0343/1117] Doctest changes from #17113 --- base/docs/helpdb/Base.jl | 5 +---- doc/devdocs/reflection.rst | 2 +- doc/stdlib/linalg.rst | 5 +---- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index db23f5bc621c7..f473b7ba402e2 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -9127,10 +9127,7 @@ on the `permute` and `scale` keyword arguments. The eigenvectors are returned co ```jldoctest julia> eig([1.0 0.0 0.0; 0.0 3.0 0.0; 0.0 0.0 18.0]) ([1.0,3.0,18.0], -3×3 Array{Float64,2}: - 1.0 0.0 0.0 - 0.0 1.0 0.0 - 0.0 0.0 1.0) +[1.0 0.0 0.0; 0.0 1.0 0.0; 0.0 0.0 1.0]) ``` `eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the diff --git a/doc/devdocs/reflection.rst b/doc/devdocs/reflection.rst index a88c34d1236f7..185016495c732 100644 --- a/doc/devdocs/reflection.rst +++ b/doc/devdocs/reflection.rst @@ -92,7 +92,7 @@ the macro will be evaluated and the result will be passed instead!). For example .. doctest:: julia> macroexpand( :(@edit println("")) ) - :(Base.edit(println,(Base.typesof)(""))) + :((Base.edit)(println,(Base.typesof)(""))) The functions :func:`Base.Meta.show_sexpr` and :func:`dump` are used to display S-expr style views and depth-nested detail views for any expression. diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index bbf6ce5893a31..12d89f8e12875 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -584,10 +584,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f julia> eig([1.0 0.0 0.0; 0.0 3.0 0.0; 0.0 0.0 18.0]) ([1.0,3.0,18.0], - 3×3 Array{Float64,2}: - 1.0 0.0 0.0 - 0.0 1.0 0.0 - 0.0 0.0 1.0) + [1.0 0.0 0.0; 0.0 1.0 0.0; 0.0 0.0 1.0]) ``eig`` is a wrapper around :func:`eigfact`\ , extracting all parts of the factorization to a tuple; where possible, using :func:`eigfact` is recommended. From 21fa49db3258cbfdbf83d4521d75315272575b3a Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 29 Jun 2016 14:05:44 -0500 Subject: [PATCH 0344/1117] Doctest changes relating to dump display --- doc/devdocs/types.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index 5a754bdf28153..ed2041cbb8b24 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -91,7 +91,7 @@ Let's look at these types a little more closely: Array{T,N} julia> dump(Array) - Array{T,N}::DataType <: DenseArray{T,N} + Array{T,N} <: DenseArray{T,N} This indicates that :obj:`Array` is a shorthand for ``Array{T,N}``. If you type this at the REPL prompt---on its own, not while defining @@ -108,7 +108,7 @@ parameters: TypeVar name: Symbol T lb: Union{} - ub: Any::DataType <: Any + ub: Any bound: Bool false A :obj:`TypeVar` is one of Julia's built-in types---it's defined in @@ -173,7 +173,7 @@ parameters. For example: TypeVar name: Symbol T lb: Union{} - ub: Any <: Any + ub: Any bound: Bool false julia> dump(p3[2].parameters[1]) From 748236801976c887e7f02a21ea0ed5a085e21b2d Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 29 Jun 2016 14:47:59 -0500 Subject: [PATCH 0345/1117] Doctest update to reflect latest code --- doc/manual/mathematical-operations.rst | 6 +++--- doc/manual/methods.rst | 9 ++++++--- doc/manual/stacktraces.rst | 12 ++++++------ doc/manual/types.rst | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index 0fb48f9b577f7..eda7ba0d29b8e 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -400,7 +400,7 @@ The following examples show the different forms. julia> Int8(128) ERROR: InexactError() - in Int8(::Int64) at ./sysimg.jl:60 + in Int8(::Int64) at ./sysimg.jl:63 ... julia> Int8(127.0) @@ -408,12 +408,12 @@ The following examples show the different forms. julia> Int8(3.14) ERROR: InexactError() - in Int8(::Float64) at ./sysimg.jl:60 + in Int8(::Float64) at ./sysimg.jl:63 ... julia> Int8(128.0) ERROR: InexactError() - in Int8(::Float64) at ./sysimg.jl:60 + in Int8(::Float64) at ./sysimg.jl:63 ... julia> 127 % Int8 diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index cb96b4dd731ad..83632a28de065 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -184,6 +184,9 @@ function ``f`` remains undefined, and applying it will still result in a julia> f() ERROR: MethodError: no method matching f() + Closest candidates are: + f(!Matched::Float64, !Matched::Float64) + f(!Matched::Number, !Matched::Number) ... You can easily see which methods exist for a function by entering the @@ -239,9 +242,9 @@ Julia language. Core operations typically have dozens of methods: +(x::Float32, y::Float32) at float.jl:206 +(x::Float64, y::Float64) at float.jl:207 +(x::Bool, z::Complex{Bool}) at complex.jl:137 - +(x::Bool, y::Bool) at bool.jl:36 - +(x::Bool) at bool.jl:33 - +{T<:AbstractFloat}(x::Bool, y::T) at bool.jl:43 + +(x::Bool, y::Bool) at bool.jl:48 + +(x::Bool) at bool.jl:45 + +{T<:AbstractFloat}(x::Bool, y::T) at bool.jl:55 +(x::Bool, z::Complex) at complex.jl:144 +(x::Bool, A::AbstractArray{Bool,N<:Any}) at arraymath.jl:105 +(x::Char, y::Integer) at char.jl:40 diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index 989b5560fd35b..62174ee078dbc 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -32,9 +32,9 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example julia> example() 6-element Array{StackFrame,1}: in example() at none:1 - in eval(::Module, ::Any) at boot.jl:231 + in eval(::Module, ::Any) at boot.jl:234 in eval_user_input(::Any, ::Bool) at client.jl:117 - in eval(::Module, ::Any) at boot.jl:231 + in eval(::Module, ::Any) at boot.jl:234 in eval_user_input(::Any, ::Bool) at client.jl:117 in _start() at client.jl:359 @@ -64,7 +64,7 @@ extra frames in the stack from ``REPL.jl``, usually looking something like this: julia> example() 5-element Array{StackFrame,1}: in example() at REPL[1]:1 - in eval(::Module, ::Any) at boot.jl:231 + in eval(::Module, ::Any) at boot.jl:234 in eval_user_input(::Any, ::Base.REPL.REPLBackend) at REPL.jl:62 in macro expansion at REPL.jl:92 [inlined] in (::Base.REPL.##1#2{Base.REPL.REPLBackend})() at event.jl:46 @@ -77,7 +77,7 @@ Each :obj:`StackFrame` contains the function name, file name, line number, lambd .. doctest:: julia> top_frame = stacktrace()[1] - in eval(::Module, ::Any) at boot.jl:231 + in eval(::Module, ::Any) at boot.jl:234 julia> top_frame.func :eval @@ -86,7 +86,7 @@ Each :obj:`StackFrame` contains the function name, file name, line number, lambd Symbol("./boot.jl") julia> top_frame.line - 231 + 234 julia> top_frame.linfo Nullable{LambdaInfo}(LambdaInfo for eval(::Module, ::Any) @@ -127,7 +127,7 @@ helpful in many places, the most obvious application is in error handling and de julia> example() 6-element Array{StackFrame,1}: in example() at none:4 - in eval(::Module, ::Any) at boot.jl:231 + in eval(::Module, ::Any) at boot.jl:234 ... You may notice that in the example above the first stack frame points points at line 4, diff --git a/doc/manual/types.rst b/doc/manual/types.rst index e272848841b14..c59781a968307 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -767,7 +767,7 @@ each field: ERROR: MethodError: Cannot `convert` an object of type Float64 to an object of type Point{Float64} This may have arisen from a call to the constructor Point{Float64}(...), since type constructors fall back to convert methods. - in Point{Float64}(::Float64) at ./sysimg.jl:60 + in Point{Float64}(::Float64) at ./sysimg.jl:63 ... julia> Point{Float64}(1.0,2.0,3.0) From 2ea7f23d687a31f5ce961422106aa0244d0d1013 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 29 Jun 2016 14:49:30 -0500 Subject: [PATCH 0346/1117] Update JuliaDoc changeset See JuliaLang/JuliaDoc#23 --- doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index c2765491f318a..e4a55f02db4bd 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,2 +1,2 @@ --e git+https://github.com/JuliaLang/JuliaDoc.git@c87dbaa47f49ecc2b70c2e9cb029b9d3541160aa#egg=JuliaDoc +-e git+https://github.com/JuliaLang/JuliaDoc.git@8dbccf7a4cec7097f005314b7b08f5bb0d3817bc#egg=JuliaDoc -e git+https://github.com/snide/sphinx_rtd_theme.git@21e875d3a53ce897089ad690d897252f6063349d#egg=sphinx_rtd_theme From 610104e04e48ebd218c4eb34614ad8d57dcb9d7a Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Fri, 8 Jul 2016 13:52:29 -0500 Subject: [PATCH 0347/1117] Skip methods(+) doctest This particular doctest ended up producing a lot of false positives as it appears the ordering from method can change from commit to commit. Since this doctest was failing so consistently I set this particular doctest to be skipped. Additionally I expanded the sample of + methods shown which was previously more restricted in an attempt to make this doctest pass more consistently. --- doc/manual/methods.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index 83632a28de065..1eca0463a7e5a 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -235,9 +235,10 @@ values is perhaps the single most powerful and central feature of the Julia language. Core operations typically have dozens of methods: .. doctest:: + :options: +SKIP julia> methods(+) - # 166 methods for generic function "+":... + # 166 methods for generic function "+": +(a::Float16, b::Float16) at float16.jl:136 +(x::Float32, y::Float32) at float.jl:206 +(x::Float64, y::Float64) at float.jl:207 @@ -248,6 +249,13 @@ Julia language. Core operations typically have dozens of methods: +(x::Bool, z::Complex) at complex.jl:144 +(x::Bool, A::AbstractArray{Bool,N<:Any}) at arraymath.jl:105 +(x::Char, y::Integer) at char.jl:40 + +{T<:Union{Int128,Int16,Int32,Int64,Int8,UInt128,UInt16,UInt32,UInt64,UInt8}}(x::T, y::T) at int.jl:32 + +(z::Complex, w::Complex) at complex.jl:115 + +(z::Complex, x::Bool) at complex.jl:134 + +(x::Real, z::Complex{Bool}) at complex.jl:140 + +(x::Real, z::Complex) at complex.jl:152 + +(z::Complex, x::Real) at complex.jl:153 + +(x::Rational, y::Rational) at rational.jl:179 ... +(a, b, c, xs...) at operators.jl:119 From 0a30f017fcd329fe857af4e9f8f88b2d341c3b3a Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Thu, 7 Jul 2016 15:12:18 -0500 Subject: [PATCH 0348/1117] Doctest fixes after rebase --- base/docs/helpdb/Base.jl | 2 +- base/test.jl | 6 +++--- doc/devdocs/types.rst | 2 ++ doc/manual/mathematical-operations.rst | 6 +++--- doc/manual/metaprogramming.rst | 9 ++++++--- doc/manual/methods.rst | 4 ++-- doc/manual/stacktraces.rst | 2 +- doc/manual/strings.rst | 8 ++++---- doc/manual/types.rst | 2 +- doc/stdlib/collections.rst | 2 +- doc/stdlib/strings.rst | 1 + doc/stdlib/test.rst | 6 +++--- 12 files changed, 28 insertions(+), 22 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index f473b7ba402e2..863fccd8dab56 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5450,7 +5450,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:571 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:511 ... ``` """ diff --git a/base/test.jl b/base/test.jl index 8eb6c1187ae1a..2681d48ee573a 100644 --- a/base/test.jl +++ b/base/test.jl @@ -895,10 +895,10 @@ Int64 julia> @code_warntype f(1,2,3) ... Body: - begin # none, line 1: - unless (Base.slt_int)(1,b::Int64)::Bool goto 4 + begin + unless (Base.slt_int)(1,b::Int64)::Bool goto 3 return 1 - 4: + 3: return 1.0 end::UNION{FLOAT64,INT64} diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index ed2041cbb8b24..7f9c089eafa25 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -329,6 +329,8 @@ the type, which is an object of type :obj:`TypeName`: max_args: Int64 0 kwsorter: #undef module: Module Core + : Int64 0 + : Int64 0 In this case, the relevant field is ``primary``, which holds a reference to the "primary" instance of the type:: diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index eda7ba0d29b8e..c08c8b115003d 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -400,7 +400,7 @@ The following examples show the different forms. julia> Int8(128) ERROR: InexactError() - in Int8(::Int64) at ./sysimg.jl:63 + in Int8(::Int64) at ./sysimg.jl:53 ... julia> Int8(127.0) @@ -408,12 +408,12 @@ The following examples show the different forms. julia> Int8(3.14) ERROR: InexactError() - in Int8(::Float64) at ./sysimg.jl:63 + in Int8(::Float64) at ./sysimg.jl:53 ... julia> Int8(128.0) ERROR: InexactError() - in Int8(::Float64) at ./sysimg.jl:63 + in Int8(::Float64) at ./sysimg.jl:53 ... julia> 127 % Int8 diff --git a/doc/manual/metaprogramming.rst b/doc/manual/metaprogramming.rst index 52e4551dea937..a2fca9211a7e2 100644 --- a/doc/manual/metaprogramming.rst +++ b/doc/manual/metaprogramming.rst @@ -629,11 +629,14 @@ Compare: Expr head: Symbol string args: Array{Any}((5,)) - 1: String "a (" + 1: String + data: Array{UInt8}((3,)) UInt8[0x61,0x20,0x28] 2: Symbol a - 3: String ") should equal b (" + 3: String + data: Array{UInt8}((18,)) UInt8[0x29,0x20,0x73,0x68,0x6f,0x75,0x6c,0x64,0x20,0x65,0x71,0x75,0x61,0x6c,0x20,0x62,0x20,0x28] 4: Symbol b - 5: String ")!" + 5: String + data: Array{UInt8}((2,)) UInt8[0x29,0x21] typ: Any So now instead of getting a plain string in ``msg_body``, the macro is diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index 1eca0463a7e5a..55a2870fdb008 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -242,11 +242,11 @@ Julia language. Core operations typically have dozens of methods: +(a::Float16, b::Float16) at float16.jl:136 +(x::Float32, y::Float32) at float.jl:206 +(x::Float64, y::Float64) at float.jl:207 - +(x::Bool, z::Complex{Bool}) at complex.jl:137 + +(x::Bool, z::Complex{Bool}) at complex.jl:126 +(x::Bool, y::Bool) at bool.jl:48 +(x::Bool) at bool.jl:45 +{T<:AbstractFloat}(x::Bool, y::T) at bool.jl:55 - +(x::Bool, z::Complex) at complex.jl:144 + +(x::Bool, z::Complex) at complex.jl:133 +(x::Bool, A::AbstractArray{Bool,N<:Any}) at arraymath.jl:105 +(x::Char, y::Integer) at char.jl:40 +{T<:Union{Int128,Int16,Int32,Int64,Int8,UInt128,UInt16,UInt32,UInt64,UInt8}}(x::T, y::T) at int.jl:32 diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index 62174ee078dbc..3886cb0f8b0da 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -264,7 +264,7 @@ by passing them into :func:`StackTraces.lookup`: julia> frame = StackTraces.lookup(pointer) 1-element Array{StackFrame,1}: - in jl_backtrace_from_here at stackwalk.c:104 + in jl_backtrace_from_here at stackwalk.c:105 julia> println("The top frame is from $(frame[1].func)!") The top frame is from jl_backtrace_from_here! diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 34508d1ee85fb..4adec34080742 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -288,15 +288,15 @@ such an invalid byte index, an error is thrown: julia> s[2] ERROR: UnicodeError: invalid character index - in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:68 - in next at ./strings/string.jl:93 [inlined] + in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:69 + in next at ./strings/string.jl:94 [inlined] in getindex(::String, ::Int64) at ./strings/basic.jl:70 ... julia> s[3] ERROR: UnicodeError: invalid character index - in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:68 - in next at ./strings/string.jl:93 [inlined] + in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:69 + in next at ./strings/string.jl:94 [inlined] in getindex(::String, ::Int64) at ./strings/basic.jl:70 ... diff --git a/doc/manual/types.rst b/doc/manual/types.rst index c59781a968307..a193c1a7c3809 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -767,7 +767,7 @@ each field: ERROR: MethodError: Cannot `convert` an object of type Float64 to an object of type Point{Float64} This may have arisen from a call to the constructor Point{Float64}(...), since type constructors fall back to convert methods. - in Point{Float64}(::Float64) at ./sysimg.jl:63 + in Point{Float64}(::Float64) at ./sysimg.jl:53 ... julia> Point{Float64}(1.0,2.0,3.0) diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 4be226fc9e9ef..54482d0e4136c 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1134,7 +1134,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:571 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:511 ... .. function:: splice!(collection, index, [replacement]) -> item diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index 40ec3e2d69347..e1ce2a06ed73d 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -472,3 +472,4 @@ .. Docstring generated from Julia source General unescaping of traditional C and Unicode escape sequences. Reverse of :func:`escape_string`\ . See also :func:`unescape_string`\ . + diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index fde0dea154d5d..4cbc2a0f9057c 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -265,10 +265,10 @@ writing new tests. julia> @code_warntype f(1,2,3) ... Body: - begin # none, line 1: - unless (Base.slt_int)(1,b::Int64)::Bool goto 4 + begin + unless (Base.slt_int)(1,b::Int64)::Bool goto 3 return 1 - 4: + 3: return 1.0 end::UNION{FLOAT64,INT64} From 2cdcdf1a117981ad9edd987b25866f469447d6b7 Mon Sep 17 00:00:00 2001 From: Oscar Blumberg <oscar.blumberg@ens.fr> Date: Thu, 2 Jun 2016 13:24:47 -0400 Subject: [PATCH 0349/1117] performance timing support code --- src/Makefile | 7 +-- src/alloc.c | 1 + src/array.c | 2 +- src/ast.c | 15 +++++- src/builtins.c | 5 +- src/codegen.cpp | 2 + src/dump.c | 6 ++- src/flisp/cvalues.c | 8 +-- src/flisp/flisp.c | 14 ++--- src/flisp/flisp.h | 36 ++++++------- src/flisp/iostream.c | 4 +- src/flisp/print.c | 6 +-- src/gc.c | 1 + src/gf.c | 4 ++ src/init.c | 17 ++++++- src/jitlayers.cpp | 2 + src/jlapi.c | 2 +- src/jltypes.c | 4 +- src/julia.h | 3 ++ src/julia_internal.h | 4 +- src/options.h | 3 ++ src/task.c | 28 +++++++++- src/timing.cpp | 62 ++++++++++++++++++++++ src/timing.h | 119 +++++++++++++++++++++++++++++++++++++++++++ 24 files changed, 306 insertions(+), 49 deletions(-) create mode 100644 src/timing.cpp create mode 100644 src/timing.h diff --git a/src/Makefile b/src/Makefile index ba6cf7298c465..47ba500a54814 100644 --- a/src/Makefile +++ b/src/Makefile @@ -37,9 +37,10 @@ endif SRCS := \ jltypes gf typemap ast builtins module interpreter \ - alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \ + alloc dlload sys init task array dump toplevel jl_uv \ simplevector APInt-C runtime_intrinsics runtime_ccall \ - threadgroup threading stackwalk gc gc-debug gc-pages safepoint jloptions + threadgroup threading stackwalk gc gc-debug gc-pages \ + jlapi signal-handling safepoint jloptions timing ifeq ($(USEMSVC), 1) SRCS += getopt @@ -61,7 +62,7 @@ SRCS += anticodegen LLVM_LIBS := support endif -HEADERS := $(addprefix $(SRCDIR)/,julia.h julia_threads.h julia_internal.h options.h) $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(LIBUV_INC)/uv.h +HEADERS := $(addprefix $(SRCDIR)/,julia.h julia_threads.h julia_internal.h options.h timing.h) $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(LIBUV_INC)/uv.h # In LLVM < 3.4, --ldflags includes both options and libraries, so use it both before and after --libs # In LLVM >= 3.4, --ldflags has only options, and --system-libs has the libraries. diff --git a/src/alloc.c b/src/alloc.c index cded29bb6e57f..e3d7225da1237 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -443,6 +443,7 @@ STATIC_INLINE jl_value_t *jl_call_staged(jl_svec_t *sparam_vals, jl_lambda_info_ static jl_lambda_info_t *jl_instantiate_staged(jl_method_t *generator, jl_tupletype_t *tt, jl_svec_t *env) { + JL_TIMING(STAGED_FUNCTION); size_t i, l; jl_expr_t *ex = NULL; jl_value_t *linenum = NULL; diff --git a/src/array.c b/src/array.c index fecf3b4e222d7..d403f450d773a 100644 --- a/src/array.c +++ b/src/array.c @@ -450,7 +450,7 @@ JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i) } static size_t array_nd_index(jl_array_t *a, jl_value_t **args, size_t nidxs, - char *fname) + const char *fname) { size_t i=0; size_t k, stride=1; diff --git a/src/ast.c b/src/ast.c index 40e3a55264f6f..4799810c41550 100644 --- a/src/ast.c +++ b/src/ast.c @@ -137,6 +137,7 @@ value_t fl_current_module_counter(fl_context_t *fl_ctx, value_t *args, uint32_t value_t fl_invoke_julia_macro(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) { + JL_TIMING(MACRO_INVOCATION); jl_ptls_t ptls = jl_get_ptls_states(); if (nargs < 1) argcount(fl_ctx, "invoke-julia-macro", nargs, 1); @@ -605,6 +606,7 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v) // this is used to parse a line of repl input JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const char *filename, size_t filename_len) { + JL_TIMING(PARSING); jl_ast_context_t *ctx = jl_ast_ctx_enter(); fl_context_t *fl_ctx = &ctx->fl; value_t s = cvalue_static_cstrn(fl_ctx, str, len); @@ -620,6 +622,7 @@ JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len, const JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len, int pos0, int greedy) { + JL_TIMING(PARSING); if (pos0 < 0 || pos0 > len) { jl_array_t *buf = jl_pchar_to_array(str, len); JL_GC_PUSH1(&buf); @@ -661,12 +664,14 @@ jl_value_t *jl_parse_eval_all(const char *fname, f = cvalue_static_cstrn(fl_ctx, fname, len); fl_gc_handle(fl_ctx, &f); if (content != NULL) { + JL_TIMING(PARSING); value_t t = cvalue_static_cstrn(fl_ctx, content, contentlen); fl_gc_handle(fl_ctx, &t); ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-string-stream")), t, f); fl_free_gc_handles(fl_ctx, 1); } else { + JL_TIMING(PARSING); assert(memchr(fname, 0, len) == NULL); // was checked already in jl_load ast = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-parse-file")), f); } @@ -691,7 +696,11 @@ jl_value_t *jl_parse_eval_all(const char *fname, assert(iscons(ast) && car_(ast) == symbol(fl_ctx,"toplevel")); ast = cdr_(ast); while (iscons(ast)) { - value_t expansion = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk")), car_(ast)); + value_t expansion; + { + JL_TIMING(LOWERING); + expansion = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk")), car_(ast)); + } form = scm_to_julia(fl_ctx, expansion, 0); jl_sym_t *head = NULL; if (jl_is_expr(form)) head = ((jl_expr_t*)form)->head; @@ -760,7 +769,7 @@ static int jl_parse_deperror(fl_context_t *fl_ctx, int err) } // returns either an expression or a thunk -jl_value_t *jl_call_scm_on_ast(char *funcname, jl_value_t *expr) +jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr) { jl_ast_context_t *ctx = jl_ast_ctx_enter(); fl_context_t *fl_ctx = &ctx->fl; @@ -775,11 +784,13 @@ jl_value_t *jl_call_scm_on_ast(char *funcname, jl_value_t *expr) JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr) { + JL_TIMING(LOWERING); return jl_call_scm_on_ast("jl-expand-to-thunk", expr); } JL_DLLEXPORT jl_value_t *jl_macroexpand(jl_value_t *expr) { + JL_TIMING(LOWERING); return jl_call_scm_on_ast("jl-macroexpand", expr); } diff --git a/src/builtins.c b/src/builtins.c index b9b4b30493929..4af852b119bb4 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -206,6 +206,9 @@ JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh) eh->defer_signal = ptls->defer_signal; eh->finalizers_inhibited = ptls->finalizers_inhibited; current_task->eh = eh; +#ifdef ENABLE_TIMINGS + eh->timing_stack = current_task->timing_stack; +#endif } JL_DLLEXPORT void jl_pop_handler(int n) @@ -1205,7 +1208,7 @@ void jl_init_primitives(void) // toys for debugging --------------------------------------------------------- -static size_t jl_show_svec(JL_STREAM *out, jl_svec_t *t, char *head, char *opn, char *cls) +static size_t jl_show_svec(JL_STREAM *out, jl_svec_t *t, const char *head, const char *opn, const char *cls) { size_t i, n=0, len = jl_svec_len(t); n += jl_printf(out, "%s", head); diff --git a/src/codegen.cpp b/src/codegen.cpp index 55013ecdb224b..98bb450a556e1 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -815,6 +815,7 @@ void jl_add_linfo_in_flight(StringRef name, jl_lambda_info_t *linfo, const DataL // objective: assign li->functionObject extern "C" void jl_compile_linfo(jl_lambda_info_t *li) { + JL_TIMING(CODEGEN); if (li->jlcall_api == 2) { // delete code for functions reduced to a constant jl_set_lambda_code_null(li); @@ -987,6 +988,7 @@ static void jl_finalize_module(std::unique_ptr<Module> uniquem, bool shadow) extern void jl_callback_triggered_linfos(void); static uint64_t getAddressForFunction(llvm::Function *llvmf) { + JL_TIMING(LLVM_EMIT); #ifdef JL_DEBUG_BUILD llvm::raw_fd_ostream out(1,false); #endif diff --git a/src/dump.c b/src/dump.c index 4780a126f3b0a..4ceb59fde78b8 100644 --- a/src/dump.c +++ b/src/dump.c @@ -876,7 +876,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) write_int32(s, li->nargs); jl_serialize_value(s, (jl_value_t*)li->def); jl_serialize_value(s, li->constval); - jl_serialize_fptr(s, li->fptr); + jl_serialize_fptr(s, (void*)(uintptr_t)li->fptr); // save functionObject pointers write_int32(s, jl_assign_functionID(li->functionObjectsDecls.functionObject)); write_int32(s, jl_assign_functionID(li->functionObjectsDecls.specFunctionObject)); @@ -1871,6 +1871,7 @@ static void jl_save_system_image_to_stream(ios_t *f) { jl_gc_collect(1); // full jl_gc_collect(0); // incremental (sweep finalizers) + JL_TIMING(SYSIMG_DUMP); JL_LOCK(&dump_lock); // Might GC int en = jl_gc_enable(0); htable_reset(&backref_table, 250000); @@ -1973,6 +1974,7 @@ JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname) static void jl_restore_system_image_from_stream(ios_t *f) { + JL_TIMING(SYSIMG_LOAD); jl_ptls_t ptls = jl_get_ptls_states(); JL_LOCK(&dump_lock); // Might GC int en = jl_gc_enable(0); @@ -2071,6 +2073,7 @@ JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) JL_DLLEXPORT jl_array_t *jl_compress_ast(jl_lambda_info_t *li, jl_array_t *ast) { + JL_TIMING(AST_COMPRESS); JL_LOCK(&li->def->writelock); // protect the roots array (Might GC) JL_LOCK(&dump_lock); // protect global structures in this file (Might GC) assert(jl_is_lambda_info(li)); @@ -2110,6 +2113,7 @@ JL_DLLEXPORT jl_array_t *jl_compress_ast(jl_lambda_info_t *li, jl_array_t *ast) JL_DLLEXPORT jl_array_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_array_t *data) { + JL_TIMING(AST_UNCOMPRESS); JL_LOCK(&li->def->writelock); // protect the roots array (Might GC) JL_LOCK(&dump_lock); // Might GC assert(jl_is_lambda_info(li)); diff --git a/src/flisp/cvalues.c b/src/flisp/cvalues.c index bbe2a6a36d150..9116835dfc721 100644 --- a/src/flisp/cvalues.c +++ b/src/flisp/cvalues.c @@ -307,7 +307,7 @@ value_t size_wrap(fl_context_t *fl_ctx, size_t sz) return mk_size(fl_ctx, sz); } -size_t tosize(fl_context_t *fl_ctx, value_t n, char *fname) +size_t tosize(fl_context_t *fl_ctx, value_t n, const char *fname) { if (isfixnum(n)) return numval(n); @@ -476,7 +476,7 @@ size_t ctype_sizeof(fl_context_t *fl_ctx, value_t type, int *palign) } // get pointer and size for any plain-old-data value -void to_sized_ptr(fl_context_t *fl_ctx, value_t v, char *fname, char **pdata, size_t *psz) +void to_sized_ptr(fl_context_t *fl_ctx, value_t v, const char *fname, char **pdata, size_t *psz) { if (iscvalue(v)) { cvalue_t *pcv = (cvalue_t*)ptr(v); @@ -708,7 +708,7 @@ value_t cvalue_compare(value_t a, value_t b) return fixnum(diff); } -static void check_addr_args(fl_context_t *fl_ctx, char *fname, value_t arr, +static void check_addr_args(fl_context_t *fl_ctx, const char *fname, value_t arr, value_t ind, char **data, size_t *index) { size_t numel; @@ -774,7 +774,7 @@ value_t fl_builtin(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) return tagptr(cv, TAG_CVALUE); } -value_t cbuiltin(fl_context_t *fl_ctx, char *name, builtin_t f) +value_t cbuiltin(fl_context_t *fl_ctx, const char *name, builtin_t f) { cvalue_t *cv = (cvalue_t*)malloc(CVALUE_NWORDS * sizeof(value_t)); cv->type = fl_ctx->builtintype; diff --git a/src/flisp/flisp.c b/src/flisp/flisp.c index 57c88b51166e4..4a75c3f6e3804 100644 --- a/src/flisp/flisp.c +++ b/src/flisp/flisp.c @@ -59,7 +59,7 @@ JL_DLLEXPORT char * dirname(char *); #include <libgen.h> #endif -static char *const builtin_names[] = +static const char *const builtin_names[] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // predicates @@ -159,14 +159,14 @@ void fl_raise(fl_context_t *fl_ctx, value_t e) longjmp(thisctx->buf, 1); } -static value_t make_error_msg(fl_context_t *fl_ctx, char *format, va_list args) +static value_t make_error_msg(fl_context_t *fl_ctx, const char *format, va_list args) { char msgbuf[512]; size_t len = vsnprintf(msgbuf, sizeof(msgbuf), format, args); return string_from_cstrn(fl_ctx, msgbuf, len); } -void lerrorf(fl_context_t *fl_ctx, value_t e, char *format, ...) +void lerrorf(fl_context_t *fl_ctx, value_t e, const char *format, ...) { va_list args; PUSH(fl_ctx, e); @@ -186,12 +186,12 @@ void lerror(fl_context_t *fl_ctx, value_t e, const char *msg) fl_raise(fl_ctx, fl_list2(fl_ctx, e, m)); } -void type_error(fl_context_t *fl_ctx, char *fname, char *expected, value_t got) +void type_error(fl_context_t *fl_ctx, const char *fname, const char *expected, value_t got) { fl_raise(fl_ctx, fl_listn(fl_ctx, 4, fl_ctx->TypeError, symbol(fl_ctx, fname), symbol(fl_ctx, expected), got)); } -void bounds_error(fl_context_t *fl_ctx, char *fname, value_t arr, value_t ind) +void bounds_error(fl_context_t *fl_ctx, const char *fname, value_t arr, value_t ind) { fl_raise(fl_ctx, fl_listn(fl_ctx, 4, fl_ctx->BoundsError, symbol(fl_ctx, fname), arr, ind)); } @@ -200,7 +200,7 @@ void bounds_error(fl_context_t *fl_ctx, char *fname, value_t arr, value_t ind) #define isstring(v) fl_isstring(fl_ctx, v) #define SAFECAST_OP(type,ctype,cnvt) \ - ctype to##type(fl_context_t *fl_ctx, value_t v, char *fname) \ + ctype to##type(fl_context_t *fl_ctx, value_t v, const char *fname) \ { \ if (is##type(v)) \ return (ctype)cnvt(v); \ @@ -261,7 +261,7 @@ static symbol_t **symtab_lookup(symbol_t **ptree, const char *str) return ptree; } -value_t symbol(fl_context_t *fl_ctx, char *str) +value_t symbol(fl_context_t *fl_ctx, const char *str) { symbol_t **pnode = symtab_lookup(&fl_ctx->symtab, str); if (*pnode == NULL) diff --git a/src/flisp/flisp.h b/src/flisp/flisp.h index d9ea2e5b8a5d1..fea46e570ea5e 100644 --- a/src/flisp/flisp.h +++ b/src/flisp/flisp.h @@ -149,7 +149,7 @@ value_t fl_applyn(fl_context_t *fl_ctx, uint32_t n, value_t f, ...); value_t fl_cons(fl_context_t *fl_ctx, value_t a, value_t b); value_t fl_list2(fl_context_t *fl_ctx, value_t a, value_t b); value_t fl_listn(fl_context_t *fl_ctx, size_t n, ...); -value_t symbol(fl_context_t *fl_ctx, char *str); +value_t symbol(fl_context_t *fl_ctx, const char *str); char *symbol_name(fl_context_t *fl_ctx, value_t v); int fl_is_keyword_name(const char *str, size_t len); value_t alloc_vector(fl_context_t *fl_ctx, size_t n, int init); @@ -161,10 +161,10 @@ uintptr_t hash_lispvalue(fl_context_t *fl_ctx, value_t a); int isnumtok_base(fl_context_t *fl_ctx, char *tok, value_t *pval, int base); /* safe casts */ -cons_t *tocons(fl_context_t *fl_ctx, value_t v, char *fname); -symbol_t *tosymbol(fl_context_t *fl_ctx, value_t v, char *fname); -fixnum_t tofixnum(fl_context_t *fl_ctx, value_t v, char *fname); -char *tostring(fl_context_t *fl_ctx, value_t v, char *fname); +cons_t *tocons(fl_context_t *fl_ctx, value_t v, const char *fname); +symbol_t *tosymbol(fl_context_t *fl_ctx, value_t v, const char *fname); +fixnum_t tofixnum(fl_context_t *fl_ctx, value_t v, const char *fname); +char *tostring(fl_context_t *fl_ctx, value_t v, const char *fname); /* error handling */ typedef struct _ectx_t { @@ -187,17 +187,17 @@ typedef struct _ectx_t { for(l__ca=1; l__ca; l__ca=0, fl_restorestate(fl_ctx, &_ctx)) #if defined(_OS_WINDOWS_) -__declspec(noreturn) void lerrorf(fl_context_t *fl_ctx, value_t e, char *format, ...); +__declspec(noreturn) void lerrorf(fl_context_t *fl_ctx, value_t e, const char *format, ...); __declspec(noreturn) void lerror(fl_context_t *fl_ctx, value_t e, const char *msg); __declspec(noreturn) void fl_raise(fl_context_t *fl_ctx, value_t e); -__declspec(noreturn) void type_error(fl_context_t *fl_ctx, char *fname, char *expected, value_t got); -__declspec(noreturn) void bounds_error(fl_context_t *fl_ctx, char *fname, value_t arr, value_t ind); +__declspec(noreturn) void type_error(fl_context_t *fl_ctx, const char *fname, const char *expected, value_t got); +__declspec(noreturn) void bounds_error(fl_context_t *fl_ctx, const char *fname, value_t arr, value_t ind); #else -void lerrorf(fl_context_t *fl_ctx, value_t e, char *format, ...) __attribute__ ((__noreturn__)); +void lerrorf(fl_context_t *fl_ctx, value_t e, const char *format, ...) __attribute__ ((__noreturn__)); void lerror(fl_context_t *fl_ctx, value_t e, const char *msg) __attribute__ ((__noreturn__)); void fl_raise(fl_context_t *fl_ctx, value_t e) __attribute__ ((__noreturn__)); -void type_error(fl_context_t *fl_ctx, char *fname, char *expected, value_t got) __attribute__ ((__noreturn__)); -void bounds_error(fl_context_t *fl_ctx, char *fname, value_t arr, value_t ind) __attribute__ ((__noreturn__)); +void type_error(fl_context_t *fl_ctx, const char *fname, const char *expected, value_t got) __attribute__ ((__noreturn__)); +void bounds_error(fl_context_t *fl_ctx, const char *fname, value_t arr, value_t ind) __attribute__ ((__noreturn__)); #endif void fl_savestate(fl_context_t *fl_ctx, fl_exception_context_t *_ctx); @@ -214,7 +214,7 @@ typedef struct { value_t relocate_lispvalue(fl_context_t *fl_ctx, value_t v); void print_traverse(fl_context_t *fl_ctx, value_t v); void fl_print_chr(fl_context_t *fl_ctx, char c, ios_t *f); -void fl_print_str(fl_context_t *fl_ctx, char *s, ios_t *f); +void fl_print_str(fl_context_t *fl_ctx, const char *s, ios_t *f); void fl_print_child(fl_context_t *fl_ctx, ios_t *f, value_t v); typedef int (*cvinitfunc_t)(fl_context_t *fl_ctx, struct _fltype_t*, value_t, void*); @@ -310,10 +310,10 @@ size_t ctype_sizeof(fl_context_t *fl_ctx, value_t type, int *palign); value_t cvalue_copy(fl_context_t *fl_ctx, value_t v); value_t cvalue_from_data(fl_context_t *fl_ctx, fltype_t *type, void *data, size_t sz); value_t cvalue_from_ref(fl_context_t *fl_ctx, fltype_t *type, void *ptr, size_t sz, value_t parent); -value_t cbuiltin(fl_context_t *fl_ctx, char *name, builtin_t f); +value_t cbuiltin(fl_context_t *fl_ctx, const char *name, builtin_t f); size_t cvalue_arraylen(value_t v); value_t size_wrap(fl_context_t *fl_ctx, size_t sz); -size_t tosize(fl_context_t *fl_ctx, value_t n, char *fname); +size_t tosize(fl_context_t *fl_ctx, value_t n, const char *fname); value_t cvalue_string(fl_context_t *fl_ctx, size_t sz); value_t cvalue_static_cstrn(fl_context_t *fl_ctx, const char *str, size_t n); value_t cvalue_static_cstring(fl_context_t *fl_ctx, const char *str); @@ -323,11 +323,11 @@ int fl_isstring(fl_context_t *fl_ctx, value_t v); int fl_isnumber(fl_context_t *fl_ctx, value_t v); int fl_isgensym(fl_context_t *fl_ctx, value_t v); int fl_isiostream(fl_context_t *fl_ctx, value_t v); -ios_t *fl_toiostream(fl_context_t *fl_ctx, value_t v, char *fname); +ios_t *fl_toiostream(fl_context_t *fl_ctx, value_t v, const char *fname); value_t cvalue_compare(value_t a, value_t b); int numeric_compare(fl_context_t *fl_ctx, value_t a, value_t b, int eq, int eqnans, char *fname); -void to_sized_ptr(fl_context_t *fl_ctx, value_t v, char *fname, char **pdata, size_t *psz); +void to_sized_ptr(fl_context_t *fl_ctx, value_t v, const char *fname, char **pdata, size_t *psz); fltype_t *get_type(fl_context_t *fl_ctx, value_t t); fltype_t *get_array_type(fl_context_t *fl_ctx, value_t eltype); @@ -343,7 +343,7 @@ value_t return_from_uint64(fl_context_t *fl_ctx, uint64_t Uaccum); value_t return_from_int64(fl_context_t *fl_ctx, int64_t Saccum); typedef struct { - char *name; + const char *name; builtin_t fptr; } builtinspec_t; @@ -482,7 +482,7 @@ struct _fl_context_t { void *jlbuf; }; -static inline void argcount(fl_context_t *fl_ctx, char *fname, uint32_t nargs, uint32_t c) +static inline void argcount(fl_context_t *fl_ctx, const char *fname, uint32_t nargs, uint32_t c) { if (__unlikely(nargs != c)) lerrorf(fl_ctx, fl_ctx->ArgError,"%s: too %s arguments", fname, nargs<c ? "few":"many"); diff --git a/src/flisp/iostream.c b/src/flisp/iostream.c index 111f9c4c4019b..015c150aec9fe 100644 --- a/src/flisp/iostream.c +++ b/src/flisp/iostream.c @@ -61,14 +61,14 @@ value_t fl_eof_objectp(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) return (fl_ctx->FL_EOF == args[0]) ? fl_ctx->T : fl_ctx->F; } -static ios_t *toiostream(fl_context_t *fl_ctx, value_t v, char *fname) +static ios_t *toiostream(fl_context_t *fl_ctx, value_t v, const char *fname) { if (!fl_isiostream(fl_ctx, v)) type_error(fl_ctx, fname, "iostream", v); return value2c(ios_t*, v); } -ios_t *fl_toiostream(fl_context_t *fl_ctx, value_t v, char *fname) +ios_t *fl_toiostream(fl_context_t *fl_ctx, value_t v, const char *fname) { return toiostream(fl_ctx, v, fname); } diff --git a/src/flisp/print.c b/src/flisp/print.c index 0d498fea117f9..d12cbf0f37338 100644 --- a/src/flisp/print.c +++ b/src/flisp/print.c @@ -8,12 +8,12 @@ static void outc(fl_context_t *fl_ctx, char c, ios_t *f) else fl_ctx->HPOS++; } -static void outs(fl_context_t *fl_ctx, char *s, ios_t *f) +static void outs(fl_context_t *fl_ctx, const char *s, ios_t *f) { ios_puts(s, f); fl_ctx->HPOS += u8_strwidth(s); } -static void outsn(fl_context_t *fl_ctx, char *s, ios_t *f, size_t n) +static void outsn(fl_context_t *fl_ctx, const char *s, ios_t *f, size_t n) { ios_write(f, s, n); fl_ctx->HPOS += u8_strwidth(s); @@ -43,7 +43,7 @@ void fl_print_chr(fl_context_t *fl_ctx, char c, ios_t *f) outc(fl_ctx, c, f); } -void fl_print_str(fl_context_t *fl_ctx, char *s, ios_t *f) +void fl_print_str(fl_context_t *fl_ctx, const char *s, ios_t *f) { outs(fl_ctx, s, f); } diff --git a/src/gc.c b/src/gc.c index 953f79a08c479..ffe93a799099c 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1623,6 +1623,7 @@ void jl_gc_sync_total_bytes(void) {last_gc_total_bytes = jl_gc_total_bytes();} // Only one thread should be running in this function static void _jl_gc_collect(jl_ptls_t ptls, int full, char *stack_hi) { + JL_TIMING(GC); uint64_t t0 = jl_hrtime(); int64_t last_perm_scanned_bytes = perm_scanned_bytes; assert(mark_sp == 0); diff --git a/src/gf.c b/src/gf.c index 6307c92fbebf3..8302c9dd16865 100644 --- a/src/gf.c +++ b/src/gf.c @@ -181,6 +181,7 @@ jl_value_t *jl_mk_builtin_func(const char *name, jl_fptr_t fptr) */ jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force) { + JL_TIMING(INFERENCE); #ifdef ENABLE_INFERENCE jl_module_t *mod = NULL; if (li->def != NULL) @@ -1309,6 +1310,7 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) // compile-time method lookup jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types) { + JL_TIMING(METHOD_LOOKUP_COMPILE); assert(jl_nparams(types) > 0); if (!jl_is_leaf_type((jl_value_t*)types) || jl_has_typevars((jl_value_t*)types)) return NULL; @@ -1978,6 +1980,7 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) } // if no method was found in the associative cache, check the full cache if (i == 4) { + JL_TIMING(METHOD_LOOKUP_FAST); jl_value_t *F = args[0]; mt = jl_gf_mtable(F); entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt)); @@ -2000,6 +2003,7 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) } else { // cache miss case + JL_TIMING(METHOD_LOOKUP_SLOW); jl_tupletype_t *tt = arg_type_tuple(args, nargs); JL_GC_PUSH1(&tt); mfunc = jl_mt_assoc_by_type(mt, tt, 1, 0); diff --git a/src/init.c b/src/init.c index 27e5d8f8c4cea..2ddaac18c79c3 100644 --- a/src/init.c +++ b/src/init.c @@ -215,6 +215,9 @@ static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdow return rv; } +void jl_init_timing(void); +void jl_destroy_timing(void); + JL_DLLEXPORT void jl_atexit_hook(int exitcode) { jl_ptls_t ptls = jl_get_ptls_states(); @@ -308,6 +311,11 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) // force libuv to spin until everything has finished closing loop->stop_flag = 0; while (uv_run(loop,UV_RUN_DEFAULT)) {} + + jl_destroy_timing(); +#ifdef ENABLE_TIMINGS + jl_print_timings(); +#endif } void jl_get_builtin_hooks(void); @@ -534,6 +542,7 @@ static void jl_set_io_wait(int v) void _julia_init(JL_IMAGE_SEARCH rel) { + jl_init_timing(); #ifdef JULIA_ENABLE_THREADING // Make sure we finalize the tls callback before starting any threads. jl_get_ptls_states_getter(); @@ -621,6 +630,10 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_init_tasks(); jl_init_root_task(ptls->stack_lo, ptls->stack_hi-ptls->stack_lo); +#ifdef ENABLE_TIMINGS + jl_root_task->timing_stack = jl_root_timing; +#endif + init_stdio(); // libuv stdio cleanup depends on jl_init_tasks() because JL_TRY is used in jl_atexit_hook() @@ -787,12 +800,12 @@ static void julia_save(void) JL_GC_POP(); } -static jl_value_t *core(char *name) +static jl_value_t *core(const char *name) { return jl_get_global(jl_core_module, jl_symbol(name)); } -static jl_value_t *basemod(char *name) +static jl_value_t *basemod(const char *name) { return jl_get_global(jl_base_module, jl_symbol(name)); } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index c0eeb33aabd72..eeb9ab93697e6 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -327,6 +327,7 @@ class JuliaOJIT { CompileLayer( ObjectLayer, [this](Module &M) { + JL_TIMING(LLVM_OPT); PM.run(M); std::unique_ptr<MemoryBuffer> ObjBuffer( new ObjectMemoryBuffer(std::move(ObjBufferSV))); @@ -411,6 +412,7 @@ class JuliaOJIT { } } #endif + JL_TIMING(LLVM_MODULE_FINISH); // We need a memory manager to allocate memory and resolve symbols for this // new module. Create one that resolves symbols by looking back into the JIT. auto Resolver = orc::createLambdaResolver( diff --git a/src/jlapi.c b/src/jlapi.c index 0cc3d8fed4ea1..f4fbd12e5461a 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -52,7 +52,7 @@ JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str) { jl_value_t *r; JL_TRY { - char *filename = "none"; + const char *filename = "none"; jl_value_t *ast = jl_parse_input_line(str, strlen(str), filename, strlen(filename)); JL_GC_PUSH1(&ast); diff --git a/src/jltypes.c b/src/jltypes.c index 5dfeeff67ef69..93da20a9c29af 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1763,7 +1763,7 @@ jl_value_t *jl_apply_type_(jl_value_t *tc, jl_value_t **params, size_t n) return tc; } size_t i; - char *tname; + const char *tname; jl_svec_t *tp; jl_datatype_t *stprimary = NULL; if (jl_is_typector(tc)) { @@ -1973,6 +1973,7 @@ static ssize_t lookup_type_idx(jl_typename_t *tn, jl_value_t **key, size_t n, in static jl_value_t *lookup_type(jl_typename_t *tn, jl_value_t **key, size_t n) { + JL_TIMING(TYPE_CACHE_LOOKUP); int ord = is_typekey_ordered(key, n); JL_LOCK(&typecache_lock); // Might GC ssize_t idx = lookup_type_idx(tn, key, n, ord); @@ -2046,6 +2047,7 @@ static void cache_insert_type(jl_value_t *type, ssize_t insert_at, int ordered) jl_value_t *jl_cache_type_(jl_datatype_t *type) { if (is_cacheable(type)) { + JL_TIMING(TYPE_CACHE_INSERT); int ord = is_typekey_ordered(jl_svec_data(type->parameters), jl_svec_len(type->parameters)); JL_LOCK(&typecache_lock); // Might GC ssize_t idx = lookup_type_idx(type->name, jl_svec_data(type->parameters), diff --git a/src/julia.h b/src/julia.h index 9c23a3eaf3ca3..7bcce4bd07cb4 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1413,6 +1413,7 @@ JL_DLLEXPORT void jl_sigatomic_end(void); // tasks and exceptions ------------------------------------------------------- +typedef struct _jl_timing_block_t jl_timing_block_t; // info describing an exception handler typedef struct _jl_handler_t { jl_jmp_buf eh_ctx; @@ -1424,6 +1425,7 @@ typedef struct _jl_handler_t { #endif sig_atomic_t defer_signal; int finalizers_inhibited; + jl_timing_block_t *timing_stack; } jl_handler_t; typedef struct _jl_task_t { @@ -1458,6 +1460,7 @@ typedef struct _jl_task_t { // This is statically initialized when the task is not holding any locks arraylist_t locks; #endif + jl_timing_block_t *timing_stack; } jl_task_t; JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize); diff --git a/src/julia_internal.h b/src/julia_internal.h index 22e40bf16f464..4c077739659dc 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -16,6 +16,8 @@ extern "C" { #endif +#include "timing.h" + #define GC_CLEAN 0 // freshly allocated #define GC_MARKED 1 // reachable and young #define GC_OLD 2 // if it is reachable it will be marked as old @@ -294,7 +296,7 @@ jl_value_t *jl_interpret_toplevel_expr(jl_value_t *e); jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, jl_lambda_info_t *li, int sparams, int allow_alloc); int jl_is_toplevel_only_expr(jl_value_t *e); -jl_value_t *jl_call_scm_on_ast(char *funcname, jl_value_t *expr); +jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr); jl_lambda_info_t *jl_method_lookup_by_type(jl_methtable_t *mt, jl_tupletype_t *types, int cache, int inexact); diff --git a/src/options.h b/src/options.h index ac641af99351b..a6445b9f1f7e8 100644 --- a/src/options.h +++ b/src/options.h @@ -123,4 +123,7 @@ # endif #endif +// Automatic Instrumenting Profiler +//#define ENABLE_TIMINGS + #endif diff --git a/src/task.c b/src/task.c index 607bc0a879d63..b1cf229bd1248 100644 --- a/src/task.c +++ b/src/task.c @@ -249,6 +249,7 @@ static void NOINLINE JL_NORETURN start_task(void) ptls->defer_signal = 0; jl_sigint_safepoint(ptls); } + JL_TIMING(ROOT); res = jl_apply(&t->start, 1); } JL_CATCH { @@ -290,6 +291,11 @@ static void ctx_switch(jl_ptls_t ptls, jl_task_t *t, jl_jmp_buf *where) { if (t == ptls->current_task) return; +#ifdef ENABLE_TIMINGS + jl_timing_block_t *blk = ptls->current_task->timing_stack; + if (blk) + jl_timing_block_stop(blk); +#endif if (!jl_setjmp(ptls->current_task->ctx, 0)) { // backtraces don't survive task switches, see e.g. issue #12485 ptls->bt_size = 0; @@ -366,6 +372,11 @@ static void ctx_switch(jl_ptls_t ptls, jl_task_t *t, jl_jmp_buf *where) jl_longjmp(*where, 1); #endif } +#ifdef ENABLE_TIMINGS + assert(blk == jl_current_task->timing_stack); + if (blk) + jl_timing_block_start(blk); +#endif } JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) @@ -497,6 +508,8 @@ static void init_task(jl_task_t *t, char *stack) #endif /* !COPY_STACKS */ +jl_timing_block_t *jl_pop_timing_block(jl_timing_block_t *cur_block); + // yield to exception handler void JL_NORETURN throw_internal(jl_value_t *e) { @@ -507,8 +520,16 @@ void JL_NORETURN throw_internal(jl_value_t *e) jl_gc_unsafe_enter(ptls); assert(e != NULL); ptls->exception_in_transit = e; - if (ptls->current_task->eh != NULL) { - jl_longjmp(ptls->current_task->eh->eh_ctx, 1); + jl_handler_t *eh = ptls->current_task->eh; + if (eh != NULL) { +#ifdef ENABLE_TIMINGS + jl_timing_block_t *cur_block = ptls->current_task->timing_stack; + while (cur_block && eh->timing_stack != cur_block) { + cur_block = jl_pop_timing_block(cur_block); + } + assert(cur_block == eh->timing_stack); +#endif + jl_longjmp(eh->eh_ctx, 1); } else { jl_printf(JL_STDERR, "fatal: error thrown and no exception handler available.\n"); @@ -569,6 +590,9 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) t->stkbuf = NULL; t->tid = 0; t->started = 0; +#ifdef ENABLE_TIMINGS + t->timing_stack = NULL; +#endif #ifdef COPY_STACKS t->bufsz = 0; diff --git a/src/timing.cpp b/src/timing.cpp new file mode 100644 index 0000000000000..8a4db8d7d3170 --- /dev/null +++ b/src/timing.cpp @@ -0,0 +1,62 @@ +#include <inttypes.h> +#include "julia.h" +#include "options.h" + +#ifdef ENABLE_TIMINGS +#include "timing.h" + +jl_timing_block_t *jl_root_timing; +uint64_t jl_timing_data[(int)JL_TIMING_LAST] = {0}; +const char *jl_timing_names[(int)JL_TIMING_LAST] = + { +#define X(name) #name + JL_TIMING_OWNERS +#undef X + }; + +extern "C" void jl_print_timings(void) +{ + uint64_t total_time = 0; + for (int i = 0; i < JL_TIMING_LAST; i++) { + total_time += jl_timing_data[i]; + } + for (int i = 0; i < JL_TIMING_LAST; i++) { + if (jl_timing_data[i] != 0) + printf("%-25s : %.2f %% %" PRIu64 "\n", jl_timing_names[i], + 100 * (((double)jl_timing_data[i]) / total_time), jl_timing_data[i]); + } +} + +extern "C" void jl_init_timing(void) +{ + jl_root_timing = new jl_timing_block_t(); +} + +extern "C" void jl_destroy_timing(void) +{ + if (jl_root_task) + delete jl_root_task->timing_stack; +} + +extern "C" jl_timing_block_t *jl_pop_timing_block(jl_timing_block_t *cur_block) +{ + cur_block->~jl_timing_block_t(); + return cur_block->prev; +} + +extern "C" void jl_timing_block_start(jl_timing_block_t *cur_block) +{ + cur_block->start(rdtscp()); +} + +extern "C" void jl_timing_block_stop(jl_timing_block_t *cur_block) +{ + cur_block->stop(rdtscp()); +} + +#else + +extern "C" void jl_init_timing(void) { } +extern "C" void jl_destroy_timing(void) { } + +#endif diff --git a/src/timing.h b/src/timing.h new file mode 100644 index 0000000000000..70a885018cdac --- /dev/null +++ b/src/timing.h @@ -0,0 +1,119 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + +#ifndef ENABLE_TIMINGS +#define JL_TIMING(owner) +#else + +#ifdef __cplusplus +extern "C" { +#endif +void jl_print_timings(void); +extern jl_timing_block_t *jl_root_timing; +void jl_timing_block_start(jl_timing_block_t *cur_block); +void jl_timing_block_stop(jl_timing_block_t *cur_block); +#ifdef __cplusplus +} +#endif + +#ifndef __cplusplus +#define JL_TIMING(owner) +#else + +static inline uint64_t rdtscp(void) +{ + uint64_t rax,rdx; + asm volatile ( "rdtscp\n" : "=a" (rax), "=d" (rdx) : : "rcx" ); + return (rdx << 32) + rax; +} + +#define JL_TIMING_OWNERS \ + X(ROOT), \ + X(GC), \ + X(LOWERING), \ + X(PARSING), \ + X(INFERENCE), \ + X(CODEGEN), \ + X(METHOD_LOOKUP_SLOW), \ + X(METHOD_LOOKUP_FAST), \ + X(LLVM_OPT), \ + X(LLVM_MODULE_FINISH), \ + X(LLVM_EMIT), \ + X(METHOD_LOOKUP_COMPILE), \ + X(TYPE_CACHE_LOOKUP), \ + X(TYPE_CACHE_INSERT), \ + X(STAGED_FUNCTION), \ + X(MACRO_INVOCATION), \ + X(AST_COMPRESS), \ + X(AST_UNCOMPRESS), \ + X(SYSIMG_LOAD), \ + X(SYSIMG_DUMP), + +enum jl_timing_owners { +#define X(name) JL_TIMING_ ## name + JL_TIMING_OWNERS +#undef X + JL_TIMING_LAST +}; + +extern uint64_t jl_timing_data[(int)JL_TIMING_LAST]; +extern const char *jl_timing_names[(int)JL_TIMING_LAST]; +#define JL_TIMING(owner) jl_timing_block_t __timing_block(JL_TIMING_ ## owner) + +struct _jl_timing_block_t { + jl_timing_block_t *prev; + uint64_t total; + uint64_t t0; + int owner; +#ifdef JL_DEBUG_BUILD + bool running; +#endif + _jl_timing_block_t(int owner) { + uint64_t t = init(owner); + jl_timing_block_t **prevp = jl_current_task ? &jl_current_task->timing_stack : &jl_root_timing; + prev = *prevp; + if (prev) + prev->stop(t); + *prevp = this; + } + _jl_timing_block_t() { + init(JL_TIMING_ROOT); + prev = NULL; + } + uint64_t init(int owner) { + uint64_t t = rdtscp(); + this->owner = owner; + total = 0; +#ifdef JL_DEBUG_BUILD + running = false; +#endif + start(t); + return t; + } + inline void stop(uint64_t t) { +#ifdef JL_DEBUG_BUILD + assert(running); + running = false; +#endif + total += t - t0; + } + inline void start(uint64_t t) { +#ifdef JL_DEBUG_BUILD + assert(!running); + running = true; +#endif + t0 = t; + } + ~_jl_timing_block_t() { + uint64_t t = rdtscp(); + stop(t); + jl_timing_data[owner] += total; + jl_timing_block_t **pcur = jl_current_task ? &jl_current_task->timing_stack : &jl_root_timing; + assert(*pcur == this); + *pcur = prev; + if (prev) + prev->start(t); + } +}; + +#endif +#endif From 2adf7ca48b0b24e6e20307454411347fd7420491 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 11 Jul 2016 13:29:47 -0400 Subject: [PATCH 0350/1117] use -O0 mode for compilation it's rarely useful to run in -O2 mode when compiling code, since that work is usually thrown away quickly, but potentially greatly slowing down the precompilation process --- base/loading.jl | 2 +- base/pkg/entry.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 2803460cadb4d..bf96a9d351436 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -455,7 +455,7 @@ function create_expr_cache(input::AbstractString, output::AbstractString) eval(Main, deserialize(STDIN)) end """ - io, pobj = open(pipeline(detach(`$(julia_cmd()) + io, pobj = open(pipeline(detach(`$(julia_cmd()) -O0 --output-ji $output --output-incremental=yes --startup-file=no --history-file=no --color=$(have_color ? "yes" : "no") diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index bb45ece3adda2..2adc1758a2693 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -591,7 +591,7 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set()) end end """ - io, pobj = open(pipeline(detach(`$(Base.julia_cmd()) + io, pobj = open(pipeline(detach(`$(Base.julia_cmd()) -O0 --compilecache=$(Bool(Base.JLOptions().use_compilecache) ? "yes" : "no") --history-file=no --color=$(Base.have_color ? "yes" : "no") From 2bf175d94b3d525c2c813523db74a7f991d11a30 Mon Sep 17 00:00:00 2001 From: Jacob Quinn <quinn.jacobd@gmail.com> Date: Mon, 11 Jul 2016 11:37:26 -0600 Subject: [PATCH 0351/1117] Remove duplicate line --- base/strings/string.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/strings/string.jl b/base/strings/string.jl index 6395c03eac5c7..9ff67987338e7 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -33,8 +33,6 @@ const utf8_trailing = [ ## required core functionality ## -is_valid_continuation(c) = ((c & 0xc0) == 0x80) - function endof(s::String) d = s.data i = length(d) From 73ee8788816f49e40084e41682446a4e7d1c6b1e Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 5 Jul 2016 14:59:51 -0400 Subject: [PATCH 0352/1117] add devdocs on julia internal locks [ci skip] --- doc/devdocs/julia.rst | 1 + doc/devdocs/locks.rst | 128 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 doc/devdocs/locks.rst diff --git a/doc/devdocs/julia.rst b/doc/devdocs/julia.rst index ce66286c99d88..32c1c212caebb 100644 --- a/doc/devdocs/julia.rst +++ b/doc/devdocs/julia.rst @@ -24,3 +24,4 @@ stdio promote-op boundscheck + locks diff --git a/doc/devdocs/locks.rst b/doc/devdocs/locks.rst new file mode 100644 index 0000000000000..653ef43115941 --- /dev/null +++ b/doc/devdocs/locks.rst @@ -0,0 +1,128 @@ +.. currentmodule:: Base + +**************************************************** +Proper maintenance and care of multi-threading locks +**************************************************** + +The following strategies are used to ensure that the code is dead-lock free +(generally by addressing the 4th Coffman condition: circular wait). + + 1. structure code such that only one lock will need to be acquired at a time + + 2. always acquire shared locks in the same order, as given by the table below + + 3. avoid constructs that expect to need unrestricted recursion + +Below are all of the locks that exist in the system and the mechanisms +for using them that avoid the potential for deadlocks (no Ostrich algorithm allowed here): + +The following are definitely leaf locks (level 1), and must not try to acquire any other lock: + + * safepoint + + Note that this lock is acquired implicitly by ``JL_LOCK`` and ``JL_UNLOCK``. + use the ``_NOGC`` variants to avoid that for level 1 locks. + + While holding this lock, the code must not do any allocation or hit any safepoints. + Note that there are safepoints when doing allocation, enabling / disabling GC, + entering / restoring exception frames, and taking / releasing locks. + + * shared_map + * finalizers + * pagealloc + * gc_perm_lock + * flisp + + flisp itself is already threadsafe, this lock only protects the ``jl_ast_context_list_t`` pool + + +The following is a leaf lock (level 2), and only acquires level 1 locks (safepoint) internally: + + * dump (this lock may be possible to eliminate by storing the state on the stack) + * typecache + +The following are level 3 locks, which can only acquire level 1 or level 2 locks internally: + + * Method->writelock + + but note that this is violated by staged functions! + +The following is a level 4 lock, which can only recurse to acquire level 1, 2, or 3 locks: + + * MethodTable->writelock + + but note that this is violated by staged functions! + +The following is a proposed level 5 lock, which can only recurse to acquire locks at lower levels: + + * staged + + this theoretical lock would create a priority inversion from the `method->writelock` (level 3), + but only prohibiting running any staging function in parallel + (thus allowing temporary release of the MethodTable and Method locks) + +The following is a level 6 lock, which can only recurse to acquire locks at lower levels: + + * codegen + +The following is an almost root lock (level end-1), meaning only the root look may be held when trying to acquire it: + + * typeinf + + this one is perhaps one of the most tricky ones, since type-inference can be invoked from many points + +The following is the root lock, meaning no other lock shall be held when trying to acquire it: + + * toplevel + + this should be held while attempting a top-level action (such as making a new type or defining a new method): + trying to obtain this lock inside a staged function will cause a deadlock condition! + + additionally, it's unclear if *any* code can safely run in parallel with an arbitrary toplevel expression, + so it may require all threads to get to a safepoint first + + +The following locks are broken: +------------------------------- + +* toplevel + + doesn't exist right now + + fix: create it + +* codegen + + recursive (through ``static_eval``), but caller might also be holding locks (due to staged functions) + + other issues? + + fix: prohibit codegen while holding any other lock (possibly by checking ``ptls->current_task->locks.len != 0`` & explicitly check the locks that are OK to hold simultaneously)? + +* typeinf + + not certain of whether there are issues here or what they are. staging functions, of course, are a source of deadlocks here. + + fix: unknown + +* staged + + possible solution to prevent staged functions from causing deadlock. + + this theoretical lock would create a priority inversion such that the Method and MethodTable write locks could be released + by ensuring that no staging functions can run in parallel allow this level 5 lock to protect staged function conflicts (a level 3 operation) + + fix: create it + +* typecache + + this only protects cache lookup and insertion but it doesn't make the lookup-and-construct atomic + + fix: lock for ``apply_type`` / global (level 2?) + + +* dump + + may be able to eliminate this by making the state thread-local + + might also need to ensure that incremental deserialize has a toplevel expression lock From fd86ed7c63e51bb03b9035163140f2ac5324b89d Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Mon, 11 Jul 2016 14:58:19 -0400 Subject: [PATCH 0353/1117] Test for ppc64le arch in the power ccall tests. --- test/ccall.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ccall.jl b/test/ccall.jl index 0129954c87bb5..8600a305ae177 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -785,7 +785,7 @@ elseif Sys.ARCH === :aarch64 @test res === expected end -elseif Sys.ARCH === :powerpc64le +elseif Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le @test_huge 1 "_ppc64" (1, (2.0, 3.0, 4.0, 5.0),) @test_huge 2 "_ppc64" ((1.0, 2.0, 3.0, 4.0), (11, 12)) @test_huge 3 "_ppc64" ((1, 2, 3, 4), (11.0, 12.0, 13.0, 14.0)) From db54e2a3b0519ff146f9d9d26cd9c8edded4836f Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Sun, 10 Jul 2016 16:07:10 -0400 Subject: [PATCH 0354/1117] fix #17354, expand `Tuple{Vararg{Any,2}}` on construction --- base/methodshow.jl | 31 +++++++------------------------ src/jltypes.c | 26 ++++++++++---------------- test/core.jl | 2 ++ 3 files changed, 19 insertions(+), 40 deletions(-) diff --git a/base/methodshow.jl b/base/methodshow.jl index fd9aa090b1f0b..fa03de7d232d1 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -2,7 +2,11 @@ # Method and method table pretty-printing -function argtype_decl(env, n, t) # -> (argname, argtype) +function argtype_decl(env, n, sig, i, nargs, isva) # -> (argname, argtype) + t = sig.parameters[i] + if i == nargs && isva && !isvarargtype(t) + t = Vararg{t,length(sig.parameters)-nargs+1} + end if isa(n,Expr) n = n.args[1] # handle n::T in arg list end @@ -30,26 +34,6 @@ function argtype_decl(env, n, t) # -> (argname, argtype) return s, string_with_env(env, t) end -function argtype_decl_vararg(env, n, t) - if isa(n, Expr) - s = string(n.args[1]) - if n.args[2].head == :... - # x... or x::T... declaration - if t.parameters[1] === Any - return string(s, "..."), "" - else - return s, string_with_env(env, t.parameters[1]) * "..." - end - elseif t == String - return s, "String" - end - end - # x::Vararg, x::Vararg{T}, or x::Vararg{T,N} declaration - s, length(n.args[2].args) < 4 ? - string_with_env(env, "Vararg{", t.parameters[1], "}") : - string_with_env(env, "Vararg{", t.parameters[1], ",", t.parameters[2], "}") -end - function arg_decl_parts(m::Method) tv = m.tvars if !isa(tv,SimpleVector) @@ -61,9 +45,8 @@ function arg_decl_parts(m::Method) file, line = "", 0 if li !== nothing argnames = li.slotnames[1:li.nargs] - s = Symbol("?") - decls = Any[argtype_decl(:tvar_env => tv, get(argnames, i, s), m.sig.parameters[i]) - for i = 1:length(m.sig.parameters)] + decls = Any[argtype_decl(:tvar_env => tv, argnames[i], m.sig, i, li.nargs, li.isva) + for i = 1:li.nargs] if isdefined(li, :def) file, line = li.def.file, li.def.line end diff --git a/src/jltypes.c b/src/jltypes.c index 93da20a9c29af..ccf8fc116a778 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2157,28 +2157,22 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i if (nt < 0) jl_errorf("apply_type: Vararg length N is negative: %zd", nt); va = jl_tparam0(va); - if (nt == 0 || jl_is_leaf_type(va)) { + if (nt == 0 || !jl_has_typevars(va)) { if (ntp == 1) return jl_tupletype_fill(nt, va); size_t i, l; + p = jl_alloc_svec(ntp - 1 + nt); for (i = 0, l = ntp - 1; i < l; i++) { - if (!jl_is_leaf_type(iparams[i])) - break; + jl_svecset(p, i, iparams[i]); } - if (i == l) { - p = jl_alloc_svec(ntp - 1 + nt); - for (i = 0, l = ntp - 1; i < l; i++) { - jl_svecset(p, i, iparams[i]); - } - l = ntp - 1 + nt; - for (; i < l; i++) { - jl_svecset(p, i, va); - } - JL_GC_PUSH1(&p); - jl_value_t *ndt = (jl_value_t*)jl_apply_tuple_type(p); - JL_GC_POP(); - return ndt; + l = ntp - 1 + nt; + for (; i < l; i++) { + jl_svecset(p, i, va); } + JL_GC_PUSH1(&p); + jl_value_t *ndt = (jl_value_t*)jl_apply_tuple_type(p); + JL_GC_POP(); + return ndt; } } } diff --git a/test/core.jl b/test/core.jl index 2863d79efb373..d98db8c25826f 100644 --- a/test/core.jl +++ b/test/core.jl @@ -139,6 +139,8 @@ let N = TypeVar(:N,true) @test is(Bottom,typeintersect(Tuple{Array{Int,N},Vararg{Int,N}}, Tuple{Vector{Int},Real,Real,Real})) @test is(Bottom,typeintersect(Tuple{Vector{Int},Real,Real,Real}, Tuple{Array{Int,N},Vararg{Int,N}})) @test Tuple{Int,Vararg{Int,2}} == Tuple{Int,Int,Int} + @test Tuple{Int,Vararg{Int,2}} === Tuple{Int,Int,Int} + @test Tuple{Any, Any} === Tuple{Vararg{Any,2}} @test Tuple{Int,Vararg{Int,2}} == Tuple{Int,Int,Vararg{Int,1}} @test Tuple{Int,Vararg{Int,2}} == Tuple{Int,Int,Int,Vararg{Int,0}} @test !(Tuple{Int,Vararg{Int,2}} <: Tuple{Int,Int,Int,Vararg{Int,1}}) From 41b9c527de24996abe9b4bedb942342890c3cf32 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Sun, 10 Jul 2016 18:13:43 +0200 Subject: [PATCH 0355/1117] Implement nested markdown lists Adds basic support for writing nested lists, as well as any other "block"-level markdown element such as code blocks, tables, quote blocks, etc. within a single list item. Since lists can now contain block elements some additional spacing has been added between items when output in the terminal to make it easier to differentiate between elements. --- base/markdown/Common/block.jl | 82 ++++++---- base/markdown/render/html.jl | 2 +- base/markdown/render/latex.jl | 6 +- base/markdown/render/plain.jl | 6 +- base/markdown/render/rst.jl | 9 +- base/markdown/render/terminal/render.jl | 2 +- test/markdown.jl | 196 ++++++++++++++++++++++-- 7 files changed, 250 insertions(+), 53 deletions(-) diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index dd91e783c1f3d..da5398380bd58 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -223,55 +223,73 @@ end List(xs...) = List(vcat(xs...)) -const bullets = "*•+-" -const num_or_bullets = r"^(\*|•|\+|-|\d+(\.|\))) " +const BULLETS = r"^ {0,3}(\*|\+|•|-)( |$)" +const NUM_OR_BULLETS = r"^ {0,3}(\*|•|\+|-|\d+(\.|\)))( |$)" -# Todo: ordered lists, inline formatting function list(stream::IO, block::MD) withstream(stream) do - eatindent(stream) || return false - b = startswith(stream, num_or_bullets) - (b === nothing || b == "") && return false - ordered = !(b[1] in bullets) - if ordered - b = b[end - 1] == '.' ? r"^\d+\. " : r"^\d+\) " - # TODO start value - end - the_list = List(ordered) + bullet = startswith(stream, NUM_OR_BULLETS; eat = false) + indent = isempty(bullet) ? (return false) : length(bullet) + # Calculate the starting number and regex to use for bullet matching. + initial, regex = + if ismatch(BULLETS, bullet) + # An unordered list. Use `-1` to flag the list as unordered. + -1, BULLETS + elseif ismatch(r"^ {0,3}\d+(\.|\))( |$)", bullet) + # An ordered list. Either with `1. ` or `1) ` style numbering. + r = contains(bullet, ".") ? r"^ {0,3}(\d+)\.( |$)" : r"^ {0,3}(\d+)\)( |$)" + Base.parse(Int, match(r, bullet).captures[1]), r + else + # Failed to match any bullets. This branch shouldn't actually be needed + # since the `NUM_OR_BULLETS` regex should cover this, but we include it + # simply for thoroughness. + return false + end - buffer = IOBuffer() - fresh_line = false + # Initialise the empty list object: either ordered or unordered. TODO: replace + # `Bool` field with `Int` to allow ordered lists to begin from a specific number. + list = List(initial >= 0) + + buffer = IOBuffer() # For capturing nested text for recursive parsing. + newline = false # For checking if we have two consecutive newlines: end of list. + count = 0 # Count of list items. Used to check if we need to push remaining + # content in `buffer` after leaving the `while` loop. while !eof(stream) - if fresh_line - sp = startswith(stream, r"^ {0,3}") - if !(startswith(stream, b) in [false, ""]) - push!(the_list.items, parseinline(takebuf_string(buffer), block)) - buffer = IOBuffer() + if startswith(stream, "\n") + if newline + # Double newline ends the current list. + pushitem!(list, buffer) + break else - # TODO write a newline here, and deal with nested - write(buffer, ' ', sp) + newline = true + println(buffer) end - fresh_line = false else - c = read(stream, Char) - if c == '\n' - eof(stream) && break - next = Char(peek(stream)) # ok since we only compare with ASCII - if next == '\n' + newline = false + if startswith(stream, " "^indent) + # Indented text that is part of the current list item. + print(buffer, readline(stream)) + else + matched = startswith(stream, regex) + if isempty(matched) + # Unindented text meaning we have left the current list. + pushitem!(list, buffer) break else - fresh_line = true + # Start of a new list item. + count += 1 + count > 1 && pushitem!(list, buffer) + print(buffer, readline(stream)) end - else - write(buffer, c) end end end - push!(the_list.items, parseinline(takebuf_string(buffer), block)) - push!(block, the_list) + count == length(list.items) || pushitem!(list, buffer) + push!(block, list) return true end end +pushitem!(list, buffer) = push!(list.items, parse(takebuf_string(buffer)).content) # –––––––––––––– # HorizontalRule diff --git a/base/markdown/render/html.jl b/base/markdown/render/html.jl index fa2dad364ae55..db5ee3aa3c05d 100644 --- a/base/markdown/render/html.jl +++ b/base/markdown/render/html.jl @@ -102,7 +102,7 @@ function html(io::IO, md::List) for item in md.items println(io) withtag(io, :li) do - htmlinline(io, item) + html(io, item) end end println(io) diff --git a/base/markdown/render/latex.jl b/base/markdown/render/latex.jl index 20d682f657ccf..a12c0ba651865 100644 --- a/base/markdown/render/latex.jl +++ b/base/markdown/render/latex.jl @@ -71,10 +71,10 @@ end function latex(io::IO, md::List) env = md.ordered ? "enumerate" : "itemize" wrapblock(io, env) do - for item in md.items + for (n, item) in enumerate(md.items) print(io, "\\item ") - latexinline(io, item) - println(io) + latex(io, item) + n < length(md.items) && println(io) end end end diff --git a/base/markdown/render/plain.jl b/base/markdown/render/plain.jl index 85d0acf01a6a0..b78abd95cf18e 100644 --- a/base/markdown/render/plain.jl +++ b/base/markdown/render/plain.jl @@ -36,7 +36,11 @@ end function plain(io::IO, list::List) for (i, item) in enumerate(list.items) print(io, list.ordered ? "$i. " : " * ") - plaininline(io, item) + lines = split(rstrip(sprint(buf -> plain(buf, item))), "\n") + for (n, line) in enumerate(lines) + print(io, (n == 1 || isempty(line)) ? "" : " ", line) + n < length(lines) && println(io) + end println(io) end end diff --git a/base/markdown/render/rst.jl b/base/markdown/render/rst.jl index 370dc8fe56601..d51a1e710c292 100644 --- a/base/markdown/render/rst.jl +++ b/base/markdown/render/rst.jl @@ -38,8 +38,13 @@ end function rst(io::IO, list::List) for (i, item) in enumerate(list.items) - print(io, list.ordered ? "$i. " : "* ") - rstinline(io, item) + bullet = list.ordered ? "$i. " : "* " + print(io, bullet) + lines = split(rstrip(sprint(buf -> rst(buf, item))), '\n') + for (n, line) in enumerate(lines) + print(io, (n == 1 || isempty(line)) ? "" : " "^length(bullet), line) + n < length(lines) && println(io) + end println(io) end end diff --git a/base/markdown/render/terminal/render.jl b/base/markdown/render/terminal/render.jl index 243a3fa3d3274..b740f60588aa3 100644 --- a/base/markdown/render/terminal/render.jl +++ b/base/markdown/render/terminal/render.jl @@ -45,7 +45,7 @@ function term(io::IO, md::List, columns) print(io, " "^2margin, md.ordered ? "$i. " : "• ") print_wrapped(io, width = columns-(4margin+2), pre = " "^(2margin+2), i = 2margin+2) do io - terminline(io, point) + term(io, point, columns - 10) end end end diff --git a/test/markdown.jl b/test/markdown.jl index 65b1db2a0c193..8c5f3e6f7f1e2 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -59,14 +59,21 @@ code_in_code = md""" @test md"[^foo]: footnote" == MD(Paragraph([Footnote("foo", Any[" footnote"])])) -@test md""" +let doc = md""" * one * two 1. pirate 2. ninja -3. zombie""" == Markdown.MD([Markdown.List(["one", "two"]), - Markdown.List(["pirate", "ninja", "zombie"], true)]) +3. zombie""" + @test isa(doc.content[1], Markdown.List) + @test isa(doc.content[2], Markdown.List) + @test doc.content[1].items[1][1].content[1] == "one" + @test doc.content[1].items[2][1].content[1] == "two" + @test doc.content[2].items[1][1].content[1] == "pirate" + @test doc.content[2].items[2][1].content[1] == "ninja" + @test doc.content[2].items[3][1].content[1] == "zombie" +end @test md"Foo [bar]" == MD(Paragraph("Foo [bar]")) @test md"Foo [bar](baz)" != MD(Paragraph("Foo [bar](baz)")) @@ -97,15 +104,20 @@ World""" |> plain == "Hello\n\n---\n\nWorld\n" # multiple whitespace is ignored @test sprint(term, md"a b") == " a b\n" -@test sprint(term, md"- a - not code") == " • a not code\n" @test sprint(term, md"[x](https://julialang.org)") == " x\n" @test sprint(term, md"![x](https://julialang.org)") == " (Image: x)\n" # enumeration is normalized -@test sprint(term, md""" - 1. a - 3. b""") == " 1. a\n 2. b\n" +let doc = Markdown.parse( + """ + 1. a + 3. b + """ + ) + @test contains(sprint(term, doc), "1. ") + @test contains(sprint(term, doc), "2. ") + @test !contains(sprint(term, doc), "3. ") +end # HTML output @@ -116,8 +128,8 @@ World""" |> plain == "Hello\n\n---\n\nWorld\n" @test md"###### h6" |> html == "<h6>h6</h6>\n" @test md"####### h7" |> html == "<p>####### h7</p>\n" @test md" >" |> html == "<blockquote>\n</blockquote>\n" -@test md"1. Hello" |> html == "<ol>\n<li>Hello</li>\n</ol>\n" -@test md"* World" |> html == "<ul>\n<li>World</li>\n</ul>\n" +@test md"1. Hello" |> html == "<ol>\n<li><p>Hello</p>\n</li>\n</ol>\n" +@test md"* World" |> html == "<ul>\n<li><p>World</p>\n</li>\n</ul>\n" @test md"# title *blah*" |> html == "<h1>title <em>blah</em></h1>\n" @test md"## title *blah*" |> html == "<h2>title <em>blah</em></h2>\n" @test md"""Hello @@ -159,7 +171,7 @@ Some **bolded** - list1 - list2 """ -@test latex(book) == "\\section{Title}\nSome discussion\n\\begin{quote}\nA quote\n\\end{quote}\n\\subsection{Section \\emph{important}}\nSome \\textbf{bolded}\n\\begin{itemize}\n\\item list1\n\\item list2\n\\end{itemize}\n" +@test latex(book) == "\\section{Title}\nSome discussion\n\\begin{quote}\nA quote\n\\end{quote}\n\\subsection{Section \\emph{important}}\nSome \\textbf{bolded}\n\\begin{itemize}\n\\item list1\n\n\\item list2\n\\end{itemize}\n" # mime output @@ -192,8 +204,10 @@ let out = <h2>Section <em>important</em></h2> <p>Some <strong>bolded</strong></p> <ul> - <li>list1</li> - <li>list2</li> + <li><p>list1</p> + </li> + <li><p>list2</p> + </li> </ul> </div>""" @test sprint(io -> show(io, "text/html", book)) == out @@ -209,6 +223,7 @@ let out = Some \\textbf{bolded} \\begin{itemize} \\item list1 + \\item list2 \\end{itemize} """ @@ -679,3 +694,158 @@ let t_1 = end end +# Nested Lists. + +let text = + """ + 1. A paragraph + with two lines. + + indented code + + > A block quote. + + - one + + two + + - one + + two + + + - baz + + + ``` + foo + ``` + + 1. foo + 2. bar + 3. baz + """, + md = Markdown.parse(text) + + # Content and structure tests. + + @test length(md.content) == 6 + @test length(md.content[1].items) == 1 + @test length(md.content[1].items[1]) == 3 + @test isa(md.content[1].items[1][1], Markdown.Paragraph) + @test isa(md.content[1].items[1][2], Markdown.Code) + @test isa(md.content[1].items[1][3], Markdown.BlockQuote) + @test length(md.content[2].items) == 1 + @test isa(md.content[2].items[1][1], Markdown.Paragraph) + @test isa(md.content[3], Markdown.Paragraph) + @test length(md.content[4].items) == 1 + @test isa(md.content[4].items[1][1], Paragraph) + @test isa(md.content[4].items[1][2], Paragraph) + @test length(md.content[5].items) == 2 + @test isa(md.content[5].items[1][1], Markdown.Paragraph) + @test isa(md.content[5].items[2][1], Markdown.Code) + @test length(md.content[6].items) == 3 + @test md.content[6].items[1][1].content[1] == "foo" + @test md.content[6].items[2][1].content[1] == "bar" + @test md.content[6].items[3][1].content[1] == "baz" + + # Rendering tests. + + let expected = + """ + 1. A paragraph with two lines. + + ``` + indented code + ``` + + > A block quote. + + * one + + two + + * one + + two + + * baz + * ``` + foo + ``` + + 1. foo + 2. bar + 3. baz + """ + @test expected == Markdown.plain(md) + end + + let expected = + """ + <ol> + <li><p>A paragraph with two lines.</p> + <pre><code>indented code</code></pre> + <blockquote> + <p>A block quote.</p> + </blockquote> + </li> + </ol> + <ul> + <li><p>one</p> + </li> + </ul> + <p>two</p> + <ul> + <li><p>one</p> + <p>two</p> + </li> + </ul> + <ul> + <li><p>baz</p> + </li> + <li><pre><code>foo</code></pre> + </li> + </ul> + <ol> + <li><p>foo</p> + </li> + <li><p>bar</p> + </li> + <li><p>baz</p> + </li> + </ol> + """ + @test expected == Markdown.html(md) + end + + let expected = + """ + 1. A paragraph with two lines. + + .. code-block:: julia + + indented code + + A block quote. + + * one + + two + + * one + + two + + * baz + * .. code-block:: julia + + foo + + 1. foo + 2. bar + 3. baz + """ + @test expected == Markdown.rst(md) + end +end + From 604d237bd0d81218f41e133d68af681093551768 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Sun, 10 Jul 2016 21:57:39 +0200 Subject: [PATCH 0356/1117] Add arbitrary starting values for lists This allows ordered lists to be numbered starting from any non-negative integer such as: 7. foo 8. bar ... --- base/markdown/Common/block.jl | 15 +++--- base/markdown/render/html.jl | 3 +- base/markdown/render/latex.jl | 19 +++++-- base/markdown/render/plain.jl | 2 +- base/markdown/render/rst.jl | 2 +- base/markdown/render/terminal/render.jl | 2 +- test/markdown.jl | 66 +++++++++++++++++++++++++ 7 files changed, 95 insertions(+), 14 deletions(-) diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index da5398380bd58..bba06e7bc3647 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -214,15 +214,17 @@ end type List items::Vector{Any} - ordered::Bool + ordered::Int # `-1` is unordered, `>= 0` is ordered. - List(x::AbstractVector, b::Bool) = new(x, b) - List(x::AbstractVector) = new(x, false) - List(b::Bool) = new(Any[], b) + List(x::AbstractVector, b::Integer) = new(x, b) + List(x::AbstractVector) = new(x, -1) + List(b::Integer) = new(Any[], b) end List(xs...) = List(vcat(xs...)) +isordered(list::List) = list.ordered >= 0 + const BULLETS = r"^ {0,3}(\*|\+|•|-)( |$)" const NUM_OR_BULLETS = r"^ {0,3}(\*|•|\+|-|\d+(\.|\)))( |$)" @@ -246,9 +248,8 @@ function list(stream::IO, block::MD) return false end - # Initialise the empty list object: either ordered or unordered. TODO: replace - # `Bool` field with `Int` to allow ordered lists to begin from a specific number. - list = List(initial >= 0) + # Initialise the empty list object: either ordered or unordered. + list = List(initial) buffer = IOBuffer() # For capturing nested text for recursive parsing. newline = false # For checking if we have two consecutive newlines: end of list. diff --git a/base/markdown/render/html.jl b/base/markdown/render/html.jl index db5ee3aa3c05d..96a3c8b08f972 100644 --- a/base/markdown/render/html.jl +++ b/base/markdown/render/html.jl @@ -98,7 +98,8 @@ function html(io::IO, md::Admonition) end function html(io::IO, md::List) - withtag(io, md.ordered ? :ol : :ul) do + maybe_attr = md.ordered > 1 ? Any[:start => string(md.ordered)] : [] + withtag(io, isordered(md) ? :ol : :ul, maybe_attr...) do for item in md.items println(io) withtag(io, :li) do diff --git a/base/markdown/render/latex.jl b/base/markdown/render/latex.jl index a12c0ba651865..ec17ce1daae7e 100644 --- a/base/markdown/render/latex.jl +++ b/base/markdown/render/latex.jl @@ -69,10 +69,23 @@ function latex(io::IO, md::Admonition) end function latex(io::IO, md::List) - env = md.ordered ? "enumerate" : "itemize" - wrapblock(io, env) do + # `\begin{itemize}` is used here for both ordered and unordered lists since providing + # custom starting numbers for enumerated lists is simpler to do by manually assigning + # each number to `\item` ourselves rather than using `\setcounter{enumi}{<start>}`. + # + # For an ordered list starting at 5 the following will be generated: + # + # \begin{itemize} + # \item[5. ] ... + # \item[6. ] ... + # ... + # \end{itemize} + # + pad = ndigits(md.ordered + length(md.items)) + 2 + fmt = n -> (isordered(md) ? "[$(rpad("$(n + md.ordered - 1).", pad))]" : "") + wrapblock(io, "itemize") do for (n, item) in enumerate(md.items) - print(io, "\\item ") + print(io, "\\item$(fmt(n)) ") latex(io, item) n < length(md.items) && println(io) end diff --git a/base/markdown/render/plain.jl b/base/markdown/render/plain.jl index b78abd95cf18e..cfd006393b902 100644 --- a/base/markdown/render/plain.jl +++ b/base/markdown/render/plain.jl @@ -35,7 +35,7 @@ end function plain(io::IO, list::List) for (i, item) in enumerate(list.items) - print(io, list.ordered ? "$i. " : " * ") + print(io, isordered(list) ? "$(i + list.ordered - 1). " : " * ") lines = split(rstrip(sprint(buf -> plain(buf, item))), "\n") for (n, line) in enumerate(lines) print(io, (n == 1 || isempty(line)) ? "" : " ", line) diff --git a/base/markdown/render/rst.jl b/base/markdown/render/rst.jl index d51a1e710c292..f50f9394b4635 100644 --- a/base/markdown/render/rst.jl +++ b/base/markdown/render/rst.jl @@ -38,7 +38,7 @@ end function rst(io::IO, list::List) for (i, item) in enumerate(list.items) - bullet = list.ordered ? "$i. " : "* " + bullet = isordered(list) ? "$(i + list.ordered - 1). " : "* " print(io, bullet) lines = split(rstrip(sprint(buf -> rst(buf, item))), '\n') for (n, line) in enumerate(lines) diff --git a/base/markdown/render/terminal/render.jl b/base/markdown/render/terminal/render.jl index b740f60588aa3..5d5ab014a751a 100644 --- a/base/markdown/render/terminal/render.jl +++ b/base/markdown/render/terminal/render.jl @@ -42,7 +42,7 @@ end function term(io::IO, md::List, columns) for (i, point) in enumerate(md.items) - print(io, " "^2margin, md.ordered ? "$i. " : "• ") + print(io, " "^2margin, isordered(md) ? "$(i + md.ordered - 1). " : "• ") print_wrapped(io, width = columns-(4margin+2), pre = " "^(2margin+2), i = 2margin+2) do io term(io, point, columns - 10) diff --git a/test/markdown.jl b/test/markdown.jl index 8c5f3e6f7f1e2..470b349b0e6e5 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -849,3 +849,69 @@ let text = end end +# Ordered list starting number. + +let text = + """ + 42. foo + 43. bar + + + 1. foo + 2. bar + + + - foo + - bar + """, + md = Markdown.parse(text) + + @test md.content[1].ordered == 42 + @test md.content[2].ordered == 1 + @test md.content[3].ordered == -1 + + let expected = + """ + <ol start="42"> + <li><p>foo</p> + </li> + <li><p>bar</p> + </li> + </ol> + <ol> + <li><p>foo</p> + </li> + <li><p>bar</p> + </li> + </ol> + <ul> + <li><p>foo</p> + </li> + <li><p>bar</p> + </li> + </ul> + """ + @test expected == Markdown.html(md) + end + + let expected = + """ + \\begin{itemize} + \\item[42. ] foo + + \\item[43. ] bar + \\end{itemize} + \\begin{itemize} + \\item[1. ] foo + + \\item[2. ] bar + \\end{itemize} + \\begin{itemize} + \\item foo + + \\item bar + \\end{itemize} + """ + @test expected == Markdown.latex(md) + end +end From 4161cd4ec347f7948f764ce676fd082a179fc1a6 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Sun, 10 Jul 2016 22:18:09 +0200 Subject: [PATCH 0357/1117] Fix rendering of two code blocks Indented code blocks appearing directly after a markdown list cannot be handled correctly since the indentation of the list contents takes precedence over the code block's indentation. To resolve this we use a fenced code block instead. --- base/docs/helpdb/Base.jl | 4 +++- base/show.jl | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 863fccd8dab56..2b23cd6f399df 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -3006,7 +3006,9 @@ Keyword arguments: * `sshflags`: specifies additional ssh options, e.g. - sshflags=`-i /home/foo/bar.pem` + ``` + sshflags=`-i /home/foo/bar.pem` + ``` * `max_parallel`: specifies the maximum number of workers connected to in parallel at a host. Defaults to 10. diff --git a/base/show.jl b/base/show.jl index 0706ed905b6b6..ad2fbc86a1a8e 100644 --- a/base/show.jl +++ b/base/show.jl @@ -25,7 +25,9 @@ Create a new entry in the IO Dictionary for the key => value pair - use `(key => value) in dict` to see if this particular combination is in the properties set - use `get(dict, key, default)` to retrieve the most recent value for a particular key - IOContext(io::IO, context::IOContext) +``` +IOContext(io::IO, context::IOContext) +``` Create a IOContext that wraps an alternate IO but inherits the keyword arguments from the context """ From c99d3a809dd95b9de3db89d8db596b5bab8b9180 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Mon, 11 Jul 2016 21:21:08 +0200 Subject: [PATCH 0358/1117] Re-run genstdlib --- doc/stdlib/base.rst | 12 ++++++------ doc/stdlib/dates.rst | 4 ++-- doc/stdlib/linalg.rst | 25 ++++++++++++------------ doc/stdlib/math.rst | 4 ++-- doc/stdlib/numbers.rst | 2 +- doc/stdlib/parallel.rst | 43 ++++++++++++++++------------------------- doc/stdlib/strings.rst | 12 ++++++------ 7 files changed, 46 insertions(+), 56 deletions(-) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 0492ef81c268c..320539cabed42 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -822,12 +822,12 @@ System Construct a new ``Cmd`` object, representing an external program and arguments, from ``cmd``\ , while changing the settings of the optional keyword arguments: - * ``ignorestatus::Bool``\ : If ``true`` (defaults to ``false``\ ), then the ``Cmd`` will not throw an error if the return code is nonzero. - * ``detach::Bool``\ : If ``true`` (defaults to ``false``\ ), then the ``Cmd`` will be run in a new process group, allowing it to outlive the ``julia`` process and not have Ctrl-C passed to it. - * ``windows_verbatim::Bool``\ : If ``true`` (defaults to ``false``\ ), then on Windows the ``Cmd`` will send a command-line string to the process with no quoting or escaping of arguments, even arguments containing spaces. (On Windows, arguments are sent to a program as a single "command-line" string, and programs are responsible for parsing it into arguments. By default, empty arguments and arguments with spaces or tabs are quoted with double quotes ``"`` in the command line, and ``\`` or ``"`` are preceded by backslashes. ``windows_verbatim=true`` is useful for launching programs that parse their command line in nonstandard ways.) Has no effect on non-Windows systems. - * ``windows_hide::Bool``\ : If ``true`` (defaults to ``false``\ ), then on Windows no new console window is displayed when the ``Cmd`` is executed. This has no effect if a console is already open or on non-Windows systems. - * ``env``\ : Set environment variables to use when running the ``Cmd``\ . ``env`` is either a dictionary mapping strings to strings, an array of strings of the form ``"var=val"``\ , an array or tuple of ``"var"=>val`` pairs, or ``nothing``\ . In order to modify (rather than replace) the existing environment, create ``env`` by ``copy(ENV)`` and then set ``env["var"]=val`` as desired. - * ``dir::AbstractString``\ : Specify a working directory for the command (instead of the current directory). + * ``ignorestatus::Bool``\ : If ``true`` (defaults to ``false``\ ), then the ``Cmd`` will not throw an error if the return code is nonzero. + * ``detach::Bool``\ : If ``true`` (defaults to ``false``\ ), then the ``Cmd`` will be run in a new process group, allowing it to outlive the ``julia`` process and not have Ctrl-C passed to it. + * ``windows_verbatim::Bool``\ : If ``true`` (defaults to ``false``\ ), then on Windows the ``Cmd`` will send a command-line string to the process with no quoting or escaping of arguments, even arguments containing spaces. (On Windows, arguments are sent to a program as a single "command-line" string, and programs are responsible for parsing it into arguments. By default, empty arguments and arguments with spaces or tabs are quoted with double quotes ``"`` in the command line, and ``\`` or ``"`` are preceded by backslashes. ``windows_verbatim=true`` is useful for launching programs that parse their command line in nonstandard ways.) Has no effect on non-Windows systems. + * ``windows_hide::Bool``\ : If ``true`` (defaults to ``false``\ ), then on Windows no new console window is displayed when the ``Cmd`` is executed. This has no effect if a console is already open or on non-Windows systems. + * ``env``\ : Set environment variables to use when running the ``Cmd``\ . ``env`` is either a dictionary mapping strings to strings, an array of strings of the form ``"var=val"``\ , an array or tuple of ``"var"=>val`` pairs, or ``nothing``\ . In order to modify (rather than replace) the existing environment, create ``env`` by ``copy(ENV)`` and then set ``env["var"]=val`` as desired. + * ``dir::AbstractString``\ : Specify a working directory for the command (instead of the current directory). For any keywords that are not specified, the current settings from ``cmd`` are used. Normally, to create a ``Cmd`` object in the first place, one uses backticks, e.g. diff --git a/doc/stdlib/dates.rst b/doc/stdlib/dates.rst index 1cc554b42b613..86ea8a182a593 100644 --- a/doc/stdlib/dates.rst +++ b/doc/stdlib/dates.rst @@ -548,8 +548,8 @@ Periods Construct a ``CompoundPeriod`` from a ``Vector`` of ``Period``\ s. The constructor will automatically simplify the periods into a canonical form according to the following rules: * All ``Period``\ s of the same type will be added together - * Any ``Period`` large enough be partially representable by a coarser ``Period`` will be broken into multiple ``Period``\ s (eg. ``Hour(30)`` becomes ``Day(1) + Hour(6)``\ ) - * ``Period``\ s with opposite signs will be combined when possible (eg. ``Hour(1) - Day(1)`` becomes ``-Hour(23)``\ ) + * Any ``Period`` large enough be partially representable by a coarser ``Period`` will be broken into multiple ``Period``\ s (eg. ``Hour(30)`` becomes ``Day(1) + Hour(6)``\ ) + * ``Period``\ s with opposite signs will be combined when possible (eg. ``Hour(1) - Day(1)`` becomes ``-Hour(23)``\ ) Due to the canonicalization, ``CompoundPeriod`` is also useful for converting time periods into more human-comprehensible forms. diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 12d89f8e12875..0cdf61ae0e7a3 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -803,7 +803,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f * ``F[:U]`` is a M-by-M orthogonal matrix, * ``F[:V]`` is a P-by-P orthogonal matrix, * ``F[:Q]`` is a N-by-N orthogonal matrix, - * ``F[:R0]`` is a (K+L)-by-N matrix whose rightmost (K+L)-by-(K+L) block is nonsingular upper block triangular, + * ``F[:R0]`` is a (K+L)-by-N matrix whose rightmost (K+L)-by-(K+L) block is nonsingular upper block triangular, * ``F[:D1]`` is a M-by-(K+L) diagonal matrix with 1s in the first K entries, * ``F[:D2]`` is a P-by-(K+L) matrix whose top right L-by-L block is diagonal, @@ -1013,7 +1013,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f Output: - * ``v`` - A unit vector being the input vector, rescaled to have norm 1. The input vector is modified in-place. + * ``v`` - A unit vector being the input vector, rescaled to have norm 1. The input vector is modified in-place. See also: @@ -1304,7 +1304,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f The following keyword arguments are supported: * ``nev``\ : Number of eigenvalues - * ``ncv``\ : Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrix ``A``\ . The default is ``ncv = max(20,2*nev+1)``\ . Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + * ``ncv``\ : Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrix ``A``\ . The default is ``ncv = max(20,2*nev+1)``\ . Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. * ``which``\ : type of eigenvalues to compute. See the note below. +-----------+-----------------------------------------------------------------------------------------------------------------------------+ @@ -1325,10 +1325,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f | ``:BE`` | compute half of the eigenvalues from each end of the spectrum, biased in favor of the high end. (real symmetric ``A`` only) | +-----------+-----------------------------------------------------------------------------------------------------------------------------+ - * ``tol``\ : parameter defining the relative tolerance for convergence of Ritz values (eigenvalue estimates). A Ritz value :math:`θ` is considered converged when its associated residual is less than or equal to the product of ``tol`` and :math:`max(ɛ^{2/3}, |θ|)`\ , where ``ɛ = eps(real(eltype(A)))/2`` is LAPACK's machine epsilon. The residual associated with :math:`θ` and its corresponding Ritz vector :math:`v` is defined as the norm :math:`||Av - vθ||`\ . The specified value of ``tol`` should be positive; otherwise, it is ignored and :math:`ɛ` is used instead. Default: :math:`ɛ`\ . - + * ``tol``\ : parameter defining the relative tolerance for convergence of Ritz values (eigenvalue estimates). A Ritz value :math:`θ` is considered converged when its associated residual is less than or equal to the product of ``tol`` and :math:`max(ɛ^{2/3}, |θ|)`\ , where ``ɛ = eps(real(eltype(A)))/2`` is LAPACK's machine epsilon. The residual associated with :math:`θ` and its corresponding Ritz vector :math:`v` is defined as the norm :math:`||Av - vθ||`\ . The specified value of ``tol`` should be positive; otherwise, it is ignored and :math:`ɛ` is used instead. Default: :math:`ɛ`\ . * ``maxiter``\ : Maximum number of iterations (default = 300) - * ``sigma``\ : Specifies the level shift used in inverse iteration. If ``nothing`` (default), defaults to ordinary (forward) iterations. Otherwise, find eigenvalues close to ``sigma`` using shift and invert iterations. + * ``sigma``\ : Specifies the level shift used in inverse iteration. If ``nothing`` (default), defaults to ordinary (forward) iterations. Otherwise, find eigenvalues close to ``sigma`` using shift and invert iterations. * ``ritzvec``\ : Returns the Ritz vectors ``v`` (eigenvectors) if ``true`` * ``v0``\ : starting vector from which to start the iterations @@ -1351,8 +1350,8 @@ Linear algebra functions in Julia are largely implemented by calling functions f For details of how the errors in the computed eigenvalues are estimated, see: - * B. N. Parlett, "The Symmetric Eigenvalue Problem", SIAM: Philadelphia, 2/e (1998), Ch. 13.2, "Accessing Accuracy in Lanczos Problems", pp. 290-292 ff. - * R. B. Lehoucq and D. C. Sorensen, "Deflation Techniques for an Implicitly Restarted Arnoldi Iteration", SIAM Journal on Matrix Analysis and Applications (1996), 17(4), 789–821. doi:10.1137/S0895479895281484 + * B. N. Parlett, "The Symmetric Eigenvalue Problem", SIAM: Philadelphia, 2/e (1998), Ch. 13.2, "Accessing Accuracy in Lanczos Problems", pp. 290-292 ff. + * R. B. Lehoucq and D. C. Sorensen, "Deflation Techniques for an Implicitly Restarted Arnoldi Iteration", SIAM Journal on Matrix Analysis and Applications (1996), 17(4), 789–821. doi:10.1137/S0895479895281484 .. function:: eigs(A, B; nev=6, ncv=max(20,2*nev+1), which="LM", tol=0.0, maxiter=300, sigma=nothing, ritzvec=true, v0=zeros((0,))) -> (d,[v,],nconv,niter,nmult,resid) @@ -1364,7 +1363,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f The following keyword arguments are supported: * ``nev``\ : Number of eigenvalues - * ``ncv``\ : Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrices ``A`` and ``B``\ . The default is ``ncv = max(20,2*nev+1)``\ . Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. + * ``ncv``\ : Number of Krylov vectors used in the computation; should satisfy ``nev+1 <= ncv <= n`` for real symmetric problems and ``nev+2 <= ncv <= n`` for other problems, where ``n`` is the size of the input matrices ``A`` and ``B``\ . The default is ``ncv = max(20,2*nev+1)``\ . Note that these restrictions limit the input matrix ``A`` to be of dimension at least 2. * ``which``\ : type of eigenvalues to compute. See the note below. +-----------+-----------------------------------------------------------------------------------------------------------------------------+ @@ -1385,9 +1384,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f | ``:BE`` | compute half of the eigenvalues from each end of the spectrum, biased in favor of the high end. (real symmetric ``A`` only) | +-----------+-----------------------------------------------------------------------------------------------------------------------------+ - * ``tol``\ : relative tolerance used in the convergence criterion for eigenvalues, similar to ``tol`` in the :func:`eigs` method for the ordinary eigenvalue problem, but effectively for the eigenvalues of :math:`B^{-1} A` instead of :math:`A`\ . See the documentation for the ordinary eigenvalue problem in :func:`eigs` and the accompanying note about ``tol``\ . + * ``tol``\ : relative tolerance used in the convergence criterion for eigenvalues, similar to ``tol`` in the :func:`eigs` method for the ordinary eigenvalue problem, but effectively for the eigenvalues of :math:`B^{-1} A` instead of :math:`A`\ . See the documentation for the ordinary eigenvalue problem in :func:`eigs` and the accompanying note about ``tol``\ . * ``maxiter``\ : Maximum number of iterations (default = 300) - * ``sigma``\ : Specifies the level shift used in inverse iteration. If ``nothing`` (default), defaults to ordinary (forward) iterations. Otherwise, find eigenvalues close to ``sigma`` using shift and invert iterations. + * ``sigma``\ : Specifies the level shift used in inverse iteration. If ``nothing`` (default), defaults to ordinary (forward) iterations. Otherwise, find eigenvalues close to ``sigma`` using shift and invert iterations. * ``ritzvec``\ : Returns the Ritz vectors ``v`` (eigenvectors) if ``true`` * ``v0``\ : starting vector from which to start the iterations @@ -1420,9 +1419,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f **Inputs** - * ``A``\ : Linear operator whose singular values are desired. ``A`` may be represented as a subtype of ``AbstractArray``\ , e.g., a sparse matrix, or any other type supporting the four methods ``size(A)``\ , ``eltype(A)``\ , ``A * vector``\ , and ``A' * vector``\ . + * ``A``\ : Linear operator whose singular values are desired. ``A`` may be represented as a subtype of ``AbstractArray``\ , e.g., a sparse matrix, or any other type supporting the four methods ``size(A)``\ , ``eltype(A)``\ , ``A * vector``\ , and ``A' * vector``\ . * ``nsv``\ : Number of singular values. Default: 6. - * ``ritzvec``\ : If ``true``\ , return the left and right singular vectors ``left_sv`` and ``right_sv``\ . If ``false``\ , omit the singular vectors. Default: ``true``\ . + * ``ritzvec``\ : If ``true``\ , return the left and right singular vectors ``left_sv`` and ``right_sv``\ . If ``false``\ , omit the singular vectors. Default: ``true``\ . * ``tol``\ : tolerance, see :func:`eigs`\ . * ``maxiter``\ : Maximum number of iterations, see :func:`eigs`\ . Default: 1000. * ``ncv``\ : Maximum size of the Krylov subspace, see :func:`eigs` (there called ``nev``\ ). Default: ``2*nsv``\ . diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index de8a2b4043ae6..5df626d3f6fd7 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1649,7 +1649,7 @@ Statistics Quantiles are computed via linear interpolation between the points ``((k-1)/(n-1), v[k])``\ , for ``k = 1:n`` where ``n = length(v)``\ . This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. - * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 .. function:: quantile!([q, ] v, p; sorted=false) @@ -1661,7 +1661,7 @@ Statistics Quantiles are computed via linear interpolation between the points ``((k-1)/(n-1), v[k])``\ , for ``k = 1:n`` where ``n = length(v)``\ . This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. - * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 .. function:: cov(x[, corrected=true]) diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index f9ffc157b7eef..1877e6c119c5c 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -552,7 +552,7 @@ As ``BigInt`` represents unbounded integers, the interval must be specified (e.g Pick a random element or array of random elements from the set of values specified by ``S``\ ; ``S`` can be * an indexable collection (for example ``1:n`` or ``['x','y','z']``\ ), or - * a type: the set of values to pick from is then equivalent to ``typemin(S):typemax(S)`` for integers (this is not applicable to ``BigInt``\ ), and to :math:`[0, 1)` for floating point numbers; + * a type: the set of values to pick from is then equivalent to ``typemin(S):typemax(S)`` for integers (this is not applicable to ``BigInt``\ ), and to :math:`[0, 1)` for floating point numbers; ``S`` defaults to ``Float64``\ . diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 65792e164ab4a..4fa5fd778760a 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -173,29 +173,21 @@ General Parallel Computing Support Keyword arguments: - * ``tunnel``\ : if ``true`` then SSH tunneling will be used to connect to the worker from the master process. Default is ``false``\ . - + * ``tunnel``\ : if ``true`` then SSH tunneling will be used to connect to the worker from the master process. Default is ``false``\ . * ``sshflags``\ : specifies additional ssh options, e.g. - .. code-block:: julia - - sshflags=`-i /home/foo/bar.pem` - - * ``max_parallel``\ : specifies the maximum number of workers connected to in parallel at a host. Defaults to 10. - - * ``dir``\ : specifies the working directory on the workers. Defaults to the host's current directory (as found by ``pwd()``\ ) - - * ``exename``\ : name of the ``julia`` executable. Defaults to ``"$JULIA_HOME/julia"`` or ``"$JULIA_HOME/julia-debug"`` as the case may be. + .. code-block:: julia + sshflags=`-i /home/foo/bar.pem` + * ``max_parallel``\ : specifies the maximum number of workers connected to in parallel at a host. Defaults to 10. + * ``dir``\ : specifies the working directory on the workers. Defaults to the host's current directory (as found by ``pwd()``\ ) + * ``exename``\ : name of the ``julia`` executable. Defaults to ``"$JULIA_HOME/julia"`` or ``"$JULIA_HOME/julia-debug"`` as the case may be. * ``exeflags``\ : additional flags passed to the worker processes. + * ``topology``\ : Specifies how the workers connect to each other. Sending a message between unconnected workers results in an error. - * ``topology``\ : Specifies how the workers connect to each other. Sending a message between unconnected workers results in an error. - - * ``topology=:all_to_all`` : All processes are connected to each other. This is the default. - - * ``topology=:master_slave`` : Only the driver process, i.e. pid 1 connects to the workers. The workers do not connect to each other. - - * ``topology=:custom`` : The ``launch`` method of the cluster manager specifes the connection topology via fields ``ident`` and ``connect_idents`` in ``WorkerConfig``\ . A worker with a cluster manager identity ``ident`` will connect to all workers specified in ``connect_idents``\ . + * ``topology=:all_to_all`` : All processes are connected to each other. This is the default. + * ``topology=:master_slave`` : Only the driver process, i.e. pid 1 connects to the workers. The workers do not connect to each other. + * ``topology=:custom`` : The ``launch`` method of the cluster manager specifes the connection topology via fields ``ident`` and ``connect_idents`` in ``WorkerConfig``\ . A worker with a cluster manager identity ``ident`` will connect to all workers specified in ``connect_idents``\ . Environment variables : @@ -335,8 +327,8 @@ General Parallel Computing Support * ``Future`` : Wait for a value to become available for the specified future. * ``Channel``\ : Wait for a value to be appended to the channel. * ``Condition``\ : Wait for ``notify`` on a condition. - * ``Process``\ : Wait for a process or process chain to exit. The ``exitcode`` field of a process can be used to determine success or failure. - * ``Task``\ : Wait for a ``Task`` to finish, returning its result value. If the task fails with an exception, the exception is propagated (re-thrown in the task that called ``wait``\ ). + * ``Process``\ : Wait for a process or process chain to exit. The ``exitcode`` field of a process can be used to determine success or failure. + * ``Task``\ : Wait for a ``Task`` to finish, returning its result value. If the task fails with an exception, the exception is propagated (re-thrown in the task that called ``wait``\ ). * ``RawFD``\ : Wait for changes on a file descriptor (see ``poll_fd`` for keyword arguments and return code) If no argument is passed, the task blocks for an undefined period. A task can only be restarted by an explicit call to ``schedule`` or ``yieldto``\ . @@ -349,8 +341,8 @@ General Parallel Computing Support Waits and fetches a value from ``x`` depending on the type of ``x``\ . Does not remove the item fetched: - * ``Future``\ : Wait for and get the value of a Future. The fetched value is cached locally. Further calls to ``fetch`` on the same reference return the cached value. If the remote value is an exception, throws a ``RemoteException`` which captures the remote exception and backtrace. - * ``RemoteChannel``\ : Wait for and get the value of a remote reference. Exceptions raised are same as for a ``Future`` . + * ``Future``\ : Wait for and get the value of a Future. The fetched value is cached locally. Further calls to ``fetch`` on the same reference return the cached value. If the remote value is an exception, throws a ``RemoteException`` which captures the remote exception and backtrace. + * ``RemoteChannel``\ : Wait for and get the value of a remote reference. Exceptions raised are same as for a ``Future`` . * ``Channel`` : Wait for and get the first available item from the channel. .. function:: remotecall_wait(func, id, args...; kwargs...) @@ -614,9 +606,8 @@ Shared Arrays Construct a ``SharedArray`` backed by the file ``filename``\ , with element type ``T`` (must be a ``bitstype``\ ) and size ``dims``\ , across the processes specified by ``pids`` - all of which have to be on the same host. This file is mmapped into the host memory, with the following consequences: - * The array data must be represented in binary format (e.g., an ASCII format like CSV cannot be supported) - - * Any changes you make to the array values (e.g., ``A[3] = 0``\ ) will also change the values on disk + * The array data must be represented in binary format (e.g., an ASCII format like CSV cannot be supported) + * Any changes you make to the array values (e.g., ``A[3] = 0``\ ) will also change the values on disk If ``pids`` is left unspecified, the shared array will be mapped across all processes on the current host, including the master. But, ``localindexes`` and ``indexpids`` will only refer to worker processes. This facilitates work distribution code to use workers for actual computation with the master process acting as a driver. @@ -812,7 +803,7 @@ between processes. It is possible for Cluster Managers to provide a different tr Implemented by cluster managers. It is called on the master process, during a worker's lifetime, with appropriate ``op`` values: * with ``:register``\ /``:deregister`` when a worker is added / removed from the Julia worker pool. - * with ``:interrupt`` when ``interrupt(workers)`` is called. The :class:`ClusterManager` should signal the appropriate worker with an interrupt signal. + * with ``:interrupt`` when ``interrupt(workers)`` is called. The :class:`ClusterManager` should signal the appropriate worker with an interrupt signal. * with ``:finalize`` for cleanup purposes. .. function:: kill(manager::FooManager, pid::Int, config::WorkerConfig) diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index e1ce2a06ed73d..b27e7616a1e1e 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -89,9 +89,9 @@ Construct a regex, such as ``r"^[a-z]*$"``\ . The regex also accepts one or more flags, listed after the ending quote, to change its behaviour: * ``i`` enables case-insensitive matching - * ``m`` treats the ``^`` and ``$`` tokens as matching the start and end of individual lines, as opposed to the whole string. + * ``m`` treats the ``^`` and ``$`` tokens as matching the start and end of individual lines, as opposed to the whole string. * ``s`` allows the ``.`` modifier to match newlines. - * ``x`` enables "comment mode": whitespace is enabled except when escaped with ``\``\ , and ``#`` is treated as starting a comment. + * ``x`` enables "comment mode": whitespace is enabled except when escaped with ``\``\ , and ``#`` is treated as starting a comment. For example, this regex has all three flags enabled: @@ -121,13 +121,13 @@ Alternatively, finer control and additional transformations may be be obtained by calling ``normalize_string(s; keywords...)``\ , where any number of the following boolean keywords options (which all default to ``false`` except for ``compose``\ ) are specified: * ``compose=false``\ : do not perform canonical composition - * ``decompose=true``\ : do canonical decomposition instead of canonical composition (``compose=true`` is ignored if present) + * ``decompose=true``\ : do canonical decomposition instead of canonical composition (``compose=true`` is ignored if present) * ``compat=true``\ : compatibility equivalents are canonicalized * ``casefold=true``\ : perform Unicode case folding, e.g. for case-insensitive string comparison - * ``newline2lf=true``\ , ``newline2ls=true``\ , or ``newline2ps=true``\ : convert various newline sequences (LF, CRLF, CR, NEL) into a linefeed (LF), line-separation (LS), or paragraph-separation (PS) character, respectively + * ``newline2lf=true``\ , ``newline2ls=true``\ , or ``newline2ps=true``\ : convert various newline sequences (LF, CRLF, CR, NEL) into a linefeed (LF), line-separation (LS), or paragraph-separation (PS) character, respectively * ``stripmark=true``\ : strip diacritical marks (e.g. accents) - * ``stripignore=true``\ : strip Unicode's "default ignorable" characters (e.g. the soft hyphen or the left-to-right marker) - * ``stripcc=true``\ : strip control characters; horizontal tabs and form feeds are converted to spaces; newlines are also converted to spaces unless a newline-conversion flag was specified + * ``stripignore=true``\ : strip Unicode's "default ignorable" characters (e.g. the soft hyphen or the left-to-right marker) + * ``stripcc=true``\ : strip control characters; horizontal tabs and form feeds are converted to spaces; newlines are also converted to spaces unless a newline-conversion flag was specified * ``rejectna=true``\ : throw an error if unassigned code points are found * ``stable=true``\ : enforce Unicode Versioning Stability From 9eba1056a693a98b2f1f0f5164367bf033eeb3a3 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 11 Jul 2016 15:22:58 -0400 Subject: [PATCH 0359/1117] need to visit the TypeMap->any list always same as with the TypeMap->linear list, because offs doesn't always start at zero need to always visit the list to catch cases where the list is a Vararg --- src/typemap.c | 4 +--- test/reflection.jl | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/typemap.c b/src/typemap.c index 466eb26d0556f..b47d8a00b1347 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -578,9 +578,7 @@ int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs, } if (!jl_typemap_intersection_node_visitor(map.node->linear, closure)) return 0; - if (ty) - return jl_typemap_intersection_visitor(map.node->any, offs+1, closure); - return 1; + return jl_typemap_intersection_visitor(map.node->any, offs+1, closure); } else { return jl_typemap_intersection_node_visitor(map.leaf, closure); diff --git a/test/reflection.jl b/test/reflection.jl index 3f34fd8c08bcd..6b4864c772ccc 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -443,8 +443,10 @@ ccall(:jl_register_newmeth_tracer, Void, (Ptr{Void},), C_NULL) for i = 1:100; @eval fLargeTable(::Val{$i}, ::Any) = 1; end for i = 1:100; @eval fLargeTable(::Any, ::Val{$i}) = 2; end fLargeTable(::Any...) = 3 +@test length(methods(fLargeTable, Tuple{})) == 1 fLargeTable(::Complex, ::Complex) = 4 fLargeTable(::Union{Complex64, Complex128}...) = 5 +@test length(methods(fLargeTable, Tuple{})) == 1 fLargeTable() = 4 @test length(methods(fLargeTable)) == 204 @test length(methods(fLargeTable, Tuple{})) == 1 From b8ea0c358718fbfc66e1039aa3c3944378aefa68 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Mon, 11 Jul 2016 15:33:49 -0400 Subject: [PATCH 0360/1117] test/perf/micro: remove dependency on Rmath (cut-n-paste edition) [ci skip] --- test/perf/micro/Makefile | 1 - test/perf/micro/perf.c | 1 + test/perf/micro/randmtzig.c | 826 ++++++++++++++++++++++++++++++++++++ 3 files changed, 827 insertions(+), 1 deletion(-) create mode 100644 test/perf/micro/randmtzig.c diff --git a/test/perf/micro/Makefile b/test/perf/micro/Makefile index 636f4885b87a8..8fba6c899b2fa 100644 --- a/test/perf/micro/Makefile +++ b/test/perf/micro/Makefile @@ -50,7 +50,6 @@ export OPENBLAS_NUM_THREADS=1 perf.h: $(JULIAHOME)/deps/Versions.make echo '#include "$(BLASDIR)cblas.h"' > $@ echo '#include "$(DSFMTDIR)/dSFMT.c"' >> $@ - echo '#include "$(RMATHDIR)/src/randmtzig.c"' >> $@ bin/perf%: perf.c perf.h $(CC) -std=c99 -O$* $< -o $@ -I$(DSFMTDIR) $(LIBBLAS) -L$(LIBMDIR) $(LIBM) $(CFLAGS) -lpthread diff --git a/test/perf/micro/perf.c b/test/perf/micro/perf.c index 7bfa4199e67a7..fa9621bbb1333 100644 --- a/test/perf/micro/perf.c +++ b/test/perf/micro/perf.c @@ -5,6 +5,7 @@ // include header file generated by make: #define DSFMT_MEXP 19937 #include "perf.h" +#include "randmtzig.c" double *myrand(int n) { double *d = (double *)malloc(n*sizeof(double)); diff --git a/test/perf/micro/randmtzig.c b/test/perf/micro/randmtzig.c new file mode 100644 index 0000000000000..05e6c27aeaaf6 --- /dev/null +++ b/test/perf/micro/randmtzig.c @@ -0,0 +1,826 @@ +/* + Parts Copyright (C) 2012 Viral B. Shah + All rights reserved. + + Modifications made for julia to support dsfmt and only __LP64__ systems. + 52-bits of randomness are used from the mantissa of random double precision + numbers generated by dsfmt. + */ + +/* + A C-program for MT19937, with initialization improved 2002/2/10. + Coded by Takuji Nishimura and Makoto Matsumoto. + This is a faster version by taking Shawn Cokus's optimization, + Matthe Bellew's simplification, Isaku Wada's real version. + David Bateman added normal and exponential distributions following + Marsaglia and Tang's Ziggurat algorithm. + + Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, + Copyright (C) 2004, David Bateman + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. The names of its contributors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER + OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + Any feedback is very welcome. + http://www.math.keio.ac.jp/matumoto/emt.html + email: matumoto@math.keio.ac.jp + +*/ + +#include <math.h> +#include <stdio.h> +#include <stddef.h> +#include <time.h> +#ifndef _MSC_VER +#include <sys/time.h> +#endif + +#include <stdlib.h> +#include <string.h> + +#define DSFMT_DO_NOT_USE_OLD_NAMES +#define DSFMT_MEXP 19937 +#include <dSFMT.h> + +typedef ptrdiff_t randmtzig_idx_type; +typedef signed char randmtzig_int8_t; +typedef unsigned char randmtzig_uint8_t; +typedef short randmtzig_int16_t; +typedef unsigned short randmtzig_uint16_t; +typedef int randmtzig_int32_t; +typedef unsigned int randmtzig_uint32_t; +typedef long long randmtzig_int64_t; +typedef unsigned long long randmtzig_uint64_t; + +/* Declarations */ + +extern double randmtzig_randn (dsfmt_t *dsfmt); +extern double randmtzig_gv_randn (void); +extern double randmtzig_gv_exprnd (void); + +/* ===== Uniform generators ===== */ + +inline static randmtzig_uint64_t gv_randi (void) +{ + double r = dsfmt_gv_genrand_close1_open2(); + return *((uint64_t *) &r) & 0x000fffffffffffff; +} + +/* generates a random number on (0,1) with 53-bit resolution */ +inline static double gv_randu (void) +{ + return dsfmt_gv_genrand_open_open(); +} + +inline static randmtzig_uint64_t randi (dsfmt_t *dsfmt) +{ + double r = dsfmt_genrand_close1_open2(dsfmt); + return *((uint64_t *) &r) & 0x000fffffffffffff; +} + +/* generates a random number on (0,1) with 53-bit resolution */ +inline static double randu (dsfmt_t *dsfmt) +{ + return dsfmt_genrand_open_open(dsfmt); +} + +/* ===== Ziggurat normal and exponential generators ===== */ +# define ZIGINT randmtzig_uint64_t +# define EMANTISSA 4503599627370496 /* 52 bit mantissa */ +# define ERANDI gv_randi() /* 52 bits for mantissa */ +# define NMANTISSA 2251799813685248 +# define NRANDI gv_randi() /* 51 bits for mantissa + 1 bit sign */ +# define RANDU gv_randu() + +#define ZIGGURAT_TABLE_SIZE 256 + +#define ZIGGURAT_NOR_R 3.6541528853610088 +#define ZIGGURAT_NOR_INV_R 0.27366123732975828 +#define NOR_SECTION_AREA 0.00492867323399 + +#define ZIGGURAT_EXP_R 7.69711747013104972 +#define ZIGGURAT_EXP_INV_R 0.129918765548341586 +#define EXP_SECTION_AREA 0.0039496598225815571993 + + +/* +This code is based on the paper Marsaglia and Tsang, "The ziggurat method +for generating random variables", Journ. Statistical Software. Code was +presented in this paper for a Ziggurat of 127 levels and using a 32 bit +integer random number generator. This version of the code, uses the +Mersenne Twister as the integer generator and uses 256 levels in the +Ziggurat. This has several advantages. + + 1) As Marsaglia and Tsang themselves states, the more levels the few + times the expensive tail algorithm must be called + 2) The cycle time of the generator is determined by the integer + generator, thus the use of a Mersenne Twister for the core random + generator makes this cycle extremely long. + 3) The license on the original code was unclear, thus rewriting the code + from the article means we are free of copyright issues. + 4) Compile flag for full 53-bit random mantissa. + +It should be stated that the authors made my life easier, by the fact that +the algorithm developed in the text of the article is for a 256 level +ziggurat, even if the code itself isn't... + +One modification to the algorithm developed in the article, is that it is +assumed that 0 <= x < Inf, and "unsigned long"s are used, thus resulting in +terms like 2^32 in the code. As the normal distribution is defined between +-Inf < x < Inf, we effectively only have 31 bit integers plus a sign. Thus +in Marsaglia and Tsang, terms like 2^32 become 2^31. We use NMANTISSA for +this term. The exponential distribution is one sided so we use the +full 32 bits. We use EMANTISSA for this term. + +It appears that I'm slightly slower than the code in the article, this +is partially due to a better generator of random integers than they +use. But might also be that the case of rapid return was optimized by +inlining the relevant code with a #define. As the basic Mersenne +Twister is only 25% faster than this code I suspect that the main +reason is just the use of the Mersenne Twister and not the inlining, +so I'm not going to try and optimize further. +*/ + + +// void randmtzig_create_ziggurat_tables (void) +// { +// int i; +// double x, x1; + +// /* Ziggurat tables for the normal distribution */ +// x1 = ZIGGURAT_NOR_R; +// wi[255] = x1 / NMANTISSA; +// fi[255] = exp (-0.5 * x1 * x1); + +// /* Index zero is special for tail strip, where Marsaglia and Tsang +// * defines this as +// * k_0 = 2^31 * r * f(r) / v, w_0 = 0.5^31 * v / f(r), f_0 = 1, +// * where v is the area of each strip of the ziggurat. +// */ +// ki[0] = (ZIGINT) (x1 * fi[255] / NOR_SECTION_AREA * NMANTISSA); +// wi[0] = NOR_SECTION_AREA / fi[255] / NMANTISSA; +// fi[0] = 1.; + +// for (i = 254; i > 0; i--) +// { +// /* New x is given by x = f^{-1}(v/x_{i+1} + f(x_{i+1})), thus +// * need inverse operator of y = exp(-0.5*x*x) -> x = sqrt(-2*ln(y)) +// */ +// x = sqrt(-2. * log(NOR_SECTION_AREA / x1 + fi[i+1])); +// ki[i+1] = (ZIGINT)(x / x1 * NMANTISSA); +// wi[i] = x / NMANTISSA; +// fi[i] = exp (-0.5 * x * x); +// x1 = x; +// } + +// ki[1] = 0; + +// /* Zigurrat tables for the exponential distribution */ +// x1 = ZIGGURAT_EXP_R; +// we[255] = x1 / EMANTISSA; +// fe[255] = exp (-x1); + +// /* Index zero is special for tail strip, where Marsaglia and Tsang +// * defines this as +// * k_0 = 2^32 * r * f(r) / v, w_0 = 0.5^32 * v / f(r), f_0 = 1, +// * where v is the area of each strip of the ziggurat. +// */ +// ke[0] = (ZIGINT) (x1 * fe[255] / EXP_SECTION_AREA * EMANTISSA); +// we[0] = EXP_SECTION_AREA / fe[255] / EMANTISSA; +// fe[0] = 1.; + +// for (i = 254; i > 0; i--) +// { +// /* New x is given by x = f^{-1}(v/x_{i+1} + f(x_{i+1})), thus +// * need inverse operator of y = exp(-x) -> x = -ln(y) +// */ +// x = - log(EXP_SECTION_AREA / x1 + fe[i+1]); +// ke[i+1] = (ZIGINT)(x / x1 * EMANTISSA); +// we[i] = x / EMANTISSA; +// fe[i] = exp (-x); +// x1 = x; +// } +// ke[1] = 0; +// } + +// Tables for randn +static ZIGINT ki[ZIGGURAT_TABLE_SIZE] = + {2104047571230236, 0,1693657211688499,1919380038164751, + 2015384402142420,2068365869415708,2101878624030987,2124958784087614, + 2141808670783638,2154644611559370,2164744887580145,2172897953690771, + 2179616279367521,2185247251864556,2190034623104318,2194154434518163, + 2197736978772008,2200880740889623,2203661538008543,2206138681107245, + 2208359231804928,2210361007256700,2212174742387166,2213825672703393, + 2215334711001466,2216719334486539,2217994262138197,2219171977964129, + 2220263139537873,2221276900116549,2222221164932202,2223102796828387, + 2223927782546019,2224701368169460,2225428170203747,2226112267247709, + 2226757276104752,2227366415327922,2227942558554233,2228488279492093, + 2229005890046815,2229497472774805,2229964908626691,2230409900758245, + 2230833995044249,2231238597815812,2231624991249884,2231994346765634, + 2232347736722468,2232686144665663,2233010474325699,2233321557544631, + 2233620161275830,2233906993781039,2234182710130112,2234447917093281, + 2234703177502812,2234949014149981,2235185913274123,2235414327692697, + 2235634679614740,2235847363174420,2236052746716668,2236251174862705, + 2236442970379808,2236628435876608,2236807855342616,2236981495548416, + 2237149607321006,2237312426707072,2237470176035519,2237623064889274, + 2237771290995262,2237915041040474,2238054491421185,2238189808931596, + 2238321151397547,2238448668260322,2238572501115061,2238692784207837, + 2238809644895031,2238923204068302,2239033576548092,2239140871448347, + 2239245192514865,2239346638439450,2239445303151863,2239541276091355, + 2239634642459413,2239725483455210,2239813876495104,2239899895417414, + 2239983610673598,2240065089506859,2240144396119109,2240221591827156, + 2240296735208897,2240369882240222,2240441086423317,2240510398906937, + 2240577868599239,2240643542273660,2240707464668327,2240769678579424, + 2240830224948918,2240889142947021,2240946470049710,2241002242111632, + 2241056493434688,2241109256832545,2241160563691345,2241210444026824, + 2241258926538069,2241306038658085,2241351806601384,2241396255408737, + 2241439408989263,2241481290159988,2241521920683014,2241561321300414, + 2241599511766981,2241636510880914,2241672336512567,2241707005631317, + 2241740534330669,2241772937851645,2241804230604542,2241834426189118, + 2241863537413270,2241891576310240,2241918554154426,2241944481475803, + 2241969368073032,2241993223025259,2242016054702647,2242037870775672, + 2242058678223188,2242078483339294,2242097291739004,2242115108362739, + 2242131937479636,2242147782689690,2242162646924702,2242176532448058, + 2242189440853303,2242201373061504,2242212329317384,2242222309184204, + 2242231311537365,2242239334556685,2242246375717338,2242252431779384, + 2242257498775863,2242261571999386,2242264645987166,2242266714504423, + 2242267770526080,2242267806216682,2242266812908434,2242264781077261, + 2242261700316790,2242257559310117,2242252345799249,2242246046552055, + 2242238647326588,2242230132832599,2242220486690050,2242209691384432, + 2242197728218658,2242184577261284,2242170217290794,2242154625735654, + 2242137778609814,2242119650443302,2242100214207531,2242079441234882, + 2242057301132111,2242033761687055,2242008788768083,2241982346215658, + 2241954395725333,2241924896721420,2241893806220494,2241861078683807, + 2241826665857576,2241790516600019,2241752576693859,2241712788642894, + 2241671091451056,2241627420382213,2241581706698751,2241533877376746, + 2241483854795259,2241431556397014,2241376894317324,2241319774977796, + 2241260098640839,2241197758920517,2241132642244683,2241064627262631, + 2240993584191722,2240919374095516,2240841848084869,2240760846432212, + 2240676197587764,2240587717084761,2240495206318733,2240398451183547, + 2240297220544145,2240191264522592,2240080312570135,2239964071293311, + 2239842221996510,2239714417896679,2239580280957705,2239439398282173, + 2239291317986176,2239135544468183,2238971532964959,2238798683265249, + 2238616332424332,2238423746288075,2238220109591870,2238004514345197, + 2237775946143192,2237533267957802,2237275200846732,2237000300869931, + 2236706931309079,2236393229029127,2236057063479481,2235695986373225, + 2235307169458838,2234887326941556,2234432617919425,2233938522519742, + 2233399683022654,2232809697779175,2232160850599794,2231443750584617, + 2230646845562145,2229755753817960,2228752329126507,2227613325162477, + 2226308442121145,2224797391720369,2223025347823800,2220915633329775, + 2218357446086993,2215184158448627,2211132412537323,2205758503851011, + 2198248265654920,2186916352102052,2167562552481677,2125549880839429}; + +static double wi[ZIGGURAT_TABLE_SIZE] = + {17367254121656703e-31,9558660348275145e-32,12708704832820278e-32, + 14909740960986864e-32,16658733630346416e-32,18136120809053487e-32, + 1942972015219358e-31,20589500627632916e-32,21646860576118966e-32, + 2262294039150043e-31,23532718913376864e-32,24387234556800803e-32, + 25194879828681465e-32,2596219977196592e-31,26694407473112964e-32, + 2739572968463095e-31,280696460019946e-30,28719058903642897e-32, + 29346417484275224e-32,29953809336344285e-32,30543030006769113e-32, + 3111563633851158e-31,3167298801818414e-31,3221628035016365e-31, + 32746570407564125e-32,33264798116476e-29,337718034169968e-30, + 34268340352771636e-32,34755088731390227e-32,3523266384567022e-31, + 3570162463362898e-31,3616248057128073e-31,36615697529342477e-32, + 3706170277693123e-31,37500889278448874e-32,3793361940125627e-31, + 38360228129389374e-32,3878102586096749e-31,3919630085297984e-31, + 39606321365983254e-32,40011337552278087e-32,4041158312387907e-31, + 4080727683070036e-31,4119862377455137e-31,41585816580575855e-32, + 41969036444492247e-32,4234845407127582e-31,42724230518658345e-32, + 43096517956924877e-32,4346546035489394e-31,4383119410062289e-31, + 4419384856424202e-31,4455354660935343e-31,4491040505860591e-31, + 4526453511835132e-31,45616042766683e-29,4596502910863464e-31, + 4631159070186941e-31,4665581985579899e-31,469978049067346e-30, + 4733763047137822e-31,4767537768070579e-31,4801112439606964e-31, + 4834494540915173e-31,4867691262722585e-31,4900709524503576e-31, + 4933555990446197e-31,4966237084303158e-31,499875900322208e-30, + 5031127730640677e-31,5063349048324261e-31,5095428547615612e-31, + 5127371639960692e-31,5159183566767805e-31,5190869408652579e-31, + 5222434094116442e-31,52538824077020155e-32,5285218997665102e-31, + 5316448383199491e-31,5347574961247755e-31,5378603012928409e-31, + 5409536709607314e-31,5440380118638932e-31,5471137208800966e-31, + 550181185544408e-30,5532407845376661e-31,5562928881503102e-31, + 5593378587232605e-31,5623760510674315e-31,5654078128633358e-31, + 5684334850421336e-31,5714534021493849e-31,5744678926926726e-31, + 5774772794741848e-31,5804818799092685e-31,5834820063319006e-31, + 5864779662879593e-31,589470062817121e-30,5924585947241581e-31, + 5954438568403615e-31,598426140275769e-30,601405732662843e-30, + 6043829183921996e-31,6073579788409578e-31,6103311925942512e-31, + 6133028356604082e-31,6162731816802865e-31,6192425021312213e-31, + 6222110665260248e-31,6251791426074554e-31,6281469965385542e-31, + 6311148930892342e-31,6340830958194888e-31,6370518672595733e-31, + 640021469087503e-30,6429921623041988e-31,645964207406601e-30, + 648937864559066e-30,6519133937633505e-31,6548910550274845e-31, + 6578711085338253e-31,6608538148065851e-31,6638394348791179e-31, + 6668282304612498e-31,6698204641069389e-31,6728163993825439e-31, + 6758163010359885e-31,6788204351671041e-31,681829069399439e-30, + 6848424730538249e-31,6878609173239948e-31,6908846754545526e-31, + 6939140229215998e-31,696949237616333e-30,6999906000319335e-31, + 7030383934540792e-31,7060929041554193e-31,7091544215943653e-31, + 7122232386185626e-31,7152996516734219e-31,7183839610161045e-31, + 7214764709353755e-31,7245774899777502e-31,72768733118038725e-32, + 7308063123111988e-31,7339347561166714e-31,7370729905779203e-31, + 7402213491755235e-31,7433801711637146e-31,7465498018545449e-31, + 7497305929126601e-31,7529229026613742e-31,7561270964007667e-31, + 7593435467385694e-31,7625726339346621e-31,7658147462600412e-31, + 7690702803711903e-31,7723396417008341e-31,7756232448661274e-31, + 778921514095401e-30,7822348836746627e-31,7855637984151357e-31, + 7889087141432085e-31,7922700982142658e-31,7956484300519808e-31, + 7990442017147628e-31,8024579184911813e-31,8058900995263265e-31, + 8093412784812165e-31,812812004227522e-30,8163028415800651e-31, + 8198143720697359e-31,8233471947596931e-31,8269019271079405e-31, + 8304792058796362e-31,834079688112767e-30,8377040521411316e-31, + 8413529986789175e-31,8450272519715296e-31,8487275610177406e-31, + 85245470086869e-29,8562094740097588e-31,8599927118319072e-31, + 86380527619967175e-32,8676480611237092e-31,8715219945465259e-31, + 8754280402508787e-31,8793671999012706e-31,8833405152300122e-31, + 88734907038049e-29,8913939944215902e-31,8954764640486935e-31, + 8995977064883017e-31,9037590026252085e-31,9079616903732087e-31, + 9122071683126914e-31,9164968996211253e-31,9208324163254476e-31, + 9252153239087913e-31,9296473063078686e-31,9341301313417584e-31, + 938665656617903e-30,9432558359669126e-31,9479027264644209e-31, + 95260849610588e-29,957375432209002e-30,962205950628746e-30, + 9671026058815726e-31,972068102289435e-30,9771053062699983e-31, + 9822172599183368e-31,9874071960473548e-31,9926785548800904e-31, + 9980350026176626e-31,10034804521429213e-31,10090190861630543e-31, + 10146553831460223e-31,10203941464676316e-31,1026240537260681e-30, + 10322001115479755e-31,10382788623508751e-31,10444832675993878e-31, + 10508203448348659e-31,1057297713900341e-30,10639236690670377e-31, + 10707072623626628e-31,107765840026618e-29,10847879564397177e-31, + 10921079038143372e-31,109963147017795e-29,11073733224929686e-31, + 11153497865847152e-31,11235791107104895e-31,11320817840158973e-31, + 11408809242576976e-31,1150002753783406e-30,11594771891443527e-31, + 11693385786905373e-31,1179626635295029e-30,11903876299277459e-31, + 1201675939253847e-30,12135560818661637e-31,12261054417445396e-31, + 12394179789158183e-31,12536093926597603e-31,1268824481425016e-30, + 12852479319091384e-31,13031206634685398e-31,13227655770190893e-31, + 13446300925006917e-31,13693606835124475e-31,13979436672771461e-31, + 14319989869657897e-31,14744848603594667e-31,1531787274160907e-30, + 16227698675312968e-31}; + +static double fi[ZIGGURAT_TABLE_SIZE] = + {1.,.9771017012827331,.9598790918124174,.9451989534530794, + .9320600759689914,.9199915050483614,.9087264400605639,.898095921906305, + .8879846607634008,.8783096558161477,.869008688043794,.8600336212030095, + .8513462584651245,.842915653118442,.8347162929929313,.8267268339520951, + .8189291916094156,.8113078743182208,.8038494831763903,.7965423304282554, + .7893761435711993,.7823418326598627,.775431304986139,.7686373158033355, + .7619533468415471,.7553735065117552,.7488924472237273,.7425052963446368, + .7362075981312672,.729995264565803,.7238645334728822,.717811932634902, + .711834248882359,.7059285013367979,.7000919181404905,.694321916130033, + .6886160830085275,.6829721616487918,.6773880362225135,.6718617199007669, + .6663913439123812,.6609751477802419,.6556114705832252,.650298743114295, + .6450354808242524,.6398202774564395,.6346517992909606,.6295287799281287, + .6244500155502747,.6194143606090396,.6144207238920772,.6094680649288958, + .6045553907005499,.599681752622168,.5948462437709915,.5900479963357922, + .5852861792663006,.5805599961036837,.5758686829752109,.5712115067380753, + .5665877632589521,.5619967758172782,.5574378936214867,.5529104904285204, + .5484139632579217,.5439477311926505,.5395112342595453,.5351039323830201, + .5307253044061945,.5263748471741873,.5220520746747954,.5177565172322012, + .5134877207497434,.5092452459981365,.5050286679458292,.5008375751284826, + .49667156905479676,.4925302636461491,.4884132847077125,.48432026942891204, + .48025086591125016,.4762047327216842,.4721815384698837,.46818096140782267, + .4642026890502793,.460246417814924,.45631185268077407,.4523987068638829, + .4485067015092144,.4446355653977281,.4407850346677702,.43695485254992955, + .43314476911457434,.42935454103134185,.42558393133990086,.4218327092313535, + .41810064983968476,.4143875340427069,.41069314827198344,.40701728433124823, + .4033597392228692,.399720314981932,.3960988185175474,.39249506146101104, + .3889088600204649,.38534003484173424,.3817884108750316,.37825381724723833, + .37473608713949164,.37123505766982157,.3677505697805964,.36428246813054976, + .36083060099117586,.3573948201472906,.35397498080156936,.35057094148288126, + .34718256395825153,.3438097131482915,.3404522570459456,.33711006663841303, + .33378301583210873,.3304709813805373,.32717384281495887,.32389148237773235, + .3206237849582305,.3173706380312227,.3141319315976305,.310907558127564, + .307697412505554,.30450139197789644,.30131939610203423,.29815132669790145, + .2949970878011627,.2918565856182811,.28872972848335393,.28561642681665805, + .28251659308484933,.27943014176276515,.2763569892967811,.27329705406967564, + .2702502563669598,.26721651834463167,.2641957639983174,.2611879191337636, + .25819291133864797,.2552106699556771,.25224112605694377,.2492842124195167, + .24633986350223877,.24340801542371202,.24048860594144916,.23758157443217368, + .2346868618732527,.23180441082524855,.22893416541557743,.22607607132326474, + .22323007576478943,.22039612748101145,.21757417672517823,.2147641752520084, + .21196607630785277,.20917983462193548,.20640540639867916,.20364274931112133, + .20089182249543117,.19815258654653795,.1954250035148854,.19270903690432864, + .19000465167119293,.18731181422451676,.18463049242750437,.18196065560021638, + .17930227452353026,.17665532144440646,.17401977008249914,.17139559563815535, + .16878277480185,.1661812857651097,.1635911082329826,.16101222343811727, + .1584446141565199,.15588826472506426,.15334316106083742,.15080929068240986, + .1482866427331284,.14577520800653765,.14327497897404687,.14078594981496803, + .138308116449064,.13584147657175705,.13338602969216254,.13094177717412792, + .12850872228047336,.12608687022065,.12367622820205106,.12127680548523516, + .11888861344334545,.11651166562603685,.11414597782825504,.1117915681642454, + .10944845714720981,.10711666777507266,.10479622562286683,.10248715894230599, + .10018949876917177,.09790327903921535,.09562853671335306,.09336531191302634, + .09111364806670041,.08887359206859394,.08664519445086755,.08442850957065445, + .0822235958134955,.08003051581494733,.07784933670237201,.07568013035919481, + .07352297371424082,.07137794905914183,.06924514439725017,.06712465382802392, + .06501657797147035,.06292102443797778,.06083810834975175,.05876795292113793, + .056710690106399425,.05466646132507786,.05263541827697361,.05061772386112175, + .04861355321603513,.04662309490208967,.044646552251446515,.042684144916619336, + .04073611065607874,.0388027074046569,.03688421568869112,.034980941461833046, + .033093219458688684,.031221417192023686,.02936593975823011,.027527235669693315, + .02570580400863265,.023902203305873237,.022117062707379908,.020351096230109344, + .018605121275783343,.016880083152595836,.015177088307982065,.013497450601780796, + .0118427578579431,.0102149714397311,.008616582769422912,.007050875471392109, + .005522403299264755,.0040379725933718715,.002609072746106362,.0012602859304985975}; + +// Tables for exprnd +static ZIGINT ke[ZIGGURAT_TABLE_SIZE] = + {3985772928715748, 0,2742928985168065,3438700186803721, + 3744780257810519,3914896975372863,4022625697542798,4096776410635450, + 4150853606149210,4192001604687417,4224344877584101,4250427292531740, + 4271901371161554,4289886428824118,4305167164135199,4318309783140431, + 4329732973408940,4339752937704679,4348612900760388,4356502988721768, + 4363573953227346,4369946852445020,4375720012348349,4380974119031481, + 4385776001930298,4390181484145305,4394237557465219,4397984061535398, + 4401454994146430,4404679543790856,4407682910787985,4410486965794400, + 4413110782053579,4415571068741702,4417882526198713,4420058138987325, + 4422109419110772,4424046609003130,4425878851844253,4427614335173868, + 4429260412563040,4430823707156475,4432310200160197,4433725306767517, + 4435073941555377,4436360575016074,4437589282595121,4438763787369085, + 4439887497305303,4440963537889317,4441994780778252,4442983869033585, + 4443933239400428,4444845142028910,4445721657973833,4446564714759241, + 4447376100252993,4448157475061632,4448910383626429,4449636264176642, + 4450336457674983,4451012215872352,4451664708573597,4452295030203006, + 4452904205747010,4453493196141906,4454062903166143,4454614173889474, + 4455147804725090,4455664545125435,4456165100957688,4456650137590828, + 4457120282722585,4457576128971459,4458018236256245,4458447133983073, + 4458863323057847,4459267277740095,4459659447352586,4460040257859578, + 4460410113325310,4460769397263133,4461118473884710,4461457689257740, + 4461787372379910,4462107836175980,4462419378424319,4462722282618581, + 4463016818769709,4463303244152965,4463581804004301,4463852732169940, + 4464116251712773,4464372575478779,4464621906626490,4464864439122178, + 4465100358203284,4465329840812355,4465553056003596,4465770165323939, + 4465981323170417,4466186677125455,4466386368271563,4466580531486827, + 4466769295722448,4466952784263502,4467131114974006,4467304400527265, + 4467472748622447,4467636262188208,4467795039574164,4467949174730939, + 4468098757379442,4468243873170018,4468384603832024,4468521027314373, + 4468653217917530,4468781246417428,4468905180181701,4469025083278642, + 4469141016579234,4469253037852582,4469361201855066,4469465560413474, + 4469566162502383,4469663054316032,4469756279334881,4469845878387080, + 4469931889704995,4470014348976986,4470093289394551,4470168741694984, + 4470240734199652,4470309292847996,4470374441227332,4470436200598525, + 4470494589917605,4470549625853344,4470601322800852,4470649692891185, + 4470694745996980,4470736489734116,4470774929459349,4470810068263924, + 4470841906963074,4470870444081369,4470895675833821,4470917596102651, + 4470936196409614,4470951465883737,4470963391224346,4470971956659198, + 4470977143897542,4470978932077904,4470977297710362,4470972214613072, + 4470963653842747,4470951583618802,4470935969240827,4470916772999009, + 4470893954077117,4470867468447603,4470837268758338,4470803304210460, + 4470765520426769,4470723859310029,4470678258890503,4470628653161980, + 4470574971905457,4470517140499614,4470455079717082,4470388705505446, + 4470317928751818,4470242655029689,4470162784326669,4470078210751556, + 4469988822219058,4469894500110287,4469795118907000,4469690545797298, + 4469580640250319,4469465253557163,4469344228335006,4469217397991048, + 4469084586142556,4468945605988875,4468800259630802,4468648337332217, + 4468489616718259,4468323861903709,4468150822544456,4467970232804102, + 4467781810226787,4467585254506222,4467380246139658,4467166444954116, + 4466943488490515,4466710990229518,4466468537640691,4466215690034133, + 4465951976190801,4465676891744455,4465389896284247,4465090410142477, + 4464777810826750,4464451429049612,4464110544301482,4463754379904174, + 4463382097472202,4462992790697122,4462585478355953,4462159096427753, + 4461712489182116,4461244399078944,4460753455289386,4460238160612098, + 4459696876515553,4459127805983956,4458528973779075,4457898203649722, + 4457233091920646,4456530976767892,4455788902331217,4455003576616607, + 4454171321891082,4453288015951104,4452349022232651,4451349106194827, + 4450282334707462,4449141954247903,4447920242480611,4446608326137821, + 4445195955871677,4443671225661690,4442020220072463,4440226566619900, + 4438270861888260,4436129927556552,4433775834104270,4431174602388627, + 4428284451100006,4425053392146958,4421415870372502,4417287970124084, + 4412560416174562,4407088078325945,4400673742272494,4393042098597073, + 4383796248451589,4372341169422858,4357740343059956,4338425130125967, + 4311541827049177,4271262897902398,4203411844498905,4061213381260384}; + +static double we[ZIGGURAT_TABLE_SIZE] = + {19311480126418366e-31,1417802848791084e-32,23278824993382457e-33, + 30487830247064326e-33,3666569771447489e-32,4217930218928974e-32, + 4722256155686277e-32,51911915446217885e-33,5632347108395505e-32, + 6051008260642765e-32,645101650967275e-31,6835264680370054e-32, + 7205993957468906e-32,7564981553739299e-32,7913664396195108e-32, + 8253223556351894e-32,8584643616885051e-32,8908755486564743e-32, + 9226267962966373e-32,9537791450529272e-32,9843856087455926e-32, + 10144925809006294e-32,10441409405585343e-32,10733669323436384e-32, + 1102202874567019e-31,11306777346479334e-32,11588176009705533e-32, + 11866460730417886e-32,1214184586569436e-31,12414526862326387e-32, + 12684682560606153e-32,12952477151912284e-32,1321806185153881e-31, + 13481576335745447e-32,13743149982367625e-32,14002902946807862e-32, + 14260947099321287e-32,14517386844829297e-32,14772319842763584e-32, + 15025837641447456e-32,15278026239101652e-32,15528966581595696e-32, + 1577873500545958e-31,1602740363335091e-31,16275040728083524e-32, + 16521711010420076e-32,16767475945078279e-32,17012393998770646e-32, + 17256520873568226e-32,17499909718432365e-32,17742611321380505e-32, + 17984674284430714e-32,18226145183195818e-32,18467068712763576e-32, + 18707487821298258e-32,18947443832625902e-32,19186976558915997e-32, + 19426124404443042e-32,19664924461299023e-32,19903412597830144e-32, + 20141623540485899e-32,20379590949693882e-32,2061734749030844e-31, + 2085492489712377e-31,21092354035891528e-32,21329664960238294e-32, + 21566886964838972e-32,2180404863516701e-31,22041177894111562e-32, + 2227830204572395e-31,2251544781633135e-31,22752641393233694e-32, + 22989908461180186e-32,23227274236804366e-32,23464763501180916e-32, + 2370240063065339e-31,23940209626069303e-32,2417821414054771e-31, + 24416437505894123e-32,24654902757768304e-32,2489363265970225e-31, + 2513264972605797e-31,2537197624400795e-31,2561163429461499e-31, + 2585164577308239e-31,26092032408240577e-32,2633281578133145e-31, + 2657401734414762e-31,2681565843657999e-31,2705776030362351e-31, + 27300344111887955e-32,27543430965657624e-32,2778704192254128e-31, + 2803119800875143e-31,28275920234049704e-32,2852122960639331e-31, + 28767147146315804e-32,29013693901073754e-32,29260890958589514e-32, + 29508759461219033e-32,2975732061937252e-31,3000659572501474e-31, + 3025660616507079e-31,3050737343476251e-31,3075891915089994e-31, + 31011265065151543e-32,3126443307731675e-31,31518445248623523e-32, + 31773323815073683e-32,32029091200858335e-32,32285770031865573e-32, + 3254338314930261e-31,3280195362345436e-31,3306150476760074e-31, + 3332206015211484e-31,33583643618764577e-32,33846279295240445e-32, + 34109991609932597e-32,34374805306980633e-32,34640745461620167e-32, + 3490783749585068e-31,3517610719444983e-31,3544558072136013e-31, + 3571628463647465e-31,35988245912849274e-32,3626149195437003e-31, + 36536050613905045e-32,36811950211971757e-32,3708921955595139e-31, + 37367887959883854e-32,3764798526487784e-31,37929541860172334e-32, + 3821258870488753e-31,38497157350504876e-32,3878327996411799e-31, + 39070989352498183e-32,3936031898702075e-31,3965130302950038e-31, + 3994397635898684e-31,40238374599574693e-32,40534534149283966e-32, + 4083249221007178e-31,41132286819038357e-32,4143395688089474e-31, + 417375422017632e-30,42043083524385856e-32,4235062256482152e-31, + 4266020205071558e-31,42971865761233266e-32,43285658568752094e-32, + 4360162648241568e-31,43919816693657415e-32,4424027762380992e-31, + 4456305897392361e-31,4488821177692617e-31,4521578845226347e-31, + 4554584286317242e-31,4587843037674623e-31,4621360792696427e-31, + 4655143408087069e-31,4689196910809916e-31,4723527505395548e-31, + 4758141581628553e-31,4793045722637247e-31,4828246713412587e-31, + 4863751549784512e-31,489956744788614e-30,4935701854138577e-31, + 4972162455791703e-31,5008957192059114e-31,5046094265888434e-31, + 5083582156411624e-31,5121429632123542e-31,5159645764841062e-31, + 5198239944499494e-31,5237221894847848e-31,5276601690109886e-31, + 531638977268369e-30,535659697195905e-30,5397234524338979e-31, + 5438314094559637e-31,547984779841163e-30,5521848226975234e-31, + 5564328472492872e-31,5607302156013967e-31,5650783456960506e-31, + 5694787144776348e-31,5739328612839635e-31,5784423914835991e-31, + 5830089803810586e-31,5876343774140057e-31,5923204106690931e-31, + 5970689917460091e-31,6018821210025236e-31,6067618932170007e-31, + 6117105037089722e-31,616730254963062e-30,6218235638068533e-31, + 6269929691993326e-31,6322411406934211e-31,6375708876439426e-31, + 6429851692413595e-31,6484871054618903e-31,6540799890364481e-31, + 6597672985544566e-31,6655527128343343e-31,6714401267106488e-31, + 677433668409101e-30,6835377187051274e-31,6897569320906848e-31, + 6960962602074885e-31,7025609778445959e-31,7091567118449584e-31, + 7158894733208553e-31,7227656936438121e-31,7297922647529085e-31, + 7369765844191243e-31,7443266072160415e-31,7518509020832513e-31, + 7595587175337749e-31,7674600557578427e-31,7755657571215791e-31, + 7838875968622858e-31,792438396157355e-30,8012321502113083e-31, + 8102841765913146e-31,8196112877806125e-31,8292319928581809e-31, + 8391667344146798e-31,849438168364877e-30,8600714963334941e-31, + 8710948629387904e-31,882539833807214e-30,8944419748519865e-31, + 9068415597131669e-31,9197844409811865e-31,9333231329422952e-31, + 9475181706524984e-31,9624398345658476e-31,978170365478442e-30, + 994806847238388e-30,1012465014428832e-30,10312843657756166e-31, + 1051435160404455e-30,10731281954224043e-31,10966288068517408e-31, + 1122277490935032e-30,11505212963006663e-31,11819635283304206e-31, + 12174462832361815e-31,12581958069755114e-31,13060984107128082e-31, + 13642786158057857e-31,14384889932178723e-31,15412190700064194e-31, + 17091034077168055e-31}; + +static double fe[ZIGGURAT_TABLE_SIZE] = + { 1.0,.9381436808621746,.9004699299257464,.8717043323812036, + .8477855006239896,.8269932966430503,.8084216515230084,.7915276369724956, + .7759568520401156,.7614633888498963,.7478686219851951,.7350380924314235, + .722867659593572,.711274760805076,.7001926550827882,.689566496117078, + .6793505722647654,.6695063167319247,.6600008410789997,.650805833414571, + .6418967164272661,.6332519942143661,.6248527387036659,.6166821809152077, + .608725382079622,.6009689663652322,.5934009016917334,.586010318477268, + .578787358602845,.5717230486648258,.5648091929124002,.5580382822625874, + .5514034165406413,.5448982376724396,.5385168720028619,.5322538802630432, + .5261042139836197,.5200631773682336,.5141263938147486,.5082897764106429, + .5025495018413477,.49690198724154955,.49134386959403253,.4858719873418849, + .4804833639304542,.4751751930373774,.46994482528396,.4647897562504262, + .4597076156421377,.45469615747461545,.449753251162755,.4448768734145485, + .4400651008423539,.4353161032156366,.43062813728845883,.42599954114303434, + .4214287289976166,.4169141864330029,.4124544659971612,.4080481831520324, + .4036940125305303,.3993906844752311,.39513698183329016,.3909317369847971, + .38677382908413765,.38266218149600983,.3785957594095808,.37457356761590216, + .370594648435146,.36665807978151416,.3627629733548178,.3589084729487498, + .35509375286678746,.35131801643748334,.347580494621637,.3438804447045024, + .34021714906678,.33658991402867755,.332998068761809,.3294409642641363, + .3259179723935562,.3224284849560891,.31897191284495724,.31554768522712895, + .31215524877417955,.3087940669345602,.30546361924459026,.3021634006756935, + .2988929210155818,.2956517042812612,.2924392881618926,.28925522348967775, + .2860990737370768,.28297041453878075,.27986883323697287,.27679392844851736, + .27374530965280297,.27072259679906,.2677254199320448,.2647534188350622, + .2618062426893629,.25888354974901623,.2559850070304154,.25311029001562946, + .2502590823688623,.24743107566532763,.2446259691318921,.24184346939887721, + .23908329026244918,.23634515245705964,.23362878343743335,.2309339171696274, + .2282602939307167,.22560766011668407,.22297576805812017,.2203643758433595, + .21777324714870053,.21520215107537868,.21265086199297828,.21011915938898826, + .20760682772422204,.2051136562938377,.20263943909370902,.20018397469191127, + .19774706610509887,.19532852067956322,.19292814997677132,.1905457696631954, + .1881811994042543,.1858342627621971,.18350478709776746,.1811926034754963, + .1788975465724783,.17661945459049488,.1743581691713535,.17211353531532006, + .16988540130252766,.1676736186172502,.165478041874936,.16329852875190182, + .16113493991759203,.1589871389693142,.15685499236936523,.15473836938446808, + .15263714202744286,.1505511850010399,.1484803756438668,.14642459387834494, + .14438372216063478,.1423576454324722,.14034625107486245,.1383494288635802, + .13636707092642886,.13439907170221363,.13244532790138752,.13050573846833077, + .12858020454522817,.12666862943751067,.12477091858083096,.12288697950954514, + .12101672182667483,.11916005717532768,.11731689921155557,.11548716357863353, + .11367076788274431,.1118676316700563,.11007767640518538,.1083008254510338, + .10653700405000166,.10478613930657017,.10304816017125772,.10132299742595363, + .09961058367063713,.0979108533114922,.0962237425504328,.09454918937605586, + .09288713355604354,.09123751663104016,.08960028191003286,.08797537446727022, + .08636274114075691,.08476233053236812,.08317409300963238,.08159798070923742, + .0800339475423199,.07848194920160642,.0769419431704805,.07541388873405841, + .07389774699236475,.07239348087570874,.07090105516237183,.06942043649872875, + .0679515934219366,.06649449638533977,.06504911778675375,.06361543199980733, + .062193415408540995,.06078304644547963,.059384305633420266,.05799717563120066, + .05662164128374288,.05525768967669704,.05390531019604609,.05256449459307169, + .05123523705512628,.04991753428270637,.0486113855733795,.04731679291318155, + .04603376107617517,.04476229773294328,.04350241356888818,.042254122413316234, + .04101744138041482,.039792391023374125,.03857899550307486,.03737728277295936, + .03618728478193142,.03500903769739741,.03384258215087433,.032687963508959535, + .03154523217289361,.030414443910466604,.029295660224637393,.028188948763978636, + .0270943837809558,.026012046645134217,.024942026419731783,.02388442051155817, + .02283933540638524,.02180688750428358,.020787204072578117,.019780424338009743, + .01878670074469603,.01780620041091136,.016839106826039948,.015885621839973163, + .014945968011691148,.014020391403181938,.013109164931254991,.012212592426255381, + .011331013597834597,.010464810181029979,.00961441364250221,.008780314985808975, + .00796307743801704,.007163353183634984,.006381905937319179,.005619642207205483, + .004877655983542392,.004157295120833795,.003460264777836904,.002788798793574076, + .0021459677437189063,.0015362997803015724,.0009672692823271745,.00045413435384149677}; + + +/* + * Here is the guts of the algorithm. As Marsaglia and Tsang state the + * algorithm in their paper + * + * 1) Calculate a random signed integer j and let i be the index + * provided by the rightmost 8-bits of j + * 2) Set x = j * w_i. If j < k_i return x + * 3) If i = 0, then return x from the tail + * 4) If [f(x_{i-1}) - f(x_i)] * U < f(x) - f(x_i), return x + * 5) goto step 1 + * + * Where f is the functional form of the distribution, which for a normal + * distribution is exp(-0.5*x*x) + */ + +/* NOTE: This is identical to randmtzig_gv_randn() below except for the random number generation */ +double randmtzig_randn (dsfmt_t *dsfmt) +{ + while (1) + { + /* arbitrary mantissa (selected by randi, with 1 bit for sign) */ + const randmtzig_uint64_t r = randi(dsfmt); + const randmtzig_int64_t rabs=r>>1; + const int idx = (int)(rabs&0xFF); + const double x = ( r&1 ? -rabs : rabs) * wi[idx]; + + if (rabs < (randmtzig_int64_t)ki[idx]) { + return x; /* 99.3% of the time we return here 1st try */ + } else if (idx == 0) { + /* As stated in Marsaglia and Tsang + * + * For the normal tail, the method of Marsaglia[5] provides: + * generate x = -ln(U_1)/r, y = -ln(U_2), until y+y > x*x, + * then return r+x. Except that r+x is always in the positive + * tail!!!! Any thing random might be used to determine the + * sign, but as we already have r we might as well use it + * + * [PAK] but not the bottom 8 bits, since they are all 0 here! + */ + double xx, yy; + do { + xx = - ZIGGURAT_NOR_INV_R * log (randu(dsfmt)); + yy = - log (randu(dsfmt)); + } + while ( yy+yy <= xx*xx); + return (rabs&0x100 ? -ZIGGURAT_NOR_R-xx : ZIGGURAT_NOR_R+xx); + } else if ((fi[idx-1] - fi[idx]) * randu(dsfmt) + fi[idx] < exp(-0.5*x*x)) { + return x; + } + + } +} + +/* NOTE: This is identical to randmtzig_randn() above except for the random number generation */ +double randmtzig_gv_randn (void) +{ + while (1) + { + /* arbitrary mantissa (selected by NRANDI, with 1 bit for sign) */ + const randmtzig_uint64_t r = NRANDI; + const randmtzig_int64_t rabs=r>>1; + const int idx = (int)(rabs&0xFF); + const double x = ( r&1 ? -rabs : rabs) * wi[idx]; + + if (rabs < (randmtzig_int64_t)ki[idx]) { + return x; /* 99.3% of the time we return here 1st try */ + } else if (idx == 0) { + /* As stated in Marsaglia and Tsang + * + * For the normal tail, the method of Marsaglia[5] provides: + * generate x = -ln(U_1)/r, y = -ln(U_2), until y+y > x*x, + * then return r+x. Except that r+x is always in the positive + * tail!!!! Any thing random might be used to determine the + * sign, but as we already have r we might as well use it + * + * [PAK] but not the bottom 8 bits, since they are all 0 here! + */ + double xx, yy; + do { + xx = - ZIGGURAT_NOR_INV_R * log (RANDU); + yy = - log (RANDU); + } + while ( yy+yy <= xx*xx); + return (rabs&0x100 ? -ZIGGURAT_NOR_R-xx : ZIGGURAT_NOR_R+xx); + } else if ((fi[idx-1] - fi[idx]) * RANDU + fi[idx] < exp(-0.5*x*x)) { + return x; + } + + } +} + +double randmtzig_gv_exprnd (void) +{ + while (1) + { + ZIGINT ri = ERANDI; + const int idx = (int)(ri & 0xFF); + const double x = ri * we[idx]; + if (ri < ke[idx]) + return x; // 98.9% of the time we return here 1st try + else if (idx == 0) + { + /* As stated in Marsaglia and Tsang + * + * For the exponential tail, the method of Marsaglia[5] provides: + * x = r - ln(U); + */ + return ZIGGURAT_EXP_R - log(RANDU); + } + else if ((fe[idx-1] - fe[idx]) * RANDU + fe[idx] < exp(-x)) + return x; + } +} + +#ifdef STANDALONE + +int main(int ac, char *av[]) { + if (ac == 1) { + printf("Usage: randmtzig <n>\n"); + return (-1); + } + + int n = atoi(av[1]); + time_t t1; + + dsfmt_gv_init_gen_rand(0); + + double *p; posix_memalign((void **)&p, 16, n*sizeof(double)); + uint32_t *u; posix_memalign((void **)&u, 16, 2*n*sizeof(uint32_t)); + + t1 = clock(); + dsfmt_gv_fill_array_close_open(p, n); + printf("Uniform fill (n): %f\n", (clock() - t1) / (double) CLOCKS_PER_SEC); + + t1 = clock(); + for (int i = 0; i < n; i++) p[i] = dsfmt_gv_genrand_close_open(); + printf("Uniform (n): %f\n", (clock() - t1) / (double) CLOCKS_PER_SEC); + + t1 = clock(); + for (int i = 0; i < 2*n; i++) u[i] = dsfmt_gv_genrand_uint32(); + printf("Uniform 32-bit ints (2*n): %f\n", (clock() - t1) / (double) CLOCKS_PER_SEC); + + memset((void *)p, 0, n*sizeof(double)); + t1 = clock(); + for (int i = 0; i < n; i++) p[i] = randmtzig_gv_randn(); + printf("Normal (n): %f\n", (clock() - t1) / (double) CLOCKS_PER_SEC); + for (int i = 0; i < 10; i++) printf("%lf\n", p[i]); + + return 0; +} + +#endif From d455a158760bfecce24da409fd73452789c29a4a Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 11 Jul 2016 16:36:56 -0400 Subject: [PATCH 0361/1117] disable gcroot creation for a value with an existing root when loading a value from a struct that didn't need a local gcroot, the loaded value also doesn't require a gcroot if the struct was immutable fix #17342 --- src/cgutils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 95cb22b4c85ff..26ec40d9be9e1 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1033,7 +1033,7 @@ static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, V builder.CreateGEP(data_pointer(strct, ctx), idx))); if ((unsigned)stt->ninitialized != nfields) null_pointer_check(fld, ctx); - *ret = mark_julia_type(fld, true, jl_any_type, ctx, true); + *ret = mark_julia_type(fld, true, jl_any_type, ctx, strct.gcroot || !strct.isimmutable); return true; } else if (is_tupletype_homogeneous(stt->types)) { @@ -1110,7 +1110,7 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, Value *fldv = tbaa_decorate(tbaa, builder.CreateLoad(emit_bitcast(addr, T_ppjlvalue))); if (idx >= (unsigned)jt->ninitialized) null_pointer_check(fldv, ctx); - jl_cgval_t ret = mark_julia_type(fldv, true, jfty, ctx, true); + jl_cgval_t ret = mark_julia_type(fldv, true, jfty, ctx, strct.gcroot || !strct.isimmutable); return ret; } else { From 5dfec5bb65c010251bd103a547f14cc309287f0c Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 11 Jul 2016 16:42:49 -0400 Subject: [PATCH 0362/1117] fix parallel test with inline=no close #16714 --- test/parallel_exec.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 9f357836cfd53..cf03ca8a2e486 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -693,7 +693,7 @@ let ex try remotecall_fetch(id_other) do @eval module AModuleLocalToOther - foo() = error("A.error") + foo() = throw(ErrorException("A.error")) foo() end end From 9a0726f5b4f41f2066f99f8e8c9aa2e71b6c6575 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Mon, 11 Jul 2016 17:36:39 -0400 Subject: [PATCH 0363/1117] dlmread: add deprecation warning for ignore_invalid_chars option Part of #16107 --- base/datafmt.jl | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index 95712b55ea869..cc4bd51dcc445 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -289,16 +289,24 @@ end const valid_opts = [:header, :has_header, :use_mmap, :quotes, :comments, :dims, :comment_char, :skipstart, :skipblanks] const valid_opt_types = [Bool, Bool, Bool, Bool, Bool, NTuple{2,Integer}, Char, Integer, Bool] const deprecated_opts = Dict(:has_header => :header) + function val_opts(opts) d = Dict{Symbol,Union{Bool,NTuple{2,Integer},Char,Integer}}() for (opt_name, opt_val) in opts - !in(opt_name, valid_opts) && throw(ArgumentError("unknown option $opt_name")) + if opt_name == :ignore_invalid_chars + Base.depwarn("the ignore_invalid_chars option is no longer supported and will be ignored", :val_opts) + continue + end + in(opt_name, valid_opts) || + throw(ArgumentError("unknown option $opt_name")) opt_typ = valid_opt_types[findfirst(valid_opts, opt_name)] - !isa(opt_val, opt_typ) && throw(ArgumentError("$opt_name should be of type $opt_typ, got $(typeof(opt_val))")) + isa(opt_val, opt_typ) || + throw(ArgumentError("$opt_name should be of type $opt_typ, got $(typeof(opt_val))")) d[opt_name] = opt_val - haskey(deprecated_opts, opt_name) && warn("$opt_name is deprecated, use $(deprecated_opts[opt_name]) instead") + haskey(deprecated_opts, opt_name) && + Base.depwarn("$opt_name is deprecated, use $(deprecated_opts[opt_name]) instead", :val_opts) end - d + return d end function dlm_fill(T::DataType, offarr::Vector{Vector{Int}}, dims::NTuple{2,Integer}, has_header::Bool, sbuff::String, auto::Bool, eol::Char) From 1fa4a9832bd36d58cd7a7a41ebab10e81eff3bd0 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Mon, 11 Jul 2016 09:57:11 -0700 Subject: [PATCH 0364/1117] Deprecate no-op transpose fallback instead of error'ing. --- NEWS.md | 8 +++++++- base/deprecated.jl | 9 +++++++++ base/operators.jl | 1 - 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index b781026011a5e..0ce4d9007b03d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -247,6 +247,9 @@ Deprecated or removed * `sub` and `slice` have been deprecated in favor of `view` ([#16972]) + * The no-op `transpose` fallback has been deprecated. Consider introducing suitable + `transpose` methods or calling `permutedims(x, [2,1])` ([#13171], [#17075], [#17374]). + [PkgDev]: https://github.com/JuliaLang/PkgDev.jl <!--- generated by NEWS-update.jl: --> [#1090]: https://github.com/JuliaLang/julia/issues/1090 @@ -261,6 +264,7 @@ Deprecated or removed [#11196]: https://github.com/JuliaLang/julia/issues/11196 [#11242]: https://github.com/JuliaLang/julia/issues/11242 [#13062]: https://github.com/JuliaLang/julia/issues/13062 +[#13171]: https://github.com/JuliaLang/julia/issues/13171 [#13232]: https://github.com/JuliaLang/julia/issues/13232 [#13338]: https://github.com/JuliaLang/julia/issues/13338 [#13387]: https://github.com/JuliaLang/julia/issues/13387 @@ -309,5 +313,7 @@ Deprecated or removed [#16645]: https://github.com/JuliaLang/julia/issues/16645 [#16731]: https://github.com/JuliaLang/julia/issues/16731 [#16972]: https://github.com/JuliaLang/julia/issues/16972 -[#17266]: https://github.com/JuliaLang/julia/issues/17266 [#17037]: https://github.com/JuliaLang/julia/issues/17037 +[#17075]: https://github.com/JuliaLang/julia/issues/17075 +[#17266]: https://github.com/JuliaLang/julia/issues/17266 +[#17374]: https://github.com/JuliaLang/julia/issues/17374 diff --git a/base/deprecated.jl b/base/deprecated.jl index 10a50952a8cf7..9f0247a809e21 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -784,6 +784,15 @@ function symperm{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, pinv::Vector{Ti}) "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) end +# Deprecate no-op transpose fallback. Please see #13171 and #17075. +function transpose(x) + depwarn(string("the no-op `transpose` fallback is deprecated, and no more specific ", + "`transpose` method for $(typeof(x)) exists. Consider `permutedims(x, [2, 1])` ", + "or writing a specific `transpose(x::$(typeof(x)))` method if appropriate."), + :transpose) + return x +end + # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/operators.jl b/base/operators.jl index 27520dcb0c01b..fd997396d3fd4 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -293,7 +293,6 @@ fldmod1{T<:Real}(x::T, y::T) = (fld1(x,y), mod1(x,y)) fldmod1{T<:Integer}(x::T, y::T) = (fld1(x,y), mod1(x,y)) # transpose -transpose(x) = throw(ArgumentError("transpose not implemented for $(typeof(x)). Consider permutedims.")) ctranspose(x) = conj(transpose(x)) conj(x) = x From 78e1dc670952a6b0032725499881ba9212adea59 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Mon, 11 Jul 2016 18:47:30 -0400 Subject: [PATCH 0365/1117] LICENSE.md: list randmtzig.c file's license (BSD-3) Although this was previously included in the Rmath sources, it was not part of Rmath and had a different, more liberal license. --- LICENSE.md | 2 ++ test/perf/micro/randmtzig.c | 16 ++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 9dd48c0581898..328802613a979 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -28,11 +28,13 @@ for exceptions. > WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Julia includes code from the following projects, which have their own licenses: + - [LDC](https://github.com/ldc-developers/ldc/blob/master/LICENSE) (for ccall/cfunction ABI definitions) [BSD-3]. The portion of code that Julia uses from LDC is [BSD-3] licensed. - [LLVM](http://llvm.org/releases/3.7.0/LICENSE.TXT) (for parts of src/jitlayers.cpp and src/disasm.cpp) [BSD-3, effectively] - [MUSL](http://git.musl-libc.org/cgit/musl/tree/COPYRIGHT) (for getopt implementation on Windows) [MIT] - [MINGW](https://sourceforge.net/p/mingw/mingw-org-wsl/ci/legacy/tree/mingwrt/mingwex/dirname.c) (for dirname implementation on Windows) [MIT] - [NetBSD](http://www.netbsd.org/about/redistribution.html) (for setjmp, longjmp, and strptime implementations on Windows) [BSD-3] +- [randmtzig.c](https://github.com/JuliaLang/julia/blob/master/test/perf/micro/randmtzig.c) for Gaussian random number generation (for C benchmarks only) [BSD-3] The Julia language links to the following external libraries, which have their own licenses: diff --git a/test/perf/micro/randmtzig.c b/test/perf/micro/randmtzig.c index 05e6c27aeaaf6..1e867d659ef32 100644 --- a/test/perf/micro/randmtzig.c +++ b/test/perf/micro/randmtzig.c @@ -1,12 +1,3 @@ -/* - Parts Copyright (C) 2012 Viral B. Shah - All rights reserved. - - Modifications made for julia to support dsfmt and only __LP64__ systems. - 52-bits of randomness are used from the mantissa of random double precision - numbers generated by dsfmt. - */ - /* A C-program for MT19937, with initialization improved 2002/2/10. Coded by Takuji Nishimura and Makoto Matsumoto. @@ -50,9 +41,14 @@ Any feedback is very welcome. http://www.math.keio.ac.jp/matumoto/emt.html email: matumoto@math.keio.ac.jp - */ +/* + Modified by Viral B. Shah for julia to support dsfmt and only __LP64__ + systems. 52-bits of randomness are used from the mantissa of random double + precision numbers generated by dsfmt. + */ + #include <math.h> #include <stdio.h> #include <stddef.h> From f9daeb1f61e3c436317b805a2c8d0bf332515f31 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 11 Jul 2016 23:44:35 -0400 Subject: [PATCH 0366/1117] fix/implement `show` of comprehension and generator expressions --- base/show.jl | 46 ++++++++++++++++++++++++++++++++++++++-------- test/show.jl | 6 ++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/base/show.jl b/base/show.jl index ad2fbc86a1a8e..fe9db2c308e44 100644 --- a/base/show.jl +++ b/base/show.jl @@ -628,6 +628,27 @@ function show_unquoted_quote_expr(io::IO, value, indent::Int, prec::Int) end end +function show_generator(io, ex, indent) + if ex.head === :flatten + fg = ex + ranges = Any[] + while isa(fg, Expr) && fg.head === :flatten + push!(ranges, fg.args[1].args[2]) + fg = fg.args[1].args[1] + end + push!(ranges, fg.args[2]) + show_unquoted(io, fg.args[1], indent) + for r in ranges + print(io, " for ") + show_unquoted(io, r, indent) + end + else + show_unquoted(io, ex.args[1], indent) + print(io, " for ") + show_unquoted(io, ex.args[2], indent) + end +end + # TODO: implement interpolated strings function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) head, args, nargs = ex.head, ex.args, length(ex.args) @@ -749,24 +770,33 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) show_call(io, head, ex.args[1], ex.args[2:end], indent) # comprehensions - elseif (head === :typed_comprehension || head === :typed_dict_comprehension) && length(args) == 3 + elseif (head === :typed_comprehension || head === :typed_dict_comprehension) && length(args) == 2 isdict = (head === :typed_dict_comprehension) isdict && print(io, '(') show_unquoted(io, args[1], indent) isdict && print(io, ')') print(io, '[') - show_unquoted(io, args[2], indent) - print(io, " for ") - show_unquoted(io, args[3], indent) + show_generator(io, args[2], indent) print(io, ']') - elseif (head === :comprehension || head === :dict_comprehension) && length(args) == 2 + elseif (head === :comprehension || head === :dict_comprehension) && length(args) == 1 print(io, '[') - show_unquoted(io, args[1], indent) - print(io, " for ") - show_unquoted(io, args[2], indent) + show_generator(io, args[1], indent) print(io, ']') + elseif head === :generator && length(args) == 2 + print(io, '(') + show_generator(io, ex, indent) + print(io, ')') + + elseif head === :filter && length(args) == 2 + show_unquoted(io, args[2], indent) + print(io, " if ") + show_unquoted(io, args[1], indent) + + elseif head === :flatten && length(args) == 1 + show_generator(io, ex, indent) + elseif is(head, :ccall) show_unquoted(io, :ccall, indent) show_enclosed_list(io, '(', args, ",", ')', indent) diff --git a/test/show.jl b/test/show.jl index ce2dab19e4d38..3eadf5e1e434c 100644 --- a/test/show.jl +++ b/test/show.jl @@ -509,3 +509,9 @@ end # issue #17338 @test repr(Core.svec(1,2)) == "svec(1,2)" + +# showing generator and comprehension expressions +@test repr(:(x for x in y for z in w)) == ":(x for x = y for z = w)" +@test repr(:(x for x in y if aa for z in w if bb)) == ":(x for x = y if aa for z = w if bb)" +@test repr(:([x for x = y])) == ":([x for x = y])" +@test repr(:([x for x = y if z])) == ":([x for x = y if z])" From f120f7e41e05282c78ca14bf973304a64a05c97c Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Tue, 12 Jul 2016 00:04:05 -0400 Subject: [PATCH 0367/1117] Remove extra `JL_GC_POP` Fix #17376 --- src/gf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gf.c b/src/gf.c index 8302c9dd16865..11d9a0ed913eb 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2015,7 +2015,6 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) if (error_en) show_call(F, args, nargs); #endif - JL_GC_POP(); jl_method_error((jl_function_t*)args[0], args, nargs); // unreachable } From 45e1eafb938ff69205b952e635dc3a4b8edd07c0 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Tue, 12 Jul 2016 00:48:26 -0400 Subject: [PATCH 0368/1117] Fix array copy write barrier check To take into account objects being promoted..... --- src/array.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/array.c b/src/array.c index d403f450d773a..f1f8123df55b5 100644 --- a/src/array.c +++ b/src/array.c @@ -965,8 +965,9 @@ JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p, // Destination is old and doesn't refer to any young object if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { jl_value_t *src_owner = jl_array_owner(src); - // Source is young or might refer to young objects - if (!(jl_astaggedvalue(src_owner)->bits.gc & GC_OLD)) { + // Source is young or being promoted or might refer to young objects + // (i.e. source is not an old object that doesn't have wb triggered) + if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) { ssize_t done; if (dest_p < src_p || dest_p > src_p + n) { done = jl_array_ptr_copy_forward(owner, src_p, dest_p, n); From 8dbdfdda4c7efb737784b24b528c4960040ada51 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 4 Jul 2016 12:01:52 -0700 Subject: [PATCH 0369/1117] remove redundant !! --- src/dump.c | 6 +++--- src/toplevel.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dump.c b/src/dump.c index 4ceb59fde78b8..c6dd9176ea905 100644 --- a/src/dump.c +++ b/src/dump.c @@ -556,10 +556,10 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) } write_int32(s, dt->size); - int has_instance = !!(dt->instance != NULL); - int has_layout = !!(dt->layout != NULL); + int has_instance = (dt->instance != NULL); + int has_layout = (dt->layout != NULL); write_uint8(s, dt->abstract | (dt->mutabl<<1) | (has_layout<<2) | (has_instance<<3) | - (dt->hastypevars<<4) | (dt->haswildcard<<5) | (dt->isleaftype<<6)); + (dt->hastypevars<<4) | (dt->haswildcard<<5) | (dt->isleaftype<<6)); write_int32(s, dt->depth); if (!dt->abstract) { write_uint16(s, dt->ninitialized); diff --git a/src/toplevel.c b/src/toplevel.c index 23db71a5e0fb6..5a0a7a682a333 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -103,7 +103,7 @@ static void jl_module_load_time_initialize(jl_module_t *m) jl_module_init_order = jl_alloc_vec_any(0); jl_array_ptr_1d_push(jl_module_init_order, (jl_value_t*)m); jl_function_t *f = jl_module_get_initializer(m); - if (f != NULL) { + if (f != NULL) { jl_value_t *tt = jl_is_type(f) ? (jl_value_t*)jl_wrap_Type(f) : jl_typeof(f); JL_GC_PUSH1(&tt); tt = (jl_value_t*)jl_apply_tuple_type_v(&tt, 1); From 61a2b19660109feac9f1545e24743e43615f2c2c Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 4 Jul 2016 12:08:47 -0700 Subject: [PATCH 0370/1117] remove signature that docstring for `base` is attached to so repl help appears for more general inputs --- base/docs/helpdb/Base.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 2b23cd6f399df..696c3e69bd2d4 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4083,7 +4083,7 @@ rand Convert an integer to a string in the given base, optionally specifying a number of digits to pad to. """ -base(base, n, pad) +base """ BoundsError([a],[i]) From d0e34cd2069f0806a2ce51ea53abf09bd15b41d0 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 4 Jul 2016 20:32:07 -0700 Subject: [PATCH 0371/1117] Add a comma to the `require` docstring --- base/docs/helpdb/Base.jl | 6 +++--- doc/stdlib/base.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 696c3e69bd2d4..39ed4d59ea485 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1455,9 +1455,9 @@ current `include` path but does not use it to search for files (see help for `in This function is typically used to load library code, and is implicitly called by `using` to load packages. -When searching for files, `require` first looks for package code under `Pkg.dir()`, then tries -paths in the global array `LOAD_PATH`. `require` is case-sensitive on all -platforms including those with case-insensitive filesystems like macOS and +When searching for files, `require` first looks for package code under `Pkg.dir()`, +then tries paths in the global array `LOAD_PATH`. `require` is case-sensitive on +all platforms, including those with case-insensitive filesystems like macOS and Windows. """ require diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 320539cabed42..31d8e48433e1b 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -126,7 +126,7 @@ Getting Around Loads a source files, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``\ ). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. - When searching for files, ``require`` first looks for package code under ``Pkg.dir()``\ , then tries paths in the global array ``LOAD_PATH``\ . ``require`` is case-sensitive on all platforms including those with case-insensitive filesystems like macOS and Windows. + When searching for files, ``require`` first looks for package code under ``Pkg.dir()``\ , then tries paths in the global array ``LOAD_PATH``\ . ``require`` is case-sensitive on all platforms, including those with case-insensitive filesystems like macOS and Windows. .. function:: Base.compilecache(module::String) From 6aa193f38e3897d593c18874cbdbbce62d4282e6 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 7 Jul 2016 12:24:50 -0700 Subject: [PATCH 0372/1117] Remove noisy 'not-64-bit' warning from test/ccall.jl --- test/ccall.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ccall.jl b/test/ccall.jl index 8600a305ae177..7f0792a87fdd5 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -845,7 +845,7 @@ elseif Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le (14, 13, 12, 11), (15, 14, 13, 12), (16, 15, 14, 13), (17, 16, 15, 14), (18, 17, 16, 15), (1024, 1023, 1022, 1021), (1025, 1024, 1023, 1022), (1026, 1025, 1024, 1023), (1027, 1026, 1025, 1024), (10028, 10027, 10026, 10025)) -else +elseif Sys.ARCH !== :i686 && Sys.ARCH !== :arm # TODO warn("ccall: no VecReg tests run for this platform") end From 01763f636dd011a4ac62a40d3e3971aee682b626 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 7 Jul 2016 14:32:22 -0700 Subject: [PATCH 0373/1117] Fix backslash in A_ldiv_B! docstring --- base/linalg/factorization.jl | 2 +- doc/stdlib/linalg.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/linalg/factorization.jl b/base/linalg/factorization.jl index 49254d2638dc1..2a2340e4c1f0c 100644 --- a/base/linalg/factorization.jl +++ b/base/linalg/factorization.jl @@ -51,7 +51,7 @@ end """ A_ldiv_B!([Y,] A, B) -> Y -Compute `A \ B` in-place and store the result in `Y`, returning the result. +Compute `A \\ B` in-place and store the result in `Y`, returning the result. If only two arguments are passed, then `A_ldiv_B!(A, B)` overwrites `B` with the result. diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 0cdf61ae0e7a3..c671d355c2775 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1473,7 +1473,7 @@ according to the usual Julia convention. .. Docstring generated from Julia source - Compute ``A B`` in-place and store the result in ``Y``\ , returning the result. If only two arguments are passed, then ``A_ldiv_B!(A, B)`` overwrites ``B`` with the result. + Compute ``A \ B`` in-place and store the result in ``Y``\ , returning the result. If only two arguments are passed, then ``A_ldiv_B!(A, B)`` overwrites ``B`` with the result. The argument ``A`` should *not* be a matrix. Rather, instead of matrices it should be a factorization object (e.g. produced by :func:`factorize` or :func:`cholfact`\ ). The reason for this is that factorization itself is both expensive and typically allocates memory (although it can also be done in-place via, e.g., :func:`lufact!`\ ), and performance-critical situations requiring ``A_ldiv_B!`` usually also require fine-grained control over the factorization of ``A``\ . From 7444d00c4a2437e6916d97ba1429242f70a38e6a Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 7 Jul 2016 14:33:08 -0700 Subject: [PATCH 0374/1117] Remove "a" in "functions filter and a filter!" in noteworthy-differences --- base/range.jl | 4 ++-- doc/manual/noteworthy-differences.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/range.jl b/base/range.jl index 335f2d89cb7d1..2c819e05c5e40 100644 --- a/base/range.jl +++ b/base/range.jl @@ -661,7 +661,7 @@ end -(r::FloatRange) = FloatRange(-r.start, -r.step, r.len, r.divisor) -(r::LinSpace) = LinSpace(-r.start, -r.stop, r.len, r.divisor) -.+(x::Real, r::AbstractUnitRange) = range(x + first(r), length(r)) +.+(x::Real, r::AbstractUnitRange) = range(x + first(r), length(r)) .+(x::Real, r::Range) = (x+first(r)):step(r):(x+last(r)) #.+(x::Real, r::StepRange) = range(x + r.start, r.step, length(r)) .+(x::Real, r::FloatRange) = FloatRange(r.divisor*x + r.start, r.step, r.len, r.divisor) @@ -678,7 +678,7 @@ function .-(x::Real, r::LinSpace) x2 = x * r.divisor / (r.len - 1) LinSpace(x2 - r.start, x2 - r.stop, r.len, r.divisor) end -.-(r::AbstractUnitRange, x::Real) = range(first(r)-x, length(r)) +.-(r::AbstractUnitRange, x::Real) = range(first(r)-x, length(r)) .-(r::StepRange , x::Real) = range(r.start-x, r.step, length(r)) .-(r::FloatRange, x::Real) = FloatRange(r.start - r.divisor*x, r.step, r.len, r.divisor) function .-(r::LinSpace, x::Real) diff --git a/doc/manual/noteworthy-differences.rst b/doc/manual/noteworthy-differences.rst index f5c539393266d..27ac14abfe3de 100644 --- a/doc/manual/noteworthy-differences.rst +++ b/doc/manual/noteworthy-differences.rst @@ -103,7 +103,7 @@ some noteworthy differences that may trip up Julia users accustomed to MATLAB: - In MATLAB, an idiomatic way to remove unwanted values is to use logical indexing, like in the expression ``x(x>3)`` or in the statement ``x(x>3) = []`` to modify ``x`` in-place. In contrast, Julia provides the - higher order functions :func:`filter` and a :func:`filter!`, allowing users + higher order functions :func:`filter` and :func:`filter!`, allowing users to write ``filter(z->z>3, x)`` and ``filter!(z->z>3, x)`` as alternatives to the corresponding transliterations ``x[x.>3]`` and ``x = x[x.>3]``. Using :func:`filter!` reduces the use of temporary arrays. @@ -235,7 +235,7 @@ noteworthy differences: - In R, an idiomatic way to remove unwanted values is to use logical indexing, like in the expression ``x[x>3]`` or in the statement ``x = x[x>3]`` to modify ``x`` in-place. In contrast, Julia provides the higher order functions - :func:`filter` and a :func:`filter!`, allowing users to write + :func:`filter` and :func:`filter!`, allowing users to write ``filter(z->z>3, x)`` and ``filter!(z->z>3, x)`` as alternatives to the corresponding transliterations ``x[x.>3]`` and ``x = x[x.>3]``. Using :func:`filter!` reduces the use of temporary arrays. From 11b971d087108b9d501dc78084d8682c5f255e73 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sun, 10 Jul 2016 10:48:37 -0700 Subject: [PATCH 0375/1117] Fix sphinx warnings about duplicate targets --- doc/manual/dates.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/dates.rst b/doc/manual/dates.rst index 08b5ba148b228..e23df78283255 100644 --- a/doc/manual/dates.rst +++ b/doc/manual/dates.rst @@ -389,7 +389,7 @@ default, the :class:`TimeType` :func:`round` method uses the ``RoundNearestTiesU rounding mode. (It's difficult to guess what breaking ties to nearest "even" :class:`TimeType` would entail.) Further details on the available ``RoundingMode`` s can be found in the -`API reference <http://docs.julialang.org/en/latest/stdlib/dates/#rounding-functions>`_. +`API reference <http://docs.julialang.org/en/latest/stdlib/dates/>`_. Rounding should generally behave as expected, but there are a few cases in which the expected behaviour is not obvious. From b1b44259d83cbb3c7e4c2cdfa19061f43330b956 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sun, 10 Jul 2016 10:49:08 -0700 Subject: [PATCH 0376/1117] Revert "Added performace tip for parentheses in expressions to docs" This reverts commit d459b1c7b715c6bbc47e94b8a5368abe1dc1bdac. The wording is misleading and the RST is malformed Revert "Double backticks and topic title fix" This reverts commit 7a27c60fa744fa0fe5719c96f952661f780e1e05. --- doc/manual/performance-tips.rst | 45 --------------------------------- 1 file changed, 45 deletions(-) diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index 2c88ae20a0703..eb092420787bc 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -947,51 +947,6 @@ Taken to its extreme, pre-allocation can make your code uglier, so performance measurements and some judgment may be required. -Use parentheses in long arithmetic operations ---------------------------------------------- - -If your code has a long arithmetic operation involving ``+`` or ``*`` operators, -then consider using parentheses to chain up to four or five operations together. -This is to avoid the splatting penalty from a longer list of arguments to function -return statement. - -Without parentheses: - - const k = zeros(20) - function test_mem() - return k[1] + k[2] + k[3] + k[4] + k[5] + 2.0 * k[6] + k[7] + k[8] + k[9] + k[10] - end - - function test(n::Int64) - ret = 0.0 - for i = 1:n - ret += test_mem() - end - ret - end - @time test(100000000) - - 5.017971 seconds (900.00 M allocations: 13.411 GB, 15.04% gc time) - -With: - - const k = zeros(20) - function test_mem() - return (k[1] + k[2] + k[3] + k[4] + k[5]) + 2.0 * k[6] + k[7] + k[8] + k[9] + k[10] - end - - function test(n::Int64) - ret = 0.0 - for i = 1:n - ret += test_mem() - end - ret - end - @time test(100000000) - - 0.302478 seconds (5.26 k allocations: 248.985 KB) - - Avoid string interpolation for I/O ---------------------------------- From 1edb00cb052021fd42f5b9dacee7a62455e4ae91 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sun, 10 Jul 2016 10:58:28 -0700 Subject: [PATCH 0377/1117] Remove at-MIME from exports, was deleted --- base/exports.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/base/exports.jl b/base/exports.jl index 8641e82cd8fba..4cdc9e4739042 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1216,7 +1216,6 @@ export TextDisplay, istextmime, MIME, - @MIME, @MIME_str, reprmime, stringmime, From fcf8e6b208d2604a044e6a9b4f9bc4c26dd0f4cd Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 8 Jul 2016 21:38:55 -0700 Subject: [PATCH 0378/1117] NEWS links and formatting up to and including 0.5-style comprehensions --- NEWS.md | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/NEWS.md b/NEWS.md index 0ce4d9007b03d..757a8f2776874 100644 --- a/NEWS.md +++ b/NEWS.md @@ -54,18 +54,19 @@ Language changes Instead of adding methods to `call`, methods are added by type using the syntax `(::ftype)(...) = ...`. `call` is deprecated ([#13412]). - * `using` and `import` are now case-sensitive even on case-insensitive filesystems (common on Mac and Windows) ([#13542]). + * `using` and `import` are now case-sensitive even on case-insensitive filesystems + (common on Mac and Windows) ([#13542]). * Relational symbols are now allowed as infix operators ([#8036]). - * A warning is always given when a method is overwritten (previously, this was done only when the new - and old definitions were in separate modules) ([#14759]). + * A warning is always given when a method is overwritten (previously, this was done + only when the new and old definitions were in separate modules) ([#14759]). - * `A <: B` is parsed as `Expr(:(<:), :A, :B)` in all cases ([#9503]). This also applies to the - `>:` operator. + * `A <: B` is parsed as `Expr(:(<:), :A, :B)` in all cases ([#9503]). + This also applies to the `>:` operator. * Simple 2-argument comparisons like `A < B` are parsed as calls instead of using the - `:comparison` expression type. + `:comparison` expression type ([#15524]). * The `if` keyword cannot be followed immediately by a line break ([#15763]). @@ -73,7 +74,7 @@ Language changes implemented internally as `Tuple{Vararg{T,N}}` ([#11242]). * Array comprehensions preserve the dimensions of the input ranges. For example, - `[ 2x for x in A]` will have the same dimensions as `A`. + `[2x for x in A]` will have the same dimensions as `A` ([#16622]). * The result type of an array comprehension depends only on the types of elements computed, instead of using type inference ([#7258]). If the result is empty, then @@ -97,7 +98,7 @@ Breaking changes * `pmap` keyword arguments `err_retry=true` and `err_stop=false` are deprecated. Action to be taken on errors can be specified via the `on_error` keyword argument. - Retry is specified via `retry_n`, `retry_on` and `retry_max_delay`. + Retry is specified via `retry_n`, `retry_on` and `retry_max_delay` ([#15409], [#15975], [#16663]). * `reshape` is now defined to always share data with the original array. If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of @@ -163,7 +164,8 @@ Library improvements * All dimensions indexed by scalars are now dropped, whereas previously only trailing scalar dimensions would be omitted from the result ([#13612]). - * Dimensions indexed by multidimensional arrays add dimensions. More generally, the dimensionality of the result is the sum of the dimensionalities of the indices ([#15431]). + * Dimensions indexed by multidimensional arrays add dimensions. More generally, the + dimensionality of the result is the sum of the dimensionalities of the indices ([#15431]). * New `normalize` and `normalize!` convenience functions for normalizing vectors ([#13681]). @@ -180,13 +182,13 @@ Library improvements vector instead of a one-column sparse matrix. ([#13440]) * Rank one update and downdate functions, `lowrankupdate`, `lowrankupdate!`, `lowrankdowndate`, - and `lowrankdowndate!`, for dense Cholesky factorizations ([#14243], [#14424]) + and `lowrankdowndate!`, for dense Cholesky factorizations ([#14243], [#14424]) * All `sparse` methods now retain provided numerical zeros as structural nonzeros; to drop numerical zeros, use `dropzeros!` ([#14798], [#15242]). * New `foreach` function for calling a function on every element of a collection when - the results are not needed. + the results are not needed ([#13774]). * `Cmd(cmd; ...)` now accepts new Windows-specific options `windows_verbatim` (to alter Windows command-line generation) and `windows_hide` (to @@ -208,10 +210,11 @@ Library improvements directory, instead of the private `<prefix>/lib/julia` directory ([#16362]). * System reflection is now more consistently exposed from Sys and not Base. - `OS_NAME` has been replaced by `Sys.KERNEL` and always reports the name of the kernel (as reported by `uname`). - The `@windows_only` and `@osx` family of macros have been replaced with functions such as `is_windows()` and - or `is_apple()`. There's now also an `@static` macro that will evaluate the condition of an if-statement at - compile time, for when a static branch is required ([#16219]). + `OS_NAME` has been replaced by `Sys.KERNEL` and always reports the name of the + kernel (as reported by `uname`). The `@windows_only` and `@osx` family of macros + have been replaced with functions such as `is_windows()` and `is_apple()`. + There's now also an `@static` macro that will evaluate the condition of an + if-statement at compile time, for when a static branch is required ([#16219]). * Prime number related functions have been moved from `Base` to the [Primes.jl package](https://github.com/JuliaMath/Primes.jl) ([#16481]). @@ -238,7 +241,8 @@ Deprecated or removed * Deprecate `chol(A,Val{:U/:L})` in favor of `chol(A)` ([#13680]). - * `issym` is deprecated in favor of `issymmetric` to match similar functions (`ishermitian`, ...) ([#15192]) + * `issym` is deprecated in favor of `issymmetric` to match similar functions + (`ishermitian`, ...) ([#15192]) * `scale` is deprecated in favor of either `α*A`, `Diagonal(x)*A`, or `A*Diagonal(x)`. ([#15258]) @@ -252,11 +256,14 @@ Deprecated or removed [PkgDev]: https://github.com/JuliaLang/PkgDev.jl <!--- generated by NEWS-update.jl: --> +[#550]: https://github.com/JuliaLang/julia/issues/550 [#1090]: https://github.com/JuliaLang/julia/issues/1090 [#4163]: https://github.com/JuliaLang/julia/issues/4163 [#4211]: https://github.com/JuliaLang/julia/issues/4211 [#4470]: https://github.com/JuliaLang/julia/issues/4470 +[#4867]: https://github.com/JuliaLang/julia/issues/4867 [#6190]: https://github.com/JuliaLang/julia/issues/6190 +[#7258]: https://github.com/JuliaLang/julia/issues/7258 [#8036]: https://github.com/JuliaLang/julia/issues/8036 [#8846]: https://github.com/JuliaLang/julia/issues/8846 [#9503]: https://github.com/JuliaLang/julia/issues/9503 @@ -277,6 +284,7 @@ Deprecated or removed [#13612]: https://github.com/JuliaLang/julia/issues/13612 [#13680]: https://github.com/JuliaLang/julia/issues/13680 [#13681]: https://github.com/JuliaLang/julia/issues/13681 +[#13774]: https://github.com/JuliaLang/julia/issues/13774 [#13780]: https://github.com/JuliaLang/julia/issues/13780 [#13824]: https://github.com/JuliaLang/julia/issues/13824 [#13897]: https://github.com/JuliaLang/julia/issues/13897 @@ -295,11 +303,14 @@ Deprecated or removed [#15242]: https://github.com/JuliaLang/julia/issues/15242 [#15244]: https://github.com/JuliaLang/julia/issues/15244 [#15258]: https://github.com/JuliaLang/julia/issues/15258 +[#15409]: https://github.com/JuliaLang/julia/issues/15409 [#15431]: https://github.com/JuliaLang/julia/issues/15431 +[#15524]: https://github.com/JuliaLang/julia/issues/15524 [#15550]: https://github.com/JuliaLang/julia/issues/15550 [#15609]: https://github.com/JuliaLang/julia/issues/15609 [#15731]: https://github.com/JuliaLang/julia/issues/15731 [#15763]: https://github.com/JuliaLang/julia/issues/15763 +[#15975]: https://github.com/JuliaLang/julia/issues/15975 [#16058]: https://github.com/JuliaLang/julia/issues/16058 [#16107]: https://github.com/JuliaLang/julia/issues/16107 [#16219]: https://github.com/JuliaLang/julia/issues/16219 @@ -310,7 +321,9 @@ Deprecated or removed [#16455]: https://github.com/JuliaLang/julia/issues/16455 [#16481]: https://github.com/JuliaLang/julia/issues/16481 [#16621]: https://github.com/JuliaLang/julia/issues/16621 +[#16622]: https://github.com/JuliaLang/julia/issues/16622 [#16645]: https://github.com/JuliaLang/julia/issues/16645 +[#16663]: https://github.com/JuliaLang/julia/issues/16663 [#16731]: https://github.com/JuliaLang/julia/issues/16731 [#16972]: https://github.com/JuliaLang/julia/issues/16972 [#17037]: https://github.com/JuliaLang/julia/issues/17037 From 7368d6173f5d26d216858dc44df842b549aef8ef Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 11 Jul 2016 16:17:12 -0700 Subject: [PATCH 0379/1117] Update contrib/add_license_to_files.jl Add test/perf/micro/randmtzig.c and src/getopt.{c,h} and rerun --- base/clusterserialize.jl | 1 + base/dates/rounding.jl | 2 ++ base/sharedarray.jl | 1 + contrib/add_license_to_files.jl | 3 +++ src/jloptions.c | 2 ++ test/cartesian.jl | 2 ++ test/dates/rounding.jl | 2 ++ 7 files changed, 13 insertions(+) diff --git a/base/clusterserialize.jl b/base/clusterserialize.jl index 11d6b1849afa6..f8505ca9b7ae5 100644 --- a/base/clusterserialize.jl +++ b/base/clusterserialize.jl @@ -1,4 +1,5 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license + import .Serializer: known_object_data, object_number, serialize_cycle, deserialize_cycle, writetag, __deserialized_types__, serialize_typename_body, deserialize_typename_body, TYPENAME_TAG, object_numbers diff --git a/base/dates/rounding.jl b/base/dates/rounding.jl index e9a08c6fc13c2..576ad44fe7fc9 100644 --- a/base/dates/rounding.jl +++ b/base/dates/rounding.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + # The epochs used for date rounding are based ISO 8601's "year zero" notation const DATEEPOCH = value(Date(0)) const DATETIMEEPOCH = value(DateTime(0)) diff --git a/base/sharedarray.jl b/base/sharedarray.jl index 894c9eb7358b5..9b527f9a1b4e6 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -1,4 +1,5 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license + import .Serializer: serialize_cycle, serialize_type, writetag, UNDEFREF_TAG type SharedArray{T,N} <: DenseArray{T,N} diff --git a/contrib/add_license_to_files.jl b/contrib/add_license_to_files.jl index 3ee803b132035..1ded19a34f31b 100644 --- a/contrib/add_license_to_files.jl +++ b/contrib/add_license_to_files.jl @@ -43,6 +43,8 @@ const skipfiles = [ "../src/abi_x86.cpp", "../src/abi_x86_64.cpp", "../src/disasm.cpp", + "../src/getopt.c", + "../src/getopt.h", "../src/support/END.h", "../src/support/ENTRY.amd64.h", "../src/support/ENTRY.i387.h", @@ -54,6 +56,7 @@ const skipfiles = [ "../src/support/strtod.c", "../src/support/tzfile.h", "../src/support/utf8.c", + "../test/perf/micro/randmtzig.c", ] const ext_prefix = Dict([ diff --git a/src/jloptions.c b/src/jloptions.c index 6265e100b3afe..ab91b109a3cb5 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -1,3 +1,5 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + #include <limits.h> #include "julia.h" diff --git a/test/cartesian.jl b/test/cartesian.jl index 907905aff13b3..3180a4f4dccfc 100644 --- a/test/cartesian.jl +++ b/test/cartesian.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + @test Base.Cartesian.exprresolve(:(1 + 3)) == 4 ex = Base.Cartesian.exprresolve(:(if 5 > 4; :x; else :y; end)) @test ex.args[2] == QuoteNode(:x) diff --git a/test/dates/rounding.jl b/test/dates/rounding.jl index 80218fcf0db47..1d6ec265b80ed 100644 --- a/test/dates/rounding.jl +++ b/test/dates/rounding.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + # Test conversion to and from the rounding epoch (ISO 8601 year 0000) @test Dates.epochdays2date(-1) == Dates.Date(-1, 12, 31) @test Dates.epochdays2date(0) == Dates.Date(0, 1, 1) From 7d1cc59e0a3b631248d5258890bc4998b631f994 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 11 Jul 2016 17:35:02 -0700 Subject: [PATCH 0380/1117] Fix compiler warning on Windows --- ui/repl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/repl.c b/ui/repl.c index db0cacf84fbd3..b0ae9c03c1ccd 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -223,7 +223,7 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) } #endif libsupport_init(); - if (argc >= 2 && strcmp(argv[1],"--lisp") == 0) { + if (argc >= 2 && strcmp((char*)argv[1],"--lisp") == 0) { jl_lisp_prompt(); return 0; } From e0aafe21b3b296db1980ba6423ed55c502b60c76 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 12 Jul 2016 01:17:09 -0700 Subject: [PATCH 0381/1117] remove last mentions of type_goto and static_typeof --- doc/devdocs/ast.rst | 8 -------- src/common_symbols2.inc | 2 -- 2 files changed, 10 deletions(-) diff --git a/doc/devdocs/ast.rst b/doc/devdocs/ast.rst index d2699c0428d62..72f1ca4774a36 100644 --- a/doc/devdocs/ast.rst +++ b/doc/devdocs/ast.rst @@ -135,14 +135,6 @@ These symbols appear in the ``head`` field of ``Expr``\s in lowered form. ``null`` has no arguments; simply yields the value ``nothing`` -``static_typeof`` - a horrible misfeature used to determine the result type of array - comprehensions. Planned to be removed. - -``type_goto`` - a virtual control flow edge used to convey type data to ``static_typeof``, - also to be removed. - ``new`` allocates a new struct-like object. First argument is the type. The ``new`` pseudo-function is lowered to this, and the type is always inserted by the diff --git a/src/common_symbols2.inc b/src/common_symbols2.inc index 8a0874f720585..a53732df4e3bb 100644 --- a/src/common_symbols2.inc +++ b/src/common_symbols2.inc @@ -70,11 +70,9 @@ jl_symbol("#print_to_string#138"), jl_symbol("Main"), jl_symbol("pointer.jl"), jl_symbol("DimensionMismatch"), -jl_symbol("type_goto"), jl_symbol("stride"), jl_symbol("uP"), jl_symbol("III"), -jl_symbol("static_typeof"), jl_symbol("uW"), jl_symbol("checked_trunc_uint"), jl_symbol("sparse/sparsematrix.jl"), From 3b81e8b72e45115a63d27b1018413e736584dacd Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 12 Jul 2016 02:21:41 -0700 Subject: [PATCH 0382/1117] Remove ignore_invalid_chars from docs, it's now deprecated --- base/docs/helpdb/Base.jl | 3 --- doc/stdlib/io-network.rst | 2 -- 2 files changed, 5 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 39ed4d59ea485..19d0b132fcbbf 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -3523,9 +3523,6 @@ If `use_mmap` is `true`, the file specified by `source` is memory mapped for pot speedups. Default is `true` except on Windows. On Windows, you may want to specify `true` if the file is large, and is only read once and not written to. -If `ignore_invalid_chars` is `true`, bytes in `source` with invalid character encoding will -be ignored. Otherwise an error is thrown indicating the offending character position. - If `quotes` is `true`, columns enclosed within double-quote (\") characters are allowed to contain new lines and column delimiters. Double-quote characters within a quoted field must be escaped with another double-quote. Specifying `dims` as a tuple of the expected rows and diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 6a78e42592b52..936258c7ac119 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -574,8 +574,6 @@ Text I/O If ``use_mmap`` is ``true``\ , the file specified by ``source`` is memory mapped for potential speedups. Default is ``true`` except on Windows. On Windows, you may want to specify ``true`` if the file is large, and is only read once and not written to. - If ``ignore_invalid_chars`` is ``true``\ , bytes in ``source`` with invalid character encoding will be ignored. Otherwise an error is thrown indicating the offending character position. - If ``quotes`` is ``true``\ , columns enclosed within double-quote (") characters are allowed to contain new lines and column delimiters. Double-quote characters within a quoted field must be escaped with another double-quote. Specifying ``dims`` as a tuple of the expected rows and columns (including header, if any) may speed up reading of large files. If ``comments`` is ``true``\ , lines beginning with ``comment_char`` and text following ``comment_char`` in any line are ignored. .. function:: readdlm(source, delim::Char, eol::Char; options...) From 78cd7762ae36250339460853b4a3ea4aa15b66ea Mon Sep 17 00:00:00 2001 From: Carlo Baldassi <carlobaldassi@gmail.com> Date: Sun, 26 Jun 2016 10:38:52 -0400 Subject: [PATCH 0383/1117] Add Pkg.update(pkgs...) functionality allows partial package updates --- base/pkg/entry.jl | 16 ++++++++++++---- base/pkg/pkg.jl | 2 +- base/pkg/query.jl | 32 ++++++++++++++++++++++++++++---- base/pkg/read.jl | 8 ++++---- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 2adc1758a2693..e8f02e4215b5b 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -349,7 +349,7 @@ function pin(pkg::AbstractString, ver::VersionNumber) pin(pkg, avail[ver].sha1) end -function update(branch::AbstractString) +function update(branch::AbstractString, upkgs::Set{String}) info("Updating METADATA...") with(GitRepo, "METADATA") do repo try @@ -390,7 +390,14 @@ function update(branch::AbstractString) end end instd = Read.installed(avail) - free = Read.free(instd) + reqs = Reqs.parse("REQUIRE") + if !isempty(upkgs) + for (pkg, (v,f)) in instd + satisfies(pkg, v, reqs) || throw(PkgError("Package $pkg: current package status does not satisfy the requirements, cannot do a partial update; use `Pkg.update()`")) + end + end + dont_update = Query.partial_update_mask(instd, avail, upkgs) + free = Read.free(instd,dont_update) for (pkg,ver) in free try Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) @@ -400,10 +407,11 @@ function update(branch::AbstractString) end end creds = LibGit2.CachedCredentials() - fixed = Read.fixed(avail,instd) + fixed = Read.fixed(avail,instd,dont_update) stopupdate = false for (pkg,ver) in fixed ispath(pkg,".git") || continue + pkg in dont_update && continue with(GitRepo, pkg) do repo if LibGit2.isattached(repo) if LibGit2.isdirty(repo) @@ -443,7 +451,7 @@ function update(branch::AbstractString) end end info("Computing changes...") - resolve(Reqs.parse("REQUIRE"), avail, instd, fixed, free) + resolve(reqs, avail, instd, fixed, free) # Don't use instd here since it may have changed updatehook(sort!(collect(keys(installed())))) diff --git a/base/pkg/pkg.jl b/base/pkg/pkg.jl index d0cc1087739fe..73d93a3376f37 100644 --- a/base/pkg/pkg.jl +++ b/base/pkg/pkg.jl @@ -204,7 +204,7 @@ Update the metadata repo – kept in `Pkg.dir("METADATA")` – then update any f that can safely be pulled from their origin; then call `Pkg.resolve()` to determine a new optimal set of packages versions. """ -update() = cd(Entry.update,Dir.getmetabranch()) +update(upkgs::AbstractString...) = cd(Entry.update,Dir.getmetabranch(),Set{String}([upkgs...])) """ resolve() diff --git a/base/pkg/query.jl b/base/pkg/query.jl index 2ae2f33c06984..1f1caece9ff49 100644 --- a/base/pkg/query.jl +++ b/base/pkg/query.jl @@ -66,6 +66,30 @@ function dependencies(avail::Dict, fix::Dict = Dict{String,Fixed}("julia"=>Fixed avail, conflicts end +function partial_update_mask(instd::Dict{String,Tuple{VersionNumber,Bool}}, avail::Dict{String,Dict{VersionNumber,Available}}, upkgs::Set{String}) + dont_update = Set{String}() + isempty(upkgs) && return dont_update + avail_new = deepcopy(avail) + for p in upkgs + haskey(instd, p) || throw(PkgError("Package $p is not installed")) + v = instd[p][1] + if haskey(avail, p) + for vn in keys(avail[p]) + vn < v && delete!(avail_new[p], vn) + end + end + end + avail_new = dependencies_subset(avail_new, upkgs) + + for p in keys(avail) + !haskey(avail_new, p) && push!(dont_update, p) + end + for (p,_) in instd + !haskey(avail_new, p) && !(p in upkgs) && push!(dont_update, p) + end + return dont_update +end + typealias PackageState Union{Void,VersionNumber} function diff(have::Dict, want::Dict, avail::Dict, fixed::Dict) @@ -324,12 +348,12 @@ end # Build a subgraph incuding only the (direct and indirect) dependencies # of a given package set function dependencies_subset(deps::Dict{String,Dict{VersionNumber,Available}}, pkgs::Set{String}) - staged = pkgs - allpkgs = copy(pkgs) + staged::Set{String} = filter(p->p in keys(deps), pkgs) + allpkgs = copy(staged) while !isempty(staged) staged_next = Set{String}() - for p in staged, a in values(deps[p]), rp in keys(a.requires) - if !(rp in allpkgs) + for p in staged, a in values(get(deps, p, Dict{VersionNumber,Available}())), rp in keys(a.requires) + if !(rp in allpkgs) && rp ≠ "julia" push!(staged_next, rp) end end diff --git a/base/pkg/read.jl b/base/pkg/read.jl index 842d083d5aa3a..faf2a91301006 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -216,11 +216,11 @@ function installed(avail::Dict=available()) return pkgs end -function fixed(avail::Dict=available(), inst::Dict=installed(avail), +function fixed(avail::Dict=available(), inst::Dict=installed(avail), dont_update::Set{String}=Set{String}(), julia_version::VersionNumber=VERSION) pkgs = Dict{String,Fixed}() for (pkg,(ver,fix)) in inst - fix || continue + (fix || pkg in dont_update) || continue ap = get(avail,pkg,Dict{VersionNumber,Available}()) pkgs[pkg] = Fixed(ver,requires_dict(pkg,ap)) end @@ -228,10 +228,10 @@ function fixed(avail::Dict=available(), inst::Dict=installed(avail), return pkgs end -function free(inst::Dict=installed()) +function free(inst::Dict=installed(), dont_update::Set{String}=Set{String}()) pkgs = Dict{String,VersionNumber}() for (pkg,(ver,fix)) in inst - fix && continue + (fix || pkg in dont_update) && continue pkgs[pkg] = ver end return pkgs From 5d84340c4473467689bb931e56eddebaa17ee633 Mon Sep 17 00:00:00 2001 From: Carlo Baldassi <carlobaldassi@gmail.com> Date: Sun, 26 Jun 2016 10:40:50 -0400 Subject: [PATCH 0384/1117] Emit a message in case partial Pkg.update fails --- base/pkg/entry.jl | 9 +++++++- base/pkg/query.jl | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index e8f02e4215b5b..16a20750019bc 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -451,7 +451,7 @@ function update(branch::AbstractString, upkgs::Set{String}) end end info("Computing changes...") - resolve(reqs, avail, instd, fixed, free) + resolve(reqs, avail, instd, fixed, free, upkgs) # Don't use instd here since it may have changed updatehook(sort!(collect(keys(installed())))) @@ -467,7 +467,9 @@ function resolve( instd :: Dict = Read.installed(avail), fixed :: Dict = Read.fixed(avail,instd), have :: Dict = Read.free(instd), + upkgs :: Set{String} = Set{String}() ) + orig_reqs = reqs reqs = Query.requirements(reqs,fixed,avail) deps, conflicts = Query.dependencies(avail,fixed) @@ -488,6 +490,11 @@ function resolve( deps = Query.prune_dependencies(reqs,deps) want = Resolve.resolve(reqs,deps) + if !isempty(upkgs) + orig_deps, _ = Query.dependencies(avail) + Query.check_partial_updates(orig_reqs,orig_deps,want,fixed,upkgs) + end + # compare what is installed with what should be changes = Query.diff(have, want, avail, fixed) isempty(changes) && return info("No packages to install, update or remove") diff --git a/base/pkg/query.jl b/base/pkg/query.jl index 1f1caece9ff49..b44db7a68b3f1 100644 --- a/base/pkg/query.jl +++ b/base/pkg/query.jl @@ -90,6 +90,62 @@ function partial_update_mask(instd::Dict{String,Tuple{VersionNumber,Bool}}, avai return dont_update end +# Try to produce some helpful message in case of a partial update which does not go all the way +# (Does not do a full analysis, it only checks requirements and direct dependents.) +function check_partial_updates(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}, want::Dict{String,VersionNumber}, fixed::Dict{String,Fixed}, upkgs::Set{String}) + for p in upkgs + if !haskey(want, p) + if !haskey(fixed, p) + warn("Something went wrong with the update of package $p, please submit a bug report") + continue + end + v = fixed[p].version + else + v = want[p] + if haskey(fixed, p) && v != fixed[p].version + warn("Something went wrong with the update of package $p, please submit a bug report") + continue + end + end + haskey(deps, p) || continue + vers = sort!(collect(keys(deps[p]))) + higher_vers = vers[vers .> v] + isempty(higher_vers) && continue # package p has been set to the highest available version + + # Determine if there are packages which depend on `p` and somehow prevent its update to + # the latest version + blocking_parents = Set{String}() + for (p1,d1) in deps + p1 in upkgs && continue # package `p1` is among the ones to be updated, skip the check + haskey(fixed, p1) || continue # if package `p1` is not fixed, it can't be blocking + r1 = fixed[p1].requires # get `p1` requirements + haskey(r1, p) || continue # check if package `p1` requires `p` + vs1 = r1[p] # get the versions of `p` allowed by `p1` requirements + any(hv in vs1 for hv in higher_vers) && continue # package `p1` would allow some of the higher versions, + # therefore it's not responsible for blocking `p` + push!(blocking_parents, p1) # package `p1` is blocking the update of `p` + end + + # Determine if the update of `p` is prevented by explicit user-provided requirements + blocking_reqs = (haskey(reqs, p) && all(hv ∉ reqs[p] for hv in higher_vers)) + + # Determine if the update of `p` is prevented by it being fixed (e.g. it's dirty, or pinned...) + isfixed = haskey(fixed, p) + + msg = "Package $p was set to version $v, but a higher version $(vers[end]) exists.\n" + if isfixed + msg *= " The package is fixed. You can try using `Pkg.free(\"$p\")` to update it." + elseif blocking_reqs + msg *= " The update is prevented by explicit requirements constraints. Edit your REQUIRE file to change this." + elseif !isempty(blocking_parents) + msg *= string(" To install the latest version, you could try updating these packages as well: ", join(blocking_parents, ", ", " and "), ".") + else + msg *= " To install the latest version, you could try doing a full update with `Pkg.update()`." + end + info(msg) + end +end + typealias PackageState Union{Void,VersionNumber} function diff(have::Dict, want::Dict, avail::Dict, fixed::Dict) From ace743c2b8cad6030d16fd43fb5cab6d3129010b Mon Sep 17 00:00:00 2001 From: Carlo Baldassi <carlobaldassi@gmail.com> Date: Tue, 12 Jul 2016 00:00:07 +0200 Subject: [PATCH 0385/1117] Add tests for partial Pkg.update() --- test/pkg.jl | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/test/pkg.jl b/test/pkg.jl index 4556e95372499..82ca3403b0ffc 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -398,4 +398,77 @@ temp_pkg_dir() do @test contains(msg, "$package is not an installed package") @test !contains(msg, "signal (15)") end + + # partial Pkg.update + begin + nothingtodomsg = "INFO: No packages to install, update or remove\n" + + ret, out, err = @grab_outputs begin + Pkg.rm("Example") + Pkg.add("Example") + end + @test ret === nothing && out == "" + @test contains(err, "INFO: Installing Example v") + + ret, out, err = @grab_outputs Pkg.update("Example") + @test ret === nothing && out == "" + @test contains(err, nothingtodomsg) + + ret, out, err = @grab_outputs begin + Pkg.rm("Example") + Pkg.add("Example", v"0", v"0.4.1-") # force version to be < 0.4.1 + end + @test ret === nothing && out == "" + @test contains(err, "INFO: Installing Example v0.4.0") + + ret, out, err = @grab_outputs Pkg.update("Example") + @test ret === nothing && out == "" + @test ismatch(r"INFO: Package Example was set to version 0\.4\.0, but a higher version \d+\.\d+\.\d+\S* exists.\n", err) + @test contains(err, "The update is prevented by explicit requirements constraints. Edit your REQUIRE file to change this.\n") + @test contains(err, nothingtodomsg) + + ret, out, err = @grab_outputs begin + Pkg.rm("Example") + Pkg.add("Example") + Pkg.pin("Example", v"0.4.0") + end + @test ret === nothing && out == "" + @test contains(err, "INFO: Installing Example") + + ret, out, err = @grab_outputs Pkg.update("Example") + @test ret === nothing && out == "" + @test contains(err, "INFO: Package Example: skipping update (pinned)...\n") + @test ismatch(r"INFO: Package Example was set to version 0\.4\.0, but a higher version \d+\.\d+\.\d+\S* exists.\n", err) + @test contains(err, "The package is fixed. You can try using `Pkg.free(\"Example\")` to update it.") + @test contains(err, nothingtodomsg) + + metadata_dir = Pkg.dir("METADATA") + const old_commit = "313bfaafa301e82d40574a778720e893c559a7e2" + + # Force a METADATA rollback to an old version, so that we will install some + # outdated versions of some packages and then update some of those + # (note that the following Pkg.update calls will update METADATA to the + # latest version even though they don't update all packages) + LibGit2.with(LibGit2.GitRepo, metadata_dir) do repo + LibGit2.reset!(repo, LibGit2.Oid(old_commit), LibGit2.Consts.RESET_HARD) + end + + ret, out, err = @grab_outputs Pkg.add("Colors") + @test ret === nothing && out == "" + @test contains(err, "INFO: Installing Colors v0.6.4") + @test contains(err, "INFO: Installing ColorTypes v0.2.2") + @test contains(err, "INFO: Installing FixedPointNumbers v0.1.3") + @test contains(err, "INFO: Installing Compat v0.7.18") + @test contains(err, "INFO: Installing Reexport v0.0.3") + + ret, out, err = @grab_outputs Pkg.update("ColorTypes") + @test ismatch(r"INFO: Upgrading ColorTypes: v0\.2\.2 => v\d+\.\d+\.\d+", err) + @test ismatch(r"INFO: Upgrading Compat: v0\.7\.18 => v\d+\.\d+\.\d+", err) + @test !contains(err, "INFO: Upgrading Colors: ") + @test Pkg.installed("Colors") == v"0.6.4" + + ret, out, err = @grab_outputs Pkg.update("FixedPointNumbers") + @test ret === nothing && out == "" + @test contains(err, nothingtodomsg) + end end From e561f2d1aeaeb2182baaa3abad1b6edb229b5914 Mon Sep 17 00:00:00 2001 From: Carlo Baldassi <carlobaldassi@gmail.com> Date: Tue, 12 Jul 2016 12:31:42 +0200 Subject: [PATCH 0386/1117] Small fixes in test/pkg.jl --- test/pkg.jl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 82ca3403b0ffc..e97861b60d481 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -56,7 +56,7 @@ macro grab_outputs(ex) end # Test basic operations: adding or removing a package, status, free -#Also test for the existence of REQUIRE and META_Branch +# Also test for the existence of REQUIRE and META_BRANCH temp_pkg_dir() do @test isfile(joinpath(Pkg.dir(),"REQUIRE")) @test isfile(joinpath(Pkg.dir(),"META_BRANCH")) @@ -219,69 +219,69 @@ temp_pkg_dir() do # Various pin/free/re-pin/change-pin patterns (issue #17176) begin ret, out, err = @grab_outputs Pkg.free("Example") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Freeing Example") ret, out, err = @grab_outputs Pkg.pin("Example") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test ismatch(r"INFO: Creating Example branch pinned\.[0-9a-f]{8}\.tmp", err) @test !contains(err, "INFO: No packages to install, update or remove") branchid = replace(err, r".*pinned\.([0-9a-f]{8})\.tmp.*"s, s"\1") vers = Pkg.installed("Example") ret, out, err = @grab_outputs Pkg.free("Example") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Freeing Example") ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Creating Example branch pinned.b1990792.tmp") @test contains(err, "INFO: No packages to install, update or remove") @test Pkg.installed("Example") == v"0.4.0" ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Package Example is already pinned to the selected commit") @test !contains(err, "INFO: No packages to install, update or remove") @test Pkg.installed("Example") == v"0.4.0" ret, out, err = @grab_outputs Pkg.pin("Example") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Package Example is already pinned") @test !contains(err, "INFO: No packages to install, update or remove") @test Pkg.installed("Example") == v"0.4.0" ret, out, err = @grab_outputs Pkg.update() - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Package Example: skipping update (pinned)...") @test Pkg.installed("Example") == v"0.4.0" ret, out, err = @grab_outputs Pkg.pin("Example", v"0.3.1") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Creating Example branch pinned.d1ef7b00.tmp") @test contains(err, "INFO: No packages to install, update or remove") @test Pkg.installed("Example") == v"0.3.1" ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Package Example: checking out existing branch pinned.b1990792.tmp") @test contains(err, "INFO: No packages to install, update or remove") @test Pkg.installed("Example") == v"0.4.0" ret, out, err = @grab_outputs Pkg.free("Example") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Freeing Example") @test contains(err, "INFO: No packages to install, update or remove") @test Pkg.installed("Example") == vers ret, out, err = @grab_outputs Pkg.pin("Example") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Package Example: checking out existing branch pinned.$branchid.tmp") @test !contains(err, "INFO: No packages to install, update or remove") @test Pkg.installed("Example") == vers ret, out, err = @grab_outputs Pkg.free("Example") - @test ret == nothing && out == "" + @test ret === nothing && out == "" @test contains(err, "INFO: Freeing Example") @test Pkg.installed("Example") == vers end From 5e484cc41782dd62b227723496426b3f62802975 Mon Sep 17 00:00:00 2001 From: Carlo Baldassi <carlobaldassi@gmail.com> Date: Tue, 12 Jul 2016 00:07:25 +0200 Subject: [PATCH 0387/1117] NEWS entry for partial Pkg.update() --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 757a8f2776874..8b62166fafd7c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -143,6 +143,9 @@ Library improvements * Package-development functions like `Pkg.tag` and `Pkg.publish` have been moved to an external [PkgDev] package ([#13387]). + * Updating only a subset of the packages is now supported, + e.g. `Pkg.update("Example")` ([#17132]) + * The `Base.Test` module now has a `@testset` feature to bundle tests together and delay throwing an error until the end ([#13062]). From 7e1c2707f8c677a7be627f6be7d53e478181110e Mon Sep 17 00:00:00 2001 From: Carlo Baldassi <carlobaldassi@gmail.com> Date: Tue, 12 Jul 2016 00:45:52 +0200 Subject: [PATCH 0388/1117] Update docs for partial Pkg.update() --- base/pkg/pkg.jl | 5 ++++- doc/manual/packages.rst | 9 +++++++++ doc/stdlib/pkg.rst | 4 +++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/base/pkg/pkg.jl b/base/pkg/pkg.jl index 73d93a3376f37..7ded34c0faaa4 100644 --- a/base/pkg/pkg.jl +++ b/base/pkg/pkg.jl @@ -198,11 +198,14 @@ Pin `pkg` at registered version `version`. pin(pkg::AbstractString, ver::VersionNumber) = cd(Entry.pin,pkg,ver) """ - update() + update(pkgs...) Update the metadata repo – kept in `Pkg.dir("METADATA")` – then update any fixed packages that can safely be pulled from their origin; then call `Pkg.resolve()` to determine a new optimal set of packages versions. + +Without arguments, updates all installed packages. When one or more package names are provided as +arguments, only those packages and their dependencies are updated. """ update(upkgs::AbstractString...) = cd(Entry.update,Dir.getmetabranch(),Set{String}([upkgs...])) diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index c241aa0e36445..b56aa2d767e44 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -209,6 +209,15 @@ A package is considered fixed if it is one of the following: If any of these are the case, the package manager cannot freely change the installed version of the package, so its requirements must be satisfied by whatever other package versions it picks. The combination of top-level requirements in ``~/.julia/v0.4/REQUIRE`` and the requirement of fixed packages are used to determine what should be installed. +You can also update only a subset of the installed packages, by providing arguments to the `Pkg.update` function. In that case, only the packages provided as arguments and their dependencies will be updated:: + + julia> Pkg.update("Example") + INFO: Updating METADATA... + INFO: Computing changes... + INFO: Upgrading Example: v0.4.0 => 0.4.1 + +This partial update process still computes the new set of package versions according to top-level requirements and "fixed" packages, but it additionally considers all other packages except those explicitly provided, and their dependencies, as fixed. + Checkout, Pin and Free ---------------------- diff --git a/doc/stdlib/pkg.rst b/doc/stdlib/pkg.rst index 37cd095ba7fa7..33d35f6ef11c3 100644 --- a/doc/stdlib/pkg.rst +++ b/doc/stdlib/pkg.rst @@ -105,12 +105,14 @@ Functions for package development (e.g. ``tag``, ``publish``, etc.) have been mo Prints out a summary of what version and state ``pkg``\ , specifically, is in. -.. function:: update() +.. function:: update(pkgs...) .. Docstring generated from Julia source Update the metadata repo – kept in ``Pkg.dir("METADATA")`` – then update any fixed packages that can safely be pulled from their origin; then call ``Pkg.resolve()`` to determine a new optimal set of packages versions. + Without arguments, updates all installed packages. When one or more package names are provided as arguments, only those packages and their dependencies are updated. + .. function:: checkout(pkg, [branch="master"]; merge=true, pull=true) .. Docstring generated from Julia source From f7e5e02883b52ba024cd190239e9ced129ebd627 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 9 Jul 2016 17:36:16 -0500 Subject: [PATCH 0389/1117] Revise checkbounds API and extend to support Array{CartesianIndex} --- base/abstractarray.jl | 53 +++++++++++------- base/multidimensional.jl | 45 ++++++++++++---- test/abstractarray.jl | 112 ++++++++++++++++++++++++++++++++------- 3 files changed, 163 insertions(+), 47 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index f95e22c9688f0..5b5df6c4f7f64 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -181,38 +181,53 @@ function checkindex(::Type{Bool}, inds::AbstractUnitRange, I::AbstractArray) end # check all indices/dimensions -# To make extension easier, avoid specializations of checkbounds on index types -# (That said, typically one does not need to specialize this function.) + +# To facilitate extension for custom array types without triggering +# ambiguities, limit the number of specializations of checkbounds on +# the types of the indices. """ checkbounds(Bool, array, indexes...) -Return `true` if the specified `indexes` are in bounds for the given `array`. Subtypes of -`AbstractArray` should specialize this method if they need to provide custom bounds checking -behaviors. +Return `true` if the specified `indexes` are in bounds for the given +`array`. Subtypes of `AbstractArray` should specialize this method if +they need to provide custom bounds checking behaviors. """ -function checkbounds(::Type{Bool}, A::AbstractArray, i::Integer) - @_inline_meta - checkindex(Bool, linearindices(A), i) -end -function checkbounds{T}(::Type{Bool}, A::Union{Array{T,1},Range{T}}, i::Integer) +function checkbounds(::Type{Bool}, A::AbstractArray, I...) @_inline_meta - (1 <= i) & (i <= length(A)) + checkbounds_indices(indices(A), I) end function checkbounds(::Type{Bool}, A::AbstractArray, I::AbstractArray{Bool}) @_inline_meta checkbounds_logical(A, I) end -function checkbounds(::Type{Bool}, A::AbstractArray, I...) - @_inline_meta - checkbounds_indices(indices(A), I) -end +# checkbounds_indices iteratively consumes elements of the +# indices-tuple of an arrray and the indices-tuple supplied by the +# caller. These two tuples are usually consumed in a 1-for-1 fashion, +# i.e., +# +# checkbounds_indices((R1, R...), (I1, I...)) = checkindex(Bool, R1, I1) & +# checkbounds_indices(R, I) +# +# However, there are two exceptions: linear indexing and CartesianIndex{N}. checkbounds_indices(::Tuple{}, ::Tuple{}) = true checkbounds_indices(::Tuple{}, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, 1:1, I[1])) -checkbounds_indices(::Tuple{}, I::Tuple) = (@_inline_meta; checkindex(Bool, 1:1, I[1]) & checkbounds_indices((), tail(I))) -checkbounds_indices(inds::Tuple{Any}, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, inds[1], I[1])) -checkbounds_indices(inds::Tuple, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, 1:prod(map(dimlength, inds)), I[1])) -checkbounds_indices(inds::Tuple, I::Tuple) = (@_inline_meta; checkindex(Bool, inds[1], I[1]) & checkbounds_indices(tail(inds), tail(I))) +function checkbounds_indices(::Tuple{}, I::Tuple) + @_inline_meta + checkindex(Bool, 1:1, I[1]) & checkbounds_indices((), tail(I)) +end +function checkbounds_indices(inds::Tuple{Any}, I::Tuple{Any}) + @_inline_meta + checkindex(Bool, inds[1], I[1]) +end +function checkbounds_indices(inds::Tuple, I::Tuple{Any}) + @_inline_meta + checkindex(Bool, 1:prod(map(dimlength, inds)), I[1]) # linear indexing +end +function checkbounds_indices(inds::Tuple, I::Tuple) + @_inline_meta + checkindex(Bool, inds[1], I[1]) & checkbounds_indices(tail(inds), tail(I)) +end # Single logical array indexing: checkbounds_logical(A::AbstractArray, I::AbstractArray{Bool}) = indices(A) == indices(I) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 3c4e3b3bef930..02daee9106e6c 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -146,16 +146,41 @@ end # IteratorsMD using .IteratorsMD ## Bounds-checking with CartesianIndex -# Ambiguity with linear indexing: -@inline _chkbnds(A::AbstractVector, checked::NTuple{1,Bool}, I::CartesianIndex) = _chkbnds(A, checked, I.I...) -@inline _chkbnds(A::AbstractArray, checked::NTuple{1,Bool}, I::CartesianIndex) = _chkbnds(A, checked, I.I...) -# Generic bounds checking -@inline _chkbnds{T,N}(A::AbstractArray{T,N}, checked::NTuple{N,Bool}, I1::CartesianIndex, I...) = _chkbnds(A, checked, I1.I..., I...) -@inline _chkbnds{T,N,M}(A::AbstractArray{T,N}, checked::NTuple{M,Bool}, I1::CartesianIndex, I...) = _chkbnds(A, checked, I1.I..., I...) - -@inline checkbounds_indices(::Tuple{}, I::Tuple{CartesianIndex,Vararg{Any}}) = checkbounds_indices((), (I[1].I..., tail(I)...)) -@inline checkbounds_indices(inds::Tuple{Any}, I::Tuple{CartesianIndex,Vararg{Any}}) = checkbounds_indices(inds, (I[1].I..., tail(I)...)) -@inline checkbounds_indices(inds::Tuple, I::Tuple{CartesianIndex,Vararg{Any}}) = checkbounds_indices(inds, (I[1].I..., tail(I)...)) +@inline checkbounds_indices(::Tuple{}, I::Tuple{CartesianIndex,Vararg{Any}}) = + checkbounds_indices((), (I[1].I..., tail(I)...)) +@inline checkbounds_indices(inds::Tuple{Any}, I::Tuple{CartesianIndex,Vararg{Any}}) = + checkbounds_indices(inds, (I[1].I..., tail(I)...)) +@inline checkbounds_indices(inds::Tuple, I::Tuple{CartesianIndex,Vararg{Any}}) = + checkbounds_indices(inds, (I[1].I..., tail(I)...)) + +# Support indexing with an array of CartesianIndex{N}s +# Here we try to consume N of the indices (if there are that many available) +# The first two simply handle ambiguities +@inline function checkbounds_indices{N}(::Tuple{}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) + checkindex(Bool, (), I[1]) & checkbounds_indices((), tail(I)) +end +@inline function checkbounds_indices{N}(inds::Tuple{Any}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) + checkindex(Bool, inds, I[1]) & checkbounds_indices((), tail(I)) +end +@inline function checkbounds_indices{N}(inds::Tuple, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) + inds1, indsrest = split(inds, Val{N}) + checkindex(Bool, inds1, I[1]) & checkbounds_indices(indsrest, tail(I)) +end + +@inline split{N}(inds, V::Type{Val{N}}) = _split((), inds, V) +@inline _split(out, indsrest, V) = _split((out..., indsrest[1]), tail(indsrest), V) +# exit either when we've exhausted the input tuple or when it has length N +@inline _split{N}(out::NTuple{N}, ::Tuple{}, ::Type{Val{N}}) = out, () # ambig. +@inline _split{N}(out, ::Tuple{}, ::Type{Val{N}}) = out, () +@inline _split{N}(out::NTuple{N}, indsrest, ::Type{Val{N}}) = out, indsrest + +function checkindex{N}(::Type{Bool}, inds::Tuple, I::AbstractArray{CartesianIndex{N}}) + b = true + for i in I + b &= checkbounds_indices(inds, (i,)) + end + b +end # Recursively compute the lengths of a list of indices, without dropping scalars # These need to be inlined for more than 3 indexes diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 6847048da7770..b3ff1b2f18c77 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -1,26 +1,102 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # Bounds checking -A = rand(3,3,3) +A = rand(5,4,3) @test checkbounds(Bool, A, 1, 1, 1) == true -@test checkbounds(Bool, A, 1, 3, 3) == true -@test checkbounds(Bool, A, 1, 4, 3) == false -@test checkbounds(Bool, A, 1, -1, 3) == false -@test checkbounds(Bool, A, 1, 9) == true # partial linear indexing -@test checkbounds(Bool, A, 1, 10) == false # partial linear indexing -@test checkbounds(Bool, A, 1) == true -@test checkbounds(Bool, A, 27) == true -@test checkbounds(Bool, A, 28) == false -@test checkbounds(Bool, A, 2, 2, 2, 1) == true +@test checkbounds(Bool, A, 5, 4, 3) == true +@test checkbounds(Bool, A, 0, 1, 1) == false +@test checkbounds(Bool, A, 1, 0, 1) == false +@test checkbounds(Bool, A, 1, 1, 0) == false +@test checkbounds(Bool, A, 6, 4, 3) == false +@test checkbounds(Bool, A, 5, 5, 3) == false +@test checkbounds(Bool, A, 5, 4, 4) == false +@test checkbounds(Bool, A, 1) == true # linear indexing +@test checkbounds(Bool, A, 60) == true +@test checkbounds(Bool, A, 61) == false +@test checkbounds(Bool, A, 2, 2, 2, 1) == true # extra indices @test checkbounds(Bool, A, 2, 2, 2, 2) == false - -@test Base.checkbounds_indices((1:5, 1:5), (2,2)) -@test !Base.checkbounds_indices((1:5, 1:5), (7,2)) -@test !Base.checkbounds_indices((1:5, 1:5), (2,0)) -@test Base.checkbounds_indices((1:5, 1:5), (13,)) -@test !Base.checkbounds_indices((1:5, 1:5), (26,)) -@test Base.checkbounds_indices((1:5, 1:5), (2,2,1)) -@test !Base.checkbounds_indices((1:5, 1:5), (2,2,2)) +@test checkbounds(Bool, A, 1, 1) == true # partial linear indexing (PLI) +@test checkbounds(Bool, A, 1, 12) == true # PLI +@test checkbounds(Bool, A, 5, 12) == true # PLI +@test checkbounds(Bool, A, 1, 13) == false # PLI +@test checkbounds(Bool, A, 6, 12) == false # PLI + +# single CartesianIndex +@test checkbounds(Bool, A, CartesianIndex((1, 1, 1))) == true +@test checkbounds(Bool, A, CartesianIndex((5, 4, 3))) == true +@test checkbounds(Bool, A, CartesianIndex((0, 1, 1))) == false +@test checkbounds(Bool, A, CartesianIndex((1, 0, 1))) == false +@test checkbounds(Bool, A, CartesianIndex((1, 1, 0))) == false +@test checkbounds(Bool, A, CartesianIndex((6, 4, 3))) == false +@test checkbounds(Bool, A, CartesianIndex((5, 5, 3))) == false +@test checkbounds(Bool, A, CartesianIndex((5, 4, 4))) == false +@test checkbounds(Bool, A, CartesianIndex((1,))) == true +@test checkbounds(Bool, A, CartesianIndex((60,))) == true +@test checkbounds(Bool, A, CartesianIndex((61,))) == false +@test checkbounds(Bool, A, CartesianIndex((2, 2, 2, 1,))) == true +@test checkbounds(Bool, A, CartesianIndex((2, 2, 2, 2,))) == false +@test checkbounds(Bool, A, CartesianIndex((1, 1,))) == true +@test checkbounds(Bool, A, CartesianIndex((1, 12,))) == true +@test checkbounds(Bool, A, CartesianIndex((5, 12,))) == true +@test checkbounds(Bool, A, CartesianIndex((1, 13,))) == false +@test checkbounds(Bool, A, CartesianIndex((6, 12,))) == false + +# mix of CartesianIndex and Int +@test checkbounds(Bool, A, CartesianIndex((1,)), 1, CartesianIndex((1,))) == true +@test checkbounds(Bool, A, CartesianIndex((5, 4)), 3) == true +@test checkbounds(Bool, A, CartesianIndex((0, 1)), 1) == false +@test checkbounds(Bool, A, 1, CartesianIndex((0, 1))) == false +@test checkbounds(Bool, A, 1, 1, CartesianIndex((0,))) == false +@test checkbounds(Bool, A, 6, CartesianIndex((4, 3))) == false +@test checkbounds(Bool, A, 5, CartesianIndex((5,)), 3) == false +@test checkbounds(Bool, A, CartesianIndex((5,)), CartesianIndex((4,)), CartesianIndex((4,))) == false + +# vector indices +@test checkbounds(Bool, A, 1:5, 1:4, 1:3) == true +@test checkbounds(Bool, A, 0:5, 1:4, 1:3) == false +@test checkbounds(Bool, A, 1:5, 0:4, 1:3) == false +@test checkbounds(Bool, A, 1:5, 1:4, 0:3) == false +@test checkbounds(Bool, A, 1:6, 1:4, 1:3) == false +@test checkbounds(Bool, A, 1:5, 1:5, 1:3) == false +@test checkbounds(Bool, A, 1:5, 1:4, 1:4) == false +@test checkbounds(Bool, A, 1:60) == true +@test checkbounds(Bool, A, 1:61) == false +@test checkbounds(Bool, A, 2, 2, 2, 1:1) == true # extra indices +@test checkbounds(Bool, A, 2, 2, 2, 1:2) == false +@test checkbounds(Bool, A, 1:5, 1:12) == true +@test checkbounds(Bool, A, 1:5, 1:13) == false +@test checkbounds(Bool, A, 1:6, 1:12) == false + +# logical +@test checkbounds(Bool, A, trues(5), trues(4), trues(3)) == true +@test checkbounds(Bool, A, trues(6), trues(4), trues(3)) == false +@test checkbounds(Bool, A, trues(5), trues(5), trues(3)) == false +@test checkbounds(Bool, A, trues(5), trues(4), trues(4)) == false +@test checkbounds(Bool, A, trues(60)) == true +@test checkbounds(Bool, A, trues(61)) == false +@test checkbounds(Bool, A, 2, 2, 2, trues(1)) == true # extra indices +@test checkbounds(Bool, A, 2, 2, 2, trues(2)) == false +@test checkbounds(Bool, A, trues(5), trues(12)) == true +@test checkbounds(Bool, A, trues(5), trues(13)) == false +@test checkbounds(Bool, A, trues(6), trues(12)) == false + +# array of CartesianIndex +@test checkbounds(Bool, A, [CartesianIndex((1, 1, 1))]) == true +@test checkbounds(Bool, A, [CartesianIndex((5, 4, 3))]) == true +@test checkbounds(Bool, A, [CartesianIndex((0, 1, 1))]) == false +@test checkbounds(Bool, A, [CartesianIndex((1, 0, 1))]) == false +@test checkbounds(Bool, A, [CartesianIndex((1, 1, 0))]) == false +@test checkbounds(Bool, A, [CartesianIndex((6, 4, 3))]) == false +@test checkbounds(Bool, A, [CartesianIndex((5, 5, 3))]) == false +@test checkbounds(Bool, A, [CartesianIndex((5, 4, 4))]) == false +@test checkbounds(Bool, A, [CartesianIndex((1, 1))], 1) == true +@test checkbounds(Bool, A, [CartesianIndex((5, 4))], 3) == true +@test checkbounds(Bool, A, [CartesianIndex((0, 1))], 1) == false +@test checkbounds(Bool, A, [CartesianIndex((1, 0))], 1) == false +@test checkbounds(Bool, A, [CartesianIndex((1, 1))], 0) == false +@test checkbounds(Bool, A, [CartesianIndex((6, 4))], 3) == false +@test checkbounds(Bool, A, [CartesianIndex((5, 5))], 3) == false +@test checkbounds(Bool, A, [CartesianIndex((5, 4))], 4) == false # sub2ind & ind2sub # 0-dimensional From e724946f6d35a89bf358c303d076f2827aa96417 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 9 Jul 2016 05:19:21 -0500 Subject: [PATCH 0390/1117] Force inlining of `linearindices` and `indices1`. Ref #17340 --- base/abstractarray.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 5b5df6c4f7f64..e52a3548326a2 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -45,8 +45,11 @@ function indices{T,N}(A::AbstractArray{T,N}) map(s->OneTo(s), size(A)) end +# Performance optimization: get rid of a branch on `d` in `indices(A, +# d)` for d=1. 1d arrays are heavily used, and the first dimension +# comes up in other applications. indices1{T}(A::AbstractArray{T,0}) = OneTo(1) -indices1{T}(A::AbstractArray{T}) = indices(A)[1] +indices1{T}(A::AbstractArray{T}) = (@_inline_meta; indices(A)[1]) """ linearindices(A) @@ -60,8 +63,8 @@ is `indices(A, 1)`. Calling this function is the "safe" way to write algorithms that exploit linear indexing. """ -linearindices(A) = 1:length(A) -linearindices(A::AbstractVector) = indices1(A) +linearindices(A) = (@_inline_meta; 1:length(A)) +linearindices(A::AbstractVector) = (@_inline_meta; indices1(A)) eltype{T}(::Type{AbstractArray{T}}) = T eltype{T,N}(::Type{AbstractArray{T,N}}) = T elsize{T}(::AbstractArray{T}) = sizeof(T) From d0e2aa819f43f0482cad056ca3bdc3fe08b86632 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 11 Jul 2016 11:22:32 -0500 Subject: [PATCH 0391/1117] Faster and simpler indices for SubArray --- base/abstractarray.jl | 6 ++++-- base/range.jl | 11 ++++++++--- base/subarray.jl | 16 +++++----------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index e52a3548326a2..266f65b1ea256 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -51,6 +51,9 @@ end indices1{T}(A::AbstractArray{T,0}) = OneTo(1) indices1{T}(A::AbstractArray{T}) = (@_inline_meta; indices(A)[1]) +unsafe_indices(A) = indices(A) +unsafe_indices(r::Range) = (OneTo(unsafe_length(r)),) # Ranges use checked_sub for size + """ linearindices(A) @@ -248,6 +251,7 @@ Throw an error if the specified `indexes` are not in bounds for the given `array function checkbounds(A::AbstractArray, I...) @_inline_meta checkbounds(Bool, A, I...) || throw_boundserror(A, I) + nothing end checkbounds(A::AbstractArray) = checkbounds(A, 1) # 0-d case @@ -1211,8 +1215,6 @@ nextL(L, l::Integer) = L*l nextL(L, r::AbstractUnitRange) = L*unsafe_length(r) offsetin(i, l::Integer) = i-1 offsetin(i, r::AbstractUnitRange) = i-first(r) -unsafe_length(r::UnitRange) = r.stop-r.start+1 -unsafe_length(r::OneTo) = length(r) ind2sub(::Tuple{}, ind::Integer) = (@_inline_meta; ind == 1 ? () : throw(BoundsError())) ind2sub(dims::DimsInteger, ind::Integer) = (@_inline_meta; _ind2sub(dims, ind-1)) diff --git a/base/range.jl b/base/range.jl index 2c819e05c5e40..48a591438ebd3 100644 --- a/base/range.jl +++ b/base/range.jl @@ -326,12 +326,17 @@ step(r::AbstractUnitRange) = 1 step(r::FloatRange) = r.step/r.divisor step{T}(r::LinSpace{T}) = ifelse(r.len <= 0, convert(T,NaN), (r.stop-r.start)/r.divisor) -function length(r::StepRange) +unsafe_length(r::Range) = length(r) # generic fallback + +function unsafe_length(r::StepRange) n = Integer(div(r.stop+r.step - r.start, r.step)) isempty(r) ? zero(n) : n end -length(r::AbstractUnitRange) = Integer(last(r) - first(r) + 1) -length(r::OneTo) = r.stop +length(r::StepRange) = unsafe_length(r) +unsafe_length(r::AbstractUnitRange) = Integer(last(r) - first(r) + 1) +unsafe_length(r::OneTo) = r.stop +length(r::AbstractUnitRange) = unsafe_length(r) +length(r::OneTo) = unsafe_length(r) length(r::FloatRange) = Integer(r.len) length(r::LinSpace) = Integer(r.len + signbit(r.len - 1)) diff --git a/base/subarray.jl b/base/subarray.jl index e9a4cf1c57765..48446fcc60cb4 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -278,17 +278,11 @@ end # they are taken from the range/vector # Since bounds-checking is performance-critical and uses # indices, it's worth optimizing these implementations thoroughly -indices(S::SubArray) = (@_inline_meta; _indices_sub(S, 1, S.indexes...)) -_indices_sub(S::SubArray, dim::Int) = () -_indices_sub(S::SubArray, dim::Int, ::Real, I...) = (@_inline_meta; _indices_sub(S, dim+1, I...)) -_indices_sub(S::SubArray, dim::Int, ::Colon, I...) = (@_inline_meta; (indices(parent(S), dim), _indices_sub(S, dim+1, I...)...)) -_indices_sub(S::SubArray, dim::Int, i1::AbstractArray, I...) = (@_inline_meta; (indices(i1)..., _indices_sub(S, dim+1, I...)...)) -indices1(S::SubArray) = (@_inline_meta; _indices1(S, 1, S.indexes...)) -_indices1(S::SubArray, dim) = OneTo(1) -_indices1(S::SubArray, dim, i1::Real, I...) = (@_inline_meta; _indices1(S, dim+1, I...)) -_indices1(S::SubArray, dim, i1::Colon, I...) = (@_inline_meta; indices(parent(S), dim)) -_indices1(S::SubArray, dim, i1::AbstractArray, I...) = (@_inline_meta; indices1(i1)) -_indices1{T}(S::SubArray, dim, i1::AbstractArray{T,0}, I...) = (@_inline_meta; _indices1(S, dim+1, I...)) +indices(S::SubArray) = (@_inline_pure_meta; _indices_sub(S, indices(S.parent), S.indexes...)) +_indices_sub(S::SubArray, pinds) = () +_indices_sub(S::SubArray, pinds, ::Real, I...) = _indices_sub(S, tail(pinds), I...) +_indices_sub(S::SubArray, pinds, ::Colon, I...) = (pinds[1], _indices_sub(S, tail(pinds), I...)...) +_indices_sub(S::SubArray, pinds, i1::AbstractArray, I...) = (unsafe_indices(i1)..., _indices_sub(S, tail(pinds), I...)...) ## Compatability # deprecate? From 000c3ceffbac18814590826c6c5ed2c28b12abae Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 11 Jul 2016 11:26:59 -0500 Subject: [PATCH 0392/1117] Don't extend split to tuples (move it into IteratorsMD) --- base/multidimensional.jl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 02daee9106e6c..f286c7c18de79 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -141,6 +141,14 @@ simd_index{I<:CartesianIndex{0}}(iter::CartesianRange{I}, ::CartesianIndex, I1:: CartesianIndex((I1+iter.start[1], Ilast.I...)) end +# Split out the first N elements of a tuple +@inline split{N}(t, V::Type{Val{N}}) = _split((), t, V) +@inline _split(tN, trest, V) = _split((tN..., trest[1]), tail(trest), V) +# exit either when we've exhausted the input tuple or when tN has length N +@inline _split{N}(tN::NTuple{N}, ::Tuple{}, ::Type{Val{N}}) = tN, () # ambig. +@inline _split{N}(tN, ::Tuple{}, ::Type{Val{N}}) = tN, () +@inline _split{N}(tN::NTuple{N}, trest, ::Type{Val{N}}) = tN, trest + end # IteratorsMD using .IteratorsMD @@ -163,17 +171,10 @@ end checkindex(Bool, inds, I[1]) & checkbounds_indices((), tail(I)) end @inline function checkbounds_indices{N}(inds::Tuple, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) - inds1, indsrest = split(inds, Val{N}) + inds1, indsrest = IteratorsMD.split(inds, Val{N}) checkindex(Bool, inds1, I[1]) & checkbounds_indices(indsrest, tail(I)) end -@inline split{N}(inds, V::Type{Val{N}}) = _split((), inds, V) -@inline _split(out, indsrest, V) = _split((out..., indsrest[1]), tail(indsrest), V) -# exit either when we've exhausted the input tuple or when it has length N -@inline _split{N}(out::NTuple{N}, ::Tuple{}, ::Type{Val{N}}) = out, () # ambig. -@inline _split{N}(out, ::Tuple{}, ::Type{Val{N}}) = out, () -@inline _split{N}(out::NTuple{N}, indsrest, ::Type{Val{N}}) = out, indsrest - function checkindex{N}(::Type{Bool}, inds::Tuple, I::AbstractArray{CartesianIndex{N}}) b = true for i in I From e0e83142ed6f33a262c6684277de6aa13ca089cd Mon Sep 17 00:00:00 2001 From: Saurav Sachidanand <sauravsachidanand@gmail.com> Date: Tue, 12 Jul 2016 19:20:04 +0530 Subject: [PATCH 0393/1117] Fix a typo in docstring for significand --- base/docs/helpdb/Base.jl | 2 +- doc/stdlib/numbers.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 19d0b132fcbbf..698602b5f1854 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -8220,7 +8220,7 @@ escape_string(str) significand(x) Extract the `significand(s)` (a.k.a. mantissa), in binary representation, of a -floating-point number or array. If `x` is a non-zero finite number, than the result will be +floating-point number or array. If `x` is a non-zero finite number, then the result will be a number of the same type on the interval ``[1,2)``. Otherwise `x` is returned. ```jldoctest diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index 1877e6c119c5c..d2048420317e6 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -100,7 +100,7 @@ Data Formats .. Docstring generated from Julia source - Extract the ``significand(s)`` (a.k.a. mantissa), in binary representation, of a floating-point number or array. If ``x`` is a non-zero finite number, than the result will be a number of the same type on the interval :math:`[1,2)`\ . Otherwise ``x`` is returned. + Extract the ``significand(s)`` (a.k.a. mantissa), in binary representation, of a floating-point number or array. If ``x`` is a non-zero finite number, then the result will be a number of the same type on the interval :math:`[1,2)`\ . Otherwise ``x`` is returned. .. doctest:: From 9144605599bbf8427ae61e73e901d31722daa912 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 6 Jul 2016 14:25:54 -0400 Subject: [PATCH 0394/1117] syntax-level fusion of nested f.(args) calls into a single broadcast call --- src/julia-syntax.scm | 76 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index aad862acd72b7..343f3705ba2d9 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1546,6 +1546,72 @@ (cadr expr) ;; eta reduce `x->f(x)` => `f` `(-> ,argname (block ,@splat ,expr))))) +(define (getfield-field? x) ; whether x from (|.| f x) is a getfield call + (or (eq? (car x) 'quote) (eq? (car x) 'inert) (eq? (car x) '$))) + +;; fuse nested calls to f.(args...) into a single broadcast call +(define (expand-fuse-broadcast f args) + (define (fuse? e) (and (pair? e) (eq? (car e) 'fuse))) + (define (anyfuse? exprs) + (if (null? exprs) #f (if (fuse? (car exprs)) #t (anyfuse? (cdr exprs))))) + (define (to-lambda f args) ; convert f to anonymous function with hygienic tuple args + (define (hygienic f) ; rename args of f == (-> (tuple args...) body) + (let* ((oldargs (cdadr f)) + (newargs (map (lambda (arg) (gensy)) oldargs)) + (renames (map cons oldargs newargs))) + `(-> (tuple ,@newargs) ,(replace-vars (caddr f) renames)))) + (if (and (pair? f) (eq? (car f) '->)) + (if (pair? (cadr f)) + (hygienic f) ; f is already anon function, nothing to do but hygienicize + (hygienic `(-> (tuple ,(cadr f)) ,(caddr f)))) ; canonicalize single arg to tuple + (let ((genargs (map (lambda (e) (gensy)) args))) ; formal parameters + `(-> ,(cons 'tuple genargs) (call ,f ,@genargs))))) + (define (from-lambda f) ; convert (-> (tuple args...) (call func args...)) back to func + (if (and (pair? f) (eq? (car f) '->) (pair? (cadr f)) (eq? (caadr f) 'tuple) + (pair? (caddr f)) (eq? (caaddr f) 'call) (equal? (cdadr f) (cdr (cdaddr f)))) + (car (cdaddr f)) + f)) + (define (fuse-args oldargs) ; replace (fuse f args) with args in oldargs list + (define (fargs newargs oldargs) + (if (null? oldargs) + newargs + (fargs (if (fuse? (car oldargs)) + (append (reverse (caddar oldargs)) newargs) + (cons (car oldargs) newargs)) + (cdr oldargs)))) + (reverse (fargs '() oldargs))) + (define (fuse-funcs f args) ; for (fuse g a) in args, merge/inline g into f + ; any argument A of f that is (fuse g a) gets replaced by let A=(body of g): + (define (fuse-lets fargs args lets) + (if (null? args) + lets + (if (fuse? (car args)) + (fuse-lets (cdr fargs) (cdr args) (cons (list '= (car fargs) (caddr (cadar args))) lets)) + (fuse-lets (cdr fargs) (cdr args) lets)))) + (let ((fargs (cdadr f)) + (fbody (caddr f))) + `(-> + (tuple ,@(fuse-args (map (lambda (oldarg arg) (if (fuse? arg) + `(fuse _ ,(cdadr (cadr arg))) + oldarg)) + fargs args))) + (let ,fbody ,@(reverse (fuse-lets fargs args '())))))) + (define (make-fuse f args) ; check for nested (fuse f args) exprs and combine + (define (expand-fuse e) + (if (and (pair? e) (eq? (car e) '|.|)) + (let ((f (cadr e)) + (x (caddr e))) + (if (getfield-field? x) + `(call (core getfield) ,(expand-forms f) ,(expand-forms x)) + (make-fuse f (cdr x)))) + (expand-forms e))) + (let ((args_ (map expand-fuse args))) + (if (anyfuse? args_) + `(fuse ,(fuse-funcs (to-lambda f args) args_) ,(fuse-args args_)) + `(fuse ,(to-lambda f args) ,args_)))) + (let ((e (make-fuse f args))) ; an expression '(fuse func args) + `(call broadcast ,(expand-forms (from-lambda (cadr e))) ,@(caddr e)))) + ;; table mapping expression head to a function expanding that form (define expand-table (table @@ -1584,11 +1650,11 @@ (lambda (e) ; e = (|.| f x) (let ((f (cadr e)) (x (caddr e))) - (if (or (eq? (car x) 'quote) (eq? (car x) 'inert) (eq? (car x) '$)) - `(call (core getfield) ,(expand-forms f) ,(expand-forms x)) - ; otherwise, came from f.(args...) --> broadcast(f, args...), - ; where x = (tuple args...) at this point: - (expand-forms `(call broadcast ,f ,@(cdr x)))))) + (if (getfield-field? x) + `(call (core getfield) ,(expand-forms f) ,(expand-forms x)) + ; otherwise, came from f.(args...) --> broadcast(f, args...), + ; where we want to fuse with any nested broadcast calls. + (expand-fuse-broadcast f (cdr x))))) '|<:| syntactic-op-to-call '|>:| syntactic-op-to-call From 50178b70e04cf4daa449bace09f408c152dcfa55 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Wed, 6 Jul 2016 17:04:40 -0400 Subject: [PATCH 0395/1117] loop fusion tests --- test/broadcast.jl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/broadcast.jl b/test/broadcast.jl index 6750e05061b75..7388facc1acb4 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -212,6 +212,22 @@ let a = sin.([1, 2]) @test a ≈ [0.8414709848078965, 0.9092974268256817] end +# PR #17300: loop fusion +@test (x->x+1).((x->x+2).((x->x+3).(1:10))) == collect(7:16) +let A = [sqrt(i)+j for i = 1:3, j=1:4] + @test atan2.(log.(A), sum(A,1)) == broadcast(atan2, broadcast(log, A), sum(A, 1)) +end +let x = sin.(1:10) + @test atan2.((x->x+1).(x), (x->x+2).(x)) == atan2(x+1, x+2) == atan2(x.+1, x.+2) +end +# Use side effects to check for loop fusion. Note that, due to #8450, +# a broadcasted function is currently called twice on the first element. +let g = Int[] + f17300(x) = begin; push!(g, x); x+1; end + f17300.(f17300.(f17300.(1:3))) + @test g == [1,2,3, 1,2,3, 2,3,4, 3,4,5] +end + # PR 16988 @test Base.promote_op(+, Bool) === Int @test isa(broadcast(+, [true]), Array{Int,1}) From fdc877762845a626117e865198a2e97c5bff21b3 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Wed, 6 Jul 2016 17:21:29 -0400 Subject: [PATCH 0396/1117] documentation for loop fusion --- NEWS.md | 4 +++- doc/manual/functions.rst | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 8b62166fafd7c..71440f4f7f935 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,7 +10,8 @@ New language features * Generators and comprehensions support filtering using `if` ([#550]) and nested iteration using multiple `for` keywords ([#4867]). - * Broadcasting syntax: ``f.(args...)`` is equivalent to ``broadcast(f, args...)`` ([#15032]). + * Broadcasting syntax: ``f.(args...)`` is equivalent to ``broadcast(f, args...)`` ([#15032]), + and nested `f.(g.(args...))` calls are fused into a single `broadcast` loop ([#17300]). * Macro expander functions are now generic, so macros can have multiple definitions (e.g. for different numbers of arguments, or optional arguments) ([#8846], [#9627]). @@ -332,4 +333,5 @@ Deprecated or removed [#17037]: https://github.com/JuliaLang/julia/issues/17037 [#17075]: https://github.com/JuliaLang/julia/issues/17075 [#17266]: https://github.com/JuliaLang/julia/issues/17266 +[#17300]: https://github.com/JuliaLang/julia/issues/17300 [#17374]: https://github.com/JuliaLang/julia/issues/17374 diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index ea6eff24eabf9..0b1b77aef8102 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -640,6 +640,22 @@ then ``f.(pi,A)`` will return a new array consisting of ``f(pi,a)`` for each consisting of ``f(vector1[i],vector2[i])`` for each index ``i`` (throwing an exception if the vectors have different length). +Moreover, *nested* ``f.(args...)`` calls are *fused* into a single ``broadcast`` +loop. For example, ``sin.(cos.(X))`` is equivalent to ``broadcast(x -> sin(cos(x)), X)``, +similar to ``[sin(cos(x)) for x in X]``: there is only a single loop over ``X``, +and a single array is allocated for the result. [In contrast, ``sin(cos(X))`` +in a typical "vectorized" language would first allocate one temporary array for ``tmp=cos(X)``, +and then compute ``sin(tmp)`` in a separate loop, allocating a second array.] +This loop fusion is not a compiler optimization that may or may not occur, it +is a *syntactic guarantee* whenever nested ``f.(args...)`` calls are encountered. Technically, +the fusion stops as soon as a "non-dot" function is encountered; for example, +in ``sin.(sort(cos.(X)))`` the ``sin`` and ``cos`` loops cannot be merged +because of the intervening ``sort`` function. + +(In future versions of Julia, operators like ``.*`` will also be handled with +the same mechanism: they will be equivalent to ``broadcast`` calls and +will be fused with other nested "dot" calls.) + Further Reading --------------- From f4c9bd52e0f6cd24103e9774b35b28f08cb35511 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Fri, 8 Jul 2016 10:26:12 -0400 Subject: [PATCH 0397/1117] correct issue number in comment --- test/broadcast.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/broadcast.jl b/test/broadcast.jl index 7388facc1acb4..99e1303d8debe 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -220,7 +220,7 @@ end let x = sin.(1:10) @test atan2.((x->x+1).(x), (x->x+2).(x)) == atan2(x+1, x+2) == atan2(x.+1, x.+2) end -# Use side effects to check for loop fusion. Note that, due to #8450, +# Use side effects to check for loop fusion. Note that, due to #17314, # a broadcasted function is currently called twice on the first element. let g = Int[] f17300(x) = begin; push!(g, x); x+1; end From f85893724ffc6caead6451b605dc3c40a5608bb0 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Fri, 8 Jul 2016 11:05:39 -0400 Subject: [PATCH 0398/1117] partial fix for loop fusion and splatting (but only handles args... if they are the last argument in the fusion) --- src/julia-syntax.scm | 27 ++++++++++++++------------- test/broadcast.jl | 1 + 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 343f3705ba2d9..9edda07871fd1 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1554,17 +1554,22 @@ (define (fuse? e) (and (pair? e) (eq? (car e) 'fuse))) (define (anyfuse? exprs) (if (null? exprs) #f (if (fuse? (car exprs)) #t (anyfuse? (cdr exprs))))) + (define (...? e) (and (pair? e) (eq? (car e) '...))) (define (to-lambda f args) ; convert f to anonymous function with hygienic tuple args + (define (genarg arg) (if (...? arg) (list '... (gensy)) (gensy))) (define (hygienic f) ; rename args of f == (-> (tuple args...) body) (let* ((oldargs (cdadr f)) - (newargs (map (lambda (arg) (gensy)) oldargs)) - (renames (map cons oldargs newargs))) + (newargs (map genarg oldargs)) + (renames (map (lambda (oldarg newarg) (if (...? oldarg) + (cons (cadr oldarg) (cadr newarg)) + (cons oldarg newarg))) + oldargs newargs))) `(-> (tuple ,@newargs) ,(replace-vars (caddr f) renames)))) (if (and (pair? f) (eq? (car f) '->)) (if (pair? (cadr f)) (hygienic f) ; f is already anon function, nothing to do but hygienicize (hygienic `(-> (tuple ,(cadr f)) ,(caddr f)))) ; canonicalize single arg to tuple - (let ((genargs (map (lambda (e) (gensy)) args))) ; formal parameters + (let ((genargs (map genarg args))) ; formal parameters `(-> ,(cons 'tuple genargs) (call ,f ,@genargs))))) (define (from-lambda f) ; convert (-> (tuple args...) (call func args...)) back to func (if (and (pair? f) (eq? (car f) '->) (pair? (cadr f)) (eq? (caadr f) 'tuple) @@ -1597,20 +1602,16 @@ fargs args))) (let ,fbody ,@(reverse (fuse-lets fargs args '())))))) (define (make-fuse f args) ; check for nested (fuse f args) exprs and combine - (define (expand-fuse e) - (if (and (pair? e) (eq? (car e) '|.|)) - (let ((f (cadr e)) - (x (caddr e))) - (if (getfield-field? x) - `(call (core getfield) ,(expand-forms f) ,(expand-forms x)) - (make-fuse f (cdr x)))) - (expand-forms e))) - (let ((args_ (map expand-fuse args))) + (define (dot-to-fuse e) ; convert e == (. f (tuple args)) to (fuse f args) + (if (and (pair? e) (eq? (car e) '|.|) (not (getfield-field? (caddr e)))) + (make-fuse (cadr e) (cdaddr e)) + e)) + (let ((args_ (map dot-to-fuse args))) (if (anyfuse? args_) `(fuse ,(fuse-funcs (to-lambda f args) args_) ,(fuse-args args_)) `(fuse ,(to-lambda f args) ,args_)))) (let ((e (make-fuse f args))) ; an expression '(fuse func args) - `(call broadcast ,(expand-forms (from-lambda (cadr e))) ,@(caddr e)))) + (expand-forms `(call broadcast ,(from-lambda (cadr e)) ,@(caddr e))))) ;; table mapping expression head to a function expanding that form (define expand-table diff --git a/test/broadcast.jl b/test/broadcast.jl index 99e1303d8debe..e433992ca9e4c 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -219,6 +219,7 @@ let A = [sqrt(i)+j for i = 1:3, j=1:4] end let x = sin.(1:10) @test atan2.((x->x+1).(x), (x->x+2).(x)) == atan2(x+1, x+2) == atan2(x.+1, x.+2) + @test sin.(atan2.([x+1,x+2]...)) == sin.(atan2.(x+1,x+2)) end # Use side effects to check for loop fusion. Note that, due to #17314, # a broadcasted function is currently called twice on the first element. From 62f141763ab40989c0aaffff280338822d63e98c Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Sun, 10 Jul 2016 16:17:46 -0400 Subject: [PATCH 0399/1117] compress fused broadcast args by eliminating literals and some (pure) duplicates --- src/julia-syntax.scm | 49 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 9edda07871fd1..9574af45482bb 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1554,13 +1554,12 @@ (define (fuse? e) (and (pair? e) (eq? (car e) 'fuse))) (define (anyfuse? exprs) (if (null? exprs) #f (if (fuse? (car exprs)) #t (anyfuse? (cdr exprs))))) - (define (...? e) (and (pair? e) (eq? (car e) '...))) (define (to-lambda f args) ; convert f to anonymous function with hygienic tuple args - (define (genarg arg) (if (...? arg) (list '... (gensy)) (gensy))) + (define (genarg arg) (if (vararg? arg) (list '... (gensy)) (gensy))) (define (hygienic f) ; rename args of f == (-> (tuple args...) body) (let* ((oldargs (cdadr f)) (newargs (map genarg oldargs)) - (renames (map (lambda (oldarg newarg) (if (...? oldarg) + (renames (map (lambda (oldarg newarg) (if (vararg? oldarg) (cons (cadr oldarg) (cadr newarg)) (cons oldarg newarg))) oldargs newargs))) @@ -1610,7 +1609,49 @@ (if (anyfuse? args_) `(fuse ,(fuse-funcs (to-lambda f args) args_) ,(fuse-args args_)) `(fuse ,(to-lambda f args) ,args_)))) - (let ((e (make-fuse f args))) ; an expression '(fuse func args) + ; given e == (fuse lambda args), compress the argument list by removing (pure) + ; duplicates in args, inlining literals, and moving any varargs to the end: + (define (compress-fuse e) + (define (findfarg arg args fargs) ; for arg in args, return corresponding farg + (if (eq? arg (car args)) + (car fargs) + (findfarg arg (cdr args) (cdr fargs)))) + (let ((f (cadr e)) + (args (caddr e))) + (define (cf old-fargs old-args new-fargs new-args renames varfarg vararg) + (if (null? old-args) + (let ((nfargs (if (null? varfarg) new-fargs (cons varfarg new-fargs))) + (nargs (if (null? vararg) new-args (cons vararg new-args)))) + `(fuse (-> (tuple ,@(reverse nfargs)) ,(replace-vars (caddr f) renames)) + ,(reverse nargs))) + (let ((farg (car old-fargs)) (arg (car old-args))) + (cond + ((and (vararg? farg) (vararg? arg)) ; arg... must be the last argument + (if (null? varfarg) + (cf (cdr old-fargs) (cdr old-args) + new-fargs new-args renames farg arg) + (if (eq? (cadr vararg) (cadr arg)) + (cf (cdr old-fargs) (cdr old-args) + new-fargs new-args (cons (cons (cadr farg) (cadr varfarg)) renames) + varfarg vararg) + (error "multiple splatted args cannot be fused into a single broadcast")))) + ((number? arg) ; inline numeric literals + (cf (cdr old-fargs) (cdr old-args) + new-fargs new-args + (cons (cons farg arg) renames) + varfarg vararg)) + ((and (symbol? arg) (memq arg new-args)) ; combine duplicate args + ; (note: calling memq for every arg is O(length(args)^2) ... + ; ... would be better to replace with a hash table if args is long) + (cf (cdr old-fargs) (cdr old-args) + new-fargs new-args + (cons (cons farg (findfarg arg new-args new-fargs)) renames) + varfarg vararg)) + (else + (cf (cdr old-fargs) (cdr old-args) + (cons farg new-fargs) (cons arg new-args) renames varfarg vararg)))))) + (cf (cdadr f) args '() '() '() '() '()))) + (let ((e (compress-fuse (make-fuse f args)))) ; an expression '(fuse func args) (expand-forms `(call broadcast ,(from-lambda (cadr e)) ,@(caddr e))))) ;; table mapping expression head to a function expanding that form From 908f978d7ed276975a4cfc3eefda3be858929164 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Sun, 10 Jul 2016 16:53:47 -0400 Subject: [PATCH 0400/1117] simplify/fix dot broadcasting of anonymous functions --- src/julia-syntax.scm | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 9574af45482bb..cfbfac08e1c9e 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1556,20 +1556,12 @@ (if (null? exprs) #f (if (fuse? (car exprs)) #t (anyfuse? (cdr exprs))))) (define (to-lambda f args) ; convert f to anonymous function with hygienic tuple args (define (genarg arg) (if (vararg? arg) (list '... (gensy)) (gensy))) - (define (hygienic f) ; rename args of f == (-> (tuple args...) body) - (let* ((oldargs (cdadr f)) - (newargs (map genarg oldargs)) - (renames (map (lambda (oldarg newarg) (if (vararg? oldarg) - (cons (cadr oldarg) (cadr newarg)) - (cons oldarg newarg))) - oldargs newargs))) - `(-> (tuple ,@newargs) ,(replace-vars (caddr f) renames)))) - (if (and (pair? f) (eq? (car f) '->)) - (if (pair? (cadr f)) - (hygienic f) ; f is already anon function, nothing to do but hygienicize - (hygienic `(-> (tuple ,(cadr f)) ,(caddr f)))) ; canonicalize single arg to tuple - (let ((genargs (map genarg args))) ; formal parameters - `(-> ,(cons 'tuple genargs) (call ,f ,@genargs))))) + ; to do: optimize the case where f is already an anonymous function, in which + ; case we only need to hygienicize the arguments? But it is quite tricky + ; to fully handle splatted args, typed args, keywords, etcetera. And probably + ; the extra function call is harmless because it will get inlined anyway. + (let ((genargs (map genarg args))) ; construct formal parameters + `(-> ,(cons 'tuple genargs) (call ,f ,@genargs)))) (define (from-lambda f) ; convert (-> (tuple args...) (call func args...)) back to func (if (and (pair? f) (eq? (car f) '->) (pair? (cadr f)) (eq? (caadr f) 'tuple) (pair? (caddr f)) (eq? (caaddr f) 'call) (equal? (cdadr f) (cdr (cdaddr f)))) From 7bf43e6a68c32a43617197ac6866d1cce6048a0b Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Mon, 11 Jul 2016 19:48:23 -0400 Subject: [PATCH 0401/1117] work around #17318 --- test/broadcast.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/broadcast.jl b/test/broadcast.jl index e433992ca9e4c..bf7d4e563c1ae 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -170,10 +170,10 @@ rt = Base.return_types(broadcast!, Tuple{Function, Array{Float64, 3}, Array{Floa @test length(rt) == 1 && rt[1] == Array{Float64, 3} # f.(args...) syntax (#15032) -let x = [1,3.2,4.7], y = [3.5, pi, 1e-4], α = 0.2342 +let x = [1,3.2,4.7], y = [3.5, pi, 1e-4], α = 0.2342, n = 3 @test sin.(x) == broadcast(sin, x) @test sin.(α) == broadcast(sin, α) - @test factorial.(3) == broadcast(factorial, 3) + @test factorial.(n) == broadcast(factorial, n) @test atan2.(x, y) == broadcast(atan2, x, y) @test atan2.(x, y') == broadcast(atan2, x, y') @test atan2.(x, α) == broadcast(atan2, x, α) From 11b479ff1eeb7f41576228f0580b55aafb68d539 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Mon, 11 Jul 2016 19:49:37 -0400 Subject: [PATCH 0402/1117] more fusion tests (splatting, literals) --- test/broadcast.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/broadcast.jl b/test/broadcast.jl index bf7d4e563c1ae..25119990556e6 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -220,6 +220,7 @@ end let x = sin.(1:10) @test atan2.((x->x+1).(x), (x->x+2).(x)) == atan2(x+1, x+2) == atan2(x.+1, x.+2) @test sin.(atan2.([x+1,x+2]...)) == sin.(atan2.(x+1,x+2)) + @test atan2.(x, 3.7) == broadcast(x -> atan2(x,3.7), x) == broadcast(atan2, x, 3.7) end # Use side effects to check for loop fusion. Note that, due to #17314, # a broadcasted function is currently called twice on the first element. @@ -228,6 +229,13 @@ let g = Int[] f17300.(f17300.(f17300.(1:3))) @test g == [1,2,3, 1,2,3, 2,3,4, 3,4,5] end +# fusion with splatted args: +let x = sin.(1:10), a = [x] + @test cos.(x) == cos.(a...) + @test atan2.(x,x) == atan2.(a..., a...) == atan2.([x, x]...) + @test atan2.(x, cos.(x)) == atan2.(a..., cos.(x)) == atan2(x, cos.(a...)) == atan2(a..., cos.(a...)) + @test ((args...)->cos(args[1])).(x) == cos.(x) == ((y,args...)->cos(y)).(x) +end # PR 16988 @test Base.promote_op(+, Bool) === Int From eee56c225fe2e576b9daa770ee4ce1561e6aa292 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Mon, 11 Jul 2016 19:50:11 -0400 Subject: [PATCH 0403/1117] correct description of #17314 --- test/broadcast.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/broadcast.jl b/test/broadcast.jl index 25119990556e6..9f767672487d9 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -223,7 +223,7 @@ let x = sin.(1:10) @test atan2.(x, 3.7) == broadcast(x -> atan2(x,3.7), x) == broadcast(atan2, x, 3.7) end # Use side effects to check for loop fusion. Note that, due to #17314, -# a broadcasted function is currently called twice on the first element. +# a broadcasted function is currently called an extra time with an argument 1. let g = Int[] f17300(x) = begin; push!(g, x); x+1; end f17300.(f17300.(f17300.(1:3))) From d858289b2fa599695e09d15bd1fec209900eb607 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Mon, 11 Jul 2016 20:07:46 -0400 Subject: [PATCH 0404/1117] undo #17318 workaround, add more constant-folding tests --- test/broadcast.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/broadcast.jl b/test/broadcast.jl index 9f767672487d9..38653fafec5e2 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -170,10 +170,10 @@ rt = Base.return_types(broadcast!, Tuple{Function, Array{Float64, 3}, Array{Floa @test length(rt) == 1 && rt[1] == Array{Float64, 3} # f.(args...) syntax (#15032) -let x = [1,3.2,4.7], y = [3.5, pi, 1e-4], α = 0.2342, n = 3 +let x = [1,3.2,4.7], y = [3.5, pi, 1e-4], α = 0.2342 @test sin.(x) == broadcast(sin, x) @test sin.(α) == broadcast(sin, α) - @test factorial.(n) == broadcast(factorial, n) + @test factorial.(3) == broadcast(factorial, 3) @test atan2.(x, y) == broadcast(atan2, x, y) @test atan2.(x, y') == broadcast(atan2, x, y') @test atan2.(x, α) == broadcast(atan2, x, α) @@ -220,6 +220,7 @@ end let x = sin.(1:10) @test atan2.((x->x+1).(x), (x->x+2).(x)) == atan2(x+1, x+2) == atan2(x.+1, x.+2) @test sin.(atan2.([x+1,x+2]...)) == sin.(atan2.(x+1,x+2)) + @test sin.(atan2.(x, 3.7)) == broadcast(x -> sin(atan2(x,3.7)), x) @test atan2.(x, 3.7) == broadcast(x -> atan2(x,3.7), x) == broadcast(atan2, x, 3.7) end # Use side effects to check for loop fusion. Note that, due to #17314, @@ -236,6 +237,7 @@ let x = sin.(1:10), a = [x] @test atan2.(x, cos.(x)) == atan2.(a..., cos.(x)) == atan2(x, cos.(a...)) == atan2(a..., cos.(a...)) @test ((args...)->cos(args[1])).(x) == cos.(x) == ((y,args...)->cos(y)).(x) end +@test atan2.(3,4) == atan2(3,4) == (() -> atan2(3,4)).() # PR 16988 @test Base.promote_op(+, Bool) === Int From eec4d67d6709de6c7476e89cf01c1b101f19e513 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Tue, 12 Jul 2016 11:21:35 -0400 Subject: [PATCH 0405/1117] handle dot calls with keyword arguments --- src/julia-syntax.scm | 35 +++++++++++++++++++++++++---------- test/broadcast.jl | 8 ++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index cfbfac08e1c9e..4cfcf09e11fb6 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1554,14 +1554,16 @@ (define (fuse? e) (and (pair? e) (eq? (car e) 'fuse))) (define (anyfuse? exprs) (if (null? exprs) #f (if (fuse? (car exprs)) #t (anyfuse? (cdr exprs))))) - (define (to-lambda f args) ; convert f to anonymous function with hygienic tuple args + (define (to-lambda f args kwargs) ; convert f to anonymous function with hygienic tuple args (define (genarg arg) (if (vararg? arg) (list '... (gensy)) (gensy))) - ; to do: optimize the case where f is already an anonymous function, in which - ; case we only need to hygienicize the arguments? But it is quite tricky - ; to fully handle splatted args, typed args, keywords, etcetera. And probably - ; the extra function call is harmless because it will get inlined anyway. - (let ((genargs (map genarg args))) ; construct formal parameters - `(-> ,(cons 'tuple genargs) (call ,f ,@genargs)))) + ; (To do: optimize the case where f is already an anonymous function, in which + ; case we only need to hygienicize the arguments? But it is quite tricky + ; to fully handle splatted args, typed args, keywords, etcetera. And probably + ; the extra function call is harmless because it will get inlined anyway.) + (let ((genargs (map genarg args))) ; hygienic formal parameters + (if (null? kwargs) + `(-> ,(cons 'tuple genargs) (call ,f ,@genargs)) ; no keyword args + `(-> ,(cons 'tuple genargs) (call ,f (parameters ,@kwargs) ,@genargs))))) (define (from-lambda f) ; convert (-> (tuple args...) (call func args...)) back to func (if (and (pair? f) (eq? (car f) '->) (pair? (cadr f)) (eq? (caadr f) 'tuple) (pair? (caddr f)) (eq? (caaddr f) 'call) (equal? (cdadr f) (cdr (cdaddr f)))) @@ -1593,14 +1595,27 @@ fargs args))) (let ,fbody ,@(reverse (fuse-lets fargs args '())))))) (define (make-fuse f args) ; check for nested (fuse f args) exprs and combine + (define (split-kwargs args) ; return (cons keyword-args positional-args) extracted from args + (define (sk args kwargs pargs) + (if (null? args) + (cons kwargs pargs) + (if (kwarg? (car args)) + (sk (cdr args) (cons (car args) kwargs) pargs) + (sk (cdr args) kwargs (cons (car args) pargs))))) + (if (has-parameters? args) + (sk (reverse (cdr args)) (cdar args) '()) + (sk (reverse args) '() '()))) (define (dot-to-fuse e) ; convert e == (. f (tuple args)) to (fuse f args) (if (and (pair? e) (eq? (car e) '|.|) (not (getfield-field? (caddr e)))) (make-fuse (cadr e) (cdaddr e)) e)) - (let ((args_ (map dot-to-fuse args))) + (let* ((kws.args (split-kwargs args)) + (kws (car kws.args)) + (args (cdr kws.args)) ; fusing occurs on positional args only + (args_ (map dot-to-fuse args))) (if (anyfuse? args_) - `(fuse ,(fuse-funcs (to-lambda f args) args_) ,(fuse-args args_)) - `(fuse ,(to-lambda f args) ,args_)))) + `(fuse ,(fuse-funcs (to-lambda f args kws) args_) ,(fuse-args args_)) + `(fuse ,(to-lambda f args kws) ,args_)))) ; given e == (fuse lambda args), compress the argument list by removing (pure) ; duplicates in args, inlining literals, and moving any varargs to the end: (define (compress-fuse e) diff --git a/test/broadcast.jl b/test/broadcast.jl index 38653fafec5e2..2668e21e68e05 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -238,6 +238,14 @@ let x = sin.(1:10), a = [x] @test ((args...)->cos(args[1])).(x) == cos.(x) == ((y,args...)->cos(y)).(x) end @test atan2.(3,4) == atan2(3,4) == (() -> atan2(3,4)).() +# fusion with keyword args: +let x = [1:4;] + f17300kw(x; y=0) = x + y + @test f17300kw.(x) == x + @test f17300kw.(x, y=1) == f17300kw.(x; y=1) == f17300kw.(x; [(:y,1)]...) == x .+ 1 + @test f17300kw.(sin.(x), y=1) == f17300kw.(sin.(x); y=1) == sin.(x) .+ 1 + @test sin.(f17300kw.(x, y=1)) == sin.(f17300kw.(x; y=1)) == sin.(x .+ 1) +end # PR 16988 @test Base.promote_op(+, Bool) === Int From 1fc7018d477a8bf7d312302d6a2bc45848f3e91a Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Tue, 12 Jul 2016 16:21:08 -0400 Subject: [PATCH 0406/1117] export and document transcode (#17323) * export and document transcode from #16974, add transcode(String, x) and transcode(T, ::String) convenience methods * docs * support UTF-32 in transcode * don't use splatting for UTF-32 to String conversion * typo * eliminate method ambiguities * re-run genstdlib * doc clarification * typo --- NEWS.md | 4 ++++ base/c.jl | 37 ++++++++++++++++++++++++++++--------- base/env.jl | 4 ++-- base/exports.jl | 1 + base/file.jl | 4 ++-- base/interactiveutil.jl | 2 +- base/libc.jl | 2 +- base/path.jl | 4 ++-- doc/manual/strings.rst | 8 +++++--- doc/stdlib/strings.rst | 11 ++++++++++- test/misc.jl | 8 +++++++- 11 files changed, 63 insertions(+), 22 deletions(-) diff --git a/NEWS.md b/NEWS.md index 71440f4f7f935..e48f3fe80e22f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -132,6 +132,9 @@ Library improvements `String(s)`, `unsafe_string(ptr)` (formerly `bytestring(ptr)`), and `unsafe_wrap(String, ptr)` (formerly `pointer_to_string`) ([#16731]). + * A `transcode(T, src)` function is now exported for converting data + between UTF-xx Unicode encodings ([#17323]). + * Most of the combinatorics functions have been moved from `Base` to the [Combinatorics.jl package](https://github.com/JuliaLang/Combinatorics.jl) ([#13897]). @@ -334,4 +337,5 @@ Deprecated or removed [#17075]: https://github.com/JuliaLang/julia/issues/17075 [#17266]: https://github.com/JuliaLang/julia/issues/17266 [#17300]: https://github.com/JuliaLang/julia/issues/17300 +[#17323]: https://github.com/JuliaLang/julia/issues/17323 [#17374]: https://github.com/JuliaLang/julia/issues/17374 diff --git a/base/c.jl b/base/c.jl index 97a1815b7d23c..8bd9b85179b24 100644 --- a/base/c.jl +++ b/base/c.jl @@ -128,20 +128,39 @@ function cwstring(s::AbstractString) end end -# transcoding between data in UTF-8 and UTF-16 for Windows APIs +# transcoding between data in UTF-8 and UTF-16 for Windows APIs, +# and also UTF-32 for APIs using Cwchar_t on other platforms. + """ - Base.transcode(T,src::Vector{U}) + transcode(T, src) + +Convert string data between Unicode encodings. `src` is either a +`String` or a `Vector{UIntXX}` of UTF-XX code units, where +`XX` is 8, 16, or 32. `T` indicates the encoding of the return value: +`String` to return a (UTF-8 encoded) `String` or `UIntXX` +to return a `Vector{UIntXX}` of UTF-`XX` data. (The alias `Cwchar_t` +can also be used as the integer type, for converting `wchar_t*` strings +used by external C libraries.) -Transcodes unicode data `src` to a different encoding, where `U` and `T` are the integers -denoting the input and output code units. Currently supported are UTF-8 and UTF-16, which -are denoted by integers `UInt8` and `UInt16`, respectively. +The `transcode` function succeeds as long as the input data can be +reasonably represented in the target encoding; it always succeeds for +conversions between UTF-XX encodings, even for invalid Unicode data. -NULs are handled like any other character (i.e. the output will be NUL-terminated if and -only if the `src` is). +Only conversion to/from UTF-8 is currently supported. """ function transcode end -transcode{T<:Union{UInt8,UInt16}}(::Type{T}, src::Vector{T}) = src -transcode(::Type{Int32}, src::Vector{UInt32}) = reinterpret(Int32, src) + +transcode{T<:Union{UInt8,UInt16,UInt32,Int32}}(::Type{T}, src::Vector{T}) = src +transcode{T<:Union{Int32,UInt32}}(::Type{T}, src::String) = T[T(c) for c in src] +transcode{T<:Union{Int32,UInt32}}(::Type{T}, src::Vector{UInt8}) = transcode(T, String(src)) +function transcode{S<:Union{Int32,UInt32}}(::Type{UInt8}, src::Vector{S}) + buf = IOBuffer() + for c in src; print(buf, Char(c)); end + takebuf_array(buf) +end +transcode(::Type{String}, src::String) = src +transcode(T, src::String) = transcode(T, src.data) +transcode(::Type{String}, src) = String(transcode(UInt8, src)) function transcode(::Type{UInt16}, src::Vector{UInt8}) dst = UInt16[] diff --git a/base/env.jl b/base/env.jl index cb21ecbe0dcf4..0f41a5bbf170e 100644 --- a/base/env.jl +++ b/base/env.jl @@ -19,7 +19,7 @@ function access_env(onError::Function, str::AbstractString) error(string("getenv: ", str, ' ', len, "-1 != ", ret, ": ", Libc.FormatMessage())) end pop!(val) # NUL - return String(transcode(UInt8, val)) + return transcode(String, val) end function _setenv(svar::AbstractString, sval::AbstractString, overwrite::Bool=true) @@ -97,7 +97,7 @@ function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) buf = Array{UInt16}(len) unsafe_copy!(pointer(buf), pos, len) - env = String(transcode(UInt8, buf)) + env = transcode(String, buf) m = match(r"^(=?[^=]+)=(.*)$"s, env) if m === nothing error("malformed environment entry: $env") diff --git a/base/exports.jl b/base/exports.jl index 4cdc9e4739042..dda16f688f0c8 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -874,6 +874,7 @@ export strip, strwidth, summary, + transcode, ucfirst, unescape_string, uppercase, diff --git a/base/file.jl b/base/file.jl index 97aa73a31aee3..396ba739f2cd4 100644 --- a/base/file.jl +++ b/base/file.jl @@ -203,7 +203,7 @@ function tempdir() error("GetTempPath failed: $(Libc.FormatMessage())") end resize!(temppath,lentemppath) - return String(transcode(UInt8, temppath)) + return transcode(String, temppath) end tempname(uunique::UInt32=UInt32(0)) = tempname(tempdir(), uunique) const temp_prefix = cwstring("jl_") @@ -216,7 +216,7 @@ function tempname(temppath::AbstractString,uunique::UInt32) error("GetTempFileName failed: $(Libc.FormatMessage())") end resize!(tname,lentname) - return String(transcode(UInt8, tname)) + return transcode(String, tname) end function mktemp(parent=tempdir()) filename = tempname(parent, UInt32(0)) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index ce4274d2551a1..6ac54783f30e4 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -150,7 +150,7 @@ elseif is_windows() len = 0 while unsafe_load(plock, len+1) != 0; len += 1; end # get Vector{UInt16}, transcode data to UTF-8, make a String of it - s = String(transcode(UInt8, unsafe_wrap(Array, plock, len))) + s = transcode(String, unsafe_wrap(Array, plock, len)) systemerror(:GlobalUnlock, 0==ccall((:GlobalUnlock, "kernel32"), stdcall, Cint, (Ptr{UInt16},), plock)) return s end diff --git a/base/libc.jl b/base/libc.jl index 6943d457ebde7..8020147d42ce1 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -277,7 +277,7 @@ if is_windows() buf = Array{UInt16}(len) unsafe_copy!(pointer(buf), p, len) ccall(:LocalFree,stdcall,Ptr{Void},(Ptr{Void},),p) - return String(transcode(UInt8, buf)) + return transcode(String, buf) end end diff --git a/base/path.jl b/base/path.jl index 494250dc29683..d19779734913a 100644 --- a/base/path.jl +++ b/base/path.jl @@ -136,7 +136,7 @@ function realpath(path::AbstractString) systemerror(:realpath, n == 0) x = n < length(buf) # is the buffer big enough? resize!(buf, n) # shrink if x, grow if !x - x && return String(transcode(UInt8, buf)) + x && return transcode(String, buf) end end @@ -150,7 +150,7 @@ function longpath(path::AbstractString) systemerror(:longpath, n == 0) x = n < length(buf) # is the buffer big enough? resize!(buf, n) # shrink if x, grow if !x - x && return String(transcode(UInt8, buf)) + x && return transcode(String, buf) end end diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 4adec34080742..5628c1d415e1d 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -352,14 +352,16 @@ exception handling required: <BLANKLINE> y -Julia uses UTF-8 encoding by default, and support for new encodings can +Julia uses the UTF-8 encoding by default, and support for new encodings can be added by packages. For example, the `LegacyStrings.jl <https://github.com/JuliaArchive/LegacyStrings.jl>`_ package implements ``UTF16String`` and ``UTF32String`` types. Additional discussion of other encodings and how to implement support for them is beyond the scope of this document for the time being. For further discussion of UTF-8 encoding issues, -see the section below on `byte array literals <#Byte+Array+Literals>`_, -which goes into some greater detail. +see the section below on `byte array literals <#Byte+Array+Literals>`_. +The :func:`transcode` function is provided to convert data between +the various UTF-xx encodings, primarily for working with external +data and libraries. .. _man-string-interpolation: diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index b27e7616a1e1e..1e6cd2a46b5cb 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -56,6 +56,16 @@ Convert a string to a contiguous byte array representation encoded as UTF-8 bytes. This representation is often appropriate for passing strings to C. +.. function:: transcode(T, src) + + .. Docstring generated from Julia source + + Convert string data between Unicode encodings. ``src`` is either a ``String`` or a ``Vector{UIntXX}`` of UTF-XX code units, where ``XX`` is 8, 16, or 32. ``T`` indicates the encoding of the return value: ``String`` to return a (UTF-8 encoded) ``String`` or ``UIntXX`` to return a ``Vector{UIntXX}`` of UTF-``XX`` data. (The alias ``Cwchar_t`` can also be used as the integer type, for converting ``wchar_t*`` strings used by external C libraries.) + + The ``transcode`` function succeeds as long as the input data can be reasonably represented in the target encoding; it always succeeds for conversions between UTF-XX encodings, even for invalid Unicode data. + + Only conversion to/from UTF-8 is currently supported. + .. function:: unsafe_string(p::Ptr{UInt8}, [length::Integer]) .. Docstring generated from Julia source @@ -472,4 +482,3 @@ .. Docstring generated from Julia source General unescaping of traditional C and Unicode escape sequences. Reverse of :func:`escape_string`\ . See also :func:`unescape_string`\ . - diff --git a/test/misc.jl b/test/misc.jl index e3ae6b72e8ce6..16066830498e9 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -209,7 +209,6 @@ whos(IOBuffer(), Tmp14173) # warm up @test @allocated(whos(IOBuffer(), Tmp14173)) < 10000 ## test conversion from UTF-8 to UTF-16 (for Windows APIs) -import Base.Libc: transcode # empty arrays @test transcode(UInt16, UInt8[]) == UInt16[] @@ -376,6 +375,13 @@ for (X,Y,Z) in ((V16,V16,V16), (I16,V16,I16), (V16,I16,V16), (V16,V16,I16), (I16 end end +let s = "abcα🐨\0x\0" + for T in (UInt8, UInt16, UInt32, Int32) + @test transcode(T, s) == transcode(T, s.data) + @test transcode(String, transcode(T, s)) == s + end +end + # clipboard functionality if is_windows() for str in ("Hello, world.", "∀ x ∃ y", "") From 17cbc751577b88c672c040742c7f20aec5260816 Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Tue, 12 Jul 2016 22:25:42 +0200 Subject: [PATCH 0407/1117] Use broadcast for array ops, closes #17254 (#17313) --- base/arraymath.jl | 42 +++++++++++++----------------------------- base/broadcast.jl | 40 +++++----------------------------------- test/arrayops.jl | 19 +++++++++++++++++++ 3 files changed, 37 insertions(+), 64 deletions(-) diff --git a/base/arraymath.jl b/base/arraymath.jl index 53a73c3b5ef66..785735d37bcb0 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -50,37 +50,21 @@ end @pure promote_array_type{S<:Integer, P}(F, ::Type{S}, ::Type{Bool}, ::Type{P}) = P for f in (:+, :-, :div, :mod, :&, :|, :$) - @eval begin - function ($f){S,T}(A::Range{S}, B::Range{T}) - F = similar(A, promote_op($f,S,T), promote_shape(size(A),size(B))) - for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) - @inbounds F[iF] = ($f)(A[iA], B[iB]) - end - return F - end - function ($f){S,T}(A::AbstractArray{S}, B::Range{T}) - F = similar(A, promote_op($f,S,T), promote_shape(A,B)) - for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) - @inbounds F[iF] = ($f)(A[iA], B[iB]) - end - return F - end - function ($f){S,T}(A::Range{S}, B::AbstractArray{T}) - F = similar(B, promote_op($f,S,T), promote_shape(A,B)) - for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) - @inbounds F[iF] = ($f)(A[iA], B[iB]) - end - return F - end - function ($f){S,T}(A::AbstractArray{S}, B::AbstractArray{T}) - F = similar(A, promote_op($f,S,T), promote_shape(A,B)) - for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) - @inbounds F[iF] = ($f)(A[iA], B[iB]) - end - return F - end + @eval ($f){S,T}(A::AbstractArray{S}, B::AbstractArray{T}) = + _elementwise($f, A, B, promote_eltype_op($f, A, B)) +end +function _elementwise{S,T}(op, A::AbstractArray{S}, B::AbstractArray{T}, ::Type{Any}) + promote_shape(A,B) # check size compatibility + return broadcast(op, A, B) +end +function _elementwise{S,T,R}(op, A::AbstractArray{S}, B::AbstractArray{T}, ::Type{R}) + F = similar(A, R, promote_shape(A,B)) + for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) + @inbounds F[iF] = op(A[iA], B[iB]) end + return F end + for f in (:.+, :.-, :.*, :./, :.\, :.^, :.÷, :.%, :.<<, :.>>, :div, :mod, :rem, :&, :|, :$) @eval begin function ($f){T}(A::Number, B::AbstractArray{T}) diff --git a/base/broadcast.jl b/base/broadcast.jl index 7eff313075d95..f3eebb2c0cac3 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -271,44 +271,14 @@ end ## elementwise operators ## -.÷(A::AbstractArray, B::AbstractArray) = broadcast(÷, A, B) -.%(A::AbstractArray, B::AbstractArray) = broadcast(%, A, B) -.<<(A::AbstractArray, B::AbstractArray) = broadcast(<<, A, B) -.>>(A::AbstractArray, B::AbstractArray) = broadcast(>>, A, B) - +# should this be deprecated? Only use in Base is in sparsematrix.jl eltype_plus(As::AbstractArray...) = promote_eltype_op(+, As...) -.+(As::AbstractArray...) = broadcast!(+, Array{eltype_plus(As...)}(to_shape(broadcast_shape(As...))), As...) - -function .-(A::AbstractArray, B::AbstractArray) - broadcast!(-, Array{promote_op(-, eltype(A), eltype(B))}(to_shape(broadcast_shape(A,B))), A, B) -end - -eltype_mul(As::AbstractArray...) = promote_eltype_op(*, As...) - -.*(As::AbstractArray...) = broadcast!(*, Array{eltype_mul(As...)}(to_shape(broadcast_shape(As...))), As...) - -function ./(A::AbstractArray, B::AbstractArray) - broadcast!(/, Array{promote_op(/, eltype(A), eltype(B))}(to_shape(broadcast_shape(A, B))), A, B) -end - -function .\(A::AbstractArray, B::AbstractArray) - broadcast!(\, Array{promote_op(\, eltype(A), eltype(B))}(to_shape(broadcast_shape(A, B))), A, B) -end - -typealias RatIntT{T<:Integer} Union{Type{Rational{T}},Type{T}} -typealias CRatIntT{T<:Integer} Union{Type{Complex{Rational{T}}},Type{Complex{T}},Type{Rational{T}},Type{T}} -type_rdiv{T<:Integer,S<:Integer}(::RatIntT{T}, ::RatIntT{S}) = - Rational{promote_type(T,S)} -type_rdiv{T<:Integer,S<:Integer}(::CRatIntT{T}, ::CRatIntT{S}) = - Complex{Rational{promote_type(T,S)}} -function .//(A::AbstractArray, B::AbstractArray) - broadcast!(//, Array{type_rdiv(eltype(A), eltype(B))}(to_shape(broadcast_shape(A, B))), A, B) -end - -function .^(A::AbstractArray, B::AbstractArray) - broadcast!(^, Array{promote_op(^, eltype(A), eltype(B))}(to_shape(broadcast_shape(A, B))), A, B) +for op in (:÷, :%, :<<, :>>, :-, :/, :\, ://, :^) + @eval $(Symbol(:., op))(A::AbstractArray, B::AbstractArray) = broadcast($(op), A, B) end +.+(As::AbstractArray...) = broadcast(+, As...) +.*(As::AbstractArray...) = broadcast(*, As...) # ## element-wise comparison operators returning BitArray ## diff --git a/test/arrayops.jl b/test/arrayops.jl index 29377c403b5a2..741fac3fb5816 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1668,3 +1668,22 @@ let A = zeros(3,3) @test size(A[:,UInt(1):UInt(2)]) == (3,2) @test size(similar(A, UInt(3), 0x3)) == size(similar(A, (UInt(3), 0x3))) == (3,3) end + +# issue 17254 +module AutoRetType + +using Base.Test + +immutable Foo end +for op in (:+, :*, :÷, :%, :<<, :>>, :-, :/, :\, ://, :^) + @eval import Base.$(op) + @eval $(op)(::Foo, ::Foo) = Foo() +end +A = fill(Foo(), 10, 10) +@test typeof(A+A) == Matrix{Foo} +@test typeof(A-A) == Matrix{Foo} +for op in (:.+, :.*, :.÷, :.%, :.<<, :.>>, :.-, :./, :.\, :.//, :.^) + @eval @test typeof($(op)(A,A)) == Matrix{Foo} +end + +end From bbdda57d9745b88209b782cecf2c34d29977ea48 Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Tue, 12 Jul 2016 20:52:40 +0200 Subject: [PATCH 0408/1117] Avoid useless OOB checks for ccall array arguments. use Ref intead of single-element arrays instead. --- base/irrationals.jl | 2 +- base/libgit2/reference.jl | 8 +-- base/math.jl | 8 +-- base/mpfr.jl | 146 +++++++++++++++++++------------------- 4 files changed, 82 insertions(+), 82 deletions(-) diff --git a/base/irrationals.jl b/base/irrationals.jl index 27699a16a2112..0d05de33b6564 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -103,7 +103,7 @@ macro irrational(sym, val, def) c = BigFloat() ccall(($(string("mpfr_const_", def)), :libmpfr), Cint, (Ptr{BigFloat}, Int32), - &c, MPFR.ROUNDING_MODE[end]) + &c, MPFR.ROUNDING_MODE[]) return c end end : quote diff --git a/base/libgit2/reference.jl b/base/libgit2/reference.jl index 8dea56222198b..ade899820cf4f 100644 --- a/base/libgit2/reference.jl +++ b/base/libgit2/reference.jl @@ -192,24 +192,24 @@ end function Base.start(bi::GitBranchIter) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - btype = Cint[0] + btype = Ref{Cint}() err = ccall((:git_branch_next, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Cint}, Ptr{Void}), ref_ptr_ptr, btype, bi.ptr) err != Int(Error.GIT_OK) && return (nothing, -1, true) - return (GitReference(ref_ptr_ptr[]), btype[1], false) + return (GitReference(ref_ptr_ptr[]), btype[], false) end Base.done(bi::GitBranchIter, state) = Bool(state[3]) function Base.next(bi::GitBranchIter, state) ref_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - btype = Cint[0] + btype = Ref{Cint}() err = ccall((:git_branch_next, :libgit2), Cint, (Ptr{Ptr{Void}}, Ptr{Cint}, Ptr{Void}), ref_ptr_ptr, btype, bi.ptr) err != Int(Error.GIT_OK) && return (state[1:2], (nothing, -1, true)) - return (state[1:2], (GitReference(ref_ptr_ptr[]), btype[1], false)) + return (state[1:2], (GitReference(ref_ptr_ptr[]), btype[], false)) end Base.iteratorsize(::Type{GitBranchIter}) = Base.SizeUnknown() diff --git a/base/math.jl b/base/math.jl index c7c0516a5658e..66b9717e80efc 100644 --- a/base/math.jl +++ b/base/math.jl @@ -278,16 +278,16 @@ end modf(x) = rem(x,one(x)), trunc(x) -const _modff_temp = Float32[0] +const _modff_temp = Ref{Float32}() function modf(x::Float32) f = ccall((:modff,libm), Float32, (Float32,Ptr{Float32}), x, _modff_temp) - f, _modff_temp[1] + f, _modff_temp[] end -const _modf_temp = Float64[0] +const _modf_temp = Ref{Float64}() function modf(x::Float64) f = ccall((:modf,libm), Float64, (Float64,Ptr{Float64}), x, _modf_temp) - f, _modf_temp[1] + f, _modf_temp[] end ^(x::Float64, y::Float64) = nan_dom_err(ccall((:pow,libm), Float64, (Float64,Float64), x, y), x+y) diff --git a/base/mpfr.jl b/base/mpfr.jl index 2ebdbe38a57fa..afd4dda83e485 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -37,7 +37,7 @@ function __init__() end end -const ROUNDING_MODE = Cint[0] +const ROUNDING_MODE = Ref{Cint}(0) const DEFAULT_PRECISION = [256] # Basic type and initialization definitions @@ -70,7 +70,7 @@ for (fJ, fC) in ((:si,:Clong), (:ui,:Culong), (:d,:Float64)) @eval begin function convert(::Type{BigFloat}, x::($fC)) z = BigFloat() - ccall(($(string(:mpfr_set_,fJ)), :libmpfr), Int32, (Ptr{BigFloat}, ($fC), Int32), &z, x, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_set_,fJ)), :libmpfr), Int32, (Ptr{BigFloat}, ($fC), Int32), &z, x, ROUNDING_MODE[]) return z end end @@ -78,7 +78,7 @@ end function convert(::Type{BigFloat}, x::BigInt) z = BigFloat() - ccall((:mpfr_set_z, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_set_z, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, ROUNDING_MODE[]) return z end @@ -92,7 +92,7 @@ convert(::Type{BigFloat}, x::Rational) = BigFloat(num(x)) / BigFloat(den(x)) function tryparse(::Type{BigFloat}, s::AbstractString, base::Int=0) z = BigFloat() - err = ccall((:mpfr_set_str, :libmpfr), Int32, (Ptr{BigFloat}, Cstring, Int32, Int32), &z, s, base, ROUNDING_MODE[end]) + err = ccall((:mpfr_set_str, :libmpfr), Int32, (Ptr{BigFloat}, Cstring, Int32, Int32), &z, s, base, ROUNDING_MODE[]) err == 0 ? Nullable(z) : Nullable{BigFloat}() end @@ -144,13 +144,13 @@ end function round{T<:Union{Signed,Unsigned}}(::Type{T}, x::BigFloat) (typemin(T) <= x <= typemax(T)) || throw(InexactError()) - unsafe_cast(T,x,ROUNDING_MODE[end]) + unsafe_cast(T,x,ROUNDING_MODE[]) end trunc(::Type{BigInt}, x::BigFloat) = unsafe_cast(BigInt, x, RoundToZero) floor(::Type{BigInt}, x::BigFloat) = unsafe_cast(BigInt, x, RoundDown) ceil(::Type{BigInt}, x::BigFloat) = unsafe_cast(BigInt, x, RoundUp) -round(::Type{BigInt}, x::BigFloat) = unsafe_cast(BigInt, x, ROUNDING_MODE[end]) +round(::Type{BigInt}, x::BigFloat) = unsafe_cast(BigInt, x, ROUNDING_MODE[]) # convert/round/trunc/floor/ceil(Integer, x) should return a BigInt trunc(::Type{Integer}, x::BigFloat) = trunc(BigInt, x) @@ -171,9 +171,9 @@ end ## BigFloat -> AbstractFloat convert(::Type{Float64}, x::BigFloat) = - ccall((:mpfr_get_d,:libmpfr), Float64, (Ptr{BigFloat},Int32), &x, ROUNDING_MODE[end]) + ccall((:mpfr_get_d,:libmpfr), Float64, (Ptr{BigFloat},Int32), &x, ROUNDING_MODE[]) convert(::Type{Float32}, x::BigFloat) = - ccall((:mpfr_get_flt,:libmpfr), Float32, (Ptr{BigFloat},Int32), &x, ROUNDING_MODE[end]) + ccall((:mpfr_get_flt,:libmpfr), Float32, (Ptr{BigFloat},Int32), &x, ROUNDING_MODE[]) # TODO: avoid double rounding convert(::Type{Float16}, x::BigFloat) = convert(Float16, convert(Float32, x)) @@ -203,14 +203,14 @@ for (fJ, fC) in ((:+,:add), (:*,:mul)) # BigFloat function ($fJ)(x::BigFloat, y::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,fC)),:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC)),:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end # Unsigned Integer function ($fJ)(x::BigFloat, c::CulongMax) z = BigFloat() - ccall(($(string(:mpfr_,fC,:_ui)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC,:_ui)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, c, ROUNDING_MODE[]) return z end ($fJ)(c::CulongMax, x::BigFloat) = ($fJ)(x,c) @@ -218,7 +218,7 @@ for (fJ, fC) in ((:+,:add), (:*,:mul)) # Signed Integer function ($fJ)(x::BigFloat, c::ClongMax) z = BigFloat() - ccall(($(string(:mpfr_,fC,:_si)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC,:_si)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, c, ROUNDING_MODE[]) return z end ($fJ)(c::ClongMax, x::BigFloat) = ($fJ)(x,c) @@ -226,7 +226,7 @@ for (fJ, fC) in ((:+,:add), (:*,:mul)) # Float32/Float64 function ($fJ)(x::BigFloat, c::CdoubleMax) z = BigFloat() - ccall(($(string(:mpfr_,fC,:_d)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cdouble, Int32), &z, &x, c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC,:_d)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cdouble, Int32), &z, &x, c, ROUNDING_MODE[]) return z end ($fJ)(c::CdoubleMax, x::BigFloat) = ($fJ)(x,c) @@ -234,7 +234,7 @@ for (fJ, fC) in ((:+,:add), (:*,:mul)) # BigInt function ($fJ)(x::BigFloat, c::BigInt) z = BigFloat() - ccall(($(string(:mpfr_,fC,:_z)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC,:_z)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &c, ROUNDING_MODE[]) return z end ($fJ)(c::BigInt, x::BigFloat) = ($fJ)(x,c) @@ -246,50 +246,50 @@ for (fJ, fC) in ((:-,:sub), (:/,:div)) # BigFloat function ($fJ)(x::BigFloat, y::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,fC)),:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC)),:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end # Unsigned Int function ($fJ)(x::BigFloat, c::CulongMax) z = BigFloat() - ccall(($(string(:mpfr_,fC,:_ui)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC,:_ui)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, c, ROUNDING_MODE[]) return z end function ($fJ)(c::CulongMax, x::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,:ui_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Culong, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,:ui_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Culong, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[]) return z end # Signed Integer function ($fJ)(x::BigFloat, c::ClongMax) z = BigFloat() - ccall(($(string(:mpfr_,fC,:_si)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC,:_si)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, c, ROUNDING_MODE[]) return z end function ($fJ)(c::ClongMax, x::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,:si_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Clong, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,:si_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Clong, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[]) return z end # Float32/Float64 function ($fJ)(x::BigFloat, c::CdoubleMax) z = BigFloat() - ccall(($(string(:mpfr_,fC,:_d)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cdouble, Int32), &z, &x, c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC,:_d)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cdouble, Int32), &z, &x, c, ROUNDING_MODE[]) return z end function ($fJ)(c::CdoubleMax, x::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,:d_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Cdouble, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,:d_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Cdouble, Ptr{BigFloat}, Int32), &z, c, &x, ROUNDING_MODE[]) return z end # BigInt function ($fJ)(x::BigFloat, c::BigInt) z = BigFloat() - ccall(($(string(:mpfr_,fC,:_z)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC,:_z)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &c, ROUNDING_MODE[]) return z end # no :mpfr_z_div function @@ -298,13 +298,13 @@ end function -(c::BigInt, x::BigFloat) z = BigFloat() - ccall((:mpfr_z_sub, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigInt}, Ptr{BigFloat}, Int32), &z, &c, &x, ROUNDING_MODE[end]) + ccall((:mpfr_z_sub, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigInt}, Ptr{BigFloat}, Int32), &z, &c, &x, ROUNDING_MODE[]) return z end function fma(x::BigFloat, y::BigFloat, z::BigFloat) r = BigFloat() - ccall(("mpfr_fma",:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &r, &x, &y, &z, ROUNDING_MODE[end]) + ccall(("mpfr_fma",:libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &r, &x, &y, &z, ROUNDING_MODE[]) return r end @@ -373,23 +373,23 @@ for (fJ, fC, fI) in ((:+, :add, 0), (:*, :mul, 1)) @eval begin function ($fJ)(a::BigFloat, b::BigFloat, c::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[end]) - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[]) return z end function ($fJ)(a::BigFloat, b::BigFloat, c::BigFloat, d::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[end]) - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[end]) - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &d, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &d, ROUNDING_MODE[]) return z end function ($fJ)(a::BigFloat, b::BigFloat, c::BigFloat, d::BigFloat, e::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[end]) - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[end]) - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &d, ROUNDING_MODE[end]) - ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &e, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &a, &b, ROUNDING_MODE[]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &c, ROUNDING_MODE[]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &d, ROUNDING_MODE[]) + ccall(($(string(:mpfr_,fC)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &z, &e, ROUNDING_MODE[]) return z end end @@ -397,14 +397,14 @@ end function -(x::BigFloat) z = BigFloat() - ccall((:mpfr_neg, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_neg, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end function sqrt(x::BigFloat) isnan(x) && return x z = BigFloat() - ccall((:mpfr_sqrt, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_sqrt, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) if isnan(z) throw(DomainError()) end @@ -415,25 +415,25 @@ sqrt(x::BigInt) = sqrt(BigFloat(x)) function ^(x::BigFloat, y::BigFloat) z = BigFloat() - ccall((:mpfr_pow, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall((:mpfr_pow, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end function ^(x::BigFloat, y::CulongMax) z = BigFloat() - ccall((:mpfr_pow_ui, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, y, ROUNDING_MODE[end]) + ccall((:mpfr_pow_ui, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, y, ROUNDING_MODE[]) return z end function ^(x::BigFloat, y::ClongMax) z = BigFloat() - ccall((:mpfr_pow_si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, y, ROUNDING_MODE[end]) + ccall((:mpfr_pow_si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, y, ROUNDING_MODE[]) return z end function ^(x::BigFloat, y::BigInt) z = BigFloat() - ccall((:mpfr_pow_z, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall((:mpfr_pow_z, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigInt}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end @@ -444,7 +444,7 @@ for f in (:exp, :exp2, :exp10, :expm1, :digamma, :erf, :erfc, :zeta, :cosh,:sinh,:tanh,:sech,:csch,:coth, :cbrt) @eval function $f(x::BigFloat) z = BigFloat() - ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end end @@ -453,7 +453,7 @@ end function big_ln2() c = BigFloat() ccall((:mpfr_const_log2, :libmpfr), Cint, (Ptr{BigFloat}, Int32), - &c, MPFR.ROUNDING_MODE[end]) + &c, MPFR.ROUNDING_MODE[]) return c end @@ -464,19 +464,19 @@ end function airyai(x::BigFloat) z = BigFloat() - ccall((:mpfr_ai, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_ai, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end airy(x::BigFloat) = airyai(x) function ldexp(x::BigFloat, n::Clong) z = BigFloat() - ccall((:mpfr_mul_2si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, n, ROUNDING_MODE[end]) + ccall((:mpfr_mul_2si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &x, n, ROUNDING_MODE[]) return z end function ldexp(x::BigFloat, n::Culong) z = BigFloat() - ccall((:mpfr_mul_2ui, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, n, ROUNDING_MODE[end]) + ccall((:mpfr_mul_2ui, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Culong, Int32), &z, &x, n, ROUNDING_MODE[]) return z end ldexp(x::BigFloat, n::ClongMax) = ldexp(x, convert(Clong, n)) @@ -485,19 +485,19 @@ ldexp(x::BigFloat, n::Integer) = x*exp2(BigFloat(n)) function besselj0(x::BigFloat) z = BigFloat() - ccall((:mpfr_j0, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_j0, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end function besselj1(x::BigFloat) z = BigFloat() - ccall((:mpfr_j1, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_j1, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end function besselj(n::Integer, x::BigFloat) z = BigFloat() - ccall((:mpfr_jn, :libmpfr), Int32, (Ptr{BigFloat}, Clong, Ptr{BigFloat}, Int32), &z, n, &x, ROUNDING_MODE[end]) + ccall((:mpfr_jn, :libmpfr), Int32, (Ptr{BigFloat}, Clong, Ptr{BigFloat}, Int32), &z, n, &x, ROUNDING_MODE[]) return z end @@ -506,7 +506,7 @@ function bessely0(x::BigFloat) throw(DomainError()) end z = BigFloat() - ccall((:mpfr_y0, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_y0, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end @@ -515,7 +515,7 @@ function bessely1(x::BigFloat) throw(DomainError()) end z = BigFloat() - ccall((:mpfr_y1, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_y1, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end @@ -524,7 +524,7 @@ function bessely(n::Integer, x::BigFloat) throw(DomainError()) end z = BigFloat() - ccall((:mpfr_yn, :libmpfr), Int32, (Ptr{BigFloat}, Clong, Ptr{BigFloat}, Int32), &z, n, &x, ROUNDING_MODE[end]) + ccall((:mpfr_yn, :libmpfr), Int32, (Ptr{BigFloat}, Clong, Ptr{BigFloat}, Int32), &z, n, &x, ROUNDING_MODE[]) return z end @@ -534,13 +534,13 @@ function factorial(x::BigFloat) end ui = convert(Culong, x) z = BigFloat() - ccall((:mpfr_fac_ui, :libmpfr), Int32, (Ptr{BigFloat}, Culong, Int32), &z, ui, ROUNDING_MODE[end]) + ccall((:mpfr_fac_ui, :libmpfr), Int32, (Ptr{BigFloat}, Culong, Int32), &z, ui, ROUNDING_MODE[]) return z end function hypot(x::BigFloat, y::BigFloat) z = BigFloat() - ccall((:mpfr_hypot, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall((:mpfr_hypot, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end @@ -550,7 +550,7 @@ for f in (:log, :log2, :log10) throw(DomainError()) end z = BigFloat() - ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end end @@ -560,19 +560,19 @@ function log1p(x::BigFloat) throw(DomainError()) end z = BigFloat() - ccall((:mpfr_log1p, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_log1p, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) return z end function max(x::BigFloat, y::BigFloat) z = BigFloat() - ccall((:mpfr_max, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall((:mpfr_max, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end function min(x::BigFloat, y::BigFloat) z = BigFloat() - ccall((:mpfr_min, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall((:mpfr_min, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end @@ -582,13 +582,13 @@ function modf(x::BigFloat) end zint = BigFloat() zfloat = BigFloat() - ccall((:mpfr_modf, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &zint, &zfloat, &x, ROUNDING_MODE[end]) + ccall((:mpfr_modf, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &zint, &zfloat, &x, ROUNDING_MODE[]) return (zfloat, zint) end function rem(x::BigFloat, y::BigFloat) z = BigFloat() - ccall((:mpfr_fmod, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall((:mpfr_fmod, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end @@ -611,7 +611,7 @@ for f in (:sin,:cos,:tan,:sec,:csc, return x end z = BigFloat() - ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[end]) + ccall(($(string(:mpfr_,f)), :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, ROUNDING_MODE[]) if isnan(z) throw(DomainError()) end @@ -624,7 +624,7 @@ end const lgamma_signp = Array{Cint}(1) function lgamma(x::BigFloat) z = BigFloat() - ccall((:mpfr_lgamma,:libmpfr), Cint, (Ptr{BigFloat}, Ptr{Cint}, Ptr{BigFloat}, Int32), &z, lgamma_signp, &x, ROUNDING_MODE[end]) + ccall((:mpfr_lgamma,:libmpfr), Cint, (Ptr{BigFloat}, Ptr{Cint}, Ptr{BigFloat}, Int32), &z, lgamma_signp, &x, ROUNDING_MODE[]) return z end @@ -632,7 +632,7 @@ lgamma_r(x::BigFloat) = (lgamma(x), lgamma_signp[1]) function atan2(y::BigFloat, x::BigFloat) z = BigFloat() - ccall((:mpfr_atan2, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &y, &x, ROUNDING_MODE[end]) + ccall((:mpfr_atan2, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &y, &x, ROUNDING_MODE[]) return z end @@ -727,15 +727,15 @@ function from_mpfr(c::Integer) RoundingMode(c) end -rounding_raw(::Type{BigFloat}) = ROUNDING_MODE[end] -setrounding_raw(::Type{BigFloat},i::Integer) = ROUNDING_MODE[end] = i +rounding_raw(::Type{BigFloat}) = ROUNDING_MODE[] +setrounding_raw(::Type{BigFloat},i::Integer) = ROUNDING_MODE[] = i rounding(::Type{BigFloat}) = from_mpfr(rounding_raw(BigFloat)) setrounding(::Type{BigFloat},r::RoundingMode) = setrounding_raw(BigFloat,to_mpfr(r)) function copysign(x::BigFloat, y::BigFloat) z = BigFloat() - ccall((:mpfr_copysign, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[end]) + ccall((:mpfr_copysign, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Ptr{BigFloat}, Int32), &z, &x, &y, ROUNDING_MODE[]) return z end @@ -749,17 +749,17 @@ end function frexp(x::BigFloat) z = BigFloat() - c = Clong[0] - ccall((:mpfr_frexp, :libmpfr), Int32, (Ptr{Clong}, Ptr{BigFloat}, Ptr{BigFloat}, Cint), c, &z, &x, ROUNDING_MODE[end]) - return (z, c[1]) + c = Ref{Clong}() + ccall((:mpfr_frexp, :libmpfr), Int32, (Ptr{Clong}, Ptr{BigFloat}, Ptr{BigFloat}, Cint), c, &z, &x, ROUNDING_MODE[]) + return (z, c[]) end function significand(x::BigFloat) z = BigFloat() - c = Clong[0] - ccall((:mpfr_frexp, :libmpfr), Int32, (Ptr{Clong}, Ptr{BigFloat}, Ptr{BigFloat}, Cint), c, &z, &x, ROUNDING_MODE[end]) + c = Ref{Clong}() + ccall((:mpfr_frexp, :libmpfr), Int32, (Ptr{Clong}, Ptr{BigFloat}, Ptr{BigFloat}, Cint), c, &z, &x, ROUNDING_MODE[]) # Double the significand to make it work as Base.significand - ccall((:mpfr_mul_si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &z, 2, ROUNDING_MODE[end]) + ccall((:mpfr_mul_si, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Clong, Int32), &z, &z, 2, ROUNDING_MODE[]) return z end @@ -779,7 +779,7 @@ end function round(x::BigFloat) z = BigFloat() - ccall((:mpfr_rint, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cint), &z, &x, ROUNDING_MODE[end]) + ccall((:mpfr_rint, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Cint), &z, &x, ROUNDING_MODE[]) return z end function round(x::BigFloat,::RoundingMode{:NearestTiesAway}) @@ -804,7 +804,7 @@ isfinite(x::BigFloat) = !isinf(x) && !isnan(x) function nextfloat(x::BigFloat) z = BigFloat() ccall((:mpfr_set, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), - &z, &x, ROUNDING_MODE[end]) + &z, &x, ROUNDING_MODE[]) ccall((:mpfr_nextabove, :libmpfr), Int32, (Ptr{BigFloat},), &z) != 0 return z end @@ -812,7 +812,7 @@ end function prevfloat(x::BigFloat) z = BigFloat() ccall((:mpfr_set, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), - &z, &x, ROUNDING_MODE[end]) + &z, &x, ROUNDING_MODE[]) ccall((:mpfr_nextbelow, :libmpfr), Int32, (Ptr{BigFloat},), &z) != 0 return z end @@ -889,7 +889,7 @@ function Base.deepcopy_internal(x::BigFloat, stackdict::ObjectIdDict) ccall((:mpfr_init2,:libmpfr), Void, (Ptr{BigFloat}, Clong), &y, N) finalizer(y, cglobal((:mpfr_clear, :libmpfr))) ccall((:mpfr_set, :libmpfr), Int32, (Ptr{BigFloat}, Ptr{BigFloat}, Int32), - &y, &x, ROUNDING_MODE[end]) + &y, &x, ROUNDING_MODE[]) stackdict[x] = y return y end From d72517b905f5841090c41c5e0672a6d804ed50d8 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Tue, 12 Jul 2016 21:01:12 -0500 Subject: [PATCH 0409/1117] Add a lot of documentation on the bounds checking hierarchy [ci skip] This also reorganizes code to make the order follow the hierarchy; makes more sense for the "big picture" documentation to be near the top node. --- base/abstractarray.jl | 150 +++++++++++++++++++++--------------- base/multidimensional.jl | 18 ++--- doc/devdocs/boundscheck.rst | 49 ++++++++++++ doc/stdlib/arrays.rst | 12 +-- 4 files changed, 153 insertions(+), 76 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 266f65b1ea256..962574a3bf5ee 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -150,53 +150,30 @@ linearindexing(::LinearFast, ::LinearFast) = LinearFast() linearindexing(::LinearIndexing, ::LinearIndexing) = LinearSlow() ## Bounds checking ## -@generated function trailingsize{T,N,n}(A::AbstractArray{T,N}, ::Type{Val{n}}) - (isa(n, Int) && isa(N, Int)) || error("Must have concrete type") - n > N && return 1 - ex = :(size(A, $n)) - for m = n+1:N - ex = :($ex * size(A, $m)) - end - Expr(:block, Expr(:meta, :inline), ex) -end -# check along a single dimension -""" - checkindex(Bool, inds::UnitRange, index) +# The overall hierarchy is +# `checkbounds(A, I...)` -> +# `checkbounds(Bool, A, I...)` -> either of: +# - `checkbounds_indices(IA, I)` which calls `checkindex(Bool, inds, i)` +# - `checkbounds_logical(A, I)` when `I` is a single logical array +# +# See the "boundscheck" devdocs for more information. +# +# Note this hierarchy has been designed to reduce the likelihood of +# method ambiguities. We try to make `checkbounds` the place to +# specialize on array type, and try to avoid specializations on index +# types; conversely, `checkindex` is intended to be specialized only +# on index type (especially, its last argument). -Return `true` if the given `index` is within the bounds of -`inds`. Custom types that would like to behave as indices for all -arrays can extend this method in order to provide a specialized bounds -checking implementation. """ -checkindex(::Type{Bool}, inds::AbstractUnitRange, i) = throw(ArgumentError("unable to check bounds for indices of type $(typeof(i))")) -checkindex(::Type{Bool}, inds::AbstractUnitRange, i::Real) = (first(inds) <= i) & (i <= last(inds)) -checkindex(::Type{Bool}, inds::AbstractUnitRange, ::Colon) = true -function checkindex(::Type{Bool}, inds::AbstractUnitRange, r::Range) - @_propagate_inbounds_meta - isempty(r) | (checkindex(Bool, inds, first(r)) & checkindex(Bool, inds, last(r))) -end -checkindex{N}(::Type{Bool}, indx::AbstractUnitRange, I::AbstractArray{Bool,N}) = N == 1 && indx == indices1(I) -function checkindex(::Type{Bool}, inds::AbstractUnitRange, I::AbstractArray) - @_inline_meta - b = true - for i in I - b &= checkindex(Bool, inds, i) - end - b -end + checkbounds(Bool, A, I...) -# check all indices/dimensions +Return `true` if the specified indices `I` are in bounds for the given +array `A`. Subtypes of `AbstractArray` should specialize this method +if they need to provide custom bounds checking behaviors; however, in +many cases one can rely on `A`'s indices and `checkindex`. -# To facilitate extension for custom array types without triggering -# ambiguities, limit the number of specializations of checkbounds on -# the types of the indices. -""" - checkbounds(Bool, array, indexes...) - -Return `true` if the specified `indexes` are in bounds for the given -`array`. Subtypes of `AbstractArray` should specialize this method if -they need to provide custom bounds checking behaviors. +See also `checkindex`. """ function checkbounds(::Type{Bool}, A::AbstractArray, I...) @_inline_meta @@ -207,35 +184,48 @@ function checkbounds(::Type{Bool}, A::AbstractArray, I::AbstractArray{Bool}) checkbounds_logical(A, I) end -# checkbounds_indices iteratively consumes elements of the -# indices-tuple of an arrray and the indices-tuple supplied by the -# caller. These two tuples are usually consumed in a 1-for-1 fashion, -# i.e., -# -# checkbounds_indices((R1, R...), (I1, I...)) = checkindex(Bool, R1, I1) & -# checkbounds_indices(R, I) -# -# However, there are two exceptions: linear indexing and CartesianIndex{N}. +""" + checkbounds_indices(IA, I) + +checks whether the "requested" indices in the tuple `I` fall within +the bounds of the "permitted" indices specified by the tuple +`IA`. This function recursively consumes elements of these tuples, +usually in a 1-for-1 fashion, + + checkbounds_indices((IA1, IA...), (I1, I...)) = checkindex(Bool, IA1, I1) & + checkbounds_indices(IA, I) + +Note that `checkindex` is being used to perform the actual +bounds-check for a single dimension of the array. + +There are two important exceptions to the 1-1 rule: linear indexing and +CartesianIndex{N}, both of which may "consume" more than one element +of `IA`. +""" +function checkbounds_indices(IA::Tuple, I::Tuple) + @_inline_meta + checkindex(Bool, IA[1], I[1]) & checkbounds_indices(tail(IA), tail(I)) +end checkbounds_indices(::Tuple{}, ::Tuple{}) = true checkbounds_indices(::Tuple{}, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, 1:1, I[1])) function checkbounds_indices(::Tuple{}, I::Tuple) @_inline_meta checkindex(Bool, 1:1, I[1]) & checkbounds_indices((), tail(I)) end -function checkbounds_indices(inds::Tuple{Any}, I::Tuple{Any}) - @_inline_meta - checkindex(Bool, inds[1], I[1]) -end -function checkbounds_indices(inds::Tuple, I::Tuple{Any}) +function checkbounds_indices(IA::Tuple{Any}, I::Tuple{Any}) @_inline_meta - checkindex(Bool, 1:prod(map(dimlength, inds)), I[1]) # linear indexing + checkindex(Bool, IA[1], I[1]) end -function checkbounds_indices(inds::Tuple, I::Tuple) +function checkbounds_indices(IA::Tuple, I::Tuple{Any}) @_inline_meta - checkindex(Bool, inds[1], I[1]) & checkbounds_indices(tail(inds), tail(I)) + checkindex(Bool, 1:prod(map(dimlength, IA)), I[1]) # linear indexing end -# Single logical array indexing: +""" + checkbounds_logical(A, I::AbstractArray{Bool}) + +tests whether the logical array `I` is consistent with the indices of `A`. +""" checkbounds_logical(A::AbstractArray, I::AbstractArray{Bool}) = indices(A) == indices(I) checkbounds_logical(A::AbstractArray, I::AbstractVector{Bool}) = length(A) == length(I) checkbounds_logical(A::AbstractVector, I::AbstractArray{Bool}) = length(A) == length(I) @@ -244,9 +234,9 @@ checkbounds_logical(A::AbstractVector, I::AbstractVector{Bool}) = indices(A) == throw_boundserror(A, I) = (@_noinline_meta; throw(BoundsError(A, I))) """ - checkbounds(array, indexes...) + checkbounds(A, I...) -Throw an error if the specified `indexes` are not in bounds for the given `array`. +Throw an error if the specified indices `I` are not in bounds for the given array `A`. """ function checkbounds(A::AbstractArray, I...) @_inline_meta @@ -255,6 +245,42 @@ function checkbounds(A::AbstractArray, I...) end checkbounds(A::AbstractArray) = checkbounds(A, 1) # 0-d case +@generated function trailingsize{T,N,n}(A::AbstractArray{T,N}, ::Type{Val{n}}) + (isa(n, Int) && isa(N, Int)) || error("Must have concrete type") + n > N && return 1 + ex = :(size(A, $n)) + for m = n+1:N + ex = :($ex * size(A, $m)) + end + Expr(:block, Expr(:meta, :inline), ex) +end + +# check along a single dimension +""" + checkindex(Bool, inds::AbstractUnitRange, index) + +Return `true` if the given `index` is within the bounds of +`inds`. Custom types that would like to behave as indices for all +arrays can extend this method in order to provide a specialized bounds +checking implementation. +""" +checkindex(::Type{Bool}, inds::AbstractUnitRange, i) = throw(ArgumentError("unable to check bounds for indices of type $(typeof(i))")) +checkindex(::Type{Bool}, inds::AbstractUnitRange, i::Real) = (first(inds) <= i) & (i <= last(inds)) +checkindex(::Type{Bool}, inds::AbstractUnitRange, ::Colon) = true +function checkindex(::Type{Bool}, inds::AbstractUnitRange, r::Range) + @_propagate_inbounds_meta + isempty(r) | (checkindex(Bool, inds, first(r)) & checkindex(Bool, inds, last(r))) +end +checkindex{N}(::Type{Bool}, indx::AbstractUnitRange, I::AbstractArray{Bool,N}) = N == 1 && indx == indices1(I) +function checkindex(::Type{Bool}, inds::AbstractUnitRange, I::AbstractArray) + @_inline_meta + b = true + for i in I + b &= checkindex(Bool, inds, i) + end + b +end + # See also specializations in multidimensional ## Constructors ## diff --git a/base/multidimensional.jl b/base/multidimensional.jl index f286c7c18de79..6fe1ca22f52d5 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -156,10 +156,10 @@ using .IteratorsMD ## Bounds-checking with CartesianIndex @inline checkbounds_indices(::Tuple{}, I::Tuple{CartesianIndex,Vararg{Any}}) = checkbounds_indices((), (I[1].I..., tail(I)...)) -@inline checkbounds_indices(inds::Tuple{Any}, I::Tuple{CartesianIndex,Vararg{Any}}) = - checkbounds_indices(inds, (I[1].I..., tail(I)...)) -@inline checkbounds_indices(inds::Tuple, I::Tuple{CartesianIndex,Vararg{Any}}) = - checkbounds_indices(inds, (I[1].I..., tail(I)...)) +@inline checkbounds_indices(IA::Tuple{Any}, I::Tuple{CartesianIndex,Vararg{Any}}) = + checkbounds_indices(IA, (I[1].I..., tail(I)...)) +@inline checkbounds_indices(IA::Tuple, I::Tuple{CartesianIndex,Vararg{Any}}) = + checkbounds_indices(IA, (I[1].I..., tail(I)...)) # Support indexing with an array of CartesianIndex{N}s # Here we try to consume N of the indices (if there are that many available) @@ -167,12 +167,12 @@ using .IteratorsMD @inline function checkbounds_indices{N}(::Tuple{}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) checkindex(Bool, (), I[1]) & checkbounds_indices((), tail(I)) end -@inline function checkbounds_indices{N}(inds::Tuple{Any}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) - checkindex(Bool, inds, I[1]) & checkbounds_indices((), tail(I)) +@inline function checkbounds_indices{N}(IA::Tuple{Any}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) + checkindex(Bool, IA, I[1]) & checkbounds_indices((), tail(I)) end -@inline function checkbounds_indices{N}(inds::Tuple, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) - inds1, indsrest = IteratorsMD.split(inds, Val{N}) - checkindex(Bool, inds1, I[1]) & checkbounds_indices(indsrest, tail(I)) +@inline function checkbounds_indices{N}(IA::Tuple, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) + IA1, IArest = IteratorsMD.split(IA, Val{N}) + checkindex(Bool, IA1, I[1]) & checkbounds_indices(IArest, tail(I)) end function checkindex{N}(::Type{Bool}, inds::Tuple, I::AbstractArray{CartesianIndex{N}}) diff --git a/doc/devdocs/boundscheck.rst b/doc/devdocs/boundscheck.rst index c6ab3e758da29..1d324e9aa546b 100644 --- a/doc/devdocs/boundscheck.rst +++ b/doc/devdocs/boundscheck.rst @@ -59,3 +59,52 @@ instance, the default ``getindex`` methods have the chain To override the "one layer of inlining" rule, a function may be marked with ``@propagate_inbounds`` to propagate an inbounds context (or out of bounds context) through one additional layer of inlining. + +The bounds checking call hierarchy +---------------------------------- + +The overall hierarchy is: + +| ``checkbounds(A, I...)`` which calls +| ``checkbounds(Bool, A, I...)`` which calls either of: +| ``checkbounds_logical(A, I)`` when ``I`` is a single logical array +| ``checkbounds_indices(indices(A), I)`` otherwise +| + +Here ``A`` is the array, and ``I`` contains the "requested" indices. +``indices(A)`` returns a tuple of "permitted" indices of ``A``. + +``checkbounds(A, I...)`` throws an error if the indices are invalid, +whereas ``checkbounds(Bool, A, I...)`` returns ``false`` in that +circumstance. ``checkbounds_indices`` discards any information about +the array other than its ``indices`` tuple, and performs a pure +indices-vs-indices comparison: this allows relatively few compiled +methods to serve a huge variety of array types. Indices are specified +as tuples, and are usually compared in a 1-1 fashion with individual +dimensions handled by calling another important function, +``checkindex``: typically, +:: + + checkbounds_indices((IA1, IA...), (I1, I...)) = checkindex(Bool, IA1, I1) & + checkbounds_indices(IA, I) + +so ``checkindex`` checks a single dimension. All of these functions, +including the unexported ``checkbounds_indices`` and +``checkbounds_logical``, have docstrings accessible with ``?`` . + +If you have to customize bounds checking for a specific array type, +you should specialize ``checkbounds(Bool, A, I...)``. However, in most +cases you should be able to rely on ``checkbounds_indices`` as long as +you supply useful ``indices`` for your array type. + +If you have novel index types, first consider specializing +``checkindex``, which handles a single index for a particular +dimension of an array. If you have a custom multidimensional index +type (similar to ``CartesianIndex``), then you may have to consider +specializing ``checkbounds_indices``. + +Note this hierarchy has been designed to reduce the likelihood of +method ambiguities. We try to make ``checkbounds`` the place to +specialize on array type, and try to avoid specializations on index +types; conversely, ``checkindex`` is intended to be specialized only +on index type (especially, the last argument). diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index e4272781be1a5..b62866876cba1 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -616,19 +616,21 @@ Indexing, Assignment, and Concatenation Check two array shapes for compatibility, allowing trailing singleton dimensions, and return whichever shape has more dimensions. -.. function:: checkbounds(array, indexes...) +.. function:: checkbounds(A, I...) .. Docstring generated from Julia source - Throw an error if the specified ``indexes`` are not in bounds for the given ``array``\ . + Throw an error if the specified indices ``I`` are not in bounds for the given array ``A``\ . -.. function:: checkbounds(Bool, array, indexes...) +.. function:: checkbounds(Bool, A, I...) .. Docstring generated from Julia source - Return ``true`` if the specified ``indexes`` are in bounds for the given ``array``\ . Subtypes of ``AbstractArray`` should specialize this method if they need to provide custom bounds checking behaviors. + Return ``true`` if the specified indices ``I`` are in bounds for the given array ``A``\ . Subtypes of ``AbstractArray`` should specialize this method if they need to provide custom bounds checking behaviors; however, in many cases one can rely on ``A``\ 's indices and ``checkindex``\ . -.. function:: checkindex(Bool, inds::UnitRange, index) + See also ``checkindex``\ . + +.. function:: checkindex(Bool, inds::AbstractUnitRange, index) .. Docstring generated from Julia source From 0859745f5ddf0d401b82a8743253e5a127c83b78 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Mon, 23 May 2016 14:55:12 -0400 Subject: [PATCH 0410/1117] LibGit2: added tree related functions & tests --- base/libgit2/tree.jl | 55 +++++++++++++++++++++++++++++++++++++++++++- test/libgit2.jl | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/base/libgit2/tree.jl b/base/libgit2/tree.jl index 321bd83708d2c..ad8e8ee9265e7 100644 --- a/base/libgit2/tree.jl +++ b/base/libgit2/tree.jl @@ -16,17 +16,19 @@ function treewalk(f::Function, tree::GitTree, payload=Any[], post::Bool = false) return cbf_payload end +"Get the filename of a tree entry." function filename(te::GitTreeEntry) str = ccall((:git_tree_entry_name, :libgit2), Cstring, (Ptr{Void},), te.ptr) str != C_NULL && return unsafe_string(str) return nothing end +"Get the UNIX file attributes of a tree entry." function filemode(te::GitTreeEntry) return ccall((:git_tree_entry_filemode, :libgit2), Cint, (Ptr{Void},), te.ptr) end - +"Convert a tree entry to the `GitAnyObject` it points to." function object(repo::GitRepo, te::GitTreeEntry) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_tree_entry_to_object, :libgit2), Cint, @@ -34,3 +36,54 @@ function object(repo::GitRepo, te::GitTreeEntry) obj_ptr_ptr, repo.ptr, te.ptr) return GitAnyObject(obj_ptr_ptr[]) end + +"Get the id of the object pointed by the entry." +function oid(te::GitTreeEntry) + return Oid(ccall((:git_tree_entry_id, :libgit2), Ptr{Oid}, (Ptr{Void},), te.ptr)) +end + +"""Retrieve a tree entry contained in a tree or in any of its subtrees, given its relative path. + +The returned tree entry is owned by the user and must be freed explicitly. +""" +function GitTreeEntry(tree::GitTree, tepath::AbstractString) + te_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + err = ccall((:git_tree_entry_bypath, :libgit2), Cint, + (Ptr{Ptr{Void}}, Ref{Void}, Cstring), + te_ptr_ptr, tree.ptr, tepath) + if err == Int(Error.ENOTFOUND) + return nothing + elseif err != Int(Error.GIT_OK) + if te_ptr_ptr[] != C_NULL + finalize(GitTreeEntry(te_ptr_ptr[])) + end + throw(Error.GitError(err)) + end + return GitTreeEntry(te_ptr_ptr[]) +end + +"""Lookup a tree entry by SHA value. + +This returns a `GitTreeEntry` that is owned by the `GitTree`. +You don't have to free it, but you must not use it after the `GitTree` is released. +""" +function GitTreeEntry(tree::GitTree, teoid::Oid) + res = ccall((:git_tree_entry_byid, :libgit2), Ptr{Void}, + (Ref{Void}, Ref{Oid}), + tree.ptr, Ref(teoid)) + res == C_NULL && return nothing + return GitTreeEntry(res) +end + +"""Lookup a tree entry by its filename. + +This returns a `GitTreeEntry` that is owned by the `GitTree`. +You don't have to free it, but you must not use it after the `GitTree` is released. +""" +function lookup(tree::GitTree, fname::AbstractString) + te_ptr_ptr = Ref{Ptr{Void}}(C_NULL) + res = ccall((:git_tree_entry_byname, :libgit2), Ptr{Void}, + (Ref{Void}, Cstring), tree.ptr, fname) + res == C_NULL && return nothing + return GitTreeEntry(res) +end diff --git a/test/libgit2.jl b/test/libgit2.jl index 7e62fb61248d4..df6fb894323ec 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -443,6 +443,51 @@ mktempdir() do dir end #end + #@testset "tree" begin + function addfile(root::Cstring, tentry::Ptr{Void}, payload::Ptr{Void}) + strarr = unsafe_pointer_to_objref(payload) + sroot = String(root) + te = LibGit2.GitTreeEntry(tentry) + fmode = LibGit2.filemode(te) + fname = LibGit2.filename(te) + if fmode == Cint(LibGit2.Consts.FILEMODE_BLOB) + push!(strarr, fname) + end + return zero(Cint) + end + + repo = LibGit2.GitRepo(test_repo) + head = LibGit2.head(repo) + ht = LibGit2.peel(LibGit2.GitTree, head) + try + # get tree entry by filename + tfte = LibGit2.lookup(ht, test_file) + + @test LibGit2.filename(tfte) == test_file + @test LibGit2.filemode(tfte) == Cint(LibGit2.Consts.FILEMODE_BLOB) + + tfoid = LibGit2.oid(tfte) + + tfte2 = LibGit2.GitTreeEntry(ht, tfoid) + @test LibGit2.filename(tfte2) == test_file + + tfcontent = LibGit2.with(LibGit2.object(repo, tfte2)) do obj + LibGit2.with(LibGit2.peel(LibGit2.GitBlob, obj)) do blob + String(convert(Cstring, LibGit2.content(blob))) + end + end + @test startswith(tfcontent, commit_msg1) + + entrs = LibGit2.treewalk(addfile, ht, String[])[] + @test length(find(e->e == test_file, entrs)) > 0 + + finally + finalize(ht) + finalize(head) + finalize(repo) + end + #end + #@testset "commits with revwalk" begin repo = LibGit2.GitRepo(test_repo) cache = LibGit2.GitRepo(cache_repo) From 1e77ac6bcf9db922772c105089c268553ebe2813 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Wed, 25 May 2016 20:12:56 -0400 Subject: [PATCH 0411/1117] text fix & removed unused variable --- base/libgit2/tree.jl | 1 - test/libgit2.jl | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/base/libgit2/tree.jl b/base/libgit2/tree.jl index ad8e8ee9265e7..28059bfe1b8aa 100644 --- a/base/libgit2/tree.jl +++ b/base/libgit2/tree.jl @@ -81,7 +81,6 @@ This returns a `GitTreeEntry` that is owned by the `GitTree`. You don't have to free it, but you must not use it after the `GitTree` is released. """ function lookup(tree::GitTree, fname::AbstractString) - te_ptr_ptr = Ref{Ptr{Void}}(C_NULL) res = ccall((:git_tree_entry_byname, :libgit2), Ptr{Void}, (Ref{Void}, Cstring), tree.ptr, fname) res == C_NULL && return nothing diff --git a/test/libgit2.jl b/test/libgit2.jl index df6fb894323ec..bc1cfc5ec2573 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -479,7 +479,7 @@ mktempdir() do dir @test startswith(tfcontent, commit_msg1) entrs = LibGit2.treewalk(addfile, ht, String[])[] - @test length(find(e->e == test_file, entrs)) > 0 + @test length(find(e->e == test_file, entrs)) == 1 finally finalize(ht) From cba36bdbf54c87b35789f194eb970c3b6a250077 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Sat, 4 Jun 2016 10:03:03 -0400 Subject: [PATCH 0412/1117] docs update --- base/libgit2/tree.jl | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/base/libgit2/tree.jl b/base/libgit2/tree.jl index 28059bfe1b8aa..a733ea7138cb8 100644 --- a/base/libgit2/tree.jl +++ b/base/libgit2/tree.jl @@ -1,7 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -""" -Traverse the entries in a tree and its subtrees in post or pre order. +"""Traverse the entries in a tree and its subtrees in post or preorder. Function parameter should have following signature: @@ -16,19 +15,19 @@ function treewalk(f::Function, tree::GitTree, payload=Any[], post::Bool = false) return cbf_payload end -"Get the filename of a tree entry." +"Returns a file name, as a `String`, of a tree entry." function filename(te::GitTreeEntry) str = ccall((:git_tree_entry_name, :libgit2), Cstring, (Ptr{Void},), te.ptr) - str != C_NULL && return unsafe_string(str) - return nothing + str == C_NULL && return "" + return String(str) end -"Get the UNIX file attributes of a tree entry." +"Returns UNIX file attributes, as a `Cint`, of a tree entry." function filemode(te::GitTreeEntry) return ccall((:git_tree_entry_filemode, :libgit2), Cint, (Ptr{Void},), te.ptr) end -"Convert a tree entry to the `GitAnyObject` it points to." +"Returns a `GitAnyObject` which is referenced by a tree entry." function object(repo::GitRepo, te::GitTreeEntry) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_tree_entry_to_object, :libgit2), Cint, @@ -37,7 +36,7 @@ function object(repo::GitRepo, te::GitTreeEntry) return GitAnyObject(obj_ptr_ptr[]) end -"Get the id of the object pointed by the entry." +"Returns an object identifier, as a `Oid`, of a tree entry." function oid(te::GitTreeEntry) return Oid(ccall((:git_tree_entry_id, :libgit2), Ptr{Oid}, (Ptr{Void},), te.ptr)) end @@ -75,7 +74,7 @@ function GitTreeEntry(tree::GitTree, teoid::Oid) return GitTreeEntry(res) end -"""Lookup a tree entry by its filename. +"""Lookup a tree entry by its file name. This returns a `GitTreeEntry` that is owned by the `GitTree`. You don't have to free it, but you must not use it after the `GitTree` is released. From 9f554565f7a50f2ccc7ad37f564b5804526a6527 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Mon, 13 Jun 2016 15:59:48 -0400 Subject: [PATCH 0413/1117] rebased & fixed incorrect string conversion --- base/libgit2/tree.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/libgit2/tree.jl b/base/libgit2/tree.jl index a733ea7138cb8..09091d6559855 100644 --- a/base/libgit2/tree.jl +++ b/base/libgit2/tree.jl @@ -19,7 +19,7 @@ end function filename(te::GitTreeEntry) str = ccall((:git_tree_entry_name, :libgit2), Cstring, (Ptr{Void},), te.ptr) str == C_NULL && return "" - return String(str) + return unsafe_string(str) end "Returns UNIX file attributes, as a `Cint`, of a tree entry." From 095e2fe09a67bc680458f3e8f766a0fca740703f Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Wed, 15 Jun 2016 15:04:15 -0400 Subject: [PATCH 0414/1117] use nullable types for values returned from tree search added `finalize` for nullable type --- base/libgit2/tree.jl | 24 ++++++++++++++---------- base/libgit2/types.jl | 29 +++++++++++++++-------------- test/libgit2.jl | 37 +++++++++++++++++++++++-------------- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/base/libgit2/tree.jl b/base/libgit2/tree.jl index 09091d6559855..795b54311a54b 100644 --- a/base/libgit2/tree.jl +++ b/base/libgit2/tree.jl @@ -18,7 +18,8 @@ end "Returns a file name, as a `String`, of a tree entry." function filename(te::GitTreeEntry) str = ccall((:git_tree_entry_name, :libgit2), Cstring, (Ptr{Void},), te.ptr) - str == C_NULL && return "" + str == C_NULL && throw(Error.GitError(Error.Tree, + LibGit2.Error.ENOTFOUND, "not found")) return unsafe_string(str) end @@ -38,7 +39,10 @@ end "Returns an object identifier, as a `Oid`, of a tree entry." function oid(te::GitTreeEntry) - return Oid(ccall((:git_tree_entry_id, :libgit2), Ptr{Oid}, (Ptr{Void},), te.ptr)) + oid_ptr = ccall((:git_tree_entry_id, :libgit2), Ptr{Oid}, (Ptr{Void},), te.ptr) + oid_ptr == C_NULL && throw(Error.GitError(Error.Tree, + LibGit2.Error.ENOTFOUND, "not found")) + return Oid(oid_ptr) end """Retrieve a tree entry contained in a tree or in any of its subtrees, given its relative path. @@ -50,15 +54,15 @@ function GitTreeEntry(tree::GitTree, tepath::AbstractString) err = ccall((:git_tree_entry_bypath, :libgit2), Cint, (Ptr{Ptr{Void}}, Ref{Void}, Cstring), te_ptr_ptr, tree.ptr, tepath) - if err == Int(Error.ENOTFOUND) - return nothing - elseif err != Int(Error.GIT_OK) + if err == Cint(Error.ENOTFOUND) + return Nullable{GitTreeEntry}() + elseif err != Cint(Error.GIT_OK) if te_ptr_ptr[] != C_NULL finalize(GitTreeEntry(te_ptr_ptr[])) end throw(Error.GitError(err)) end - return GitTreeEntry(te_ptr_ptr[]) + return Nullable(GitTreeEntry(te_ptr_ptr[])) end """Lookup a tree entry by SHA value. @@ -70,8 +74,8 @@ function GitTreeEntry(tree::GitTree, teoid::Oid) res = ccall((:git_tree_entry_byid, :libgit2), Ptr{Void}, (Ref{Void}, Ref{Oid}), tree.ptr, Ref(teoid)) - res == C_NULL && return nothing - return GitTreeEntry(res) + res == C_NULL && return Nullable{GitTreeEntry}() + return Nullable(GitTreeEntry(res)) end """Lookup a tree entry by its file name. @@ -82,6 +86,6 @@ You don't have to free it, but you must not use it after the `GitTree` is releas function lookup(tree::GitTree, fname::AbstractString) res = ccall((:git_tree_entry_byname, :libgit2), Ptr{Void}, (Ref{Void}, Cstring), tree.ptr, fname) - res == C_NULL && return nothing - return GitTreeEntry(res) + res == C_NULL && return Nullable{GitTreeEntry}() + return Nullable(GitTreeEntry(res)) end diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 919c2866bc4f3..dde76d423f515 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -625,24 +625,25 @@ for (typ, ref, sup, fnc) in ( (:GitTag, :Void, :GitObject, nothing) ) - @eval type $typ <: $sup - ptr::Ptr{$ref} - function $typ(ptr::Ptr{$ref}) - @assert ptr != C_NULL - obj = new(ptr) - return obj + @eval begin + type $typ <: $sup + ptr::Ptr{$ref} + function $typ(ptr::Ptr{$ref}) + @assert ptr != C_NULL + obj = new(ptr) + return obj + end end - end - - if fnc !== nothing - @eval function Base.finalize(obj::$typ) - if obj.ptr != C_NULL - ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) - obj.ptr = C_NULL + if $fnc !== nothing + function Base.finalize(obj::$typ) + if obj.ptr != C_NULL + ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) + obj.ptr = C_NULL + end end + Base.finalize(obj::Nullable{$typ}) = !isnull(obj) && Base.finalize(Base.get(obj)) end end - end # Structure has the same layout as SignatureStruct diff --git a/test/libgit2.jl b/test/libgit2.jl index bc1cfc5ec2573..cd1ce4891b3bc 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -446,7 +446,7 @@ mktempdir() do dir #@testset "tree" begin function addfile(root::Cstring, tentry::Ptr{Void}, payload::Ptr{Void}) strarr = unsafe_pointer_to_objref(payload) - sroot = String(root) + sroot = unsafe_string(root) te = LibGit2.GitTreeEntry(tentry) fmode = LibGit2.filemode(te) fname = LibGit2.filename(te) @@ -461,22 +461,31 @@ mktempdir() do dir ht = LibGit2.peel(LibGit2.GitTree, head) try # get tree entry by filename - tfte = LibGit2.lookup(ht, test_file) - - @test LibGit2.filename(tfte) == test_file - @test LibGit2.filemode(tfte) == Cint(LibGit2.Consts.FILEMODE_BLOB) - - tfoid = LibGit2.oid(tfte) - - tfte2 = LibGit2.GitTreeEntry(ht, tfoid) - @test LibGit2.filename(tfte2) == test_file + @test isnull(LibGit2.lookup(ht, "no such name")) + tfte = get(LibGit2.lookup(ht, test_file)) + try + @test LibGit2.filename(tfte) == test_file + @test LibGit2.filemode(tfte) == Cint(LibGit2.Consts.FILEMODE_BLOB) - tfcontent = LibGit2.with(LibGit2.object(repo, tfte2)) do obj - LibGit2.with(LibGit2.peel(LibGit2.GitBlob, obj)) do blob - String(convert(Cstring, LibGit2.content(blob))) + @test isnull(LibGit2.GitTreeEntry(ht, LibGit2.Oid())) + tfoid = LibGit2.oid(tfte) + tfte2 = LibGit2.GitTreeEntry(ht, tfoid) + try + @test !isnull(tfte2) + @test LibGit2.filename(get(tfte2)) == test_file + + tfcontent = LibGit2.with(LibGit2.object(repo, get(tfte2))) do obj + LibGit2.with(LibGit2.peel(LibGit2.GitBlob, obj)) do blob + unsafe_string(convert(Cstring, LibGit2.content(blob))) + end + end + @test startswith(tfcontent, commit_msg1) + finally + finalize(tfte2) end + finally + finalize(tfte) end - @test startswith(tfcontent, commit_msg1) entrs = LibGit2.treewalk(addfile, ht, String[])[] @test length(find(e->e == test_file, entrs)) == 1 From 104ab14698a12eb2a5a8d51e80feb7fa76a9ba44 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 12 Jul 2016 20:51:13 -0700 Subject: [PATCH 0415/1117] Something is wrong with inference of promote_op on Float16 please revert this if you can fix it --- test/numbers.jl | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/test/numbers.jl b/test/numbers.jl index 64b3174fe484f..019ffcffcb695 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2770,7 +2770,12 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Complex{Int}, Complex{UInt}, Complex32, Complex64, Complex128) for S in types for op in (+, -) - T = @inferred Base.promote_op(op, S) + if S === Float16 # type instability here? + @test_broken @inferred Base.promote_op(op, S) + T = Base.promote_op(op, S) + else + T = @inferred Base.promote_op(op, S) + end t = @inferred op(one(S)) @test T === typeof(t) end @@ -2780,8 +2785,15 @@ let types = (Base.BitInteger_types..., BigInt, Bool, for R in types, S in types for op in (+, -, *, /, ^) - T = @inferred Base.promote_op(op, R, S) - t = @inferred op(one(R), one(S)) + if R in (Float16, Complex{Float16}) || S in (Float16, Complex{Float16}) + @test_broken @inferred Base.promote_op(op, R, S) + T = Base.promote_op(op, R, S) + @test_broken @inferred op(one(R), one(S)) + t = op(one(R), one(S)) + else + T = @inferred Base.promote_op(op, R, S) + t = @inferred op(one(R), one(S)) + end @test T === typeof(t) end end @@ -2792,7 +2804,12 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Float16, Float32, Float64, BigFloat) for S in types, T in types for op in (<, >, <=, >=, (==)) - @test @inferred(Base.promote_op(op, S, T)) === Bool + if S === Float16 || T === Float16 + @test_broken @inferred(Base.promote_op(op, S, T)) + @test Base.promote_op(op, S, T) === Bool + else + @test @inferred(Base.promote_op(op, S, T)) === Bool + end end end end From 5c2ee55ced0bf2ec723937315b9899b5c0eb3f4d Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Tue, 12 Jul 2016 23:58:37 -0400 Subject: [PATCH 0416/1117] fix tree tests & error msgs --- base/libgit2/tree.jl | 4 ++-- test/libgit2.jl | 35 +++++++++++++++++------------------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/base/libgit2/tree.jl b/base/libgit2/tree.jl index 795b54311a54b..f1d73119a4aa0 100644 --- a/base/libgit2/tree.jl +++ b/base/libgit2/tree.jl @@ -19,7 +19,7 @@ end function filename(te::GitTreeEntry) str = ccall((:git_tree_entry_name, :libgit2), Cstring, (Ptr{Void},), te.ptr) str == C_NULL && throw(Error.GitError(Error.Tree, - LibGit2.Error.ENOTFOUND, "not found")) + LibGit2.Error.ENOTFOUND, "filename not found")) return unsafe_string(str) end @@ -41,7 +41,7 @@ end function oid(te::GitTreeEntry) oid_ptr = ccall((:git_tree_entry_id, :libgit2), Ptr{Oid}, (Ptr{Void},), te.ptr) oid_ptr == C_NULL && throw(Error.GitError(Error.Tree, - LibGit2.Error.ENOTFOUND, "not found")) + LibGit2.Error.ENOTFOUND, "oid not found")) return Oid(oid_ptr) end diff --git a/test/libgit2.jl b/test/libgit2.jl index cd1ce4891b3bc..9211cfc58b18f 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -460,31 +460,30 @@ mktempdir() do dir head = LibGit2.head(repo) ht = LibGit2.peel(LibGit2.GitTree, head) try - # get tree entry by filename @test isnull(LibGit2.lookup(ht, "no such name")) tfte = get(LibGit2.lookup(ht, test_file)) - try - @test LibGit2.filename(tfte) == test_file - @test LibGit2.filemode(tfte) == Cint(LibGit2.Consts.FILEMODE_BLOB) - @test isnull(LibGit2.GitTreeEntry(ht, LibGit2.Oid())) - tfoid = LibGit2.oid(tfte) - tfte2 = LibGit2.GitTreeEntry(ht, tfoid) - try - @test !isnull(tfte2) - @test LibGit2.filename(get(tfte2)) == test_file + # get tree entry by filename + @test isnull(LibGit2.GitTreeEntry(ht, "no such name")) + @test LibGit2.filename(tfte) == test_file + @test LibGit2.filemode(tfte) == Cint(LibGit2.Consts.FILEMODE_BLOB) - tfcontent = LibGit2.with(LibGit2.object(repo, get(tfte2))) do obj - LibGit2.with(LibGit2.peel(LibGit2.GitBlob, obj)) do blob - unsafe_string(convert(Cstring, LibGit2.content(blob))) - end + tfoid = LibGit2.oid(tfte) + + @test isnull(LibGit2.GitTreeEntry(ht, LibGit2.Oid())) + tfte2 = LibGit2.GitTreeEntry(ht, tfoid) + try + @test !isnull(tfte2) + @test LibGit2.filename(get(tfte2)) == test_file + + tfcontent = LibGit2.with(LibGit2.object(repo, get(tfte2))) do obj + LibGit2.with(LibGit2.peel(LibGit2.GitBlob, obj)) do blob + unsafe_string(convert(Cstring, LibGit2.content(blob))) end - @test startswith(tfcontent, commit_msg1) - finally - finalize(tfte2) end + @test startswith(tfcontent, commit_msg1) finally - finalize(tfte) + finalize(tfte2) end entrs = LibGit2.treewalk(addfile, ht, String[])[] From 360a22a51afbeba87832df2ad69e1598c500f130 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 12 Jul 2016 22:12:23 -0700 Subject: [PATCH 0417/1117] Use test_throws instead of test_broken in Float16 numbers tests Since test_broken is not good for things that currently error but when fixed will return non-boolean. If you know how to fix the inference issue and get tests to pass with #17394 (377dede7c7e08b3a442579c59c86fcf3db5fa390) reverted, then please do so and ignore this change. --- test/numbers.jl | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/test/numbers.jl b/test/numbers.jl index 019ffcffcb695..c447a730b4f01 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2771,7 +2771,8 @@ let types = (Base.BitInteger_types..., BigInt, Bool, for S in types for op in (+, -) if S === Float16 # type instability here? - @test_broken @inferred Base.promote_op(op, S) + # broken, fixme then remove this branch + @test_throws ErrorException @inferred(Base.promote_op(op, S)) T = Base.promote_op(op, S) else T = @inferred Base.promote_op(op, S) @@ -2785,11 +2786,31 @@ let types = (Base.BitInteger_types..., BigInt, Bool, for R in types, S in types for op in (+, -, *, /, ^) - if R in (Float16, Complex{Float16}) || S in (Float16, Complex{Float16}) - @test_broken @inferred Base.promote_op(op, R, S) + if R === Float16 || S === Float16 + # broken, fixme then remove this branch + @test_throws ErrorException @inferred(Base.promote_op(op, R, S)) T = Base.promote_op(op, R, S) - @test_broken @inferred op(one(R), one(S)) - t = op(one(R), one(S)) + if ((R === Bool || S === Bool) && op in (+, *)) || + ((S in (Rational{Int}, Complex{Float16})) && op === ^) || + (R === Complex{Float16} && op === ^) + @test_throws ErrorException @inferred(op(one(R), one(S))) + t = op(one(R), one(S)) + else + t = @inferred op(one(R), one(S)) + end + elseif R === Complex{Float16} || S === Complex{Float16} + # broken, fixme then remove this branch too + if (R === Bool && op in (+, *, /, ^)) || + (S === Bool && op in (+, *)) || + (S in (R, Rational{Int}) && op === ^) + @test_throws ErrorException @inferred(Base.promote_op(op, R, S)) + T = Base.promote_op(op, R, S) + @test_throws ErrorException @inferred(op(one(R), one(S))) + t = op(one(R), one(S)) + else + T = @inferred Base.promote_op(op, R, S) + t = @inferred op(one(R), one(S)) + end else T = @inferred Base.promote_op(op, R, S) t = @inferred op(one(R), one(S)) @@ -2805,7 +2826,8 @@ let types = (Base.BitInteger_types..., BigInt, Bool, for S in types, T in types for op in (<, >, <=, >=, (==)) if S === Float16 || T === Float16 - @test_broken @inferred(Base.promote_op(op, S, T)) + # broken, fixme then remove this branch + @test_throws ErrorException @inferred(Base.promote_op(op, S, T)) @test Base.promote_op(op, S, T) === Bool else @test @inferred(Base.promote_op(op, S, T)) === Bool From c8167644057a59265b76e2a0a083b22c77ccfb10 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 13 Jul 2016 01:54:36 -0700 Subject: [PATCH 0418/1117] consolidate logic slightly --- test/numbers.jl | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/test/numbers.jl b/test/numbers.jl index c447a730b4f01..cc14ff0156848 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2798,19 +2798,15 @@ let types = (Base.BitInteger_types..., BigInt, Bool, else t = @inferred op(one(R), one(S)) end - elseif R === Complex{Float16} || S === Complex{Float16} + elseif (R === Complex{Float16} || S === Complex{Float16}) && + ((R === Bool && op in (+, *, /, ^)) || + (S === Bool && op in (+, *)) || + (S in (R, Rational{Int}) && op === ^)) # broken, fixme then remove this branch too - if (R === Bool && op in (+, *, /, ^)) || - (S === Bool && op in (+, *)) || - (S in (R, Rational{Int}) && op === ^) - @test_throws ErrorException @inferred(Base.promote_op(op, R, S)) - T = Base.promote_op(op, R, S) - @test_throws ErrorException @inferred(op(one(R), one(S))) - t = op(one(R), one(S)) - else - T = @inferred Base.promote_op(op, R, S) - t = @inferred op(one(R), one(S)) - end + @test_throws ErrorException @inferred(Base.promote_op(op, R, S)) + T = Base.promote_op(op, R, S) + @test_throws ErrorException @inferred(op(one(R), one(S))) + t = op(one(R), one(S)) else T = @inferred Base.promote_op(op, R, S) t = @inferred op(one(R), one(S)) From f7695f5ceea991c9c8bf9a9aa0f08f10397a795d Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 5 Jul 2016 22:10:20 -0700 Subject: [PATCH 0419/1117] Remove command-line Git from Mac and Windows binaries --- DISTRIBUTING.md | 9 +- LICENSE.md | 1 - Makefile | 18 +--- README.md | 2 - README.windows.md | 6 +- base/exports.jl | 1 - base/pkg/git.jl | 124 ----------------------- base/pkg/pkg.jl | 4 +- base/sysimg.jl | 1 - contrib/mac/app/Makefile | 3 - contrib/mac/juliarc.jl | 4 +- contrib/windows/juliarc.jl | 2 +- contrib/windows/msys_build.sh | 6 +- deps/Makefile | 3 +- deps/Versions.make | 1 - deps/checksums/git-1.8.5.6.tar.gz/md5 | 1 - deps/checksums/git-1.8.5.6.tar.gz/sha512 | 1 - deps/git.mk | 37 ------- doc/manual/packages.rst | 2 +- test/cmdlineargs.jl | 12 +-- test/spawn.jl | 10 +- 21 files changed, 25 insertions(+), 223 deletions(-) delete mode 100644 base/pkg/git.jl delete mode 100644 deps/checksums/git-1.8.5.6.tar.gz/md5 delete mode 100644 deps/checksums/git-1.8.5.6.tar.gz/sha512 delete mode 100644 deps/git.mk diff --git a/DISTRIBUTING.md b/DISTRIBUTING.md index 4c70468d9c49a..da615a1546580 100644 --- a/DISTRIBUTING.md +++ b/DISTRIBUTING.md @@ -12,8 +12,8 @@ separated most of the notes by OS. Note that while the code for Julia is [MIT-licensed, with a few exceptions](https://github.com/JuliaLang/julia/blob/master/LICENSE.md), the distribution created by the techniques described herein will be -GPL licensed, as various dependent libraries such as `FFTW`, -`SuiteSparse`, and `git` are GPL licensed. We do hope to have a +GPL licensed, as various dependent libraries such as `FFTW` and +`SuiteSparse` are GPL licensed. We do hope to have a non-GPL distribution of Julia in the future. Versioning and Git @@ -84,9 +84,8 @@ to force the installation into a temporary directory. By default, Julia loads `$prefix/etc/julia/juliarc.jl` as an installation-wide initialization file. This file can be used by -distribution managers to provide paths to various binaries such as a -bundled `git` executable (as we do on OS X), or to setup paths (as -we do on Windows). For Linux distribution packages, if `$prefix` is +distribution managers to set up custom paths or initialization code. +For Linux distribution packages, if `$prefix` is set to `/usr`, there is no `/usr/etc` to look into. This requires the path to Julia's private `etc` directory to be changed. This can be done via the `sysconfdir` make variable when building. Simply diff --git a/LICENSE.md b/LICENSE.md index 328802613a979..fe82764039f2f 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -81,7 +81,6 @@ Julia's build process uses the following external tools: Julia bundles the following external programs and libraries on some platforms: - [7-Zip](http://www.7-zip.org/license.txt) -- [GIT](http://git-scm.com/about/free-and-open-source) - [ZLIB](http://zlib.net/zlib_license.html) - [LIBEXPAT](http://expat.cvs.sourceforge.net/viewvc/expat/expat/README) - [OPENSSL](https://github.com/openssl/openssl/blob/master/LICENSE) diff --git a/Makefile b/Makefile index 9d7e02534915c..d987bb95d7dde 100644 --- a/Makefile +++ b/Makefile @@ -426,12 +426,6 @@ distclean dist-clean: dist: @echo \'dist\' target is deprecated: use \'binary-dist\' instead. -ifeq ($(ARCH),x86_64) -GITCONFIG := $(BUILDROOT)/julia-$(JULIA_COMMIT)/Git/mingw64/etc/gitconfig -else -GITCONFIG := $(BUILDROOT)/julia-$(JULIA_COMMIT)/Git/mingw32/etc/gitconfig -endif - binary-dist: distclean ifeq ($(USE_SYSTEM_BLAS),0) ifeq ($(ISX86),1) @@ -468,11 +462,7 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && \ - mkdir $(BUILDROOT)/julia-$(JULIA_COMMIT)/Git && \ - 7z x PortableGit.7z -o"$(BUILDROOT)/julia-$(JULIA_COMMIT)/Git" && \ - echo "[core] eol = lf" >> "$(GITCONFIG)" && \ - sed -i "s/\bautocrlf = true$$/autocrlf = input/" "$(GITCONFIG)" ) + cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. @@ -605,8 +595,7 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686)) $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libssp0-5.3.0-1.1.noarch.rpm && \ for i in *.rpm; do 7z x -y $$i; done && \ for i in *.cpio; do 7z x -y $$i; done && \ - cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . && \ - $(JLDOWNLOAD) PortableGit.7z https://github.com/git-for-windows/git/releases/download/v2.6.1.windows.1/PortableGit-2.6.1-32-bit.7z.exe + cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . else ifeq ($(ARCH),x86_64) cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) 7z920-x64.msi http://downloads.sourceforge.net/sevenzip/7z920-x64.msi && \ @@ -622,8 +611,7 @@ else ifeq ($(ARCH),x86_64) $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libssp0-5.3.0-1.1.noarch.rpm && \ for i in *.rpm; do 7z x -y $$i; done && \ for i in *.cpio; do 7z x -y $$i; done && \ - cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . && \ - $(JLDOWNLOAD) PortableGit.7z https://github.com/git-for-windows/git/releases/download/v2.6.1.windows.1/PortableGit-2.6.1-64-bit.7z.exe + cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . else $(error no win-extras target for ARCH=$(ARCH)) endif diff --git a/README.md b/README.md index 98ecf2852596e..263f69f4040fe 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,6 @@ Building Julia requires that the following software be installed: - **[GNU make]** — building dependencies. - **[gcc & g++][gcc]** (>= 4.7) or **[Clang][clang]** (>= 3.1, Xcode 4.3.3 on OS X) — compiling and linking C, C++ - **[gfortran]** — compiling and linking Fortran libraries -- **[git]** — version control and package management (version 1.7.3+ required) - **[perl]** — preprocessing of header files of libraries. - **[wget]**, **[curl]**, or **[fetch]** (FreeBSD) — to automatically download external libraries. - **[m4]** — needed to build GMP. @@ -298,7 +297,6 @@ For a longer overview of Julia's dependencies, see these [slides](https://github [gfortran]: https://gcc.gnu.org/fortran/ [curl]: http://curl.haxx.se [fetch]: http://www.freebsd.org/cgi/man.cgi?fetch(1) -[git]: http://git-scm.com [perl]: http://www.perl.org [cmake]: http://www.cmake.org [OpenLibm]: https://github.com/JuliaLang/openlibm diff --git a/README.windows.md b/README.windows.md index 4d2dd90539205..8198db61a291a 100644 --- a/README.windows.md +++ b/README.windows.md @@ -309,7 +309,11 @@ replace `config.vm.provision :shell, privileged: false, :inline => $script_msys2 ## After compiling -Compiling using one of the options above creates a basic Julia build, but not some extra components (like a built-in git installation) that are included if you run the full Julia binary installer. If you need these components, the easiest way to get them is to build the installer yourself using ```make win-extras``` followed by ```make binary-dist```, and then running the resulting installer. +Compiling using one of the options above creates a basic Julia build, but not some +extra components that are included if you run the full Julia binary installer. +If you need these components, the easiest way to get them is to build the installer +yourself using ```make win-extras``` followed by ```make binary-dist```, and then +running the resulting installer. ## Windows Build Debugging diff --git a/base/exports.jl b/base/exports.jl index dda16f688f0c8..2e83da356f2b2 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -7,7 +7,6 @@ export Meta, Operators, Pkg, - Git, LibGit2, StackTraces, Profile, diff --git a/base/pkg/git.jl b/base/pkg/git.jl deleted file mode 100644 index 3a34fb2aa3f29..0000000000000 --- a/base/pkg/git.jl +++ /dev/null @@ -1,124 +0,0 @@ -# This file is a part of Julia. License is MIT: http://julialang.org/license - -module Git -# -# some utility functions for working with git repos -# -import Base: shell_escape - -function dir(d) - g = joinpath(d,".git") - isdir(g) && return g - normpath(d, Base.readchomp(setenv(`git rev-parse --git-dir`, dir=d))) -end - -function git(d) - isempty(d) && return `git` - work_tree = abspath(d) - git_dir = joinpath(work_tree, dir(work_tree)) - normpath(work_tree, ".") == normpath(git_dir, ".") ? # is it a bare repo? - `git --git-dir=$work_tree` : `git --work-tree=$work_tree --git-dir=$git_dir` -end - -cmd(args::Cmd; dir="") = `$(git(dir)) $args` -run(args::Cmd; dir="", out=STDOUT) = Base.run(pipeline(cmd(args,dir=dir), out)) -readstring(args::Cmd; dir="") = Base.readstring(cmd(args,dir=dir)) -readchomp(args::Cmd; dir="") = Base.readchomp(cmd(args,dir=dir)) - -function success(args::Cmd; dir="") - g = git(dir) - Base.readchomp(`$g rev-parse --is-bare-repository`) == "false" && - Base.run(`$g update-index -q --really-refresh`) - Base.success(`$g $args`) -end - -function version() - vs = split(readchomp(`version`), ' ')[3] - ns = split(vs, '.') - if length(ns) > 3 - VersionNumber(join(ns[1:3], '.')) - else - VersionNumber(join(ns, '.')) - end -end - -modules(args::Cmd; dir="") = readchomp(`config -f .gitmodules $args`, dir=dir) -different(verA::AbstractString, verB::AbstractString, path::AbstractString; dir="") = - !success(`diff-tree --quiet $verA $verB -- $path`, dir=dir) - -dirty(; dir="") = !success(`diff-index --quiet HEAD`, dir=dir) -staged(; dir="") = !success(`diff-index --quiet --cached HEAD`, dir=dir) -unstaged(; dir="") = !success(`diff-files --quiet`, dir=dir) -dirty(paths; dir="") = !success(`diff-index --quiet HEAD -- $paths`, dir=dir) -staged(paths; dir="") = !success(`diff-index --quiet --cached HEAD -- $paths`, dir=dir) -unstaged(paths; dir="") = !success(`diff-files --quiet -- $paths`, dir=dir) - -iscommit(name; dir="") = success(`cat-file commit $name`, dir=dir) -attached(; dir="") = success(`symbolic-ref -q HEAD`, dir=dir) -branch(; dir="") = readchomp(`rev-parse --symbolic-full-name --abbrev-ref HEAD`, dir=dir) -head(; dir="") = readchomp(`rev-parse HEAD`, dir=dir) - -function iscommit(sha1s::Vector; dir="") - indexin(sha1s,split(readchomp(`log --all --format=%H`, dir=dir),"\n")).!=0 -end - -immutable State - head::String - index::String - work::String -end - -function snapshot(; dir="") - head = readchomp(`rev-parse HEAD`, dir=dir) - index = readchomp(`write-tree`, dir=dir) - work = try - if length(readdir(abspath(dir))) > 1 - run(`add --all`, dir=dir) - run(`add .`, dir=dir) - end - readchomp(`write-tree`, dir=dir) - finally - run(`read-tree $index`, dir=dir) # restore index - end - State(head, index, work) -end - -function restore(s::State; dir="") - run(`reset -q --`, dir=dir) # unstage everything - run(`read-tree $(s.work)`, dir=dir) # move work tree to index - run(`checkout-index -fa`, dir=dir) # check the index out to work - run(`clean -qdf`, dir=dir) # remove everything else - run(`read-tree $(s.index)`, dir=dir) # restore index - run(`reset -q --soft $(s.head)`, dir=dir) # restore head -end - -function transact(f::Function; dir="") - state = snapshot(dir=dir) - try f() catch - restore(state, dir=dir) - rethrow() - end -end - -function is_ancestor_of(a::AbstractString, b::AbstractString; dir="") - A = readchomp(`rev-parse $a`, dir=dir) - readchomp(`merge-base $A $b`, dir=dir) == A -end - -const GITHUB_REGEX = - r"^(?:git@|git://|https://(?:[\w\.\+\-]+@)?)github.com[:/](([^/].+)/(.+?))(?:\.git)?$"i - -function set_remote_url(url::AbstractString; remote::AbstractString="origin", dir="") - run(`config remote.$remote.url $url`, dir=dir) - m = match(GITHUB_REGEX,url) - m === nothing && return - push = "git@github.com:$(m.captures[1]).git" - push != url && run(`config remote.$remote.pushurl $push`, dir=dir) -end - -function normalize_url(url::AbstractString) - m = match(GITHUB_REGEX,url) - m === nothing ? url : "git://github.com/$(m.captures[1]).git" -end - -end # module diff --git a/base/pkg/pkg.jl b/base/pkg/pkg.jl index 7ded34c0faaa4..828a7d9bef50e 100644 --- a/base/pkg/pkg.jl +++ b/base/pkg/pkg.jl @@ -2,7 +2,7 @@ module Pkg -export Dir, Types, Reqs, Cache, Read, Query, Resolve, Write, Entry, Git +export Dir, Types, Reqs, Cache, Read, Query, Resolve, Write, Entry export dir, init, rm, add, available, installed, status, clone, checkout, update, resolve, test, build, free, pin, PkgError, setprotocol! @@ -30,7 +30,7 @@ function Base.showerror(io::IO, pkgerr::PkgError) end end -for file in split("dir types reqs cache read query resolve write entry git") +for file in split("dir types reqs cache read query resolve write entry") include("$file.jl") end const cd = Dir.cd diff --git a/base/sysimg.jl b/base/sysimg.jl index 85dfeaf5ef75e..a2ec4b59a0cd5 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -325,7 +325,6 @@ include("libgit2/libgit2.jl") # package manager include("pkg/pkg.jl") -const Git = Pkg.Git # Stack frames and traces include("stacktraces.jl") diff --git a/contrib/mac/app/Makefile b/contrib/mac/app/Makefile index 1a10ce5def3ed..52adebdaa873d 100644 --- a/contrib/mac/app/Makefile +++ b/contrib/mac/app/Makefile @@ -31,13 +31,10 @@ $(DMG_NAME): dmg/$(APP_NAME) sudo hdiutil create $(DMG_NAME) -size 500m -ov -volname "$(VOL_NAME)" -imagekey zlib-level=9 -srcfolder dmg dmg/$(APP_NAME): julia.icns - make -C ../../../deps install-git make -C ../../.. binary-dist tar zxf ../../../julia-*.tar.gz mv julia-* julia -mkdir -p ./julia/libexec ./julia/share - -cp -a $(build_libexecdir)/git* ./julia/libexec - -cp -a $(build_datarootdir)/git* ./julia/share rm -f julia/lib/*.{a,la} -mkdir dmg platypus -a Julia -p /bin/bash -V $(JULIA_VERSION) -R -u "The Julia Project" -i julia.icns -Q julia.icns -o "None" -I org.julialang.julia -x -f julia script ./dmg/$(APP_NAME) diff --git a/contrib/mac/juliarc.jl b/contrib/mac/juliarc.jl index c7568fede6d79..da5bc9a95dd4c 100644 --- a/contrib/mac/juliarc.jl +++ b/contrib/mac/juliarc.jl @@ -3,9 +3,7 @@ # Set up environment for Julia OSX binary distribution let ROOT = abspath(JULIA_HOME,"..") - ENV["PATH"]="$JULIA_HOME:$(joinpath(ROOT, "libexec", "git-core")):$(ENV["PATH"])" + ENV["PATH"]="$JULIA_HOME:$(ENV["PATH"])" ENV["FONTCONFIG_PATH"] = joinpath(ROOT, "etc", "fonts") - ENV["GIT_EXEC_PATH"] = joinpath(ROOT, "libexec", "git-core") - ENV["GIT_TEMPLATE_DIR"] = joinpath(ROOT, "share", "git-core") ENV["TK_LIBRARY"] = "/System/Library/Frameworks/Tk.framework/Versions/8.5/Resources/Scripts" end diff --git a/contrib/windows/juliarc.jl b/contrib/windows/juliarc.jl index 0f04c6729f5cf..cfa10a98d7111 100644 --- a/contrib/windows/juliarc.jl +++ b/contrib/windows/juliarc.jl @@ -1,2 +1,2 @@ # Set up environment for Julia Windows binary distribution -ENV["PATH"] = JULIA_HOME*";"*joinpath(JULIA_HOME,"..","Git","bin")*";"*ENV["PATH"] +ENV["PATH"] = JULIA_HOME*";"*ENV["PATH"] diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 2c635dab4cb7c..c89cf379326a9 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -86,8 +86,6 @@ case $(uname) in esac # Download most recent Julia binary for dependencies -# Fix directory not found error during decompression on msys2 -mkdir -p usr/Git/usr if ! [ -e julia-installer.exe ]; then f=julia-latest-win$bits.exe echo "Downloading $f" @@ -95,7 +93,7 @@ if ! [ -e julia-installer.exe ]; then echo "Extracting $f" $SEVENZIP x -y $f >> get-deps.log fi -for i in bin/*.dll Git/usr/bin/*.dll Git/usr/bin/*.exe; do +for i in bin/*.dll; do $SEVENZIP e -y julia-installer.exe "$i" \ -ousr\\`dirname $i | sed -e 's|/julia||' -e 's|/|\\\\|g'` >> get-deps.log done @@ -103,8 +101,6 @@ for i in share/julia/base/pcre_h.jl; do $SEVENZIP e -y julia-installer.exe "$i" -obase >> get-deps.log done echo "override PCRE_INCL_PATH =" >> Make.user -# suppress "bash.exe: warning: could not find /tmp, please create!" -mkdir -p usr/Git/tmp # Remove libjulia.dll if it was copied from downloaded binary rm -f usr/bin/libjulia.dll rm -f usr/bin/libjulia-debug.dll diff --git a/deps/Makefile b/deps/Makefile index e376b07671922..00d2f0fab4547 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -17,7 +17,7 @@ include $(SRCDIR)/llvm-ver.make # additionally all targets should be listed in the getall target for easier off-line compilation # if you are adding a new target, it can help to copy an similar, existing target # -# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv git +# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv # custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind # entirely custom: virtualenv # CMake libs: libgit2 @@ -260,7 +260,6 @@ include $(SRCDIR)/unwind.mk include $(SRCDIR)/gmp.mk include $(SRCDIR)/mpfr.mk include $(SRCDIR)/patchelf.mk -include $(SRCDIR)/git.mk include $(SRCDIR)/libgit2.mk include $(SRCDIR)/virtualenv.mk diff --git a/deps/Versions.make b/deps/Versions.make index 3cf63f3c7eaf4..76cfab2af787d 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -11,5 +11,4 @@ OSXUNWIND_VER = 0.0.3 GMP_VER = 6.1.0 MPFR_VER = 3.1.4 PATCHELF_VER = 0.9 -GIT_VER = 1.8.5.6 VIRTUALENV_VER = 15.0.0 diff --git a/deps/checksums/git-1.8.5.6.tar.gz/md5 b/deps/checksums/git-1.8.5.6.tar.gz/md5 deleted file mode 100644 index 75986d77a39e1..0000000000000 --- a/deps/checksums/git-1.8.5.6.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -c437b3485cb157f8e84e5b2ccb135885 diff --git a/deps/checksums/git-1.8.5.6.tar.gz/sha512 b/deps/checksums/git-1.8.5.6.tar.gz/sha512 deleted file mode 100644 index 96e132813d0bd..0000000000000 --- a/deps/checksums/git-1.8.5.6.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -fd004a5a99e8ff80556e669ef6e297a4eea9c9f9c58ef9162eef1c66580c4156165c81371ff5832e56ad729d6ed3fb1459a14d682cfd2b8c37c9ceb298be2ef3 diff --git a/deps/git.mk b/deps/git.mk deleted file mode 100644 index 03794068e0865..0000000000000 --- a/deps/git.mk +++ /dev/null @@ -1,37 +0,0 @@ -## Git -# only used for the mac binaries in contrib/mac/app/Makefile - -GIT_SOURCE := $(BUILDDIR)/git-$(GIT_VER)/git -GIT_TARGET := $(build_libexecdir)/git - -$(SRCDIR)/srccache/git-$(GIT_VER).tar.gz: | $(SRCDIR)/srccache - $(JLDOWNLOAD) $@ https://www.kernel.org/pub/software/scm/git/git-$(GIT_VER).tar.gz -$(BUILDDIR)/git-$(GIT_VER)/configure: $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz - $(JLCHECKSUM) $< - cd $(BUILDDIR) && $(TAR) zxf $< - touch -c $@ -$(BUILDDIR)/git-$(GIT_VER)/config.status: $(BUILDDIR)/git-$(GIT_VER)/configure - cd $(dir $@) && \ - ./configure $(CONFIGURE_COMMON) --bindir="$(build_libexecdir)" - touch -c $@ -$(GIT_SOURCE): $(BUILDDIR)/git-$(GIT_VER)/config.status - $(MAKE) -C $(dir $<) - touch -c $@ -$(BUILDDIR)/git-$(GIT_VER)/checked: $(GIT_SOURCE) - echo 1 > $@ -$(GIT_TARGET): $(GIT_SOURCE) - $(call make-install,git-$(GIT_VER),NO_INSTALL_HARDLINKS=1 bindir="$(build_libexecdir)") - touch -c $@ - -clean-git: - -$(MAKE) -C $(BUILDDIR)/git-$(GIT_VER) clean - -rm -f $(GIT_OBJ_TARGET) -distclean-git: - -rm -rf $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz \ - $(BUILDDIR)/git-$(GIT_VER) - -get-git: $(SRCDIR)/srccache/git-$(GIT_VER).tar.gz -configure-git: $(BUILDDIR)/git-$(GIT_VER)/config.status -compile-git: $(GIT_SOURCE) -check-git: $(BUILDDIR)/git-$(GIT_VER)/checked -install-git: $(GIT_TARGET) diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index b56aa2d767e44..a2db87603aa00 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -130,7 +130,7 @@ Once again, this is equivalent to editing the ``REQUIRE`` file to remove the lin While :func:`Pkg.add` and :func:`Pkg.rm` are convenient for adding and removing requirements for a single package, when you want to add or remove multiple packages, you can call :func:`Pkg.edit` to manually change the contents of ``REQUIRE`` and then update your packages accordingly. :func:`Pkg.edit` does not roll back the contents of ``REQUIRE`` if :func:`Pkg.resolve` fails – rather, you have to run :func:`Pkg.edit` again to fix the files contents yourself. -Because the package manager uses git internally to manage the package git repositories, users may run into protocol issues (if behind a firewall, for example), when running :func:`Pkg.add`. By default, all GitHub-hosted packages wil be accessed via 'https'; this default can be modified by calling :func:`Pkg.setprotocol!`. The following command can be run from the command line in order to tell git to use 'https' instead of the 'git' protocol when cloning all repositories, wherever they are hosted:: +Because the package manager uses libgit2 internally to manage the package git repositories, users may run into protocol issues (if behind a firewall, for example), when running :func:`Pkg.add`. By default, all GitHub-hosted packages wil be accessed via 'https'; this default can be modified by calling :func:`Pkg.setprotocol!`. The following command can be run from the command line in order to tell git to use 'https' instead of the 'git' protocol when cloning all repositories, wherever they are hosted:: git config --global url."https://".insteadOf git:// diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 1dfb0aa9dd1f9..87ceb6d931046 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -246,13 +246,11 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` @test readchomp(`$exename -e 'println(ARGS);' ''`) == "String[\"\"]" # issue #12679 - extrapath = is_windows() ? joinpath(JULIA_HOME, "..", "Git", "usr", "bin") * ";" : "" - withenv("PATH" => extrapath * ENV["PATH"]) do - @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --compile=yes -ioo`),stderr=`cat`)) == "ERROR: unknown option `-o`" - @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -p`),stderr=`cat`)) == "ERROR: option `-p/--procs` is missing an argument" - @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --inline`),stderr=`cat`)) == "ERROR: option `--inline` is missing an argument" - @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -e "@show ARGS" -now -- julia RUN.jl`),stderr=`cat`)) == "ERROR: unknown option `-n`" - end + # TODO fix this on windows via busybox-w32 + @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --compile=yes -ioo`),stderr=`cat`)) == "ERROR: unknown option `-o`" + @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -p`),stderr=`cat`)) == "ERROR: option `-p/--procs` is missing an argument" + @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --inline`),stderr=`cat`)) == "ERROR: option `--inline` is missing an argument" + @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -e "@show ARGS" -now -- julia RUN.jl`),stderr=`cat`)) == "ERROR: unknown option `-n`" # --compilecache={yes|no} @test readchomp(`$exename -E "Bool(Base.JLOptions().use_compilecache)"`) == "true" diff --git a/test/spawn.jl b/test/spawn.jl index 84046640e727f..9d081976b9129 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -10,11 +10,7 @@ #TODO: # - Windows: -# - Add a test whether coreutils are available and skip tests if not -if is_windows() - oldpath = ENV["PATH"] - ENV["PATH"] = joinpath(JULIA_HOME, "..", "Git", "usr", "bin") * ";" * oldpath -end +# - fix coreutils dependent tests using busybox-w32, skip tests for no-gpl build valgrind_off = ccall(:jl_running_on_valgrind, Cint, ()) == 0 @@ -346,10 +342,6 @@ end # make sure Cmd is nestable @test string(Cmd(Cmd(`ls`, detach=true))) == "`ls`" -if is_windows() - ENV["PATH"] = oldpath -end - # equality tests for Cmd @test Base.Cmd(``) == Base.Cmd(``) @test Base.Cmd(`lsof -i :9090`) == Base.Cmd(`lsof -i :9090`) From 0ed701cecb9803991ff72a9dd5b58dabe0819c5d Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 13 Jul 2016 03:23:39 -0700 Subject: [PATCH 0420/1117] Reinstate downloading of busybox-w32 in make win-extras We may want to make this conditional on USE_GPL_LIBS, though since we're only running it as a command line executable and not linking to it, it probably falls under the aggregation clause --- LICENSE.md | 1 + Makefile | 6 ++++-- appveyor.yml | 1 + contrib/windows/msys_build.sh | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index fe82764039f2f..067fd16490f7c 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -81,6 +81,7 @@ Julia's build process uses the following external tools: Julia bundles the following external programs and libraries on some platforms: - [7-Zip](http://www.7-zip.org/license.txt) +- [BUSYBOX](https://github.com/rmyorston/busybox-w32/blob/master/LICENSE) - [ZLIB](http://zlib.net/zlib_license.html) - [LIBEXPAT](http://expat.cvs.sourceforge.net/viewvc/expat/expat/README) - [OPENSSL](https://github.com/openssl/openssl/blob/master/LICENSE) diff --git a/Makefile b/Makefile index d987bb95d7dde..3e9518b1e03e4 100644 --- a/Makefile +++ b/Makefile @@ -462,7 +462,7 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) + cp 7z.exe 7z.dll busybox.exe libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. @@ -618,10 +618,12 @@ endif cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920_extra.7z && \ $(JLDOWNLOAD) https://unsis.googlecode.com/files/nsis-2.46.5-Unicode-setup.exe && \ + $(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-483-g31277ab.exe && \ chmod a+x 7z.exe && \ chmod a+x 7z.dll && \ $(call spawn,./7z.exe) x -y -onsis nsis-2.46.5-Unicode-setup.exe && \ - chmod a+x ./nsis/makensis.exe + chmod a+x ./nsis/makensis.exe && \ + chmod a+x busybox.exe # various statistics about the build that may interest the user ifeq ($(USE_SYSTEM_LLVM), 1) diff --git a/appveyor.yml b/appveyor.yml index b417243521f8b..0d573f4c25f73 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -35,6 +35,7 @@ cache: - x86_64-4.9.2-release-win32-seh-rt_v4-rev3.7z - llvm-3.7.1-i686-w64-mingw32-juliadeps-r09.7z - llvm-3.7.1-x86_64-w64-mingw32-juliadeps-r09.7z + - usr/bin/busybox.exe build_script: # If there's a newer build queued for the same PR, cancel this one diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index c89cf379326a9..369bb793be3eb 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -156,6 +156,10 @@ if [ -z "`which make 2>/dev/null`" ]; then export PATH=$PWD/bin:$PATH fi +f=busybox-w32-FRP-483-g31277ab.exe +echo "Downloading $f" +$curlflags -o usr/bin/busybox.exe http://frippery.org/files/busybox/$f + for lib in SUITESPARSE ARPACK BLAS LAPACK FFTW \ GMP MPFR PCRE LIBUNWIND OPENSPECFUN; do echo "USE_SYSTEM_$lib = 1" >> Make.user From cfee150fc9018f8186c31af1fe895fb3fe01b605 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 13 Jul 2016 03:56:41 -0700 Subject: [PATCH 0421/1117] Use busybox-w32 in spawn and cmdlineargs tests on Windows note that one test involving perl is being skipped, but it had been disabled until recently - ref e412c69a6e77b0751e0fa0303633e52154937a46 --- test/cmdlineargs.jl | 10 ++-- test/spawn.jl | 121 +++++++++++++++++++++++--------------------- 2 files changed, 69 insertions(+), 62 deletions(-) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 87ceb6d931046..88457975a8cfe 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -246,11 +246,11 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` @test readchomp(`$exename -e 'println(ARGS);' ''`) == "String[\"\"]" # issue #12679 - # TODO fix this on windows via busybox-w32 - @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --compile=yes -ioo`),stderr=`cat`)) == "ERROR: unknown option `-o`" - @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -p`),stderr=`cat`)) == "ERROR: option `-p/--procs` is missing an argument" - @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --inline`),stderr=`cat`)) == "ERROR: option `--inline` is missing an argument" - @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -e "@show ARGS" -now -- julia RUN.jl`),stderr=`cat`)) == "ERROR: unknown option `-n`" + catcmd = is_unix() ? `cat` : `busybox cat` + @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --compile=yes -ioo`),stderr=catcmd)) == "ERROR: unknown option `-o`" + @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -p`),stderr=catcmd)) == "ERROR: option `-p/--procs` is missing an argument" + @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --inline`),stderr=catcmd)) == "ERROR: option `--inline` is missing an argument" + @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -e "@show ARGS" -now -- julia RUN.jl`),stderr=catcmd)) == "ERROR: unknown option `-n`" # --compilecache={yes|no} @test readchomp(`$exename -E "Bool(Base.JLOptions().use_compilecache)"`) == "true" diff --git a/test/spawn.jl b/test/spawn.jl index 9d081976b9129..b04786d892ca0 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -4,30 +4,31 @@ # Cross Plaform tests for spawn. # ################################## -# Assumes the following are available in the -# - GNU coreutils (or equivalent) -# - perl - -#TODO: -# - Windows: -# - fix coreutils dependent tests using busybox-w32, skip tests for no-gpl build - valgrind_off = ccall(:jl_running_on_valgrind, Cint, ()) == 0 -yes = `perl -le 'while (1) {print STDOUT "y"}'` +# use busybox-w32 on windows +yes = is_unix() ? `perl -le 'while (1) {print STDOUT "y"}'` : `busybox yes` +echo = is_unix() ? `echo` : `busybox echo` +sortcmd = is_unix() ? `sort` : `busybox sort` +printf = is_unix() ? `printf` : `busybox printf` +truecmd = is_unix() ? `true` : `busybox true` +falsecmd = is_unix() ? `false` : `busybox false` +catcmd = is_unix() ? `cat` : `busybox cat` +shcmd = is_unix() ? `sh` : `busybox sh` +sleepcmd = is_unix() ? `sleep` : `busybox sleep` #### Examples used in the manual #### -@test readstring(`echo hello | sort`) == "hello | sort\n" -@test readstring(pipeline(`echo hello`, `sort`)) == "hello\n" -@test length(spawn(pipeline(`echo hello`, `sort`)).processes) == 2 +@test readstring(`$echo hello | sort`) == "hello | sort\n" +@test readstring(pipeline(`$echo hello`, sortcmd)) == "hello\n" +@test length(spawn(pipeline(`$echo hello`, sortcmd)).processes) == 2 -out = readstring(`echo hello` & `echo world`) +out = readstring(`$echo hello` & `$echo world`) @test search(out,"world") != 0:-1 @test search(out,"hello") != 0:-1 -@test readstring(pipeline(`echo hello` & `echo world`, `sort`)) == "hello\nworld\n" +@test readstring(pipeline(`$echo hello` & `$echo world`, sortcmd)) == "hello\nworld\n" -@test (run(`printf " \033[34m[stdio passthrough ok]\033[0m\n"`); true) +@test (run(`$printf " \033[34m[stdio passthrough ok]\033[0m\n"`); true) # Test for SIGPIPE being treated as normal termination (throws an error if broken) is_unix() && run(pipeline(yes, `head`, DevNull)) @@ -49,28 +50,30 @@ if valgrind_off @test_throws Base.UVError run(`foo_is_not_a_valid_command`) end -prefixer(prefix, sleep) = `perl -nle '$|=1; print "'$prefix' ", $_; sleep '$sleep';'` -@test success(pipeline(`perl -le '$|=1; for(0..2){ print; sleep 1 }'`, +if is_unix() + prefixer(prefix, sleep) = `perl -nle '$|=1; print "'$prefix' ", $_; sleep '$sleep';'` + @test success(pipeline(`perl -le '$|=1; for(0..2){ print; sleep 1 }'`, prefixer("A",2) & prefixer("B",2))) -@test success(pipeline(`perl -le '$|=1; for(0..2){ print; sleep 1 }'`, + @test success(pipeline(`perl -le '$|=1; for(0..2){ print; sleep 1 }'`, prefixer("X",3) & prefixer("Y",3) & prefixer("Z",3), prefixer("A",2) & prefixer("B",2))) +end -@test success(`true`) -@test !success(`false`) -@test success(pipeline(`true`, `true`)) -@test_broken success(ignorestatus(`false`)) -@test_broken success(pipeline(ignorestatus(`false`), `true`)) -@test !success(pipeline(ignorestatus(`false`), `false`)) -@test !success(ignorestatus(`false`) & `false`) -@test_broken success(ignorestatus(pipeline(`false`, `false`))) -@test_broken success(ignorestatus(`false` & `false`)) +@test success(truecmd) +@test !success(falsecmd) +@test success(pipeline(truecmd, truecmd)) +@test_broken success(ignorestatus(falsecmd)) +@test_broken success(pipeline(ignorestatus(falsecmd), truecmd)) +@test !success(pipeline(ignorestatus(falsecmd), falsecmd)) +@test !success(ignorestatus(falsecmd) & falsecmd) +@test_broken success(ignorestatus(pipeline(falsecmd, falsecmd))) +@test_broken success(ignorestatus(falsecmd & falsecmd)) # STDIN Redirection file = tempname() -run(pipeline(`echo hello world`, file)) -@test readstring(pipeline(file, `cat`)) == "hello world\n" -@test open(readstring, pipeline(file, `cat`), "r") == "hello world\n" +run(pipeline(`$echo hello world`, file)) +@test readstring(pipeline(file, catcmd)) == "hello world\n" +@test open(readstring, pipeline(file, catcmd), "r") == "hello world\n" rm(file) # Stream Redirection @@ -80,23 +83,23 @@ if !is_windows() # WINNT reports operation not supported on socket (ENOTSUP) for port, server = listenany(2326) put!(r, port) client = accept(server) - @test readstring(pipeline(client, `cat`)) == "hello world\n" + @test readstring(pipeline(client, catcmd)) == "hello world\n" close(server) end @async begin sock = connect(fetch(r)) - run(pipeline(`echo hello world`, sock)) + run(pipeline(`$echo hello world`, sock)) close(sock) end end -@test readstring(setenv(`sh -c "echo \$TEST"`,["TEST=Hello World"])) == "Hello World\n" -@test readstring(setenv(`sh -c "echo \$TEST"`,Dict("TEST"=>"Hello World"))) == "Hello World\n" -@test readstring(setenv(`sh -c "echo \$TEST"`,"TEST"=>"Hello World")) == "Hello World\n" +@test readstring(setenv(`$shcmd -c "echo \$TEST"`,["TEST=Hello World"])) == "Hello World\n" +@test readstring(setenv(`$shcmd -c "echo \$TEST"`,Dict("TEST"=>"Hello World"))) == "Hello World\n" +@test readstring(setenv(`$shcmd -c "echo \$TEST"`,"TEST"=>"Hello World")) == "Hello World\n" @test (withenv("TEST"=>"Hello World") do - readstring(`sh -c "echo \$TEST"`); end) == "Hello World\n" -pathA = readchomp(setenv(`sh -c "pwd -P"`;dir="..")) -pathB = readchomp(setenv(`sh -c "cd .. && pwd -P"`)) + readstring(`$shcmd -c "echo \$TEST"`); end) == "Hello World\n" +pathA = readchomp(setenv(`$shcmd -c "pwd -P"`;dir="..")) +pathB = readchomp(setenv(`$shcmd -c "cd .. && pwd -P"`)) if is_windows() # on windows, sh returns posix-style paths that are not valid according to ispath @test pathA == pathB @@ -109,7 +112,7 @@ str = "" for i=1:1000 str = "$str\n $(randstring(10))" end -stdout, stdin, proc = readandwrite(`cat -`) +stdout, stdin, proc = readandwrite(`$catcmd -`) write(stdin, str) close(stdin) str2 = readstring(stdout) @@ -117,7 +120,7 @@ str2 = readstring(stdout) # This test hangs if the end of run walk across uv streams calls shutdown on a stream that is shutting down. file = tempname() -open(pipeline(`cat -`, file), "w") do io +open(pipeline(`$catcmd -`, file), "w") do io write(io, str) end rm(file) @@ -129,7 +132,7 @@ t = @async begin try wait(r) end - p = spawn(`sleep 1`); wait(p) + p = spawn(`$sleepcmd 1`); wait(p) @test p.exitcode == 0 end yield() @@ -172,7 +175,7 @@ exename = Base.julia_cmd() if valgrind_off # If --trace-children=yes is passed to valgrind, we will get a # valgrind banner here, not "Hello World\n". - @test readstring(pipeline(`$exename --startup-file=no -e 'println(STDERR,"Hello World")'`, stderr=`cat`)) == "Hello World\n" + @test readstring(pipeline(`$exename --startup-file=no -e 'println(STDERR,"Hello World")'`, stderr=catcmd)) == "Hello World\n" out = Pipe() proc = spawn(pipeline(`$exename --startup-file=no -e 'println(STDERR,"Hello World")'`, stderr = out)) close(out.in) @@ -181,10 +184,10 @@ if valgrind_off end # issue #6310 -@test readstring(pipeline(`echo "2+2"`, `$exename --startup-file=no`)) == "4\n" +@test readstring(pipeline(`$echo "2+2"`, `$exename --startup-file=no`)) == "4\n" # issue #5904 -@test run(pipeline(ignorestatus(`false`), `true`)) === nothing +@test run(pipeline(ignorestatus(falsecmd), truecmd)) === nothing # issue #6010 @@ -244,9 +247,9 @@ end # issue #10994: libuv can't handle strings containing NUL let bad = "bad\0name" @test_throws ArgumentError run(`$bad`) - @test_throws ArgumentError run(`echo $bad`) - @test_throws ArgumentError run(setenv(`echo hello`, bad=>"good")) - @test_throws ArgumentError run(setenv(`echo hello`, "good"=>bad)) + @test_throws ArgumentError run(`$echo $bad`) + @test_throws ArgumentError run(setenv(`$echo hello`, bad=>"good")) + @test_throws ArgumentError run(setenv(`$echo hello`, "good"=>bad)) end # issue #12829 @@ -300,10 +303,14 @@ let fname = tempname() write(fname, "test\n") code = """ for line in eachline(STDIN) - run(pipeline(`echo asdf`,`cat`)) + if is_unix() + run(pipeline(`echo asdf`,`cat`)) + else + run(pipeline(`busybox echo asdf`,`busybox cat`)) + end end """ - @test success(pipeline(`cat $fname`, `$exename -e $code`)) + @test success(pipeline(`$catcmd $fname`, `$exename -e $code`)) rm(fname) end @@ -332,7 +339,7 @@ end @test Base.shell_split("\"\\\\\"") == ["\\"] # issue #13616 -@test_throws ErrorException collect(eachline(pipeline(`cat _doesnt_exist__111_`, stderr=DevNull))) +@test_throws ErrorException collect(eachline(pipeline(`$catcmd _doesnt_exist__111_`, stderr=DevNull))) # make sure windows_verbatim strips quotes if is_windows() @@ -345,19 +352,19 @@ end # equality tests for Cmd @test Base.Cmd(``) == Base.Cmd(``) @test Base.Cmd(`lsof -i :9090`) == Base.Cmd(`lsof -i :9090`) -@test Base.Cmd(`echo test`) == Base.Cmd(`echo test`) -@test Base.Cmd(``) != Base.Cmd(`echo test`) +@test Base.Cmd(`$echo test`) == Base.Cmd(`$echo test`) +@test Base.Cmd(``) != Base.Cmd(`$echo test`) @test Base.Cmd(``, ignorestatus=true) != Base.Cmd(``, ignorestatus=false) @test Base.Cmd(``, dir="TESTS") != Base.Cmd(``, dir="TEST") @test Base.Set([``, ``]) == Base.Set([``]) -@test Set([``, `echo`]) != Set([``, ``]) -@test Set([`echo`, ``, ``, `echo`]) == Set([`echo`, ``]) +@test Set([``, echo]) != Set([``, ``]) +@test Set([echo, ``, ``, echo]) == Set([echo, ``]) # equality tests for AndCmds -@test Base.AndCmds(`echo abc`, `echo def`) == Base.AndCmds(`echo abc`, `echo def`) -@test Base.AndCmds(`echo abc`, `echo def`) != Base.AndCmds(`echo abc`, `echo xyz`) +@test Base.AndCmds(`$echo abc`, `$echo def`) == Base.AndCmds(`$echo abc`, `$echo def`) +@test Base.AndCmds(`$echo abc`, `$echo def`) != Base.AndCmds(`$echo abc`, `$echo xyz`) # tests for reducing over collection of Cmd @test_throws ArgumentError reduce(&, Base.AbstractCmd[]) @test_throws ArgumentError reduce(&, Base.Cmd[]) -@test reduce(&, [`echo abc`, `echo def`, `echo hij`]) == `echo abc` & `echo def` & `echo hij` +@test reduce(&, [`$echo abc`, `$echo def`, `$echo hij`]) == `$echo abc` & `$echo def` & `$echo hij` From 555856ee2fe96c0f391e5e93f6b54e21d8cf1deb Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 13 Jul 2016 04:31:22 -0700 Subject: [PATCH 0422/1117] fall back to using coreutils if busybox is not present it usually won't be in source builds --- test/cmdlineargs.jl | 8 +++++++- test/spawn.jl | 46 ++++++++++++++++++++++++++++++--------------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 88457975a8cfe..d1c6b7df4ab38 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -246,7 +246,13 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` @test readchomp(`$exename -e 'println(ARGS);' ''`) == "String[\"\"]" # issue #12679 - catcmd = is_unix() ? `cat` : `busybox cat` + catcmd = `cat` + if is_windows() + try # use busybox-w32 on windows + success(`busybox`) + catcmd = `busybox cat` + end + end @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --compile=yes -ioo`),stderr=catcmd)) == "ERROR: unknown option `-o`" @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -p`),stderr=catcmd)) == "ERROR: option `-p/--procs` is missing an argument" @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --inline`),stderr=catcmd)) == "ERROR: option `--inline` is missing an argument" diff --git a/test/spawn.jl b/test/spawn.jl index b04786d892ca0..fe59cd7148461 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -6,16 +6,29 @@ valgrind_off = ccall(:jl_running_on_valgrind, Cint, ()) == 0 -# use busybox-w32 on windows -yes = is_unix() ? `perl -le 'while (1) {print STDOUT "y"}'` : `busybox yes` -echo = is_unix() ? `echo` : `busybox echo` -sortcmd = is_unix() ? `sort` : `busybox sort` -printf = is_unix() ? `printf` : `busybox printf` -truecmd = is_unix() ? `true` : `busybox true` -falsecmd = is_unix() ? `false` : `busybox false` -catcmd = is_unix() ? `cat` : `busybox cat` -shcmd = is_unix() ? `sh` : `busybox sh` -sleepcmd = is_unix() ? `sleep` : `busybox sleep` +yes = `perl -le 'while (1) {print STDOUT "y"}'` +echo = `echo` +sortcmd = `sort` +printf = `printf` +truecmd = `true` +falsecmd = `false` +catcmd = `cat` +shcmd = `sh` +sleepcmd = `sleep` +if is_windows() + try # use busybox-w32 on windows + success(`busybox`) + yes = `busybox yes` + echo = `busybox echo` + sortcmd = `busybox sort` + printf = `busybox printf` + truecmd = `busybox true` + falsecmd = `busybox false` + catcmd = `busybox cat` + shcmd = `busybox sh` + sleepcmd = `busybox sleep` + end +end #### Examples used in the manual #### @@ -302,13 +315,16 @@ end let fname = tempname() write(fname, "test\n") code = """ - for line in eachline(STDIN) - if is_unix() - run(pipeline(`echo asdf`,`cat`)) - else - run(pipeline(`busybox echo asdf`,`busybox cat`)) + cmd = pipeline(`echo asdf`,`cat`) + if is_windows() + try + success(`busybox`) + cmd = pipeline(`busybox echo asdf`,`busybox cat`) end end + for line in eachline(STDIN) + run(cmd) + end """ @test success(pipeline(`$catcmd $fname`, `$exename -e $code`)) rm(fname) From e81ead7fbcb43a9249cb45d0024cc5d0418df355 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 13 Jul 2016 13:20:12 -0400 Subject: [PATCH 0423/1117] updated to utf8proc v2.0.1, gives support for Unicode 9 --- base/strings/utf8proc.jl | 20 ++++++++++++++------ deps/utf8proc.version | 4 ++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/base/strings/utf8proc.jl b/base/strings/utf8proc.jl index 5b3ff7c430f17..6bc7d94329079 100644 --- a/base/strings/utf8proc.jl +++ b/base/strings/utf8proc.jl @@ -184,6 +184,12 @@ end isgraphemebreak(c1::Char, c2::Char) = ccall(:utf8proc_grapheme_break, Bool, (UInt32, UInt32), c1, c2) +# Stateful grapheme break required by Unicode-9 rules: the string +# must be processed in sequence, with state initialized to Ref{Int32}(0). +# Requires utf8proc v2.0 or later. +isgraphemebreak(c1::Char, c2::Char, state::Ref{Int32}) = + ccall(:utf8proc_grapheme_break_stateful, Bool, (UInt32, UInt32, Ref{Int32}), c1, c2, state) + immutable GraphemeIterator{S<:AbstractString} s::S # original string (for generation of SubStrings) end @@ -194,28 +200,30 @@ eltype{S}(::Type{GraphemeIterator{S}}) = SubString{S} function length(g::GraphemeIterator) c0 = Char(0x00ad) # soft hyphen (grapheme break always allowed after this) n = 0 + state = Ref{Int32}(0) for c in g.s - n += isgraphemebreak(c0, c) + n += isgraphemebreak(c0, c, state) c0 = c end return n end -start(g::GraphemeIterator) = start(g.s) -done(g::GraphemeIterator, i) = done(g.s, i) +start(g::GraphemeIterator) = (start(g.s), Ref{Int32}(0)) +done(g::GraphemeIterator, i) = done(g.s, i[1]) -function next(g::GraphemeIterator, i) +function next(g::GraphemeIterator, i_) s = g.s + i, state = i_ j = i c0, k = next(s, i) while !done(s, k) # loop until next grapheme is s[i:j] c, ℓ = next(s, k) - isgraphemebreak(c0, c) && break + isgraphemebreak(c0, c, state) && break j = k k = ℓ c0 = c end - return (SubString(s, i, j), k) + return (SubString(s, i, j), (k, state)) end ==(g1::GraphemeIterator, g2::GraphemeIterator) = g1.s == g2.s diff --git a/deps/utf8proc.version b/deps/utf8proc.version index 6f56e47295c9b..c194a01b80cf6 100644 --- a/deps/utf8proc.version +++ b/deps/utf8proc.version @@ -1,2 +1,2 @@ -UTF8PROC_BRANCH=v1.3 -UTF8PROC_SHA1=85789180158ac7fff85b9f008828d6ac44f072ea +UTF8PROC_BRANCH=v2.0.1 +UTF8PROC_SHA1=a1fe9955bbc75ffb923c1219bf58befd2688e34c From 0d8bb5e65a566f6dccfdc4cda0185335ec88f27e Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 13 Jul 2016 13:24:12 -0400 Subject: [PATCH 0424/1117] NEWS update --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index e48f3fe80e22f..b96a6f1f6568d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -135,6 +135,8 @@ Library improvements * A `transcode(T, src)` function is now exported for converting data between UTF-xx Unicode encodings ([#17323]). + * Support for Unicode 9 ([#17402]). + * Most of the combinatorics functions have been moved from `Base` to the [Combinatorics.jl package](https://github.com/JuliaLang/Combinatorics.jl) ([#13897]). @@ -335,7 +337,9 @@ Deprecated or removed [#16972]: https://github.com/JuliaLang/julia/issues/16972 [#17037]: https://github.com/JuliaLang/julia/issues/17037 [#17075]: https://github.com/JuliaLang/julia/issues/17075 +[#17132]: https://github.com/JuliaLang/julia/issues/17132 [#17266]: https://github.com/JuliaLang/julia/issues/17266 [#17300]: https://github.com/JuliaLang/julia/issues/17300 [#17323]: https://github.com/JuliaLang/julia/issues/17323 [#17374]: https://github.com/JuliaLang/julia/issues/17374 +[#17402]: https://github.com/JuliaLang/julia/issues/17402 From 1ccc5513f66202d682a1bb0d3f0cb47c108a9858 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 13 Jul 2016 13:45:38 -0400 Subject: [PATCH 0425/1117] update utf8proc checksums --- .../utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 | 1 + .../sha512 | 1 + 2 files changed, 2 insertions(+) create mode 100644 deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 create mode 100644 deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/sha512 diff --git a/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 b/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 new file mode 100644 index 0000000000000..46106d558178e --- /dev/null +++ b/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 @@ -0,0 +1 @@ +9b562e952121df2438496d878ae086af diff --git a/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/sha512 b/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/sha512 new file mode 100644 index 0000000000000..a3ca40fbb0bad --- /dev/null +++ b/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/sha512 @@ -0,0 +1 @@ +790a103fd00501822dcb8744fc52e87ed1065b829d1f14aed74204b32186ffec5784c4f4901c4f3bd8b5529dc508461cf9374fd7bc3eadff0a183bb443a7abb2 From a4575f0ef1ef10f57cc160a6fb36815bc3a4580b Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 13 Jul 2016 13:46:52 -0400 Subject: [PATCH 0426/1117] note utf8proc 2.0 dependency in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 263f69f4040fe..3c07cf02d49e8 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,7 @@ Julia uses the following external libraries, which are automatically downloaded - **[GMP]** (>= 5.0) — GNU multiple precision arithmetic library, needed for `BigInt` support. - **[MPFR]** (>= 3.0) — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support. - **[libgit2]** (>= 0.21) — Git linkable library, used by Julia's package manager -- **[utf8proc]** (>= 1.3) — a library for processing UTF-8 encoded Unicode strings +- **[utf8proc]** (>= 2.0) — a library for processing UTF-8 encoded Unicode strings - **[libosxunwind]** — clone of [libunwind], a library that determines the call-chain of a program For a longer overview of Julia's dependencies, see these [slides](https://github.com/tkelman/BAJUtalk-Dec2014/blob/master/BAJUtalkDec2014.pdf?raw=true). From 28493629ac0f798f3ccb4f9bd4d005396371fe9e Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 13 Jul 2016 13:49:58 -0400 Subject: [PATCH 0427/1117] delete old utf8proc checksums --- .../utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/md5 | 1 - .../sha512 | 1 - 2 files changed, 2 deletions(-) delete mode 100644 deps/checksums/utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/md5 delete mode 100644 deps/checksums/utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/sha512 diff --git a/deps/checksums/utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/md5 b/deps/checksums/utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/md5 deleted file mode 100644 index d78c36ec7dfc7..0000000000000 --- a/deps/checksums/utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -9dab2320faeb9b7ca8623737b108a815 diff --git a/deps/checksums/utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/sha512 b/deps/checksums/utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/sha512 deleted file mode 100644 index e068ca65daf93..0000000000000 --- a/deps/checksums/utf8proc-85789180158ac7fff85b9f008828d6ac44f072ea.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -3e0b7cb61d6fb8161d9890ed03485c445eee761ea228c4196af9fe190ca78ef9893f41f4a76ee2ecb49bcd42e69541cf6e763730970a130ab592e2956d1e8316 From 864bfd464b29a7c36e23f067bfbd02a77fdd67ef Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 13 Jul 2016 16:26:18 -0400 Subject: [PATCH 0428/1117] update w.del_msgs array atomically in flush_gc_msgs fix #16880, tentatively --- base/multi.jl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/base/multi.jl b/base/multi.jl index 2b52b033a8dca..6d21904b24892 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -260,15 +260,18 @@ function flush_gc_msgs(w::Worker) return end w.gcflag = false - msgs = copy(w.add_msgs) + new_array = Any[] + msgs = w.add_msgs + w.add_msgs = new_array if !isempty(msgs) - empty!(w.add_msgs) remote_do(add_clients, w, msgs) end - msgs = copy(w.del_msgs) + # del_msgs gets populated by finalizers, so be very careful here about ordering of allocations + new_array = Any[] + msgs = w.del_msgs + w.del_msgs = new_array if !isempty(msgs) - empty!(w.del_msgs) #print("sending delete of $msgs\n") remote_do(del_clients, w, msgs) end From 15a2b3af52cfd0bc6f0f93a039e93f1d10a8e3d4 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Wed, 13 Jul 2016 20:48:06 -0500 Subject: [PATCH 0429/1117] Non-throwing boundscheck functions are written as f(Bool, args...) --- base/abstractarray.jl | 79 +++++++++++++++++++++++-------------- base/multidimensional.jl | 26 ++++++------ doc/devdocs/boundscheck.rst | 8 ++-- doc/stdlib/arrays.rst | 1 - test/replutil.jl | 3 ++ 5 files changed, 69 insertions(+), 48 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 962574a3bf5ee..3e82aeb2245fa 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -154,8 +154,8 @@ linearindexing(::LinearIndexing, ::LinearIndexing) = LinearSlow() # The overall hierarchy is # `checkbounds(A, I...)` -> # `checkbounds(Bool, A, I...)` -> either of: -# - `checkbounds_indices(IA, I)` which calls `checkindex(Bool, inds, i)` -# - `checkbounds_logical(A, I)` when `I` is a single logical array +# - `checkbounds_logical(Bool, A, I)` when `I` is a single logical array +# - `checkbounds_indices(Bool, IA, I)` otherwise (uses `checkindex`) # # See the "boundscheck" devdocs for more information. # @@ -177,23 +177,35 @@ See also `checkindex`. """ function checkbounds(::Type{Bool}, A::AbstractArray, I...) @_inline_meta - checkbounds_indices(indices(A), I) + checkbounds_indices(Bool, indices(A), I) end function checkbounds(::Type{Bool}, A::AbstractArray, I::AbstractArray{Bool}) @_inline_meta - checkbounds_logical(A, I) + checkbounds_logical(Bool, A, I) end """ - checkbounds_indices(IA, I) + checkbounds(A, I...) + +Throw an error if the specified indices `I` are not in bounds for the given array `A`. +""" +function checkbounds(A::AbstractArray, I...) + @_inline_meta + checkbounds(Bool, A, I...) || throw_boundserror(A, I) + nothing +end +checkbounds(A::AbstractArray) = checkbounds(A, 1) # 0-d case -checks whether the "requested" indices in the tuple `I` fall within +""" + checkbounds_indices(Bool, IA, I) + +Return `true` if the "requested" indices in the tuple `I` fall within the bounds of the "permitted" indices specified by the tuple `IA`. This function recursively consumes elements of these tuples, usually in a 1-for-1 fashion, - checkbounds_indices((IA1, IA...), (I1, I...)) = checkindex(Bool, IA1, I1) & - checkbounds_indices(IA, I) + checkbounds_indices(Bool, (IA1, IA...), (I1, I...)) = checkindex(Bool, IA1, I1) & + checkbounds_indices(Bool, IA, I) Note that `checkindex` is being used to perform the actual bounds-check for a single dimension of the array. @@ -202,48 +214,55 @@ There are two important exceptions to the 1-1 rule: linear indexing and CartesianIndex{N}, both of which may "consume" more than one element of `IA`. """ -function checkbounds_indices(IA::Tuple, I::Tuple) +function checkbounds_indices(::Type{Bool}, IA::Tuple, I::Tuple) @_inline_meta - checkindex(Bool, IA[1], I[1]) & checkbounds_indices(tail(IA), tail(I)) + checkindex(Bool, IA[1], I[1]) & checkbounds_indices(Bool, tail(IA), tail(I)) end -checkbounds_indices(::Tuple{}, ::Tuple{}) = true -checkbounds_indices(::Tuple{}, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, 1:1, I[1])) -function checkbounds_indices(::Tuple{}, I::Tuple) +checkbounds_indices(::Type{Bool}, ::Tuple{}, ::Tuple{}) = true +checkbounds_indices(::Type{Bool}, ::Tuple{}, I::Tuple{Any}) = (@_inline_meta; checkindex(Bool, 1:1, I[1])) +function checkbounds_indices(::Type{Bool}, ::Tuple{}, I::Tuple) @_inline_meta - checkindex(Bool, 1:1, I[1]) & checkbounds_indices((), tail(I)) + checkindex(Bool, 1:1, I[1]) & checkbounds_indices(Bool, (), tail(I)) end -function checkbounds_indices(IA::Tuple{Any}, I::Tuple{Any}) +function checkbounds_indices(::Type{Bool}, IA::Tuple{Any}, I::Tuple{Any}) @_inline_meta checkindex(Bool, IA[1], I[1]) end -function checkbounds_indices(IA::Tuple, I::Tuple{Any}) +function checkbounds_indices(::Type{Bool}, IA::Tuple, I::Tuple{Any}) @_inline_meta checkindex(Bool, 1:prod(map(dimlength, IA)), I[1]) # linear indexing end """ - checkbounds_logical(A, I::AbstractArray{Bool}) + checkbounds_logical(Bool, A, I::AbstractArray{Bool}) -tests whether the logical array `I` is consistent with the indices of `A`. +Return `true` if the logical array `I` is consistent with the indices +of `A`. `I` and `A` should have the same size and compatible indices. """ -checkbounds_logical(A::AbstractArray, I::AbstractArray{Bool}) = indices(A) == indices(I) -checkbounds_logical(A::AbstractArray, I::AbstractVector{Bool}) = length(A) == length(I) -checkbounds_logical(A::AbstractVector, I::AbstractArray{Bool}) = length(A) == length(I) -checkbounds_logical(A::AbstractVector, I::AbstractVector{Bool}) = indices(A) == indices(I) - -throw_boundserror(A, I) = (@_noinline_meta; throw(BoundsError(A, I))) +function checkbounds_logical(::Type{Bool}, A::AbstractArray, I::AbstractArray{Bool}) + indices(A) == indices(I) +end +function checkbounds_logical(::Type{Bool}, A::AbstractArray, I::AbstractVector{Bool}) + length(A) == length(I) +end +function checkbounds_logical(::Type{Bool}, A::AbstractVector, I::AbstractArray{Bool}) + length(A) == length(I) +end +function checkbounds_logical(::Type{Bool}, A::AbstractVector, I::AbstractVector{Bool}) + indices(A) == indices(I) +end """ - checkbounds(A, I...) + checkbounds_logical(A, I::AbstractArray{Bool}) -Throw an error if the specified indices `I` are not in bounds for the given array `A`. +Throw an error if the logical array `I` is inconsistent with the indices of `A`. """ -function checkbounds(A::AbstractArray, I...) - @_inline_meta - checkbounds(Bool, A, I...) || throw_boundserror(A, I) +function checkbounds_logical(A, I::AbstractVector{Bool}) + checkbounds_logical(Bool, A, I) || throw_boundserror(A, I) nothing end -checkbounds(A::AbstractArray) = checkbounds(A, 1) # 0-d case + +throw_boundserror(A, I) = (@_noinline_meta; throw(BoundsError(A, I))) @generated function trailingsize{T,N,n}(A::AbstractArray{T,N}, ::Type{Val{n}}) (isa(n, Int) && isa(N, Int)) || error("Must have concrete type") diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 6fe1ca22f52d5..199ef18cf5afc 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -154,31 +154,31 @@ end # IteratorsMD using .IteratorsMD ## Bounds-checking with CartesianIndex -@inline checkbounds_indices(::Tuple{}, I::Tuple{CartesianIndex,Vararg{Any}}) = - checkbounds_indices((), (I[1].I..., tail(I)...)) -@inline checkbounds_indices(IA::Tuple{Any}, I::Tuple{CartesianIndex,Vararg{Any}}) = - checkbounds_indices(IA, (I[1].I..., tail(I)...)) -@inline checkbounds_indices(IA::Tuple, I::Tuple{CartesianIndex,Vararg{Any}}) = - checkbounds_indices(IA, (I[1].I..., tail(I)...)) +@inline checkbounds_indices(::Type{Bool}, ::Tuple{}, I::Tuple{CartesianIndex,Vararg{Any}}) = + checkbounds_indices(Bool, (), (I[1].I..., tail(I)...)) +@inline checkbounds_indices(::Type{Bool}, IA::Tuple{Any}, I::Tuple{CartesianIndex,Vararg{Any}}) = + checkbounds_indices(Bool, IA, (I[1].I..., tail(I)...)) +@inline checkbounds_indices(::Type{Bool}, IA::Tuple, I::Tuple{CartesianIndex,Vararg{Any}}) = + checkbounds_indices(Bool, IA, (I[1].I..., tail(I)...)) # Support indexing with an array of CartesianIndex{N}s # Here we try to consume N of the indices (if there are that many available) # The first two simply handle ambiguities -@inline function checkbounds_indices{N}(::Tuple{}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) - checkindex(Bool, (), I[1]) & checkbounds_indices((), tail(I)) +@inline function checkbounds_indices{N}(::Type{Bool}, ::Tuple{}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) + checkindex(Bool, (), I[1]) & checkbounds_indices(Bool, (), tail(I)) end -@inline function checkbounds_indices{N}(IA::Tuple{Any}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) - checkindex(Bool, IA, I[1]) & checkbounds_indices((), tail(I)) +@inline function checkbounds_indices{N}(::Type{Bool}, IA::Tuple{Any}, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) + checkindex(Bool, IA, I[1]) & checkbounds_indices(Bool, (), tail(I)) end -@inline function checkbounds_indices{N}(IA::Tuple, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) +@inline function checkbounds_indices{N}(::Type{Bool}, IA::Tuple, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg{Any}}) IA1, IArest = IteratorsMD.split(IA, Val{N}) - checkindex(Bool, IA1, I[1]) & checkbounds_indices(IArest, tail(I)) + checkindex(Bool, IA1, I[1]) & checkbounds_indices(Bool, IArest, tail(I)) end function checkindex{N}(::Type{Bool}, inds::Tuple, I::AbstractArray{CartesianIndex{N}}) b = true for i in I - b &= checkbounds_indices(inds, (i,)) + b &= checkbounds_indices(Bool, inds, (i,)) end b end diff --git a/doc/devdocs/boundscheck.rst b/doc/devdocs/boundscheck.rst index 1d324e9aa546b..6a85da5c0225e 100644 --- a/doc/devdocs/boundscheck.rst +++ b/doc/devdocs/boundscheck.rst @@ -67,8 +67,8 @@ The overall hierarchy is: | ``checkbounds(A, I...)`` which calls | ``checkbounds(Bool, A, I...)`` which calls either of: -| ``checkbounds_logical(A, I)`` when ``I`` is a single logical array -| ``checkbounds_indices(indices(A), I)`` otherwise +| ``checkbounds_logical(Bool, A, I)`` when ``I`` is a single logical array +| ``checkbounds_indices(Bool, indices(A), I)`` otherwise | Here ``A`` is the array, and ``I`` contains the "requested" indices. @@ -85,8 +85,8 @@ dimensions handled by calling another important function, ``checkindex``: typically, :: - checkbounds_indices((IA1, IA...), (I1, I...)) = checkindex(Bool, IA1, I1) & - checkbounds_indices(IA, I) + checkbounds_indices(Bool, (IA1, IA...), (I1, I...)) = checkindex(Bool, IA1, I1) & + checkbounds_indices(Bool, IA, I) so ``checkindex`` checks a single dimension. All of these functions, including the unexported ``checkbounds_indices`` and diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index b62866876cba1..7b222dea1416c 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1080,4 +1080,3 @@ dense counterparts. The following functions are specific to sparse arrays. For additional (algorithmic) information, and for versions of these methods that forgo argument checking, see (unexported) parent methods :func:`Base.SparseArrays.unchecked_noalias_permute!` and :func:`Base.SparseArrays.unchecked_aliasing_permute!`\ . See also: :func:`Base.SparseArrays.permute` - diff --git a/test/replutil.jl b/test/replutil.jl index 3bd83c8f7f29a..c0a34b29df5e9 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -222,6 +222,9 @@ let undefvar err_str = @except_strbt (-1)^0.25 DomainError @test contains(err_str, "Exponentiation yielding a complex result requires a complex argument") + err_str = @except_str (1,2,3)[4] BoundsError + @test err_str == "BoundsError: attempt to access (1,2,3)\n at index [4]" + err_str = @except_str [5,4,3][-2,1] BoundsError @test err_str == "BoundsError: attempt to access 3-element Array{$Int,1} at index [-2,1]" err_str = @except_str [5,4,3][1:5] BoundsError From fd3751b4223c8d9ca2a4559b4ee5c51d4fc73d3c Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 13 Jul 2016 21:25:54 -0700 Subject: [PATCH 0430/1117] Revert "LibGit2: added tree related functions & tests" --- base/libgit2/tree.jl | 65 ++++--------------------------------------- base/libgit2/types.jl | 29 ++++++++++--------- test/libgit2.jl | 53 ----------------------------------- 3 files changed, 19 insertions(+), 128 deletions(-) diff --git a/base/libgit2/tree.jl b/base/libgit2/tree.jl index f1d73119a4aa0..321bd83708d2c 100644 --- a/base/libgit2/tree.jl +++ b/base/libgit2/tree.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -"""Traverse the entries in a tree and its subtrees in post or preorder. +""" +Traverse the entries in a tree and its subtrees in post or pre order. Function parameter should have following signature: @@ -15,20 +16,17 @@ function treewalk(f::Function, tree::GitTree, payload=Any[], post::Bool = false) return cbf_payload end -"Returns a file name, as a `String`, of a tree entry." function filename(te::GitTreeEntry) str = ccall((:git_tree_entry_name, :libgit2), Cstring, (Ptr{Void},), te.ptr) - str == C_NULL && throw(Error.GitError(Error.Tree, - LibGit2.Error.ENOTFOUND, "filename not found")) - return unsafe_string(str) + str != C_NULL && return unsafe_string(str) + return nothing end -"Returns UNIX file attributes, as a `Cint`, of a tree entry." function filemode(te::GitTreeEntry) return ccall((:git_tree_entry_filemode, :libgit2), Cint, (Ptr{Void},), te.ptr) end -"Returns a `GitAnyObject` which is referenced by a tree entry." + function object(repo::GitRepo, te::GitTreeEntry) obj_ptr_ptr = Ref{Ptr{Void}}(C_NULL) @check ccall((:git_tree_entry_to_object, :libgit2), Cint, @@ -36,56 +34,3 @@ function object(repo::GitRepo, te::GitTreeEntry) obj_ptr_ptr, repo.ptr, te.ptr) return GitAnyObject(obj_ptr_ptr[]) end - -"Returns an object identifier, as a `Oid`, of a tree entry." -function oid(te::GitTreeEntry) - oid_ptr = ccall((:git_tree_entry_id, :libgit2), Ptr{Oid}, (Ptr{Void},), te.ptr) - oid_ptr == C_NULL && throw(Error.GitError(Error.Tree, - LibGit2.Error.ENOTFOUND, "oid not found")) - return Oid(oid_ptr) -end - -"""Retrieve a tree entry contained in a tree or in any of its subtrees, given its relative path. - -The returned tree entry is owned by the user and must be freed explicitly. -""" -function GitTreeEntry(tree::GitTree, tepath::AbstractString) - te_ptr_ptr = Ref{Ptr{Void}}(C_NULL) - err = ccall((:git_tree_entry_bypath, :libgit2), Cint, - (Ptr{Ptr{Void}}, Ref{Void}, Cstring), - te_ptr_ptr, tree.ptr, tepath) - if err == Cint(Error.ENOTFOUND) - return Nullable{GitTreeEntry}() - elseif err != Cint(Error.GIT_OK) - if te_ptr_ptr[] != C_NULL - finalize(GitTreeEntry(te_ptr_ptr[])) - end - throw(Error.GitError(err)) - end - return Nullable(GitTreeEntry(te_ptr_ptr[])) -end - -"""Lookup a tree entry by SHA value. - -This returns a `GitTreeEntry` that is owned by the `GitTree`. -You don't have to free it, but you must not use it after the `GitTree` is released. -""" -function GitTreeEntry(tree::GitTree, teoid::Oid) - res = ccall((:git_tree_entry_byid, :libgit2), Ptr{Void}, - (Ref{Void}, Ref{Oid}), - tree.ptr, Ref(teoid)) - res == C_NULL && return Nullable{GitTreeEntry}() - return Nullable(GitTreeEntry(res)) -end - -"""Lookup a tree entry by its file name. - -This returns a `GitTreeEntry` that is owned by the `GitTree`. -You don't have to free it, but you must not use it after the `GitTree` is released. -""" -function lookup(tree::GitTree, fname::AbstractString) - res = ccall((:git_tree_entry_byname, :libgit2), Ptr{Void}, - (Ref{Void}, Cstring), tree.ptr, fname) - res == C_NULL && return Nullable{GitTreeEntry}() - return Nullable(GitTreeEntry(res)) -end diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index dde76d423f515..919c2866bc4f3 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -625,25 +625,24 @@ for (typ, ref, sup, fnc) in ( (:GitTag, :Void, :GitObject, nothing) ) - @eval begin - type $typ <: $sup - ptr::Ptr{$ref} - function $typ(ptr::Ptr{$ref}) - @assert ptr != C_NULL - obj = new(ptr) - return obj - end + @eval type $typ <: $sup + ptr::Ptr{$ref} + function $typ(ptr::Ptr{$ref}) + @assert ptr != C_NULL + obj = new(ptr) + return obj end - if $fnc !== nothing - function Base.finalize(obj::$typ) - if obj.ptr != C_NULL - ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) - obj.ptr = C_NULL - end + end + + if fnc !== nothing + @eval function Base.finalize(obj::$typ) + if obj.ptr != C_NULL + ccall(($fnc, :libgit2), Void, (Ptr{$ref},), obj.ptr) + obj.ptr = C_NULL end - Base.finalize(obj::Nullable{$typ}) = !isnull(obj) && Base.finalize(Base.get(obj)) end end + end # Structure has the same layout as SignatureStruct diff --git a/test/libgit2.jl b/test/libgit2.jl index 9211cfc58b18f..7e62fb61248d4 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -443,59 +443,6 @@ mktempdir() do dir end #end - #@testset "tree" begin - function addfile(root::Cstring, tentry::Ptr{Void}, payload::Ptr{Void}) - strarr = unsafe_pointer_to_objref(payload) - sroot = unsafe_string(root) - te = LibGit2.GitTreeEntry(tentry) - fmode = LibGit2.filemode(te) - fname = LibGit2.filename(te) - if fmode == Cint(LibGit2.Consts.FILEMODE_BLOB) - push!(strarr, fname) - end - return zero(Cint) - end - - repo = LibGit2.GitRepo(test_repo) - head = LibGit2.head(repo) - ht = LibGit2.peel(LibGit2.GitTree, head) - try - @test isnull(LibGit2.lookup(ht, "no such name")) - tfte = get(LibGit2.lookup(ht, test_file)) - - # get tree entry by filename - @test isnull(LibGit2.GitTreeEntry(ht, "no such name")) - @test LibGit2.filename(tfte) == test_file - @test LibGit2.filemode(tfte) == Cint(LibGit2.Consts.FILEMODE_BLOB) - - tfoid = LibGit2.oid(tfte) - - @test isnull(LibGit2.GitTreeEntry(ht, LibGit2.Oid())) - tfte2 = LibGit2.GitTreeEntry(ht, tfoid) - try - @test !isnull(tfte2) - @test LibGit2.filename(get(tfte2)) == test_file - - tfcontent = LibGit2.with(LibGit2.object(repo, get(tfte2))) do obj - LibGit2.with(LibGit2.peel(LibGit2.GitBlob, obj)) do blob - unsafe_string(convert(Cstring, LibGit2.content(blob))) - end - end - @test startswith(tfcontent, commit_msg1) - finally - finalize(tfte2) - end - - entrs = LibGit2.treewalk(addfile, ht, String[])[] - @test length(find(e->e == test_file, entrs)) == 1 - - finally - finalize(ht) - finalize(head) - finalize(repo) - end - #end - #@testset "commits with revwalk" begin repo = LibGit2.GitRepo(test_repo) cache = LibGit2.GitRepo(cache_repo) From a8f9f6f275a610a468757f5b576656389e5f124d Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 13 Jul 2016 13:08:33 -0400 Subject: [PATCH 0431/1117] split JIT out of codegen --- src/Makefile | 5 +- src/ccall.cpp | 2 +- src/cgutils.cpp | 28 +- src/codegen.cpp | 184 ++++------- src/codegen_internal.h | 5 - src/jitlayers.cpp | 717 ++++++++++++++++++++++------------------- src/jitlayers.h | 207 ++++++++++++ 7 files changed, 654 insertions(+), 494 deletions(-) create mode 100644 src/jitlayers.h diff --git a/src/Makefile b/src/Makefile index 47ba500a54814..8e783ef471cc1 100644 --- a/src/Makefile +++ b/src/Makefile @@ -49,7 +49,7 @@ endif LLVMLINK := ifeq ($(JULIACODEGEN),LLVM) -SRCS += codegen disasm debuginfo llvm-simdloop llvm-gcroot cgmemmgr +SRCS += codegen jitlayers disasm debuginfo llvm-simdloop llvm-gcroot cgmemmgr FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir) LLVM_LIBS := all ifeq ($(USE_POLLY),1) @@ -139,10 +139,11 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm \ # additional dependency links $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h -$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp jitlayers.cpp intrinsics.h codegen_internal.h cgutils.cpp ccall.cpp abi_*.cpp) +$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp jitlayers.h intrinsics.h codegen_internal.h cgutils.cpp ccall.cpp abi_*.cpp) $(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h $(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(SRCDIR)/codegen_internal.h $(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/codegen_internal.h +$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c $(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc.h $(BUILDDIR)/gc-debug.o $(BUILDDIR)/gc-debug.dbg.obj: $(SRCDIR)/gc.h diff --git a/src/ccall.cpp b/src/ccall.cpp index 76a9757050984..c6eb11fe434c5 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -271,7 +271,7 @@ static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, } builder.SetInsertPoint(old); builder.SetCurrentDebugLocation(olddl); - jl_finalize_module(std::unique_ptr<Module>(M), true); + jl_finalize_module(M, true); auto shadowgot = cast<GlobalVariable>(shadow_output->getNamedValue(gname)); auto shadowplt = cast<Function>(shadow_output->getNamedValue(fname)); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 26ec40d9be9e1..e5ab5f75eaed5 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1,7 +1,5 @@ // This file is a part of Julia. License is MIT: http://julialang.org/license -#include <iostream> - // utility procedures used in code generation static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store) @@ -10,17 +8,6 @@ static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store) return load_or_store; } -static GlobalVariable *prepare_global(GlobalVariable *G, Module *M) -{ - if (G->getParent() == M) - return G; - GlobalValue *local = M->getNamedValue(G->getName()); - if (!local) { - local = global_proto(G, M); - } - return cast<GlobalVariable>(local); -} - static llvm::Value *prepare_call(llvm::Value *Callee) { if (Function *F = dyn_cast<Function>(Callee)) { @@ -189,20 +176,7 @@ static Value *literal_static_pointer_val(const void *p, Type *t) static Value *julia_gv(const char *cname, void *addr) { // emit a GlobalVariable for a jl_value_t named "cname" - std::map<void*, jl_value_llvm>::iterator it; - // first see if there already is a GlobalVariable for this address - it = jl_value_to_llvm.find(addr); - if (it != jl_value_to_llvm.end()) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global((llvm::GlobalVariable*)it->second.gv))); - - std::stringstream gvname; - gvname << cname << globalUnique++; - // no existing GlobalVariable, create one and store it - GlobalVariable *gv = new GlobalVariable(*jl_builderModule, T_pjlvalue, - false, GlobalVariable::ExternalLinkage, - NULL, gvname.str()); - addComdat(gv); - *(void**)jl_emit_and_add_to_shadow(gv, addr) = addr; + GlobalVariable *gv = jl_get_global_for(cname, addr, jl_builderModule); return tbaa_decorate(tbaa_const, builder.CreateLoad(gv)); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 98bb450a556e1..af6c6ea34a3a4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -22,89 +22,69 @@ #define __STDC_CONSTANT_MACROS #endif -#include <llvm/ExecutionEngine/ExecutionEngine.h> -#include <llvm/ExecutionEngine/JITEventListener.h> -#include <llvm/IR/IntrinsicInst.h> -#ifdef LLVM38 -#include <llvm/Analysis/BasicAliasAnalysis.h> -#include <llvm/Analysis/TypeBasedAliasAnalysis.h> -#endif -#ifdef LLVM37 -#include "llvm/IR/LegacyPassManager.h" -#else -#include <llvm/PassManager.h> -#endif +#include <setjmp.h> +#include <string> +#include <sstream> +#include <fstream> +#include <map> +#include <vector> +#include <set> +#include <cstdio> +#include <cassert> +#include <iostream> + +// target machine computation #include <llvm/Target/TargetSubtargetInfo.h> #include <llvm/Support/TargetRegistry.h> -#include <llvm/Analysis/Passes.h> -#include <llvm/Bitcode/ReaderWriter.h> -#ifdef LLVM35 -#include <llvm/Bitcode/BitcodeWriterPass.h> +#ifndef LLVM37 +#include <llvm/Target/TargetLibraryInfo.h> #endif +#include <llvm/Target/TargetOptions.h> +#include <llvm/Support/Host.h> +#include <llvm/Support/TargetSelect.h> #ifdef LLVM37 -#include <llvm/Analysis/TargetTransformInfo.h> #include <llvm/Analysis/TargetLibraryInfo.h> +#endif + +#ifdef LLVM37 #include <llvm/Object/SymbolSize.h> -#else -#include <llvm/Target/TargetLibraryInfo.h> #endif + +// IR building +#include <llvm/IR/IntrinsicInst.h> #ifdef LLVM35 -#include <llvm/IR/Verifier.h> #include <llvm/Object/ObjectFile.h> #include <llvm/IR/DIBuilder.h> -#include <llvm/Target/TargetMachine.h> #include <llvm/AsmParser/Parser.h> #else #include <llvm/Assembly/Parser.h> -#include <llvm/Analysis/Verifier.h> #endif #include <llvm/DebugInfo/DIContext.h> #include <llvm/IR/DerivedTypes.h> -#include <llvm/IR/LLVMContext.h> -#include <llvm/IR/Module.h> #include <llvm/IR/Intrinsics.h> #include <llvm/IR/Attributes.h> #include <llvm/IR/IRBuilder.h> #include <llvm/IR/MDBuilder.h> -#include <llvm/IR/Value.h> #ifndef LLVM35 #include <llvm/DebugInfo.h> #include <llvm/DIBuilder.h> #endif -#include <llvm/Target/TargetOptions.h> -#include <llvm/Transforms/Scalar.h> -#include <llvm/Transforms/Utils/BasicBlockUtils.h> -#include <llvm/Transforms/Instrumentation.h> -#include <llvm/Transforms/Vectorize.h> -#ifdef LLVM39 -#include <llvm/Transforms/Scalar/GVN.h> -#endif -#include <llvm/Support/Host.h> -#include <llvm/Support/TargetSelect.h> + +// support #include <llvm/Support/raw_ostream.h> #include <llvm/Support/FormattedStream.h> -#include <llvm/Support/DynamicLibrary.h> -#include <llvm/Support/PrettyStackTrace.h> -#include <llvm/Support/SourceMgr.h> -#include <llvm/Support/CommandLine.h> -#include <llvm/Transforms/Utils/Cloning.h> - -#if defined(USE_ORCJIT) -#include "llvm/ExecutionEngine/Orc/CompileUtils.h" -#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" -#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" -#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" -#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" -#include "llvm/ExecutionEngine/ObjectMemoryBuffer.h" -#elif defined(USE_MCJIT) -#include <llvm/ExecutionEngine/MCJIT.h> -#include <llvm/ADT/DenseMapInfo.h> -#include <llvm/Object/ObjectFile.h> +#include <llvm/Support/SourceMgr.h> // for llvmcall +#include <llvm/Transforms/Utils/Cloning.h> // for llvmcall inlining +#ifdef LLVM35 +#include <llvm/IR/Verifier.h> // for llvmcall validation #else -#include <llvm/ExecutionEngine/JIT.h> -#include <llvm/ExecutionEngine/JITMemoryManager.h> -#include <llvm/ExecutionEngine/Interpreter.h> +#include <llvm/Analysis/Verifier.h> #endif + +// for configuration options +#include <llvm/Support/PrettyStackTrace.h> +#include <llvm/Support/CommandLine.h> + #if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) || \ (defined(LLVM37) && defined(JULIA_ENABLE_THREADING)) # include <llvm/IR/InlineAsm.h> @@ -115,6 +95,9 @@ #endif using namespace llvm; +namespace llvm { + extern bool annotateSimdLoop(BasicBlock *latch); +} #if defined(_OS_WINDOWS_) && !defined(NOMINMAX) #define NOMINMAX @@ -122,25 +105,16 @@ using namespace llvm; #include "julia.h" #include "julia_internal.h" +#include "jitlayers.h" #include "codegen_internal.h" -#include <setjmp.h> - -#include <string> -#include <sstream> -#include <fstream> -#include <map> -#include <vector> -#include <set> -#include <cstdio> -#include <cassert> - // LLVM version compatibility macros #ifdef LLVM37 -using namespace llvm::legacy; +legacy::PassManager *jl_globalPM; #define LLVM37_param(x) (x), #else #define LLVM37_param(x) +PassManager *jl_globalPM; #endif #ifndef LLVM35 @@ -204,27 +178,17 @@ JL_DLLEXPORT LLVMContext &jl_LLVMContext = getGlobalContext(); #endif static IRBuilder<> builder(jl_LLVMContext); static bool nested_compile = false; -static TargetMachine *jl_TargetMachine; +TargetMachine *jl_TargetMachine; extern JITEventListener *CreateJuliaJITEventListener(); -namespace llvm { - extern Pass *createLowerSimdLoopPass(); - extern bool annotateSimdLoop(BasicBlock *latch); -} - // for image reloading -static bool imaging_mode = false; +bool imaging_mode = false; -static Module *shadow_output; +Module *shadow_output; #define jl_Module ctx->f->getParent() #define jl_builderModule builder.GetInsertBlock()->getParent()->getParent() static MDBuilder *mbuilder; -#ifdef LLVM38 -static legacy::PassManager *PM; -#else -static PassManager *PM; -#endif #ifdef LLVM37 // No DataLayout pass needed anymore. @@ -349,7 +313,7 @@ static GlobalVariable *jltls_states_var; #else // Imaging mode only static GlobalVariable *jltls_states_func_ptr = NULL; -static size_t jltls_states_func_idx = 0; +size_t jltls_states_func_idx = 0; #endif // important functions @@ -416,7 +380,7 @@ static Function *jlgetnthfieldchecked_func; #ifdef _OS_WINDOWS_ static Function *resetstkoflw_func; #if defined(_CPU_X86_64_) -static Function *juliapersonality_func; +Function *juliapersonality_func; #endif #endif static Function *diff_gc_total_bytes_func; @@ -439,8 +403,6 @@ extern "C" { int globalUnique = 0; } -#include "jitlayers.cpp" - // metadata tracking for a llvm Value* during codegen struct jl_cgval_t { Value *V; // may be of type T* or T, or set to NULL if ghost (or if the value has not been initialized yet, for a variable definition) @@ -591,7 +553,6 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol, MDNode *tbaa); static Value *emit_condition(jl_value_t *cond, const std::string &msg, jl_codectx_t *ctx); static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx); -static void jl_finalize_module(std::unique_ptr<Module> m, bool shadow); static GlobalVariable *prepare_global(GlobalVariable *G, Module *M = jl_builderModule); @@ -883,7 +844,7 @@ extern "C" void jl_compile_linfo(jl_lambda_info_t *li) li->fptr = NULL; // success. add the result to the execution engine now - jl_finalize_module(std::move(m), !toplevel); + jl_finalize_module(m.release(), !toplevel); // if not inlineable, code won't be needed again if (JL_DELETE_NON_INLINEABLE && @@ -957,32 +918,6 @@ static void jl_setup_module(Module *m) #endif } -// this takes ownership of a module after code emission is complete -// and will add it to the execution engine when required (by jl_finalize_function) -static void finalize_gc_frame(Module *m); -static void jl_finalize_module(std::unique_ptr<Module> uniquem, bool shadow) -{ - Module *m = uniquem.release(); // unique_ptr won't be able track what we do with this (the invariant is recovered by jl_finalize_function) - finalize_gc_frame(m); -#if !defined(USE_ORCJIT) - PM->run(*m); -#endif -#ifdef USE_MCJIT - // record the function names that are part of this Module - // so it can be added to the JIT when needed - for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { - Function *F = &*I; - if (!F->isDeclaration()) - module_for_fname[F->getName()] = m; - } -#endif -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - // in the newer JITs, the shadow module is separate from the execution module - if (shadow) -#endif - jl_add_to_shadow(m); -} - // this ensures that llvmf has been emitted to the execution engine, // returning the function pointer to it extern void jl_callback_triggered_linfos(void); @@ -1147,7 +1082,7 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) jl_llvm_functions_t declarations; std::unique_ptr<Module> m = emit_function(temp ? temp : linfo, &declarations); finalize_gc_frame(m.get()); - PM->run(*m.get()); + jl_globalPM->run(*m.get()); f = (llvm::Function*)declarations.functionObject; specf = (llvm::Function*)declarations.specFunctionObject; // swap declarations for definitions and destroy declarations @@ -3506,7 +3441,7 @@ static void finalize_gc_frame(Function *F) #endif } -static void finalize_gc_frame(Module *m) +void finalize_gc_frame(Module *m) { for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { Function *F = &*I; @@ -3589,7 +3524,6 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t Function *cw = Function::Create(FunctionType::get(sret ? T_void : prt, fargt_sig, false), GlobalVariable::ExternalLinkage, funcName.str(), M); - addComdat(cw); cw->setAttributes(attrs); #ifdef LLVM37 cw->addFnAttr("no-frame-pointer-elim", "true"); @@ -3839,7 +3773,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t else builder.CreateRet(r); - jl_finalize_module(std::unique_ptr<Module>(M), true); + jl_finalize_module(M, true); return cw_proto; } @@ -3946,7 +3880,6 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, Function *f, bool sre Function *w = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, funcName.str(), M); - addComdat(w); #ifdef LLVM37 w->addFnAttr("no-frame-pointer-elim", "true"); #endif @@ -4181,7 +4114,6 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func f->addAttribute(1, Attribute::StructRet); f->addAttribute(1, Attribute::NoAlias); } - addComdat(f); #ifdef LLVM37 f->addFnAttr("no-frame-pointer-elim", "true"); #endif @@ -4193,7 +4125,6 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func f = Function::Create(needsparams ? jl_func_sig_sparams : jl_func_sig, GlobalVariable::ExternalLinkage, funcName.str(), M); - addComdat(f); #ifdef LLVM37 f->addFnAttr("no-frame-pointer-elim", "true"); #endif @@ -5135,6 +5066,7 @@ static void init_julia_llvm_env(Module *m) four_pvalue_llvmt.push_back(T_pjlvalue); four_pvalue_llvmt.push_back(T_pjlvalue); V_null = Constant::getNullValue(T_pjlvalue); + jl_init_jit(T_pjlvalue); std::vector<Type*> ftargs(0); ftargs.push_back(T_pjlvalue); // linfo->sparam_vals @@ -5674,20 +5606,20 @@ static void init_julia_llvm_env(Module *m) jl_data_layout = new DataLayout(*jl_ExecutionEngine->getDataLayout()); #endif -#ifdef LLVM38 - PM = new legacy::PassManager(); +#ifdef LLVM37 + jl_globalPM = new legacy::PassManager(); #else - PM = new PassManager(); + jl_globalPM = new PassManager(); #endif #ifndef LLVM37 - PM->add(new TargetLibraryInfo(Triple(jl_TargetMachine->getTargetTriple()))); + jl_globalPM->add(new TargetLibraryInfo(Triple(jl_TargetMachine->getTargetTriple()))); #else - PM->add(new TargetLibraryInfoWrapperPass(Triple(jl_TargetMachine->getTargetTriple()))); + jl_globalPM->add(new TargetLibraryInfoWrapperPass(Triple(jl_TargetMachine->getTargetTriple()))); #endif #ifndef LLVM37 - PM->add(jl_data_layout); + jl_globalPM->add(jl_data_layout); #endif - addOptimizationPasses(PM); + addOptimizationPasses(jl_globalPM); } // Helper to figure out what features to set for the LLVM target diff --git a/src/codegen_internal.h b/src/codegen_internal.h index af172d8cc363a..12ab6fb2bf645 100644 --- a/src/codegen_internal.h +++ b/src/codegen_internal.h @@ -35,11 +35,6 @@ extern bool jl_dylib_DI_for_fptr(size_t pointer, const object::ObjectFile **obje bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename); #ifdef USE_ORCJIT -JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, - const object::ObjectFile &obj, - const object::ObjectFile &debugObj, - const RuntimeDyld::LoadedObjectInfo &L, - RTDyldMemoryManager *memmgr); #ifdef _OS_WINDOWS_ void *lookupWriteAddressFor(RTDyldMemoryManager *memmgr, void *rt_addr); #endif diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index eeb9ab93697e6..bcab549ac9073 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1,13 +1,101 @@ // This file is a part of Julia. License is MIT: http://julialang.org/license +#include "llvm-version.h" +#include "platform.h" +#include "options.h" #include <iostream> -#include <llvm/ExecutionEngine/SectionMemoryManager.h> +#include <sstream> + +// analysis passes +#include <llvm/Analysis/Passes.h> +#ifdef LLVM38 +#include <llvm/Analysis/BasicAliasAnalysis.h> +#include <llvm/Analysis/TypeBasedAliasAnalysis.h> +#endif +#ifdef LLVM37 +#include <llvm/Analysis/TargetTransformInfo.h> +#include <llvm/Analysis/TargetLibraryInfo.h> +#endif +#ifdef LLVM35 +#include <llvm/IR/Verifier.h> +#else +#include <llvm/Analysis/Verifier.h> +#endif +#if defined(USE_POLLY) +#include <polly/RegisterPasses.h> +#endif + +#include <llvm/Transforms/Scalar.h> +#include <llvm/Transforms/Utils/BasicBlockUtils.h> +#include <llvm/Transforms/Instrumentation.h> +#include <llvm/Transforms/Vectorize.h> +#ifdef LLVM39 +#include <llvm/Transforms/Scalar/GVN.h> +#endif + +namespace llvm { + extern Pass *createLowerSimdLoopPass(); +} + +#include <llvm/Bitcode/ReaderWriter.h> +#ifdef LLVM35 +#include <llvm/Bitcode/BitcodeWriterPass.h> +#endif + +#include <llvm/Transforms/Utils/Cloning.h> +#include <llvm/ExecutionEngine/JITEventListener.h> + +// target support +#include <llvm/ADT/Triple.h> +#include <llvm/Support/TargetRegistry.h> +#ifndef LLVM37 +#include <llvm/Target/TargetLibraryInfo.h> +#endif +#include <llvm/IR/DataLayout.h> +#include <llvm/Support/DynamicLibrary.h> + + +#include <llvm/Support/raw_ostream.h> +#include <llvm/Support/FormattedStream.h> + +using namespace llvm; + +#include "julia.h" +#include "julia_internal.h" +#include "jitlayers.h" +#ifdef USE_MCJIT +RTDyldMemoryManager* createRTDyldMemoryManager(void); +#endif + +static Type *T_void; +static IntegerType *T_uint32; +static IntegerType *T_uint64; +static IntegerType *T_size; +static Type *T_psize; +static Type *T_pvoidfunc; +static Type *T_pjlvalue; +void jl_init_jit(Type *T_pjlvalue_) +{ + T_void = Type::getVoidTy(jl_LLVMContext); + T_uint32 = Type::getInt32Ty(jl_LLVMContext); + T_uint64 = Type::getInt64Ty(jl_LLVMContext); + if (sizeof(size_t) == 8) + T_size = T_uint64; + else + T_size = T_uint32; + T_psize = PointerType::get(T_size, 0); + T_pvoidfunc = FunctionType::get(T_void, /*isVarArg*/false)->getPointerTo(); + T_pjlvalue = T_pjlvalue_; +} // Except for parts of this file which were copied from LLVM, under the UIUC license (marked below). // this defines the set of optimization passes defined for Julia at various optimization levels -template <class T> -static void addOptimizationPasses(T *PM) +#ifdef LLVM37 +void addOptimizationPasses(legacy::PassManager *PM) +#else +void addOptimizationPasses(PassManager *PM) +#endif { #ifdef JL_DEBUG_BUILD PM->add(createVerifierPass()); @@ -214,320 +302,290 @@ static uint64_t resolve_atomic(const char *name) } #endif -// A simplified model of the LLVM ExecutionEngine that implements only the methods that Julia needs -// but tries to roughly match the API anyways so that compatibility is easier -class JuliaOJIT { - // Custom object emission notification handler for the JuliaOJIT - // TODO: hook up RegisterJITEventListener, instead of hard-coding the GDB and JuliaListener targets - class DebugObjectRegistrar { - public: - DebugObjectRegistrar(JuliaOJIT &JIT) - : JuliaListener(CreateJuliaJITEventListener()), - JIT(JIT) {} - - template <typename ObjSetT, typename LoadResult> - void operator()(ObjectLinkingLayerBase::ObjSetHandleT H, const ObjSetT &Objects, - const LoadResult &LOS) - { +// Custom object emission notification handler for the JuliaOJIT +extern JITEventListener *CreateJuliaJITEventListener(); +JuliaOJIT::DebugObjectRegistrar::DebugObjectRegistrar(JuliaOJIT &JIT) + : JuliaListener(CreateJuliaJITEventListener()), + JIT(JIT) {} + +JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, + const object::ObjectFile &obj, + const object::ObjectFile &debugObj, + const RuntimeDyld::LoadedObjectInfo &L, + RTDyldMemoryManager *memmgr); + +// TODO: hook up RegisterJITEventListener, instead of hard-coding the GDB and JuliaListener targets +template <typename ObjSetT, typename LoadResult> +void JuliaOJIT::DebugObjectRegistrar::operator()(ObjectLinkingLayerBase::ObjSetHandleT H, const ObjSetT &Objects, + const LoadResult &LOS) +{ #ifndef LLVM38 - notifyObjectLoaded(JIT.MemMgr, H); + notifyObjectLoaded(JIT.MemMgr, H); #endif - auto oit = Objects.begin(); - auto lit = LOS.begin(); - for (; oit != Objects.end(); ++oit, ++lit) { + auto oit = Objects.begin(); + auto lit = LOS.begin(); + for (; oit != Objects.end(); ++oit, ++lit) { #ifdef LLVM39 - const auto &Object = (*oit)->getBinary(); + const auto &Object = (*oit)->getBinary(); #else - auto &Object = *oit; + auto &Object = *oit; #endif - auto &LO = *lit; + auto &LO = *lit; - OwningBinary<object::ObjectFile> SavedObject = LO->getObjectForDebug(*Object); + OwningBinary<object::ObjectFile> SavedObject = LO->getObjectForDebug(*Object); - // If the debug object is unavailable, save (a copy of) the original object - // for our backtraces - if (!SavedObject.getBinary()) { - // This is unfortunate, but there doesn't seem to be a way to take - // ownership of the original buffer - auto NewBuffer = MemoryBuffer::getMemBufferCopy(Object->getData(), Object->getFileName()); - auto NewObj = ObjectFile::createObjectFile(NewBuffer->getMemBufferRef()); - assert(NewObj); - SavedObject = OwningBinary<object::ObjectFile>(std::move(*NewObj),std::move(NewBuffer)); - } - else { - NotifyGDB(SavedObject); - } + // If the debug object is unavailable, save (a copy of) the original object + // for our backtraces + if (!SavedObject.getBinary()) { + // This is unfortunate, but there doesn't seem to be a way to take + // ownership of the original buffer + auto NewBuffer = MemoryBuffer::getMemBufferCopy(Object->getData(), Object->getFileName()); + auto NewObj = ObjectFile::createObjectFile(NewBuffer->getMemBufferRef()); + assert(NewObj); + SavedObject = OwningBinary<object::ObjectFile>(std::move(*NewObj),std::move(NewBuffer)); + } + else { + NotifyGDB(SavedObject); + } - SavedObjects.push_back(std::move(SavedObject)); - - ORCNotifyObjectEmitted(JuliaListener.get(), - *Object, - *SavedObjects.back().getBinary(), - *LO, JIT.MemMgr); - - // record all of the exported symbols defined in this object - // in the primary hash table for the enclosing JIT - for (auto &Symbol : Object->symbols()) { - auto Flags = Symbol.getFlags(); - if (Flags & object::BasicSymbolRef::SF_Undefined) - continue; - if (!(Flags & object::BasicSymbolRef::SF_Exported)) - continue; - auto NameOrError = Symbol.getName(); - assert(NameOrError); - auto Name = NameOrError.get(); - orc::JITSymbol Sym = JIT.CompileLayer.findSymbolIn(H, Name, true); - assert(Sym); - // note: calling getAddress here eagerly finalizes H - // as an alternative, we could store the JITSymbol instead - // (which would present a lazy-initializer functor interface instead) - JIT.LocalSymbolTable[Name] = (void*)(uintptr_t)Sym.getAddress(); - } - } + SavedObjects.push_back(std::move(SavedObject)); + + ORCNotifyObjectEmitted(JuliaListener.get(), + *Object, + *SavedObjects.back().getBinary(), + *LO, JIT.MemMgr); + + // record all of the exported symbols defined in this object + // in the primary hash table for the enclosing JIT + for (auto &Symbol : Object->symbols()) { + auto Flags = Symbol.getFlags(); + if (Flags & object::BasicSymbolRef::SF_Undefined) + continue; + if (!(Flags & object::BasicSymbolRef::SF_Exported)) + continue; + auto NameOrError = Symbol.getName(); + assert(NameOrError); + auto Name = NameOrError.get(); + orc::JITSymbol Sym = JIT.CompileLayer.findSymbolIn(H, Name, true); + assert(Sym); + // note: calling getAddress here eagerly finalizes H + // as an alternative, we could store the JITSymbol instead + // (which would present a lazy-initializer functor interface instead) + JIT.LocalSymbolTable[Name] = (void*)(uintptr_t)Sym.getAddress(); } + } +} - private: - void NotifyGDB(OwningBinary<object::ObjectFile> &DebugObj) - { - const char *Buffer = DebugObj.getBinary()->getMemoryBufferRef().getBufferStart(); - size_t Size = DebugObj.getBinary()->getMemoryBufferRef().getBufferSize(); +void JuliaOJIT::DebugObjectRegistrar::NotifyGDB(OwningBinary<object::ObjectFile> &DebugObj) +{ + const char *Buffer = DebugObj.getBinary()->getMemoryBufferRef().getBufferStart(); + size_t Size = DebugObj.getBinary()->getMemoryBufferRef().getBufferSize(); - assert(Buffer && "Attempt to register a null object with a debugger."); - jit_code_entry *JITCodeEntry = new jit_code_entry(); + assert(Buffer && "Attempt to register a null object with a debugger."); + jit_code_entry *JITCodeEntry = new jit_code_entry(); - if (!JITCodeEntry) { - jl_printf(JL_STDERR, "WARNING: Allocation failed when registering a JIT entry!\n"); - } - else { - JITCodeEntry->symfile_addr = Buffer; - JITCodeEntry->symfile_size = Size; + if (!JITCodeEntry) { + jl_printf(JL_STDERR, "WARNING: Allocation failed when registering a JIT entry!\n"); + } + else { + JITCodeEntry->symfile_addr = Buffer; + JITCodeEntry->symfile_size = Size; - NotifyDebugger(JITCodeEntry); - } - } + NotifyDebugger(JITCodeEntry); + } +} - std::vector<OwningBinary<object::ObjectFile>> SavedObjects; - std::unique_ptr<JITEventListener> JuliaListener; - JuliaOJIT &JIT; - }; - -public: - typedef orc::ObjectLinkingLayer<DebugObjectRegistrar> ObjLayerT; - typedef orc::IRCompileLayer<ObjLayerT> CompileLayerT; - typedef CompileLayerT::ModuleSetHandleT ModuleHandleT; - typedef StringMap<void*> SymbolTableT; - typedef object::OwningBinary<object::ObjectFile> OwningObj; - - JuliaOJIT(TargetMachine &TM) - : TM(TM), - DL(TM.createDataLayout()), - ObjStream(ObjBufferSV), - MemMgr(createRTDyldMemoryManager()), - ObjectLayer(DebugObjectRegistrar(*this)), - CompileLayer( - ObjectLayer, - [this](Module &M) { - JL_TIMING(LLVM_OPT); - PM.run(M); - std::unique_ptr<MemoryBuffer> ObjBuffer( - new ObjectMemoryBuffer(std::move(ObjBufferSV))); - auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); - - if (!Obj) { - M.dump(); +JuliaOJIT::JuliaOJIT(TargetMachine &TM) + : TM(TM), + DL(TM.createDataLayout()), + ObjStream(ObjBufferSV), + MemMgr(createRTDyldMemoryManager()), + ObjectLayer(DebugObjectRegistrar(*this)), + CompileLayer( + ObjectLayer, + [this](Module &M) { + JL_TIMING(LLVM_OPT); + PM.run(M); + std::unique_ptr<MemoryBuffer> ObjBuffer( + new ObjectMemoryBuffer(std::move(ObjBufferSV))); + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); + + if (!Obj) { + M.dump(); #ifdef LLVM39 - std::string Buf; - raw_string_ostream OS(Buf); - logAllUnhandledErrors(Obj.takeError(), OS, ""); - OS.flush(); - llvm::report_fatal_error("FATAL: Unable to compile LLVM Module: '" + Buf + "'\n" - "The module's content was printed above. Please file a bug report"); + std::string Buf; + raw_string_ostream OS(Buf); + logAllUnhandledErrors(Obj.takeError(), OS, ""); + OS.flush(); + llvm::report_fatal_error("FATAL: Unable to compile LLVM Module: '" + Buf + "'\n" + "The module's content was printed above. Please file a bug report"); #else - llvm::report_fatal_error("FATAL: Unable to compile LLVM Module.\n" - "The module's content was printed above. Please file a bug report"); + llvm::report_fatal_error("FATAL: Unable to compile LLVM Module.\n" + "The module's content was printed above. Please file a bug report"); #endif - } - - return OwningObj(std::move(*Obj), std::move(ObjBuffer)); } - ) - { - addOptimizationPasses(&PM); - if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) - llvm_unreachable("Target does not support MC emission."); - - // Make sure SectionMemoryManager::getSymbolAddressInProcess can resolve - // symbols in the program as well. The nullptr argument to the function - // tells DynamicLibrary to load the program, not a library. - std::string *ErrorStr = nullptr; - if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, ErrorStr)) - report_fatal_error("FATAL: unable to dlopen self\n" + *ErrorStr); - } - void addGlobalMapping(StringRef Name, uint64_t Addr) - { - bool successful = GlobalSymbolTable.insert(std::make_pair(Name, (void*)Addr)).second; - (void)successful; - assert(successful); - } + return OwningObj(std::move(*Obj), std::move(ObjBuffer)); + } + ) +{ + addOptimizationPasses(&PM); + if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) + llvm_unreachable("Target does not support MC emission."); + + // Make sure SectionMemoryManager::getSymbolAddressInProcess can resolve + // symbols in the program as well. The nullptr argument to the function + // tells DynamicLibrary to load the program, not a library. + std::string *ErrorStr = nullptr; + if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, ErrorStr)) + report_fatal_error("FATAL: unable to dlopen self\n" + *ErrorStr); +} - void addGlobalMapping(const GlobalValue *GV, void *Addr) - { - addGlobalMapping(getMangledName(GV), (uintptr_t)Addr); - } +void JuliaOJIT::addGlobalMapping(StringRef Name, uint64_t Addr) +{ + bool successful = GlobalSymbolTable.insert(std::make_pair(Name, (void*)Addr)).second; + (void)successful; + assert(successful); +} - void *getPointerToGlobalIfAvailable(StringRef S) - { - SymbolTableT::const_iterator pos = GlobalSymbolTable.find(S); - if (pos != GlobalSymbolTable.end()) - return pos->second; - return nullptr; - } +void JuliaOJIT::addGlobalMapping(const GlobalValue *GV, void *Addr) +{ + addGlobalMapping(getMangledName(GV), (uintptr_t)Addr); +} - void *getPointerToGlobalIfAvailable(const GlobalValue *GV) - { - return getPointerToGlobalIfAvailable(getMangledName(GV)); - } +void *JuliaOJIT::getPointerToGlobalIfAvailable(StringRef S) +{ + SymbolTableT::const_iterator pos = GlobalSymbolTable.find(S); + if (pos != GlobalSymbolTable.end()) + return pos->second; + return nullptr; +} +void *JuliaOJIT::getPointerToGlobalIfAvailable(const GlobalValue *GV) +{ + return getPointerToGlobalIfAvailable(getMangledName(GV)); +} - void addModule(std::unique_ptr<Module> M) - { + +void JuliaOJIT::addModule(std::unique_ptr<Module> M) +{ #ifndef NDEBUG - // validate the relocations for M - for (Module::iterator I = M->begin(), E = M->end(); I != E; ) { - Function *F = &*I; - ++I; - if (F->isDeclaration()) { - if (F->use_empty()) - F->eraseFromParent(); - else if (!(F->isIntrinsic() || - findUnmangledSymbol(F->getName()) || - SectionMemoryManager::getSymbolAddressInProcess( - F->getName()))) { - std::cerr << "FATAL ERROR: " - << "Symbol \"" << F->getName().str() << "\"" - << "not found"; - abort(); - } + // validate the relocations for M + for (Module::iterator I = M->begin(), E = M->end(); I != E; ) { + Function *F = &*I; + ++I; + if (F->isDeclaration()) { + if (F->use_empty()) + F->eraseFromParent(); + else if (!(F->isIntrinsic() || + findUnmangledSymbol(F->getName()) || + SectionMemoryManager::getSymbolAddressInProcess( + F->getName()))) { + std::cerr << "FATAL ERROR: " + << "Symbol \"" << F->getName().str() << "\"" + << "not found"; + abort(); } } + } #endif - JL_TIMING(LLVM_MODULE_FINISH); - // We need a memory manager to allocate memory and resolve symbols for this - // new module. Create one that resolves symbols by looking back into the JIT. - auto Resolver = orc::createLambdaResolver( - [&](const std::string &Name) { - // TODO: consider moving the FunctionMover resolver here - // Step 0: ObjectLinkingLayer has checked whether it is in the current module - // Step 1: See if it's something known to the ExecutionEngine - if (auto Sym = findSymbol(Name, true)) - return RuntimeDyld::SymbolInfo(Sym.getAddress(), - Sym.getFlags()); - // Step 2: Search the program symbols - if (uint64_t addr = SectionMemoryManager::getSymbolAddressInProcess(Name)) - return RuntimeDyld::SymbolInfo(addr, JITSymbolFlags::Exported); + JL_TIMING(LLVM_MODULE_FINISH); + // We need a memory manager to allocate memory and resolve symbols for this + // new module. Create one that resolves symbols by looking back into the JIT. + auto Resolver = orc::createLambdaResolver( + [&](const std::string &Name) { + // TODO: consider moving the FunctionMover resolver here + // Step 0: ObjectLinkingLayer has checked whether it is in the current module + // Step 1: See if it's something known to the ExecutionEngine + if (auto Sym = findSymbol(Name, true)) + return RuntimeDyld::SymbolInfo(Sym.getAddress(), + Sym.getFlags()); + // Step 2: Search the program symbols + if (uint64_t addr = SectionMemoryManager::getSymbolAddressInProcess(Name)) + return RuntimeDyld::SymbolInfo(addr, JITSymbolFlags::Exported); #ifdef _OS_LINUX_ - if (uint64_t addr = resolve_atomic(Name.c_str())) - return RuntimeDyld::SymbolInfo(addr, JITSymbolFlags::Exported); -#endif - // Return failure code - return RuntimeDyld::SymbolInfo(nullptr); - }, - [](const std::string &S) { return nullptr; } - ); - SmallVector<std::unique_ptr<Module>,1> Ms; - Ms.push_back(std::move(M)); - auto modset = CompileLayer.addModuleSet(std::move(Ms), MemMgr, - std::move(Resolver)); - // Force LLVM to emit the module so that we can register the symbols - // in our lookup table. - CompileLayer.emitAndFinalize(modset); - } - - void removeModule(ModuleHandleT H) - { - CompileLayer.removeModuleSet(H); - } - - orc::JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) - { - void *Addr = nullptr; - if (ExportedSymbolsOnly) { - // Step 1: Check against list of known external globals - Addr = getPointerToGlobalIfAvailable(Name); - } - // Step 2: Search all previously emitted symbols - if (Addr == nullptr) - Addr = LocalSymbolTable[Name]; - return orc::JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); - } + if (uint64_t addr = resolve_atomic(Name.c_str())) + return RuntimeDyld::SymbolInfo(addr, JITSymbolFlags::Exported); +#endif + // Return failure code + return RuntimeDyld::SymbolInfo(nullptr); + }, + [](const std::string &S) { return nullptr; } + ); + SmallVector<std::unique_ptr<Module>,1> Ms; + Ms.push_back(std::move(M)); + auto modset = CompileLayer.addModuleSet(std::move(Ms), MemMgr, + std::move(Resolver)); + // Force LLVM to emit the module so that we can register the symbols + // in our lookup table. + CompileLayer.emitAndFinalize(modset); +} - orc::JITSymbol findUnmangledSymbol(const std::string Name) - { - return findSymbol(getMangledName(Name), true); - } +void JuliaOJIT::removeModule(ModuleHandleT H) +{ + CompileLayer.removeModuleSet(H); +} - uint64_t getGlobalValueAddress(const std::string &Name) - { - return findSymbol(getMangledName(Name), false).getAddress(); +orc::JITSymbol JuliaOJIT::findSymbol(const std::string &Name, bool ExportedSymbolsOnly) +{ + void *Addr = nullptr; + if (ExportedSymbolsOnly) { + // Step 1: Check against list of known external globals + Addr = getPointerToGlobalIfAvailable(Name); } + // Step 2: Search all previously emitted symbols + if (Addr == nullptr) + Addr = LocalSymbolTable[Name]; + return orc::JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); +} - uint64_t getFunctionAddress(const std::string &Name) - { - return findSymbol(getMangledName(Name), false).getAddress(); - } +orc::JITSymbol JuliaOJIT::findUnmangledSymbol(const std::string Name) +{ + return findSymbol(getMangledName(Name), true); +} - Function *FindFunctionNamed(const std::string &Name) - { - return shadow_output->getFunction(Name); - } +uint64_t JuliaOJIT::getGlobalValueAddress(const std::string &Name) +{ + return findSymbol(getMangledName(Name), false).getAddress(); +} - void RegisterJITEventListener(JITEventListener *L) - { - // TODO - } +uint64_t JuliaOJIT::getFunctionAddress(const std::string &Name) +{ + return findSymbol(getMangledName(Name), false).getAddress(); +} - const DataLayout& getDataLayout() const - { - return DL; - } +Function *JuliaOJIT::FindFunctionNamed(const std::string &Name) +{ + return shadow_output->getFunction(Name); +} - const Triple& getTargetTriple() const - { - return TM.getTargetTriple(); - } +void JuliaOJIT::RegisterJITEventListener(JITEventListener *L) +{ + // TODO +} -private: - std::string getMangledName(const std::string &Name) - { - SmallString<128> FullName; - Mangler::getNameWithPrefix(FullName, Name, DL); - return FullName.str(); - } +const DataLayout& JuliaOJIT::getDataLayout() const +{ + return DL; +} - std::string getMangledName(const GlobalValue *GV) - { - return getMangledName(GV->getName()); - } +const Triple& JuliaOJIT::getTargetTriple() const +{ + return TM.getTargetTriple(); +} - TargetMachine &TM; - const DataLayout DL; - // Should be big enough that in the common case, The - // object fits in its entirety - SmallVector<char, 4096> ObjBufferSV; - raw_svector_ostream ObjStream; - legacy::PassManager PM; - MCContext *Ctx; - RTDyldMemoryManager *MemMgr; - ObjLayerT ObjectLayer; - CompileLayerT CompileLayer; - SymbolTableT GlobalSymbolTable; - SymbolTableT LocalSymbolTable; -}; +std::string JuliaOJIT::getMangledName(const std::string &Name) +{ + SmallString<128> FullName; + Mangler::getNameWithPrefix(FullName, Name, DL); + return FullName.str(); +} -#endif +std::string JuliaOJIT::getMangledName(const GlobalValue *GV) +{ + return getMangledName(GV->getName()); +} -#ifdef USE_ORCJIT JuliaOJIT *jl_ExecutionEngine; #else ExecutionEngine *jl_ExecutionEngine; @@ -667,7 +725,7 @@ static void jl_merge_module(Module *dest, std::unique_ptr<Module> src) // (which aliases the engine module), so this is unneeded #ifdef USE_MCJIT static StringMap<Module*> module_for_fname; -static void jl_finalize_function(const std::string &F, Module *collector = NULL) +static void jl_finalize_function(const std::string &F, Module *collector) { std::unique_ptr<Module> m(module_for_fname.lookup(F)); if (m) { @@ -712,68 +770,44 @@ static void jl_finalize_function(const std::string &F, Module *collector = NULL) } } } -static void jl_finalize_function(Function *F, Module *collector = NULL) +void jl_finalize_function(Function *F, Module *collector) { jl_finalize_function(F->getName().str(), collector); } #endif -// Connect Modules via prototypes, each owned by module `M` -static GlobalVariable *global_proto(GlobalVariable *G, Module *M = NULL) +// this takes ownership of a module after code emission is complete +// and will add it to the execution engine when required (by jl_finalize_function) +void jl_finalize_module(Module *m, bool shadow) { - // Copy the GlobalVariable, but without the initializer, so it becomes a declaration - GlobalVariable *proto = new GlobalVariable(G->getType()->getElementType(), - G->isConstant(), GlobalVariable::ExternalLinkage, - NULL, G->getName(), G->getThreadLocalMode()); - proto->copyAttributesFrom(G); -#ifdef LLVM35 - // DLLImport only needs to be set for the shadow module - // it just gets annoying in the JIT - proto->setDLLStorageClass(GlobalValue::DefaultStorageClass); + finalize_gc_frame(m); +#if !defined(USE_ORCJIT) + jl_globalPM->run(*m); #endif - if (M) - M->getGlobalList().push_back(proto); - return proto; -} - -static Function *function_proto(Function *F, Module *M = NULL) -{ - // Copy the declaration characteristics of the Function (not the body) - Function *NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), M); - // FunctionType does not include any attributes. Copy them over manually - // as codegen may make decisions based on the presence of certain attributes - NewF->copyAttributesFrom(F); - -#ifdef LLVM37 - // Declarations are not allowed to have personality routines, but - // copyAttributesFrom sets them anyway, so clear them again manually - NewF->setPersonalityFn(nullptr); +#ifdef USE_MCJIT + // record the function names that are part of this Module + // so it can be added to the JIT when needed + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) + module_for_fname[F->getName()] = m; + } #endif - -#ifdef LLVM35 - // DLLImport only needs to be set for the shadow module - // it just gets annoying in the JIT - NewF->setDLLStorageClass(GlobalValue::DefaultStorageClass); +#if defined(USE_ORCJIT) || defined(USE_MCJIT) + // in the newer JITs, the shadow module is separate from the execution module + if (shadow) #endif - - return NewF; + jl_add_to_shadow(m); } // helper function for adding a DLLImport (dlsym) address to the execution engine // (for values created locally or in the sysimage, jl_emit_and_add_to_shadow is generally preferable) -template<typename T> #ifdef LLVM35 -static inline void add_named_global(GlobalObject *gv, T *_addr, bool dllimport = true) +void add_named_global(GlobalObject *gv, void *addr, bool dllimport) #else -static inline void add_named_global(GlobalValue *gv, T *_addr, bool dllimport = true) +void add_named_global(GlobalValue *gv, void *addr, bool dllimport) #endif { - // cast through integer to avoid c++ pedantic warning about casting between - // data and code pointers - void *addr = (void*)(uintptr_t)_addr; - #ifdef _OS_WINDOWS_ // setting JL_DLLEXPORT correctly only matters when building a binary // (global_proto will strip this from the JIT) @@ -799,7 +833,6 @@ static inline void add_named_global(GlobalValue *gv, T *_addr, bool dllimport = static std::vector<Constant*> jl_sysimg_gvars; static std::vector<Constant*> jl_sysimg_fvars; -typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing static std::map<void*, jl_value_llvm> jl_value_to_llvm; // global variables to pointers are pretty common, @@ -812,7 +845,7 @@ static std::map<void*, jl_value_llvm> jl_value_to_llvm; // // then add a global mapping to the current value (usually from calloc'd space) // to the execution engine to make it valid for the current session (with the current value) -static void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL) +void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit) { PointerType *T = cast<PointerType>(gv->getType()->getElementType()); // pointer is the only supported type here @@ -846,12 +879,11 @@ static void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL #endif } -#ifdef JULIA_ENABLE_THREADING // Emit a slot in the system image to be filled at sysimg init time. // Returns the global var. Fill `idx` with 1-base index in the sysimg gv. // Use as an optimization for runtime constant addresses to have one less // load. (Used only by threading). -static GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name, +GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name, uintptr_t init, size_t &idx) { assert(imaging_mode); @@ -874,9 +906,8 @@ static GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *nam idx = jl_sysimg_gvars.size(); return gv; } -#endif -static void* jl_get_global(GlobalVariable *gv) +void* jl_get_global(GlobalVariable *gv) { #if defined(USE_MCJIT) || defined(USE_ORCJIT) void *p = (void*)(intptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable(gv); @@ -890,7 +921,7 @@ static void* jl_get_global(GlobalVariable *gv) // clones the contents of the module `m` to the shadow_output collector // in the old JIT, this is equivalent to also adding it to the execution engine -static void jl_add_to_shadow(Module *m) +void jl_add_to_shadow(Module *m) { #if defined(USE_MCJIT) || defined(USE_ORCJIT) #ifndef KEEP_BODIES @@ -966,10 +997,10 @@ static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, jl_cpuid((int32_t*)info, 1); addComdat(new GlobalVariable(*mod, - T_int64, + T_uint64, true, GlobalVariable::ExternalLinkage, - ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), + ConstantInt::get(T_uint64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), "jl_sysimg_cpu_cpuid")); } #endif @@ -996,13 +1027,14 @@ void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sys // it uses the large code model and we may potentially // want less optimizations there. Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); -#if defined(_OS_WINDOWS_) && defined(FORCE_ELF) + // make sure to emit the native object format, even if FORCE_ELF was set in codegen +#if defined(_OS_WINDOWS_) #ifdef LLVM35 TheTriple.setObjectFormat(Triple::COFF); #else TheTriple.setEnvironment(Triple::UnknownEnvironment); #endif -#elif defined(_OS_DARWIN_) && defined(FORCE_ELF) +#elif defined(_OS_DARWIN_) #ifdef LLVM35 TheTriple.setObjectFormat(Triple::MachO); #else @@ -1030,7 +1062,7 @@ void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sys CodeGenOpt::Aggressive // -O3 TODO: respect command -O0 flag? )); -#ifdef LLVM38 +#ifdef LLVM37 legacy::PassManager PM; #else PassManager PM; @@ -1163,3 +1195,22 @@ extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) return 0; return it->second.index; } + +GlobalVariable *jl_get_global_for(const char *cname, void *addr, Module *M) +{ + // emit a GlobalVariable for a jl_value_t named "cname" + std::map<void*, jl_value_llvm>::iterator it; + // first see if there already is a GlobalVariable for this address + it = jl_value_to_llvm.find(addr); + if (it != jl_value_to_llvm.end()) + return prepare_global((llvm::GlobalVariable*)it->second.gv, M); + + std::stringstream gvname; + gvname << cname << globalUnique++; + // no existing GlobalVariable, create one and store it + GlobalVariable *gv = new GlobalVariable(*M, T_pjlvalue, + false, GlobalVariable::ExternalLinkage, + NULL, gvname.str()); + *(void**)jl_emit_and_add_to_shadow(gv, addr) = addr; + return gv; +} diff --git a/src/jitlayers.h b/src/jitlayers.h new file mode 100644 index 0000000000000..e9488124667d4 --- /dev/null +++ b/src/jitlayers.h @@ -0,0 +1,207 @@ +#include <llvm/IR/LLVMContext.h> +#include <llvm/IR/Constants.h> +#include <llvm/IR/Module.h> +#include <llvm/IR/Value.h> + +#include <llvm/ExecutionEngine/SectionMemoryManager.h> +#if defined(USE_ORCJIT) +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" +#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/ObjectMemoryBuffer.h" +#elif defined(USE_MCJIT) +#include <llvm/ExecutionEngine/MCJIT.h> +#include <llvm/ADT/DenseMapInfo.h> +#include <llvm/Object/ObjectFile.h> +#else +#include <llvm/ExecutionEngine/JIT.h> +#include <llvm/ExecutionEngine/JITMemoryManager.h> +#include <llvm/ExecutionEngine/Interpreter.h> +#include <llvm/ExecutionEngine/ExecutionEngine.h> +#include <llvm/ExecutionEngine/JITEventListener.h> +#endif + +#ifdef LLVM37 +#include "llvm/IR/LegacyPassManager.h" +extern legacy::PassManager *jl_globalPM; +#else +#include <llvm/PassManager.h> +extern PassManager *jl_globalPM; +#endif + +#ifdef LLVM35 +#include <llvm/Target/TargetMachine.h> +#else +#endif + +extern "C" { + extern int globalUnique; +} +extern TargetMachine *jl_TargetMachine; +extern Module *shadow_output; +extern bool imaging_mode; +#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) +extern Function *juliapersonality_func; +#endif + + +#ifdef JULIA_ENABLE_THREADING +extern size_t jltls_states_func_idx; +#endif + +typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing + +#ifdef LLVM37 +void addOptimizationPasses(legacy::PassManager *PM); +#else +void addOptimizationPasses(PassManager *PM); +#endif +void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL); +GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name, + uintptr_t init, size_t &idx); +void* jl_get_global(GlobalVariable *gv); +GlobalVariable *jl_get_global_for(const char *cname, void *addr, Module *M); +void jl_add_to_shadow(Module *m); +void finalize_gc_frame(Module *m); +void jl_finalize_function(Function *F, Module *collector = NULL); +void jl_finalize_module(Module *m, bool shadow); + +// Connect Modules via prototypes, each owned by module `M` +static inline GlobalVariable *global_proto(GlobalVariable *G, Module *M = NULL) +{ + // Copy the GlobalVariable, but without the initializer, so it becomes a declaration + GlobalVariable *proto = new GlobalVariable(G->getType()->getElementType(), + G->isConstant(), GlobalVariable::ExternalLinkage, + NULL, G->getName(), G->getThreadLocalMode()); + proto->copyAttributesFrom(G); +#ifdef LLVM35 + // DLLImport only needs to be set for the shadow module + // it just gets annoying in the JIT + proto->setDLLStorageClass(GlobalValue::DefaultStorageClass); +#endif + if (M) + M->getGlobalList().push_back(proto); + return proto; +} + +static inline Function *function_proto(Function *F, Module *M = NULL) +{ + // Copy the declaration characteristics of the Function (not the body) + Function *NewF = Function::Create(F->getFunctionType(), + Function::ExternalLinkage, + F->getName(), M); + // FunctionType does not include any attributes. Copy them over manually + // as codegen may make decisions based on the presence of certain attributes + NewF->copyAttributesFrom(F); + +#ifdef LLVM37 + // Declarations are not allowed to have personality routines, but + // copyAttributesFrom sets them anyway, so clear them again manually + NewF->setPersonalityFn(nullptr); +#endif + +#ifdef LLVM35 + // DLLImport only needs to be set for the shadow module + // it just gets annoying in the JIT + NewF->setDLLStorageClass(GlobalValue::DefaultStorageClass); +#endif + + return NewF; +} + +static inline GlobalVariable *prepare_global(GlobalVariable *G, Module *M) +{ + if (G->getParent() == M) + return G; + GlobalValue *local = M->getNamedValue(G->getName()); + if (!local) { + local = global_proto(G, M); + } + return cast<GlobalVariable>(local); +} + +#ifdef LLVM35 +void add_named_global(GlobalObject *gv, void *addr, bool dllimport); +template<typename T> +static inline void add_named_global(GlobalObject *gv, T *addr, bool dllimport = true) +#else +void add_named_global(GlobalValue *gv, void *addr, bool dllimport); +template<typename T> +static inline void add_named_global(GlobalValue *gv, T *addr, bool dllimport = true) +#endif +{ + // cast through integer to avoid c++ pedantic warning about casting between + // data and code pointers + add_named_global(gv, (void*)(uintptr_t)addr, dllimport); +} + +void jl_init_jit(Type *T_pjlvalue_); +#ifdef USE_ORCJIT +class JuliaOJIT { + // Custom object emission notification handler for the JuliaOJIT + // TODO: hook up RegisterJITEventListener, instead of hard-coding the GDB and JuliaListener targets + class DebugObjectRegistrar { + public: + DebugObjectRegistrar(JuliaOJIT &JIT); + template <typename ObjSetT, typename LoadResult> + void operator()(orc::ObjectLinkingLayerBase::ObjSetHandleT H, const ObjSetT &Objects, + const LoadResult &LOS); + private: + void NotifyGDB(object::OwningBinary<object::ObjectFile> &DebugObj); + std::vector<object::OwningBinary<object::ObjectFile>> SavedObjects; + std::unique_ptr<JITEventListener> JuliaListener; + JuliaOJIT &JIT; + }; + +public: + typedef orc::ObjectLinkingLayer<DebugObjectRegistrar> ObjLayerT; + typedef orc::IRCompileLayer<ObjLayerT> CompileLayerT; + typedef CompileLayerT::ModuleSetHandleT ModuleHandleT; + typedef StringMap<void*> SymbolTableT; + typedef object::OwningBinary<object::ObjectFile> OwningObj; + + JuliaOJIT(TargetMachine &TM); + + void addGlobalMapping(StringRef Name, uint64_t Addr); + void addGlobalMapping(const GlobalValue *GV, void *Addr); + void *getPointerToGlobalIfAvailable(StringRef S); + void *getPointerToGlobalIfAvailable(const GlobalValue *GV); + void addModule(std::unique_ptr<Module> M); + void removeModule(ModuleHandleT H); + orc::JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly); + orc::JITSymbol findUnmangledSymbol(const std::string Name); + uint64_t getGlobalValueAddress(const std::string &Name); + uint64_t getFunctionAddress(const std::string &Name); + Function *FindFunctionNamed(const std::string &Name); + void RegisterJITEventListener(JITEventListener *L); + const DataLayout& getDataLayout() const; + const Triple& getTargetTriple() const; +private: + std::string getMangledName(const std::string &Name); + std::string getMangledName(const GlobalValue *GV); + + TargetMachine &TM; + const DataLayout DL; + // Should be big enough that in the common case, The + // object fits in its entirety + SmallVector<char, 4096> ObjBufferSV; + raw_svector_ostream ObjStream; + legacy::PassManager PM; + MCContext *Ctx; + RTDyldMemoryManager *MemMgr; + ObjLayerT ObjectLayer; + CompileLayerT CompileLayer; + SymbolTableT GlobalSymbolTable; + SymbolTableT LocalSymbolTable; +}; +extern JuliaOJIT *jl_ExecutionEngine; +#else +extern ExecutionEngine *jl_ExecutionEngine; +#endif +#ifdef LLVM39 +JL_DLLEXPORT extern LLVMContext jl_LLVMContext; +#else +JL_DLLEXPORT extern LLVMContext &jl_LLVMContext; +#endif From 92212bfebf2bd2acfaf550f6464eeca607ff095c Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 14 Jul 2016 02:48:33 -0400 Subject: [PATCH 0432/1117] don't download busybox-w32 every time use the cached copy on appveyor if present --- contrib/windows/msys_build.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 369bb793be3eb..41258a21f414d 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -156,9 +156,11 @@ if [ -z "`which make 2>/dev/null`" ]; then export PATH=$PWD/bin:$PATH fi -f=busybox-w32-FRP-483-g31277ab.exe -echo "Downloading $f" -$curlflags -o usr/bin/busybox.exe http://frippery.org/files/busybox/$f +if ! [ -e usr/bin/busybox.exe ]; then + f=busybox-w32-FRP-483-g31277ab.exe + echo "Downloading $f" + $curlflags -o usr/bin/busybox.exe http://frippery.org/files/busybox/$f +fi for lib in SUITESPARSE ARPACK BLAS LAPACK FFTW \ GMP MPFR PCRE LIBUNWIND OPENSPECFUN; do From d4d507b10c0a9dd1d9812991fd483b57690e32cb Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Wed, 6 Jul 2016 15:55:02 +0200 Subject: [PATCH 0433/1117] Use promote_op in SparseMatrixCSC operators --- base/sparse/sparse.jl | 4 ++-- base/sparse/sparsematrix.jl | 14 ++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/base/sparse/sparse.jl b/base/sparse/sparse.jl index 7a7a5a82fc905..6409eb18c9a5b 100644 --- a/base/sparse/sparse.jl +++ b/base/sparse/sparse.jl @@ -2,7 +2,7 @@ module SparseArrays -using Base: ReshapedArray, setindex_shape_check, to_shape +using Base: ReshapedArray, promote_op, setindex_shape_check, to_shape using Base.Sort: Forward using Base.LinAlg: AbstractTriangular, PosDefException @@ -26,7 +26,7 @@ import Base: @get!, acos, acosd, acot, acotd, acsch, asech, asin, asind, asinh, rotl90, rotr90, round, scale!, setindex!, similar, size, transpose, tril, triu, vcat, vec, permute! -import Base.Broadcast: eltype_plus, broadcast_shape +import Base.Broadcast: broadcast_shape export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, SparseMatrixCSC, SparseVector, blkdiag, dense, droptol!, dropzeros!, dropzeros, etree, diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 62d50096a880e..3c0c5c47e5a92 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1666,13 +1666,7 @@ broadcast_zpreserving{Tv,Ti}(f::Function, A_1::Union{Array,BitArray,Number}, A_2 ## Binary arithmetic and boolean operators -for (op, pro) in ((+, :eltype_plus), - (-, :eltype_plus), - (min, :promote_eltype), - (max, :promote_eltype), - (&, :promote_eltype), - (|, :promote_eltype), - ($, :promote_eltype)) +for op in (+, -, min, max, &, |, $) body = gen_broadcast_body_sparse(op, true) OP = Symbol(string(op)) @eval begin @@ -1680,7 +1674,7 @@ for (op, pro) in ((+, :eltype_plus), if size(A_1,1) != size(A_2,1) || size(A_1,2) != size(A_2,2) throw(DimensionMismatch("")) end - Tv = ($pro)(A_1, A_2) + Tv = promote_op($op, Tv1, Tv2) B = spzeros(Tv, promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))) $body B @@ -1723,10 +1717,10 @@ end # macro (.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, full(B)) .+{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = - broadcast!(+, spzeros(eltype_plus(A_1, A_2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) + broadcast!(+, spzeros(promote_op(+, Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) function .-{Tva,Tia,Tvb,Tib}(A::SparseMatrixCSC{Tva,Tia}, B::SparseMatrixCSC{Tvb,Tib}) - broadcast!(-, spzeros(eltype_plus(A, B), promote_type(Tia, Tib), to_shape(broadcast_shape(A, B))), A, B) + broadcast!(-, spzeros(promote_op(-, Tva, Tvb), promote_type(Tia, Tib), to_shape(broadcast_shape(A, B))), A, B) end ## element-wise comparison operators returning SparseMatrixCSC ## From f55697eafcf3b7dd897d04690fbbb07f0536522e Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Wed, 13 Jul 2016 09:45:37 +0200 Subject: [PATCH 0434/1117] Remove eltype_plus --- base/broadcast.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index f3eebb2c0cac3..d7e11afcd1a9d 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -271,9 +271,6 @@ end ## elementwise operators ## -# should this be deprecated? Only use in Base is in sparsematrix.jl -eltype_plus(As::AbstractArray...) = promote_eltype_op(+, As...) - for op in (:÷, :%, :<<, :>>, :-, :/, :\, ://, :^) @eval $(Symbol(:., op))(A::AbstractArray, B::AbstractArray) = broadcast($(op), A, B) end From 1599a265e5c8773437ed416ad7df4547b04d16de Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 14 Jul 2016 08:12:39 -0400 Subject: [PATCH 0435/1117] Handle ccall callee in inlining pass Fix #17403 --- base/inference.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index cfa9ef1094b54..bb11810f68621 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2965,7 +2965,8 @@ function inlining_pass(e::Expr, sv, linfo) # by the interpreter and inlining might put in something it can't handle, # like another ccall (or try to move the variables out into the function) if is_known_call(e, Core.Intrinsics.ccall, sv) - i0 = 5 + # 4 is rewrite to 2 below to handle the callee. + i0 = 4 isccall = true elseif is_known_call(e, Core.Intrinsics.llvmcall, sv) i0 = 5 @@ -2975,20 +2976,23 @@ function inlining_pass(e::Expr, sv, linfo) isccall = false end has_stmts = false # needed to preserve order-of-execution - for i=length(eargs):-1:i0 + for _i=length(eargs):-1:i0 + i = (isccall && _i == 4) ? 2 : _i ei = eargs[i] if isa(ei,Expr) + ei = ei::Expr if ei.head === :& - argloc = (ei::Expr).args + argloc = ei.args i = 1 ei = argloc[1] if !isa(ei,Expr) continue end + ei = ei::Expr else argloc = eargs end - res = inlining_pass(ei::Expr, sv, linfo) + res = inlining_pass(ei, sv, linfo) res1 = res[1] if has_stmts && !effect_free(res1, sv, false) restype = exprtype(res1,sv) From 19fe6ed9e4d6c111ff47ad30b7a3357ddca07a7a Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Tue, 24 May 2016 00:42:41 -0400 Subject: [PATCH 0436/1117] build libgit2 with libssh2 support - Linux: OpenSSL & mbedTLS cryptolib for libssh2 - macOS: Native SSL & mbedTLS cryptolib for libssh2 - Windows: Native SSL & cryptolib --- .travis.yml | 8 +- Make.inc | 2 + Makefile | 7 ++ base/libgit2/libgit2.jl | 22 +++--- contrib/windows/msys_build.sh | 6 +- deps/Makefile | 14 +++- deps/Versions.make | 1 + .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + deps/checksums/mbedtls-2.2.1-apache.tgz/md5 | 1 + .../checksums/mbedtls-2.2.1-apache.tgz/sha512 | 1 + deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 | 1 + deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 | 1 + deps/libgit2.mk | 20 +++-- deps/libgit2.version | 4 +- deps/libssh2.mk | 56 +++++++++++++ deps/libssh2.version | 2 + deps/mbedtls.mk | 78 +++++++++++++++++++ deps/patches/libgit2-custom-tls.patch | 65 ++++++++++++++++ deps/patches/libgit2-ssh.patch | 23 ++++++ deps/patches/mbedtls.patch | 27 +++++++ test/libgit2.jl | 8 +- 26 files changed, 327 insertions(+), 26 deletions(-) create mode 100644 deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/md5 create mode 100644 deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/sha512 delete mode 100644 deps/checksums/libgit2-785d8c48ea8725691da3c50e7dae8751523d4c30.tar.gz/md5 delete mode 100644 deps/checksums/libgit2-785d8c48ea8725691da3c50e7dae8751523d4c30.tar.gz/sha512 create mode 100644 deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/md5 create mode 100644 deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/sha512 create mode 100644 deps/checksums/mbedtls-2.2.1-apache.tgz/md5 create mode 100644 deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 create mode 100644 deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 create mode 100644 deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 create mode 100644 deps/libssh2.mk create mode 100644 deps/libssh2.version create mode 100644 deps/mbedtls.mk create mode 100644 deps/patches/libgit2-custom-tls.patch create mode 100644 deps/patches/libgit2-ssh.patch create mode 100644 deps/patches/mbedtls.patch diff --git a/.travis.yml b/.travis.yml index 5749b77932f45..187edb2ab12cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,9 @@ matrix: apt: sources: - ubuntu-toolchain-r-test + - kalakris-cmake packages: + - cmake - bar - time - binutils @@ -18,7 +20,7 @@ matrix: - gcc-5-multilib - g++-5-multilib - make:i386 - - libssl-dev:i386 + - libssl-dev:i386s - gfortran-5 - gfortran-5-multilib - os: linux @@ -28,7 +30,9 @@ matrix: apt: sources: - ubuntu-toolchain-r-test + - kalakris-cmake packages: + - cmake - bar - time - g++-5 @@ -79,7 +83,7 @@ before_install: BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm37-julia)/bin/llvm-config-3.7.1 LLVM_SIZE=$(brew --prefix llvm37-julia)/bin/llvm-size-3.7.1"; BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1 STAGE2_DEPS=utf8proc"; BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas"; - for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND LIBGIT2; do + for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND; do export BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1"; done; export CXXFLAGS=-DUSE_ORCJIT; diff --git a/Make.inc b/Make.inc index 8d34d4be08880..f13ad5e4d75f4 100644 --- a/Make.inc +++ b/Make.inc @@ -36,6 +36,8 @@ USE_SYSTEM_ARPACK:=0 USE_SYSTEM_SUITESPARSE:=0 USE_SYSTEM_LIBUV:=0 USE_SYSTEM_UTF8PROC:=0 +USE_SYSTEM_MBEDTLS:=0 +USE_SYSTEM_LIBSSH2:=0 USE_SYSTEM_LIBGIT2:=0 USE_SYSTEM_PATCHELF:=0 diff --git a/Makefile b/Makefile index 3e9518b1e03e4..23e9e26919068 100644 --- a/Makefile +++ b/Makefile @@ -271,6 +271,12 @@ endif ifeq ($(USE_SYSTEM_MPFR),0) JL_PRIVATE_LIBS += mpfr endif +ifeq ($(USE_SYSTEM_MBEDTLS),0) +JL_PRIVATE_LIBS += mbedtls mbedcrypto mbedx509 +endif +ifeq ($(USE_SYSTEM_LIBSSH2),0) +JL_PRIVATE_LIBS += ssh2 +endif ifeq ($(USE_SYSTEM_LIBGIT2),0) JL_PRIVATE_LIBS += git2 endif @@ -358,6 +364,7 @@ ifeq ($(OS),Linux) -$(INSTALL_M) $(build_libdir)/libssl*.so* $(DESTDIR)$(private_libdir) -$(INSTALL_M) $(build_libdir)/libcrypto*.so* $(DESTDIR)$(private_libdir) endif + endif ifeq ($(USE_SYSTEM_LIBUV),0) diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index 89f05da2d84c6..b8d5e78bde40f 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -511,6 +511,7 @@ function set_ssl_cert_locations(cert_loc) ccall((:git_libgit2_opts, :libgit2), Cint, (Cint, Cstring, Cstring), Cint(Consts.SET_SSL_CERT_LOCATIONS), cert_file, cert_dir) + end function __init__() @@ -520,16 +521,19 @@ function __init__() ccall((:git_libgit2_shutdown, :libgit2), Cint, ()) end - # Look for OpenSSL env variable for CA bundle - cert_loc = if "SSL_CERT_DIR" in keys(ENV) - ENV["SSL_CERT_DIR"] - elseif "SSL_CERT_FILE" in keys(ENV) - ENV["SSL_CERT_FILE"] - else - # If we have a bundled ca cert file, point libgit2 at that so SSL connections work. - abspath(ccall(:jl_get_julia_home, Any, ()),Base.DATAROOTDIR,"julia","cert.pem") + # Look for OpenSSL env variable for CA bundle (linux only) + # windows and macOS are use their security backends + @static if is_linux() + cert_loc = if "SSL_CERT_DIR" in keys(ENV) + ENV["SSL_CERT_DIR"] + elseif "SSL_CERT_FILE" in keys(ENV) + ENV["SSL_CERT_FILE"] + else + # If we have a bundled ca cert file, point libgit2 at that so SSL connections work. + abspath(ccall(:jl_get_julia_home, Any, ()),Base.DATAROOTDIR,"julia","cert.pem") + end + set_ssl_cert_locations(cert_loc) end - set_ssl_cert_locations(cert_loc) end diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 41258a21f414d..2c31c77473dff 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -173,9 +173,9 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user # libuv since its static lib is no longer included in the binaries # openlibm since we need it as a static library to work properly # utf8proc since its headers are not in the binary download -echo 'override STAGE1_DEPS = libuv' >> Make.user -echo 'override STAGE2_DEPS = utf8proc' >> Make.user -echo 'override STAGE3_DEPS = ' >> Make.user +echo 'override STAGE1_DEPS = libuv mbedtls' >> Make.user +echo 'override STAGE2_DEPS = utf8proc libssh2' >> Make.user +echo 'override STAGE3_DEPS = libgit2' >> Make.user if [ -n "$USEMSVC" ]; then # Openlibm doesn't build well with MSVC right now diff --git a/deps/Makefile b/deps/Makefile index 00d2f0fab4547..fc337f49dfbb7 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -162,8 +162,16 @@ ifeq ($(USE_SYSTEM_GMP), 0) STAGE1_DEPS += gmp endif +ifeq ($(USE_SYSTEM_MBEDTLS), 0) +STAGE1_DEPS += mbedtls +endif + +ifeq ($(USE_SYSTEM_LIBSSH2), 0) +STAGE2_DEPS += libssh2 +endif + ifeq ($(USE_SYSTEM_LIBGIT2), 0) -STAGE1_DEPS += libgit2 +STAGE3_DEPS += libgit2 endif ifeq ($(USE_SYSTEM_MPFR), 0) @@ -213,7 +221,7 @@ install: $(addprefix install-, $(DEP_LIBS)) cleanall: $(addprefix clean-, $(DEP_LIBS)) distcleanall: $(addprefix distclean-, $(DEP_LIBS)) rm -rf $(build_prefix) -getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-libgit2 +getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-libgit2 ## PATHS ## # sort is used to remove potential duplicates @@ -260,6 +268,8 @@ include $(SRCDIR)/unwind.mk include $(SRCDIR)/gmp.mk include $(SRCDIR)/mpfr.mk include $(SRCDIR)/patchelf.mk +include $(SRCDIR)/mbedtls.mk +include $(SRCDIR)/libssh2.mk include $(SRCDIR)/libgit2.mk include $(SRCDIR)/virtualenv.mk diff --git a/deps/Versions.make b/deps/Versions.make index 76cfab2af787d..5a662941892c8 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -12,3 +12,4 @@ GMP_VER = 6.1.0 MPFR_VER = 3.1.4 PATCHELF_VER = 0.9 VIRTUALENV_VER = 15.0.0 +MBEDTLS_VER = 2.2.1 diff --git a/deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/md5 b/deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/md5 new file mode 100644 index 0000000000000..99237ab85b12d --- /dev/null +++ b/deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/md5 @@ -0,0 +1 @@ +eb46efa15014aaec7e574dfdc5921a36 diff --git a/deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/sha512 b/deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/sha512 new file mode 100644 index 0000000000000..6a1a3f0542e15 --- /dev/null +++ b/deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/sha512 @@ -0,0 +1 @@ +20aa4ca35e78873961ad45734db8af36fe600d523a8ef8c77dba66944c7e33efbe239c49f3258e81cf6a7f7b6afa3b867ad62fc1fb507377ad8e9fa2bb629472 diff --git a/deps/checksums/libgit2-785d8c48ea8725691da3c50e7dae8751523d4c30.tar.gz/md5 b/deps/checksums/libgit2-785d8c48ea8725691da3c50e7dae8751523d4c30.tar.gz/md5 deleted file mode 100644 index 59a7366c9f144..0000000000000 --- a/deps/checksums/libgit2-785d8c48ea8725691da3c50e7dae8751523d4c30.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -fd5c285856ae59d3aea3e316fb31885b diff --git a/deps/checksums/libgit2-785d8c48ea8725691da3c50e7dae8751523d4c30.tar.gz/sha512 b/deps/checksums/libgit2-785d8c48ea8725691da3c50e7dae8751523d4c30.tar.gz/sha512 deleted file mode 100644 index d274b61af61b1..0000000000000 --- a/deps/checksums/libgit2-785d8c48ea8725691da3c50e7dae8751523d4c30.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -a5d7b03c337e770194ed83497fa1c895cdd02ab518e0b1c87b42115f9d26b31de0bf3a54cbe80a6ed58fcb5d5440c71325714b5cb607892f67d0c6269c302211 diff --git a/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/md5 b/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/md5 new file mode 100644 index 0000000000000..cf69a675640e7 --- /dev/null +++ b/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/md5 @@ -0,0 +1 @@ +791327566b5d17fac1c4740b7ef52777 diff --git a/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/sha512 b/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/sha512 new file mode 100644 index 0000000000000..04ca540a5cd7b --- /dev/null +++ b/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/sha512 @@ -0,0 +1 @@ +871bf2469f6011d22f1f683353a98ce484396b4ed3df3f3d7e28ef0ac8060c3861a74c31860f189491a4e46dc402b082f8794cc578317258017dc902456f81ae diff --git a/deps/checksums/mbedtls-2.2.1-apache.tgz/md5 b/deps/checksums/mbedtls-2.2.1-apache.tgz/md5 new file mode 100644 index 0000000000000..6a06d0ebc2b38 --- /dev/null +++ b/deps/checksums/mbedtls-2.2.1-apache.tgz/md5 @@ -0,0 +1 @@ +77751c0e370ed2ab01934e4c5e1d380f diff --git a/deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 b/deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 new file mode 100644 index 0000000000000..f8b91b02ac065 --- /dev/null +++ b/deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 @@ -0,0 +1 @@ +6a74abc4ea225eb6bcf20894bb1a6faa82dbaff11129c41849151e2654570609efeee70d0644ce63c4d2c11e6142b2db262b88f3a22fdceff0a215a64a5d6eb0 diff --git a/deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 b/deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 new file mode 100644 index 0000000000000..5b8fbecd5b1d7 --- /dev/null +++ b/deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 @@ -0,0 +1 @@ +b6909d94600fc5f644e9bca52e96fb27 diff --git a/deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 b/deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 new file mode 100644 index 0000000000000..79f2f47dbb2d8 --- /dev/null +++ b/deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 @@ -0,0 +1 @@ +03c32b46369d35e61acfeac83f417d09307d2f1e4358f9ea6af275d9a9a2319ca7a244733b5950a48d59f6305e4adcad6a66e589b42ee7417d1d4362c2595a4f diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 6160739a36279..c343f14ae0bff 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -7,9 +7,9 @@ $(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,build/libgit2.$(SHLIB_ LIBGIT2_OBJ_SOURCE := $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/libgit2.$(SHLIB_EXT) LIBGIT2_OBJ_TARGET := $(build_shlibdir)/libgit2.$(SHLIB_EXT) -LIBGIT2_OPTS := $(CMAKE_COMMON) -DTHREADSAFE=ON +LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DTHREADSAFE=ON -DCMAKE_PREFIX_PATH=$(build_prefix) ifeq ($(OS),WINNT) -LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON -DUSE_SSH=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo +LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON ifneq ($(ARCH),x86_64) LIBGIT2_OPTS += -DCMAKE_C_FLAGS="-mincoming-stack-boundary=2" endif @@ -20,26 +20,36 @@ LIBGIT2_OPTS += -DBUILD_CLAR=OFF -DDLLTOOL=`which $(CROSS_COMPILE)dlltool` LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY endif else -LIBGIT2_OPTS += -DCMAKE_BUILD_TYPE=Release +LIBGIT2_OPTS += -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE endif -$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(LIBSSH2_OBJ_TARGET) +ifeq ($(OS),WINNT) + -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p0 -f < $(SRCDIR)/patches/libgit2-ssh.patch +else -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch +endif mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) touch -c $@ + $(LIBGIT2_OBJ_SOURCE): $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile $(MAKE) -C $(dir $<) touch -c $@ + $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/checked: $(LIBGIT2_OBJ_SOURCE) ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) test endif echo 1 > $@ + $(LIBGIT2_OBJ_TARGET): $(LIBGIT2_OBJ_SOURCE) | $(build_shlibdir) +ifeq ($(BUILD_OS),WINNT) cp $< $@ - @#$$(call make-install,$(LIBGIT2_SRC_DIR),) # currently don't need the full install +else + $(call make-install,$(LIBGIT2_SRC_DIR),) +endif ifeq ($(OS),Linux) @# If we're on linux, copy over libssl and libcrypto for libgit2 -LIBGIT_LIBS=$$(ldd "$@" | tail -n +2 | awk '{print $$(NF-1)}'); \ diff --git a/deps/libgit2.version b/deps/libgit2.version index 66898b3d70c4b..8b72ed4066012 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -1,2 +1,2 @@ -LIBGIT2_BRANCH=v0.24.0 -LIBGIT2_SHA1=785d8c48ea8725691da3c50e7dae8751523d4c30 +LIBGIT2_BRANCH=v0.24.1 +LIBGIT2_SHA1=211e117a0590583a720c53172406f34186c543bd diff --git a/deps/libssh2.mk b/deps/libssh2.mk new file mode 100644 index 0000000000000..74decfa9682ec --- /dev/null +++ b/deps/libssh2.mk @@ -0,0 +1,56 @@ +## libssh2 + +LIBSSH2_GIT_URL := git://github.com/wildart/libssh2.git +LIBSSH2_TAR_URL = https://api.github.com/repos/wildart/libssh2/tarball/$1 +$(eval $(call git-external,libssh2,LIBSSH2,CMakeLists.txt,build/libssh2.$(SHLIB_EXT),$(SRCDIR)/srccache)) + +LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT) +LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT) + +LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \ + -DCMAKE_PREFIX_PATH=$(build_prefix) \ + -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \ + -DCMAKE_BUILD_TYPE=Release + +ifeq ($(OS),WINNT) +LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF +ifeq ($(BUILD_OS),WINNT) +LIBSSH2_OPTS += -G"MSYS Makefiles" +endif +else +LIBSSH2_OPTS += -DCRYPTO_BACKEND=mbedTLS -DENABLE_ZLIB_COMPRESSION=ON +endif + +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(MBEDTLS_OBJ_TARGET) + mkdir -p $(dir $@) + cd $(dir $@) && \ + $(CMAKE) $(dir $<) $(LIBSSH2_OPTS) + touch -c $@ + +$(LIBSSH2_OBJ_SOURCE): $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile + $(MAKE) -C $(dir $<) libssh2 + touch -c $@ + +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/checked: $(LIBSSH2_OBJ_SOURCE) +ifeq ($(OS),$(BUILD_OS)) + $(MAKE) -C $(dir $@) test +endif + echo 1 > $@ + +$(LIBSSH2_OBJ_TARGET): $(LIBSSH2_OBJ_SOURCE) | $(build_shlibdir) +ifeq ($(OS),WINNT) + $(MAKE) -C $(BUILDDIR)/$(LIBSSH2_SRC_DIR) install +else + $(call make-install,$(LIBSSH2_SRC_DIR),) +endif + touch -c $(LIBSSH2_OBJ_TARGET) + +clean-libssh2: + -rm -rf $(BUILDDIR)/$(LIBSSH2_SRC_DIR) + -rm -f $(LIBSSH2_OBJ_TARGET) + +get-libssh2: $(LIBSSH2_SRC_FILE) +configure-libssh2: $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile +compile-libssh2: $(LIBSSH2_OBJ_SOURCE) +check-libssh2: $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/checked +install-libssh2: $(LIBSSH2_OBJ_TARGET) diff --git a/deps/libssh2.version b/deps/libssh2.version new file mode 100644 index 0000000000000..dcdf4f563a0eb --- /dev/null +++ b/deps/libssh2.version @@ -0,0 +1,2 @@ +LIBSSH2_BRANCH=mbedtls +LIBSSH2_SHA1=a486c43e4a699d319a5afdb139545843769bcc0d diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk new file mode 100644 index 0000000000000..93121d9a59eb5 --- /dev/null +++ b/deps/mbedtls.mk @@ -0,0 +1,78 @@ +## mbedtls + +ifeq ($(USE_GPL_LIBS), 1) + MBEDTLS_SRC = mbedtls-$(MBEDTLS_VER)-gpl +else + MBEDTLS_SRC = mbedtls-$(MBEDTLS_VER)-apache +endif +MBEDTLS_URL = https://tls.mbed.org/download/$(MBEDTLS_SRC).tgz + +MBEDTLS_OBJ_SOURCE := $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedtls.$(SHLIB_EXT) \ + $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedx509.$(SHLIB_EXT) \ + $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedcrypto.$(SHLIB_EXT) +MBEDTLS_OBJ_TARGET := $(build_shlibdir)/libmbedtls.$(SHLIB_EXT) \ + $(build_shlibdir)/libmbedx509.$(SHLIB_EXT) \ + $(build_shlibdir)/libmbedcrypto.$(SHLIB_EXT) + +MBEDTLS_OPTS := $(CMAKE_COMMON) -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ + -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE + +ifeq ($(OS),WINNT) +MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=OFF +ifeq ($(BUILD_OS),WINNT) +MBEDTLS_OPTS += -G"MSYS Makefiles" -DENABLE_TESTING=OFF +endif +else +MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=ON +endif + +$(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz: | $(SRCDIR)/srccache + $(JLDOWNLOAD) $@ $(MBEDTLS_URL) + +$(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt: $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz + $(JLCHECKSUM) $< + mkdir -p $(dir $@) && \ + $(TAR) -C $(dir $@) --strip-components 1 -xf $< + touch -c $@ + +$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt + -cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls.patch + mkdir -p $(dir $@) + cd $(dir $@) && \ + $(CMAKE) $(dir $<) $(MBEDTLS_OPTS) + touch -c $@ + +$(MBEDTLS_OBJ_SOURCE): $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile + $(MAKE) -C $(dir $<) + touch -c $@ + +$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/checked: $(MBEDTLS_OBJ_SOURCE) +ifeq ($(OS),$(BUILD_OS)) + $(MAKE) -C $(dir $@) test +endif + echo 1 > $@ + +$(MBEDTLS_OBJ_TARGET): $(MBEDTLS_OBJ_SOURCE) | $(build_shlibdir) +ifeq ($(OS), WINNT) + cp $^ $(build_shlibdir) +else + $(call make-install,mbedtls-$(MBEDTLS_VER),) +endif + touch -c $(MBEDTLS_OBJ_TARGET) + +clean-mbedtls: + -rm -rf $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) + -rm -f $(MBEDTLS_OBJ_TARGET) + +distclean-mbedtls: + -rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz + -rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC) + -rm -rf $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) + + +get-mbedtls: $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz +configure-mbedtls: $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile +compile-mbedtls: $(MBEDTLS_OBJ_SOURCE) +check-mbedtls: $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/checked +install-mbedtls: $(MBEDTLS_OBJ_TARGET) diff --git a/deps/patches/libgit2-custom-tls.patch b/deps/patches/libgit2-custom-tls.patch new file mode 100644 index 0000000000000..cb06fe3e512a6 --- /dev/null +++ b/deps/patches/libgit2-custom-tls.patch @@ -0,0 +1,65 @@ +diff --git a/src/settings.c b/src/settings.c +index 00a3ef0..21430bc 100644 +--- a/src/settings.c ++++ b/src/settings.c +@@ -29,9 +29,7 @@ int git_libgit2_features(void) + #ifdef GIT_THREADS + | GIT_FEATURE_THREADS + #endif +-#if defined(GIT_OPENSSL) || defined(GIT_WINHTTP) || defined(GIT_SECURE_TRANSPORT) + | GIT_FEATURE_HTTPS +-#endif + #if defined(GIT_SSH) + | GIT_FEATURE_SSH + #endif +diff --git a/src/transport.c b/src/transport.c +index 327052f..32f8464 100644 +--- a/src/transport.c ++++ b/src/transport.c +@@ -29,9 +29,7 @@ static transport_definition local_transport_definition = { "file://", git_transp + static transport_definition transports[] = { + { "git://", git_transport_smart, &git_subtransport_definition }, + { "http://", git_transport_smart, &http_subtransport_definition }, +-#if defined(GIT_OPENSSL) || defined(GIT_WINHTTP) || defined(GIT_SECURE_TRANSPORT) + { "https://", git_transport_smart, &http_subtransport_definition }, +-#endif + { "file://", git_transport_local, NULL }, + #ifdef GIT_SSH + { "ssh://", git_transport_smart, &ssh_subtransport_definition }, +diff --git a/src/transports/http.c b/src/transports/http.c +index 4fbbfbb..30520a0 100644 +--- a/src/transports/http.c ++++ b/src/transports/http.c +@@ -620,7 +620,6 @@ static int http_connect(http_subtransport *t) + + error = git_stream_connect(t->io); + +-#if defined(GIT_OPENSSL) || defined(GIT_SECURE_TRANSPORT) || defined(GIT_CURL) + if ((!error || error == GIT_ECERTIFICATE) && t->owner->certificate_check_cb != NULL && + git_stream_is_encrypted(t->io)) { + git_cert *cert; +@@ -640,7 +639,7 @@ static int http_connect(http_subtransport *t) + return error; + } + } +-#endif ++ + if (error < 0) + return error; + +diff --git a/tests/core/features.c b/tests/core/features.c +index 85cddfe..cf5e190 100644 +--- a/tests/core/features.c ++++ b/tests/core/features.c +@@ -17,11 +17,7 @@ void test_core_features__0(void) + cl_assert((caps & GIT_FEATURE_THREADS) == 0); + #endif + +-#if defined(GIT_OPENSSL) || defined(GIT_WINHTTP) || defined(GIT_SECURE_TRANSPORT) + cl_assert((caps & GIT_FEATURE_HTTPS) != 0); +-#else +- cl_assert((caps & GIT_FEATURE_HTTPS) == 0); +-#endif + + #if defined(GIT_SSH) + cl_assert((caps & GIT_FEATURE_SSH) != 0); diff --git a/deps/patches/libgit2-ssh.patch b/deps/patches/libgit2-ssh.patch new file mode 100644 index 0000000000000..4c7081e37e2bb --- /dev/null +++ b/deps/patches/libgit2-ssh.patch @@ -0,0 +1,23 @@ +--- CMakeLists.txt 2016-05-26 23:13:48.000000000 -0400 ++++ CMakeLists.txt 2016-07-02 23:20:55.842735529 -0400 +@@ -354,14 +354,16 @@ + + # Optional external dependency: libssh2 + IF (USE_SSH) +- PKG_CHECK_MODULES(LIBSSH2 libssh2) ++ FIND_PACKAGE(Libssh2) ++ GET_TARGET_PROPERTY(LIBSSH2_INCLUDE_DIRS Libssh2::libssh2 INTERFACE_INCLUDE_DIRECTORIES) ++ GET_TARGET_PROPERTY(LIBSSH2_LOCATION Libssh2::libssh2 IMPORTED_LOCATION_RELEASE) ++ GET_FILENAME_COMPONENT(LIBSSH2_LIBRARY_DIRS ${LIBSSH2_LOCATION} DIRECTORY) ++ SET(LIBSSH2_LIBRARIES "-lssh2") + ENDIF() +-IF (LIBSSH2_FOUND) ++IF (Libssh2_FOUND) + ADD_DEFINITIONS(-DGIT_SSH) + INCLUDE_DIRECTORIES(${LIBSSH2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${LIBSSH2_LIBRARY_DIRS}) +- LIST(APPEND LIBGIT2_PC_LIBS ${LIBSSH2_LDFLAGS}) +- #SET(LIBGIT2_PC_LIBS "${LIBGIT2_PC_LIBS} ${LIBSSH2_LDFLAGS}") + SET(SSH_LIBRARIES ${LIBSSH2_LIBRARIES}) + + CHECK_LIBRARY_EXISTS("${LIBSSH2_LIBRARIES}" libssh2_userauth_publickey_frommemory "${LIBSSH2_LIBRARY_DIRS}" HAVE_LIBSSH2_MEMORY_CREDENTIALS) diff --git a/deps/patches/mbedtls.patch b/deps/patches/mbedtls.patch new file mode 100644 index 0000000000000..1f774ec61ac33 --- /dev/null +++ b/deps/patches/mbedtls.patch @@ -0,0 +1,27 @@ +diff --git include/mbedtls/config.h include/mbedtls/config.h +index 0efee04..787cd8c 100644 +--- include/mbedtls/config.h ++++ include/mbedtls/config.h +@@ -2434,19 +2434,19 @@ + + /* MPI / BIGNUM options */ + //#define MBEDTLS_MPI_WINDOW_SIZE 6 /**< Maximum windows size used. */ +-//#define MBEDTLS_MPI_MAX_SIZE 1024 /**< Maximum number of bytes for usable MPIs. */ ++#define MBEDTLS_MPI_MAX_SIZE 2048 /**< Maximum number of bytes for usable MPIs. */ + + /* CTR_DRBG options */ + //#define MBEDTLS_CTR_DRBG_ENTROPY_LEN 48 /**< Amount of entropy used per seed by default (48 with SHA-512, 32 with SHA-256) */ + //#define MBEDTLS_CTR_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ + //#define MBEDTLS_CTR_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ +-//#define MBEDTLS_CTR_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ ++#define MBEDTLS_CTR_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ + //#define MBEDTLS_CTR_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ + + /* HMAC_DRBG options */ + //#define MBEDTLS_HMAC_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ + //#define MBEDTLS_HMAC_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ +-//#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ ++#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ + //#define MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ + + /* ECP options */ diff --git a/test/libgit2.jl b/test/libgit2.jl index 7e62fb61248d4..35032040ea099 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,7 +2,7 @@ #@testset "libgit2" begin -const LIBGIT2_VER = v"0.23.0" +const LIBGIT2_VER = v"0.24.0" ######### # TESTS # @@ -13,6 +13,12 @@ const LIBGIT2_VER = v"0.23.0" @test v.major == LIBGIT2_VER.major && v.minor >= LIBGIT2_VER.minor #end +#@testset "Check library features" begin + f = LibGit2.features() + @test findfirst(f, LibGit2.Consts.FEATURE_SSH) > 0 + @test findfirst(f, LibGit2.Consts.FEATURE_HTTPS) > 0 +#end + #@testset "OID" begin z = LibGit2.Oid() @test LibGit2.iszero(z) From 0fd18473cad0412c6ae88012688920f9359782e6 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Tue, 12 Jul 2016 20:48:36 -0400 Subject: [PATCH 0437/1117] fix typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 187edb2ab12cf..94edf04dbd460 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ matrix: - gcc-5-multilib - g++-5-multilib - make:i386 - - libssl-dev:i386s + - libssl-dev:i386 - gfortran-5 - gfortran-5-multilib - os: linux From dd2619f6d7dfab7c13e448459172e26076577923 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Tue, 12 Jul 2016 21:00:55 -0400 Subject: [PATCH 0438/1117] get libssh2 from official repo & patch it --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/libssh2.mk | 5 +- deps/libssh2.version | 4 +- deps/patches/libssh2-mbedtls.patch | 1176 +++++++++++++++++ 7 files changed, 1183 insertions(+), 6 deletions(-) create mode 100644 deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/md5 create mode 100644 deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/sha512 delete mode 100644 deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/md5 delete mode 100644 deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/sha512 create mode 100644 deps/patches/libssh2-mbedtls.patch diff --git a/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/md5 b/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/md5 new file mode 100644 index 0000000000000..8be6221cb0950 --- /dev/null +++ b/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/md5 @@ -0,0 +1 @@ +41eb49bcd2800bcf2cc12263be16cdfb diff --git a/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/sha512 b/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/sha512 new file mode 100644 index 0000000000000..35f599b6b6d17 --- /dev/null +++ b/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/sha512 @@ -0,0 +1 @@ +455a853076b94944a09965c57a97454989a0f6c87cd9927a75f3bce69ceff56cbe4a34b40deadcf70923434dd20381f6f4fae1564b41d3d8a970bee5f50efee0 diff --git a/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/md5 b/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/md5 deleted file mode 100644 index cf69a675640e7..0000000000000 --- a/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -791327566b5d17fac1c4740b7ef52777 diff --git a/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/sha512 b/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/sha512 deleted file mode 100644 index 04ca540a5cd7b..0000000000000 --- a/deps/checksums/libssh2-a486c43e4a699d319a5afdb139545843769bcc0d.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -871bf2469f6011d22f1f683353a98ce484396b4ed3df3f3d7e28ef0ac8060c3861a74c31860f189491a4e46dc402b082f8794cc578317258017dc902456f81ae diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 74decfa9682ec..92deec707b34f 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -1,7 +1,7 @@ ## libssh2 -LIBSSH2_GIT_URL := git://github.com/wildart/libssh2.git -LIBSSH2_TAR_URL = https://api.github.com/repos/wildart/libssh2/tarball/$1 +LIBSSH2_GIT_URL := git://github.com/libssh2/libssh2.git +LIBSSH2_TAR_URL = https://api.github.com/repos/libssh2/libssh2/tarball/$1 $(eval $(call git-external,libssh2,LIBSSH2,CMakeLists.txt,build/libssh2.$(SHLIB_EXT),$(SRCDIR)/srccache)) LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT) @@ -22,6 +22,7 @@ LIBSSH2_OPTS += -DCRYPTO_BACKEND=mbedTLS -DENABLE_ZLIB_COMPRESSION=ON endif $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(MBEDTLS_OBJ_TARGET) + -cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-mbedtls.patch mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBSSH2_OPTS) diff --git a/deps/libssh2.version b/deps/libssh2.version index dcdf4f563a0eb..5e0bbbfa8c85c 100644 --- a/deps/libssh2.version +++ b/deps/libssh2.version @@ -1,2 +1,2 @@ -LIBSSH2_BRANCH=mbedtls -LIBSSH2_SHA1=a486c43e4a699d319a5afdb139545843769bcc0d +LIBSSH2_BRANCH=master +LIBSSH2_SHA1=7934c9ce2a029c43e3642a492d3b9e494d1542be diff --git a/deps/patches/libssh2-mbedtls.patch b/deps/patches/libssh2-mbedtls.patch new file mode 100644 index 0000000000000..ffab8f7786685 --- /dev/null +++ b/deps/patches/libssh2-mbedtls.patch @@ -0,0 +1,1176 @@ +diff --git a/.travis.yml b/.travis.yml +index 9d6c0be..8bc1292 100644 +--- a/.travis.yml ++++ b/.travis.yml +@@ -49,6 +49,10 @@ env: + - ADDRESS_SIZE=64 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF + - ADDRESS_SIZE=64 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON + - ADDRESS_SIZE=64 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=ON ++ - ADDRESS_SIZE=64 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=OFF ++ - ADDRESS_SIZE=64 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF ++ - ADDRESS_SIZE=64 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON ++ - ADDRESS_SIZE=64 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=ON + - ADDRESS_SIZE=32 CRYPTO_BACKEND=OpenSSL BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=OFF + - ADDRESS_SIZE=32 CRYPTO_BACKEND=OpenSSL BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF + - ADDRESS_SIZE=32 CRYPTO_BACKEND=OpenSSL BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON +@@ -57,6 +61,11 @@ env: + - ADDRESS_SIZE=32 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF + - ADDRESS_SIZE=32 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON + - ADDRESS_SIZE=32 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=ON ++ - ADDRESS_SIZE=32 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=OFF ++ - ADDRESS_SIZE=32 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF ++ - ADDRESS_SIZE=32 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON ++ - ADDRESS_SIZE=32 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=ON ++ + + before_install: + - sudo add-apt-repository --yes ppa:kalakris/cmake +@@ -66,6 +75,9 @@ before_install: + - if [ $ADDRESS_SIZE = '32' ]; then sudo apt-get install -y linux-libc-dev linux-libc-dev:i386; fi + - if [ $ADDRESS_SIZE = '32' ]; then sudo apt-get install -y gcc-multilib libgcrypt11-dev:i386 libssl-dev:i386 zlib1g-dev:i386; fi + - if [ $ADDRESS_SIZE = '32' ]; then export TOOLCHAIN_OPTION="-DCMAKE_TOOLCHAIN_FILE=../cmake/Toolchain-Linux-32.cmake"; fi ++ - if [ $CRYPTO_BACKEND = 'mbedTLS' ]; then mbedtls/buildmbedtls; fi ++ - if [ $CRYPTO_BACKEND = 'mbedTLS' ]; then export TOOLCHAIN_OPTION="-DCMAKE_PREFIX_PATH=../mbedtls/install"; fi ++ - if [ $CRYPTO_BACKEND = 'mbedTLS' ]; then export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/mbedtls/install/lib; fi + + install: + - mkdir bin +diff --git a/cmake/FindmbedTLS.cmake b/cmake/FindmbedTLS.cmake +new file mode 100644 +index 0000000..2f4adbc +--- /dev/null ++++ b/cmake/FindmbedTLS.cmake +@@ -0,0 +1,64 @@ ++# - Try to find mbedTLS ++# Once done this will define ++# ++# Read-Only variables ++# MBEDTLS_FOUND - system has mbedTLS ++# MBEDTLS_INCLUDE_DIR - the mbedTLS include directory ++# MBEDTLS_LIBRARY_DIR - the mbedTLS library directory ++# MBEDTLS_LIBRARIES - Link these to use mbedTLS ++# MBEDTLS_LIBRARY - path to mbedTLS library ++# MBEDX509_LIBRARY - path to mbedTLS X.509 library ++# MBEDCRYPTO_LIBRARY - path to mbedTLS Crypto library ++ ++FIND_PATH(MBEDTLS_INCLUDE_DIR mbedtls/version.h) ++ ++IF(MBEDTLS_INCLUDE_DIR AND MBEDTLS_LIBRARIES) ++ # Already in cache, be silent ++ SET(MBEDTLS_FIND_QUIETLY TRUE) ++ENDIF() ++ ++FIND_LIBRARY(MBEDTLS_LIBRARY NAMES mbedtls libmbedtls libmbedx509) ++FIND_LIBRARY(MBEDX509_LIBRARY NAMES mbedx509 libmbedx509) ++FIND_LIBRARY(MBEDCRYPTO_LIBRARY NAMES mbedcrypto libmbedcrypto) ++ ++IF(MBEDTLS_INCLUDE_DIR AND MBEDTLS_LIBRARY AND MBEDX509_LIBRARY AND MBEDCRYPTO_LIBRARY) ++ SET(MBEDTLS_FOUND TRUE) ++ENDIF() ++ ++IF(MBEDTLS_FOUND) ++ # split mbedTLS into -L and -l linker options, so we can set them for pkg-config ++ GET_FILENAME_COMPONENT(MBEDTLS_LIBRARY_DIR ${MBEDTLS_LIBRARY} PATH) ++ GET_FILENAME_COMPONENT(MBEDTLS_LIBRARY_FILE ${MBEDTLS_LIBRARY} NAME_WE) ++ GET_FILENAME_COMPONENT(MBEDX509_LIBRARY_FILE ${MBEDX509_LIBRARY} NAME_WE) ++ GET_FILENAME_COMPONENT(MBEDCRYPTO_LIBRARY_FILE ${MBEDCRYPTO_LIBRARY} NAME_WE) ++ STRING(REGEX REPLACE "^lib" "" MBEDTLS_LIBRARY_FILE ${MBEDTLS_LIBRARY_FILE}) ++ STRING(REGEX REPLACE "^lib" "" MBEDX509_LIBRARY_FILE ${MBEDX509_LIBRARY_FILE}) ++ STRING(REGEX REPLACE "^lib" "" MBEDCRYPTO_LIBRARY_FILE ${MBEDCRYPTO_LIBRARY_FILE}) ++ SET(MBEDTLS_LIBRARIES "-L${MBEDTLS_LIBRARY_DIR} -l${MBEDTLS_LIBRARY_FILE} -l${MBEDX509_LIBRARY_FILE} -l${MBEDCRYPTO_LIBRARY_FILE}") ++ ++ IF(NOT MBEDTLS_FIND_QUIETLY) ++ MESSAGE(STATUS "Found mbedTLS:") ++ FILE(READ ${MBEDTLS_INCLUDE_DIR}/mbedtls/version.h MBEDTLSCONTENT) ++ STRING(REGEX MATCH "MBEDTLS_VERSION_STRING +\"[0-9|.]+\"" MBEDTLSMATCH ${MBEDTLSCONTENT}) ++ IF (MBEDTLSMATCH) ++ STRING(REGEX REPLACE "MBEDTLS_VERSION_STRING +\"([0-9|.]+)\"" "\\1" MBEDTLS_VERSION ${MBEDTLSMATCH}) ++ MESSAGE(STATUS " version ${MBEDTLS_VERSION}") ++ ENDIF(MBEDTLSMATCH) ++ MESSAGE(STATUS " TLS: ${MBEDTLS_LIBRARY}") ++ MESSAGE(STATUS " X509: ${MBEDX509_LIBRARY}") ++ MESSAGE(STATUS " Crypto: ${MBEDCRYPTO_LIBRARY}") ++ ENDIF(NOT MBEDTLS_FIND_QUIETLY) ++ELSE(MBEDTLS_FOUND) ++ IF(MBEDTLS_FIND_REQUIRED) ++ MESSAGE(FATAL_ERROR "Could not find mbedTLS") ++ ENDIF(MBEDTLS_FIND_REQUIRED) ++ENDIF(MBEDTLS_FOUND) ++ ++MARK_AS_ADVANCED( ++ MBEDTLS_INCLUDE_DIR ++ MBEDTLS_LIBRARY_DIR ++ MBEDTLS_LIBRARIES ++ MBEDTLS_LIBRARY ++ MBEDX509_LIBRARY ++ MBEDCRYPTO_LIBRARY ++) +diff --git a/mbedtls/buildmbedtls b/mbedtls/buildmbedtls +new file mode 100755 +index 0000000..2385a8b +--- /dev/null ++++ b/mbedtls/buildmbedtls +@@ -0,0 +1,18 @@ ++#!/bin/sh ++set -ex ++ ++MBEDTLS_SRC_ARCH=mbedtls-2.2.1 ++ ++# download mbedtls sources ++cd mbedtls ++wget https://github.com/ARMmbed/mbedtls/archive/$MBEDTLS_SRC_ARCH.tar.gz ++tar -xzf $MBEDTLS_SRC_ARCH.tar.gz ++ ++# patch mbedtls configuration ++cd mbedtls-$MBEDTLS_SRC_ARCH ++patch -p0 < ../mbedtls.patch ++ ++# build library ++cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ ++ -DCMAKE_INSTALL_PREFIX:PATH=../install ++make install +diff --git a/mbedtls/mbedtls.patch b/mbedtls/mbedtls.patch +new file mode 100644 +index 0000000..1f774ec +--- /dev/null ++++ b/mbedtls/mbedtls.patch +@@ -0,0 +1,27 @@ ++diff --git include/mbedtls/config.h include/mbedtls/config.h ++index 0efee04..787cd8c 100644 ++--- include/mbedtls/config.h +++++ include/mbedtls/config.h ++@@ -2434,19 +2434,19 @@ ++ ++ /* MPI / BIGNUM options */ ++ //#define MBEDTLS_MPI_WINDOW_SIZE 6 /**< Maximum windows size used. */ ++-//#define MBEDTLS_MPI_MAX_SIZE 1024 /**< Maximum number of bytes for usable MPIs. */ +++#define MBEDTLS_MPI_MAX_SIZE 2048 /**< Maximum number of bytes for usable MPIs. */ ++ ++ /* CTR_DRBG options */ ++ //#define MBEDTLS_CTR_DRBG_ENTROPY_LEN 48 /**< Amount of entropy used per seed by default (48 with SHA-512, 32 with SHA-256) */ ++ //#define MBEDTLS_CTR_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ ++ //#define MBEDTLS_CTR_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ ++-//#define MBEDTLS_CTR_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ +++#define MBEDTLS_CTR_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ ++ //#define MBEDTLS_CTR_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ ++ ++ /* HMAC_DRBG options */ ++ //#define MBEDTLS_HMAC_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ ++ //#define MBEDTLS_HMAC_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ ++-//#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ +++#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ ++ //#define MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ ++ ++ /* ECP options */ +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index c29a2d8..987dfe0 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -48,7 +48,7 @@ set(CRYPTO_BACKEND + "" + CACHE + STRING +- "The backend to use for cryptography: OpenSSL, Libgcrypt or WinCNG, ++ "The backend to use for cryptography: OpenSSL, Libgcrypt or WinCNG, mbedTLS + or empty to try any available") + + # If the crypto backend was given, rather than searching for the first +@@ -151,6 +151,21 @@ if(CRYPTO_BACKEND STREQUAL "WinCNG" OR NOT CRYPTO_BACKEND) + endif() + endif() + ++if(CRYPTO_BACKEND STREQUAL "mbedTLS" OR NOT CRYPTO_BACKEND) ++ ++ find_package(mbedTLS ${SPECIFIC_CRYPTO_REQUIREMENT}) ++ ++ if(MBEDTLS_FOUND) ++ set(CRYPTO_BACKEND "mbedTLS") ++ set(CRYPTO_SOURCES mbedtls.c mbedtls.h) ++ list(APPEND PRIVATE_COMPILE_DEFINITIONS LIBSSH2_MBEDTLS) ++ list(APPEND PRIVATE_INCLUDE_DIRECTORIES ${MBEDTLS_INCLUDE_DIR}) ++ list(APPEND LIBRARIES ${MBEDTLS_LIBRARIES}) ++ list(APPEND PC_LIBS -lmbedcrypto) ++ link_directories(${MBEDTLS_LIBRARY_DIR}) ++ endif() ++endif() ++ + if(NOT CRYPTO_BACKEND) + message(FATAL_ERROR "No suitable cryptography backend found.") + endif() +@@ -209,6 +224,11 @@ target_include_directories(libssh2 + + ## Options + ++option(CLEAR_MEMORY "Enable clearing of memory before being freed" OFF) ++if(CLEAR_MEMORY) ++ add_definitions(-DLIBSSH2_CLEAR_MEMORY) ++endif(CLEAR_MEMORY) ++ + add_feature_info("Shared library" BUILD_SHARED_LIBS + "creating libssh2 as a shared library (.so/.dll)") + +diff --git a/src/crypto.h b/src/crypto.h +index caad19f..aa997a3 100644 +--- a/src/crypto.h ++++ b/src/crypto.h +@@ -54,6 +54,10 @@ + #include "os400qc3.h" + #endif + ++#ifdef LIBSSH2_MBEDTLS ++#include "mbedtls.h" ++#endif ++ + int _libssh2_rsa_new(libssh2_rsa_ctx ** rsa, + const unsigned char *edata, + unsigned long elen, +diff --git a/src/mbedtls.c b/src/mbedtls.c +new file mode 100644 +index 0000000..98bc549 +--- /dev/null ++++ b/src/mbedtls.c +@@ -0,0 +1,570 @@ ++#include "libssh2_priv.h" ++ ++#ifdef LIBSSH2_MBEDTLS /* compile only if we build with mbedtls */ ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: Generic functions ++ */ ++ ++void ++_libssh2_mbedtls_init(void) ++{ ++ int ret; ++ ++ mbedtls_entropy_init(&_libssh2_mbedtls_entropy); ++ mbedtls_ctr_drbg_init(&_libssh2_mbedtls_ctr_drbg); ++ ++ ret = mbedtls_ctr_drbg_seed(&_libssh2_mbedtls_ctr_drbg, ++ mbedtls_entropy_func, ++ &_libssh2_mbedtls_entropy, NULL, 0); ++ if (ret != 0) ++ mbedtls_ctr_drbg_free(&_libssh2_mbedtls_ctr_drbg); ++} ++ ++void ++_libssh2_mbedtls_free(void) ++{ ++ mbedtls_ctr_drbg_free(&_libssh2_mbedtls_ctr_drbg); ++ mbedtls_entropy_free(&_libssh2_mbedtls_entropy); ++} ++ ++int ++_libssh2_mbedtls_random(unsigned char *buf, int len) ++{ ++ int ret; ++ ret = mbedtls_ctr_drbg_random(&_libssh2_mbedtls_ctr_drbg, buf, len); ++ return ret == 0 ? 0 : -1; ++} ++ ++static void ++_libssh2_mbedtls_safe_free(void *buf, int len) ++{ ++#ifndef LIBSSH2_CLEAR_MEMORY ++ (void)len; ++#endif ++ ++ if (!buf) ++ return; ++ ++#ifdef LIBSSH2_CLEAR_MEMORY ++ if (len > 0) ++ memset(buf, 0, len); ++#endif ++ ++ mbedtls_free(buf); ++} ++ ++int ++_libssh2_mbedtls_cipher_init(_libssh2_cipher_ctx *ctx, ++ _libssh2_cipher_type(algo), ++ unsigned char *iv, ++ unsigned char *secret, ++ int encrypt) ++{ ++ const mbedtls_cipher_info_t *cipher_info; ++ int ret, op; ++ ++ if (!ctx) ++ return -1; ++ ++ op = encrypt == 0 ? MBEDTLS_ENCRYPT : MBEDTLS_DECRYPT; ++ ++ cipher_info = mbedtls_cipher_info_from_type(algo); ++ if(!cipher_info) ++ return -1; ++ ++ mbedtls_cipher_init(ctx); ++ ret = mbedtls_cipher_setup(ctx, cipher_info); ++ if(!ret) ++ ret = mbedtls_cipher_setkey(ctx, secret, cipher_info->key_bitlen, op); ++ ++ if(!ret) ++ ret = mbedtls_cipher_set_iv(ctx, iv, cipher_info->iv_size); ++ ++ return ret == 0 ? 0 : -1; ++} ++ ++int ++_libssh2_mbedtls_cipher_crypt(_libssh2_cipher_ctx *ctx, ++ _libssh2_cipher_type(algo), ++ int encrypt, ++ unsigned char *block, ++ size_t blocklen) ++{ ++ int ret; ++ unsigned char *output; ++ size_t osize, olen, finish_olen; ++ ++ (void) encrypt; ++ (void) algo; ++ ++ osize = blocklen+mbedtls_cipher_get_block_size(ctx); ++ ++ output = (unsigned char *)mbedtls_calloc(osize, sizeof(char)); ++ if(output) ++ { ++ ret = mbedtls_cipher_reset(ctx); ++ ++ if(!ret) ++ ret = mbedtls_cipher_update(ctx, block, blocklen, output, &olen); ++ ++ if(!ret) ++ ret = mbedtls_cipher_finish(ctx, output + olen, &finish_olen); ++ ++ olen += finish_olen; ++ ++ if (!ret) ++ memcpy(block, output, olen); ++ ++ _libssh2_mbedtls_safe_free(output, osize); ++ } ++ else ++ ret = -1; ++ ++ return ret == 0 ? 0 : -1; ++} ++ ++void ++_libssh2_mbedtls_cipher_dtor(_libssh2_cipher_ctx *ctx) ++{ ++ mbedtls_cipher_free(ctx); ++} ++ ++ ++int ++_libssh2_mbedtls_hash_init(mbedtls_md_context_t *ctx, ++ mbedtls_md_type_t mdtype, ++ const unsigned char *key, unsigned long keylen) ++{ ++ const mbedtls_md_info_t *md_info; ++ int ret, hmac; ++ ++ md_info = mbedtls_md_info_from_type(mdtype); ++ hmac = key == NULL ? 0 : 1; ++ ++ mbedtls_md_init(ctx); ++ ret = mbedtls_md_setup(ctx, md_info, hmac); ++ if (!ret){ ++ if (hmac) ++ ret = mbedtls_md_hmac_starts(ctx, key, keylen); ++ else ++ ret = mbedtls_md_starts(ctx); ++ } ++ ++ return ret == 0 ? 1 : 0; ++} ++ ++int ++_libssh2_mbedtls_hash_final(mbedtls_md_context_t *ctx, unsigned char *hash) ++{ ++ int ret; ++ ++ ret = mbedtls_md_finish(ctx, hash); ++ mbedtls_md_free(ctx); ++ ++ return ret == 0 ? 0 : -1; ++} ++ ++int ++_libssh2_mbedtls_hash(const unsigned char *data, unsigned long datalen, ++ mbedtls_md_type_t mdtype, unsigned char *hash) ++{ ++ const mbedtls_md_info_t *md_info; ++ int ret; ++ ++ md_info = mbedtls_md_info_from_type(mdtype); ++ ret = mbedtls_md(md_info, data, datalen, hash); ++ ++ return ret == 0 ? 0 : -1; ++} ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: BigNumber functions ++ */ ++ ++_libssh2_bn * ++_libssh2_mbedtls_bignum_init(void) ++{ ++ _libssh2_bn *bignum; ++ ++ bignum = (_libssh2_bn *)mbedtls_calloc(1, sizeof(_libssh2_bn)); ++ if (bignum) { ++ mbedtls_mpi_init(bignum); ++ } ++ ++ return bignum; ++} ++ ++void ++_libssh2_mbedtls_bignum_free(_libssh2_bn *bn) ++{ ++ if (bn) ++ { ++ mbedtls_mpi_free(bn); ++#ifdef LIBSSH2_CLEAR_MEMORY ++ memset(bn, 0, sizeof(_libssh2_bn)); ++#endif ++ } ++ mbedtls_free(bn); ++} ++ ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: RSA functions ++ */ ++ ++int ++_libssh2_mbedtls_rsa_new(libssh2_rsa_ctx **rsa, ++ const unsigned char *edata, ++ unsigned long elen, ++ const unsigned char *ndata, ++ unsigned long nlen, ++ const unsigned char *ddata, ++ unsigned long dlen, ++ const unsigned char *pdata, ++ unsigned long plen, ++ const unsigned char *qdata, ++ unsigned long qlen, ++ const unsigned char *e1data, ++ unsigned long e1len, ++ const unsigned char *e2data, ++ unsigned long e2len, ++ const unsigned char *coeffdata, ++ unsigned long coefflen) ++{ ++ int ret; ++ libssh2_rsa_ctx *ctx; ++ ++ ctx = (libssh2_rsa_ctx *) mbedtls_calloc(1, sizeof(libssh2_rsa_ctx)); ++ if (ctx != NULL) { ++ mbedtls_rsa_init(ctx, MBEDTLS_RSA_PKCS_V15, 0); ++ } ++ else ++ return -1; ++ ++ if( (ret = mbedtls_mpi_read_binary(&(ctx->E), edata, elen) ) != 0 || ++ (ret = mbedtls_mpi_read_binary(&(ctx->N), ndata, nlen) ) != 0 ) ++ { ++ ret = -1; ++ } ++ ++ if (!ret) ++ { ++ ctx->len = mbedtls_mpi_size(&(ctx->N)); ++ } ++ ++ if (!ret && ddata) ++ { ++ if( (ret = mbedtls_mpi_read_binary(&(ctx->D) , ddata, dlen) ) != 0 || ++ (ret = mbedtls_mpi_read_binary(&(ctx->P) , pdata, plen) ) != 0 || ++ (ret = mbedtls_mpi_read_binary(&(ctx->Q) , qdata, qlen) ) != 0 || ++ (ret = mbedtls_mpi_read_binary(&(ctx->DP), e1data, e1len) ) != 0 || ++ (ret = mbedtls_mpi_read_binary(&(ctx->DQ), e2data, e2len) ) != 0 || ++ (ret = mbedtls_mpi_read_binary(&(ctx->QP), coeffdata, coefflen) ) != 0 ) ++ { ++ ret = -1; ++ } ++ ret = mbedtls_rsa_check_privkey(ctx); ++ } ++ else if (!ret) ++ { ++ ret = mbedtls_rsa_check_pubkey(ctx); ++ } ++ ++ if (ret && ctx) { ++ _libssh2_mbedtls_rsa_free(ctx); ++ ctx = NULL; ++ } ++ *rsa = ctx; ++ return ret; ++} ++ ++int ++_libssh2_mbedtls_rsa_new_private(libssh2_rsa_ctx **rsa, ++ LIBSSH2_SESSION *session, ++ const char *filename, ++ const unsigned char *passphrase) ++{ ++ int ret; ++ mbedtls_pk_context pkey; ++ ++ *rsa = (libssh2_rsa_ctx *) LIBSSH2_ALLOC(session, sizeof(libssh2_rsa_ctx)); ++ if (*rsa == NULL) ++ return -1; ++ ++ mbedtls_rsa_init(*rsa, MBEDTLS_RSA_PKCS_V15, 0); ++ mbedtls_pk_init(&pkey); ++ ++ ret = mbedtls_pk_parse_keyfile(&pkey, filename, (char *)passphrase); ++ if( ret != 0 ) ++ { ++ mbedtls_pk_free(&pkey); ++ mbedtls_rsa_free(*rsa); ++ LIBSSH2_FREE(session, *rsa); ++ *rsa = NULL; ++ return -1; ++ } ++ ++ mbedtls_rsa_context *pk_rsa = mbedtls_pk_rsa(pkey); ++ mbedtls_rsa_copy(*rsa, pk_rsa); ++ mbedtls_pk_free(&pkey); ++ ++ return 0; ++} ++ ++int ++_libssh2_mbedtls_rsa_new_private_frommemory(libssh2_rsa_ctx **rsa, ++ LIBSSH2_SESSION *session, ++ const char *filedata, ++ size_t filedata_len, ++ unsigned const char *passphrase) ++{ ++ int ret; ++ mbedtls_pk_context pkey; ++ ++ *rsa = (libssh2_rsa_ctx *) mbedtls_calloc( 1, sizeof( libssh2_rsa_ctx ) ); ++ if (*rsa == NULL) ++ return -1; ++ ++ mbedtls_pk_init(&pkey); ++ ++ ret = mbedtls_pk_parse_key(&pkey, (unsigned char *)filedata, ++ filedata_len, NULL, 0); ++ if( ret != 0 ) ++ { ++ mbedtls_pk_free(&pkey); ++ mbedtls_rsa_free(*rsa); ++ LIBSSH2_FREE(session, *rsa); ++ *rsa = NULL; ++ return -1; ++ } ++ ++ mbedtls_rsa_context *pk_rsa = mbedtls_pk_rsa(pkey); ++ mbedtls_rsa_copy(*rsa, pk_rsa); ++ mbedtls_pk_free(&pkey); ++ ++ return 0; ++} ++ ++int ++_libssh2_mbedtls_rsa_sha1_verify(libssh2_rsa_ctx *rsa, ++ const unsigned char *sig, ++ unsigned long sig_len, ++ const unsigned char *m, ++ unsigned long m_len) ++{ ++ unsigned char hash[SHA_DIGEST_LENGTH]; ++ int ret; ++ ++ ret = _libssh2_mbedtls_hash(m, m_len, MBEDTLS_MD_SHA1, hash); ++ if(ret) ++ return -1; /* failure */ ++ ++ ret = mbedtls_rsa_pkcs1_verify(rsa, NULL, NULL, MBEDTLS_RSA_PUBLIC, ++ MBEDTLS_MD_SHA1, sig_len, hash, sig); ++ ++ return (ret == 0) ? 0 : -1; ++} ++ ++int ++_libssh2_mbedtls_rsa_sha1_sign(LIBSSH2_SESSION *session, ++ libssh2_rsa_ctx *rsa, ++ const unsigned char *hash, ++ size_t hash_len, ++ unsigned char **signature, ++ size_t *signature_len) ++{ ++ int ret; ++ unsigned char *sig; ++ unsigned int sig_len; ++ ++ (void)hash_len; ++ ++ sig_len = rsa->len; ++ sig = LIBSSH2_ALLOC(session, sig_len); ++ if (!sig) { ++ return -1; ++ } ++ ++ ret = mbedtls_rsa_pkcs1_sign(rsa, NULL, NULL, MBEDTLS_RSA_PRIVATE, ++ MBEDTLS_MD_SHA1, SHA_DIGEST_LENGTH, ++ hash, sig); ++ if (ret) { ++ LIBSSH2_FREE(session, sig); ++ return -1; ++ } ++ ++ *signature = sig; ++ *signature_len = sig_len; ++ ++ return (ret == 0) ? 0 : -1; ++} ++ ++void ++_libssh2_mbedtls_rsa_free(libssh2_rsa_ctx *ctx) ++{ ++ mbedtls_rsa_free(ctx); ++ mbedtls_free(ctx); ++} ++ ++static unsigned char * ++gen_publickey_from_rsa(LIBSSH2_SESSION *session, ++ mbedtls_rsa_context *rsa, ++ size_t *keylen) ++{ ++ int e_bytes, n_bytes; ++ unsigned long len; ++ unsigned char* key; ++ unsigned char* p; ++ ++ e_bytes = mbedtls_mpi_size(&rsa->E); ++ n_bytes = mbedtls_mpi_size(&rsa->N); ++ ++ /* Key form is "ssh-rsa" + e + n. */ ++ len = 4 + 7 + 4 + e_bytes + 4 + n_bytes; ++ ++ key = LIBSSH2_ALLOC(session, len); ++ if (!key) { ++ return NULL; ++ } ++ ++ /* Process key encoding. */ ++ p = key; ++ ++ _libssh2_htonu32(p, 7); /* Key type. */ ++ p += 4; ++ memcpy(p, "ssh-rsa", 7); ++ p += 7; ++ ++ _libssh2_htonu32(p, e_bytes); ++ p += 4; ++ mbedtls_mpi_write_binary(&rsa->E, p, e_bytes); ++ ++ _libssh2_htonu32(p, n_bytes); ++ p += 4; ++ mbedtls_mpi_write_binary(&rsa->N, p, n_bytes); ++ ++ *keylen = (size_t)(p - key); ++ return key; ++} ++ ++static int ++_libssh2_mbedtls_pub_priv_key(LIBSSH2_SESSION *session, ++ unsigned char **method, ++ size_t *method_len, ++ unsigned char **pubkeydata, ++ size_t *pubkeydata_len, ++ mbedtls_pk_context *pkey) ++{ ++ unsigned char *key = NULL, *mth = NULL; ++ size_t keylen = 0, mthlen = 0; ++ int ret; ++ ++ if( mbedtls_pk_get_type(pkey) != MBEDTLS_PK_RSA ) ++ { ++ mbedtls_pk_free(pkey); ++ return _libssh2_error(session, LIBSSH2_ERROR_FILE, ++ "Key type not supported"); ++ } ++ ++ // write method ++ mthlen = 7; ++ mth = LIBSSH2_ALLOC(session, mthlen); ++ if (mth) { ++ memcpy(mth, "ssh-rsa", mthlen); ++ } else { ++ ret = -1; ++ } ++ ++ mbedtls_rsa_context *rsa = mbedtls_pk_rsa(*pkey); ++ key = gen_publickey_from_rsa(session, rsa, &keylen); ++ if (key == NULL) { ++ ret = -1; ++ } ++ ++ // write output ++ if (ret) { ++ if (mth) ++ LIBSSH2_FREE(session, mth); ++ if (key) ++ LIBSSH2_FREE(session, key); ++ } else { ++ *method = mth; ++ *method_len = mthlen; ++ *pubkeydata = key; ++ *pubkeydata_len = keylen; ++ } ++ ++ return ret; ++} ++ ++int ++_libssh2_mbedtls_pub_priv_keyfile(LIBSSH2_SESSION *session, ++ unsigned char **method, ++ size_t *method_len, ++ unsigned char **pubkeydata, ++ size_t *pubkeydata_len, ++ const char *privatekey, ++ const char *passphrase) ++{ ++ mbedtls_pk_context pkey; ++ char buf[1024]; ++ int ret; ++ ++ mbedtls_pk_init(&pkey); ++ ret = mbedtls_pk_parse_keyfile(&pkey, privatekey, passphrase); ++ if( ret != 0 ) ++ { ++ mbedtls_strerror(ret, (char *)buf, sizeof(buf)); ++ mbedtls_pk_free(&pkey); ++ return _libssh2_error(session, LIBSSH2_ERROR_FILE, buf); ++ } ++ ++ ret = _libssh2_mbedtls_pub_priv_key(session, method, method_len, ++ pubkeydata, pubkeydata_len, &pkey); ++ ++ mbedtls_pk_free(&pkey); ++ ++ return ret; ++} ++ ++int ++_libssh2_mbedtls_pub_priv_keyfilememory(LIBSSH2_SESSION *session, ++ unsigned char **method, ++ size_t *method_len, ++ unsigned char **pubkeydata, ++ size_t *pubkeydata_len, ++ const char *privatekeydata, ++ size_t privatekeydata_len, ++ const char *passphrase) ++{ ++ mbedtls_pk_context pkey; ++ char buf[1024]; ++ int ret; ++ ++ mbedtls_pk_init(&pkey); ++ ret = mbedtls_pk_parse_key(&pkey, (unsigned char *)privatekeydata, ++ privatekeydata_len, NULL, 0); ++ if( ret != 0 ) ++ { ++ mbedtls_strerror(ret, (char *)buf, sizeof(buf)); ++ mbedtls_pk_free(&pkey); ++ return _libssh2_error(session, LIBSSH2_ERROR_FILE, buf); ++ } ++ ++ ret = _libssh2_mbedtls_pub_priv_key(session, method, method_len, ++ pubkeydata, pubkeydata_len, &pkey); ++ ++ mbedtls_pk_free(&pkey); ++ ++ return ret; ++} ++ ++void _libssh2_init_aes_ctr(void) ++{ ++ /* no implementation */ ++} ++#endif /* LIBSSH2_MBEDTLS */ +diff --git a/src/mbedtls.h b/src/mbedtls.h +new file mode 100644 +index 0000000..f594575 +--- /dev/null ++++ b/src/mbedtls.h +@@ -0,0 +1,368 @@ ++#include <stdlib.h> ++#include <string.h> ++ ++#include <mbedtls/platform.h> ++#include <mbedtls/md.h> ++#include <mbedtls/rsa.h> ++#include <mbedtls/bignum.h> ++#include <mbedtls/cipher.h> ++#include <mbedtls/entropy.h> ++#include <mbedtls/ctr_drbg.h> ++#include <mbedtls/pk.h> ++#include <mbedtls/error.h> ++ ++/* Define which features are supported. */ ++#define LIBSSH2_MD5 1 ++ ++#define LIBSSH2_HMAC_RIPEMD 1 ++#define LIBSSH2_HMAC_SHA256 1 ++#define LIBSSH2_HMAC_SHA512 1 ++ ++#define LIBSSH2_AES 1 ++#define LIBSSH2_AES_CTR 1 ++#define LIBSSH2_BLOWFISH 1 ++#define LIBSSH2_RC4 1 ++#define LIBSSH2_CAST 0 ++#define LIBSSH2_3DES 1 ++ ++#define LIBSSH2_RSA 1 ++#define LIBSSH2_DSA 0 ++ ++#define MD5_DIGEST_LENGTH 16 ++#define SHA_DIGEST_LENGTH 20 ++#define SHA256_DIGEST_LENGTH 32 ++#define SHA512_DIGEST_LENGTH 64 ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: Global context handles ++ */ ++ ++mbedtls_entropy_context _libssh2_mbedtls_entropy; ++mbedtls_ctr_drbg_context _libssh2_mbedtls_ctr_drbg; ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: Generic functions ++ */ ++ ++#define libssh2_crypto_init() \ ++ _libssh2_mbedtls_init() ++#define libssh2_crypto_exit() \ ++ _libssh2_mbedtls_free() ++ ++#define _libssh2_random(buf, len) \ ++ _libssh2_mbedtls_random(buf, len) ++ ++#define libssh2_prepare_iovec(vec, len) /* Empty. */ ++ ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: HMAC functions ++ */ ++ ++#define libssh2_hmac_ctx mbedtls_md_context_t ++ ++#define libssh2_hmac_ctx_init(ctx) ++#define libssh2_hmac_cleanup(pctx) \ ++ mbedtls_md_free(pctx) ++#define libssh2_hmac_update(ctx, data, datalen) \ ++ mbedtls_md_hmac_update(&ctx, (unsigned char *) data, datalen) ++#define libssh2_hmac_final(ctx, hash) \ ++ mbedtls_md_hmac_finish(&ctx, hash) ++ ++#define libssh2_hmac_sha1_init(pctx, key, keylen) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_SHA1, key, keylen) ++#define libssh2_hmac_md5_init(pctx, key, keylen) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_MD5, key, keylen) ++#define libssh2_hmac_ripemd160_init(pctx, key, keylen) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_RIPEMD160, key, keylen) ++#define libssh2_hmac_sha256_init(pctx, key, keylen) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_SHA256, key, keylen) ++#define libssh2_hmac_sha512_init(pctx, key, keylen) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_SHA512, key, keylen) ++ ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: SHA1 functions ++ */ ++ ++#define libssh2_sha1_ctx mbedtls_md_context_t ++ ++#define libssh2_sha1_init(pctx) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_SHA1, NULL, 0) ++#define libssh2_sha1_update(ctx, data, datalen) \ ++ mbedtls_md_update(&ctx, (unsigned char *) data, datalen) ++#define libssh2_sha1_final(ctx, hash) \ ++ _libssh2_mbedtls_hash_final(&ctx, hash) ++#define libssh2_sha1(data, datalen, hash) \ ++ _libssh2_mbedtls_hash(data, datalen, MBEDTLS_MD_SHA1, hash) ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: SHA256 functions ++ */ ++ ++#define libssh2_sha256_ctx mbedtls_md_context_t ++ ++#define libssh2_sha256_init(pctx) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_SHA256, NULL, 0) ++#define libssh2_sha256_update(ctx, data, datalen) \ ++ mbedtls_md_update(&ctx, (unsigned char *) data, datalen) ++#define libssh2_sha256_final(ctx, hash) \ ++ _libssh2_mbedtls_hash_final(&ctx, hash) ++#define libssh2_sha256(data, datalen, hash) \ ++ _libssh2_mbedtls_hash(data, datalen, MBEDTLS_MD_SHA256, hash) ++ ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: SHA512 functions ++ */ ++ ++#define libssh2_sha512_ctx mbedtls_md_context_t ++ ++#define libssh2_sha512_init(pctx) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_SHA512, NULL, 0) ++#define libssh2_sha512_update(ctx, data, datalen) \ ++ mbedtls_md_update(&ctx, (unsigned char *) data, datalen) ++#define libssh2_sha512_final(ctx, hash) \ ++ _libssh2_mbedtls_hash_final(&ctx, hash) ++#define libssh2_sha512(data, datalen, hash) \ ++ _libssh2_mbedtls_hash(data, datalen, MBEDTLS_MD_SHA512, hash) ++ ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: MD5 functions ++ */ ++ ++#define libssh2_md5_ctx mbedtls_md_context_t ++ ++#define libssh2_md5_init(pctx) \ ++ _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_MD5, NULL, 0) ++#define libssh2_md5_update(ctx, data, datalen) \ ++ mbedtls_md_update(&ctx, (unsigned char *) data, datalen) ++#define libssh2_md5_final(ctx, hash) \ ++ _libssh2_mbedtls_hash_final(&ctx, hash) ++#define libssh2_md5(data, datalen, hash) \ ++ _libssh2_mbedtls_hash(data, datalen, MBEDTLS_MD_MD5, hash) ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: RSA structure ++ */ ++ ++#define libssh2_rsa_ctx mbedtls_rsa_context ++ ++#define _libssh2_rsa_new(rsactx, e, e_len, n, n_len, \ ++ d, d_len, p, p_len, q, q_len, \ ++ e1, e1_len, e2, e2_len, c, c_len) \ ++ _libssh2_mbedtls_rsa_new(rsactx, e, e_len, n, n_len, \ ++ d, d_len, p, p_len, q, q_len, \ ++ e1, e1_len, e2, e2_len, c, c_len) ++ ++#define _libssh2_rsa_new_private(rsactx, s, filename, passphrase) \ ++ _libssh2_mbedtls_rsa_new_private(rsactx, s, filename, passphrase) ++ ++#define _libssh2_rsa_new_private_frommemory(rsactx, s, filedata, \ ++ filedata_len, passphrase) \ ++ _libssh2_mbedtls_rsa_new_private_frommemory(rsactx, s, filedata, \ ++ filedata_len, passphrase) ++ ++#define _libssh2_rsa_sha1_sign(s, rsactx, hash, hash_len, sig, sig_len) \ ++ _libssh2_mbedtls_rsa_sha1_sign(s, rsactx, hash, hash_len, sig, sig_len) ++ ++#define _libssh2_rsa_sha1_verify(rsactx, sig, sig_len, m, m_len) \ ++ _libssh2_mbedtls_rsa_sha1_verify(rsactx, sig, sig_len, m, m_len) ++ ++#define _libssh2_rsa_free(rsactx) \ ++ _libssh2_mbedtls_rsa_free(rsactx) ++ ++/* ++ * mbedTLS backend: Key functions ++ */ ++ ++#define _libssh2_pub_priv_keyfile(s, m, m_len, p, p_len, pk, pw) \ ++ _libssh2_mbedtls_pub_priv_keyfile(s, m, m_len, p, p_len, pk, pw) ++#define _libssh2_pub_priv_keyfilememory(s, m, m_len, p, p_len, \ ++ pk, pk_len, pw) \ ++ _libssh2_mbedtls_pub_priv_keyfilememory(s, m, m_len, p, p_len, \ ++ pk, pk_len, pw) ++ ++ ++ /*******************************************************************/ ++/* ++ * mbedTLS backend: Cipher Context structure ++ */ ++#define _libssh2_cipher_ctx mbedtls_cipher_context_t ++ ++#define _libssh2_cipher_type(algo) mbedtls_cipher_type_t algo ++ ++#define _libssh2_cipher_aes256ctr MBEDTLS_CIPHER_AES_256_CTR ++#define _libssh2_cipher_aes192ctr MBEDTLS_CIPHER_AES_192_CTR ++#define _libssh2_cipher_aes128ctr MBEDTLS_CIPHER_AES_128_CTR ++#define _libssh2_cipher_aes256 MBEDTLS_CIPHER_AES_256_CBC ++#define _libssh2_cipher_aes192 MBEDTLS_CIPHER_AES_192_CBC ++#define _libssh2_cipher_aes128 MBEDTLS_CIPHER_AES_128_CBC ++#define _libssh2_cipher_blowfish MBEDTLS_CIPHER_BLOWFISH_CBC ++#define _libssh2_cipher_arcfour MBEDTLS_CIPHER_ARC4_128 ++#define _libssh2_cipher_cast5 MBEDTLS_CIPHER_NULL ++#define _libssh2_cipher_3des MBEDTLS_CIPHER_DES_EDE3_CBC ++ ++/* ++ * mbedTLS backend: Cipher functions ++ */ ++ ++#define _libssh2_cipher_init(ctx, type, iv, secret, encrypt) \ ++ _libssh2_mbedtls_cipher_init(ctx, type, iv, secret, encrypt) ++#define _libssh2_cipher_crypt(ctx, type, encrypt, block, blocklen) \ ++ _libssh2_mbedtls_cipher_crypt(ctx, type, encrypt, block, blocklen) ++#define _libssh2_cipher_dtor(ctx) \ ++ _libssh2_mbedtls_cipher_dtor(ctx) ++ ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: BigNumber Support ++ */ ++ ++#define _libssh2_bn_ctx int /* not used */ ++#define _libssh2_bn_ctx_new() 0 /* not used */ ++#define _libssh2_bn_ctx_free(bnctx) ((void)0) /* not used */ ++ ++#define _libssh2_bn mbedtls_mpi ++ ++#define _libssh2_bn_init() \ ++ _libssh2_mbedtls_bignum_init() ++#define _libssh2_bn_init_from_bin() \ ++ _libssh2_mbedtls_bignum_init() ++#define _libssh2_bn_rand(bn, bytes, top, bottom) \ ++ mbedtls_mpi_fill_random(bn, bytes, mbedtls_ctr_drbg_random, &_libssh2_mbedtls_ctr_drbg) ++#define _libssh2_bn_mod_exp(r, a, p, m, ctx) \ ++ mbedtls_mpi_exp_mod(r, a, p, m, NULL) ++#define _libssh2_bn_set_word(bn, word) \ ++ mbedtls_mpi_lset(bn, word) ++#define _libssh2_bn_from_bin(bn, len, bin) \ ++ mbedtls_mpi_read_binary(bn, bin, len) ++#define _libssh2_bn_to_bin(bn, bin) \ ++ mbedtls_mpi_write_binary(bn, bin, mbedtls_mpi_size(bn)) ++#define _libssh2_bn_bytes(bn) \ ++ mbedtls_mpi_size(bn) ++#define _libssh2_bn_bits(bn) \ ++ mbedtls_mpi_bitlen(bn) ++#define _libssh2_bn_free(bn) \ ++ _libssh2_mbedtls_bignum_free(bn) ++ ++ ++/*******************************************************************/ ++/* ++ * mbedTLS backend: forward declarations ++ */ ++void ++_libssh2_mbedtls_init(void); ++ ++void ++_libssh2_mbedtls_free(void); ++ ++int ++_libssh2_mbedtls_random(unsigned char *buf, int len); ++ ++int ++_libssh2_mbedtls_cipher_init(_libssh2_cipher_ctx *ctx, ++ _libssh2_cipher_type(type), ++ unsigned char *iv, ++ unsigned char *secret, ++ int encrypt); ++int ++_libssh2_mbedtls_cipher_crypt(_libssh2_cipher_ctx *ctx, ++ _libssh2_cipher_type(type), ++ int encrypt, ++ unsigned char *block, ++ size_t blocklen); ++void ++_libssh2_mbedtls_cipher_dtor(_libssh2_cipher_ctx *ctx); ++ ++int ++_libssh2_mbedtls_hash_init(mbedtls_md_context_t *ctx, ++ mbedtls_md_type_t mdtype, ++ const unsigned char *key, unsigned long keylen); ++ ++int ++_libssh2_mbedtls_hash_final(mbedtls_md_context_t *ctx, unsigned char *hash); ++int ++_libssh2_mbedtls_hash(const unsigned char *data, unsigned long datalen, ++ mbedtls_md_type_t mdtype, unsigned char *hash); ++ ++_libssh2_bn * ++_libssh2_mbedtls_bignum_init(void); ++ ++void ++_libssh2_mbedtls_bignum_free(_libssh2_bn *bn); ++ ++int ++_libssh2_mbedtls_rsa_new(libssh2_rsa_ctx **rsa, ++ const unsigned char *edata, ++ unsigned long elen, ++ const unsigned char *ndata, ++ unsigned long nlen, ++ const unsigned char *ddata, ++ unsigned long dlen, ++ const unsigned char *pdata, ++ unsigned long plen, ++ const unsigned char *qdata, ++ unsigned long qlen, ++ const unsigned char *e1data, ++ unsigned long e1len, ++ const unsigned char *e2data, ++ unsigned long e2len, ++ const unsigned char *coeffdata, ++ unsigned long coefflen); ++ ++int ++_libssh2_mbedtls_rsa_new_private(libssh2_rsa_ctx **rsa, ++ LIBSSH2_SESSION *session, ++ const char *filename, ++ const unsigned char *passphrase); ++ ++int ++_libssh2_mbedtls_rsa_new_private_frommemory(libssh2_rsa_ctx **rsa, ++ LIBSSH2_SESSION *session, ++ const char *filedata, ++ size_t filedata_len, ++ unsigned const char *passphrase); ++int ++_libssh2_mbedtls_rsa_sha1_verify(libssh2_rsa_ctx *rsa, ++ const unsigned char *sig, ++ unsigned long sig_len, ++ const unsigned char *m, ++ unsigned long m_len); ++int ++_libssh2_mbedtls_rsa_sha1_sign(LIBSSH2_SESSION *session, ++ libssh2_rsa_ctx *rsa, ++ const unsigned char *hash, ++ size_t hash_len, ++ unsigned char **signature, ++ size_t *signature_len); ++void ++_libssh2_mbedtls_rsa_free(libssh2_rsa_ctx *rsa); ++ ++int ++_libssh2_mbedtls_pub_priv_keyfile(LIBSSH2_SESSION *session, ++ unsigned char **method, ++ size_t *method_len, ++ unsigned char **pubkeydata, ++ size_t *pubkeydata_len, ++ const char *privatekey, ++ const char *passphrase); ++int ++_libssh2_mbedtls_pub_priv_keyfilememory(LIBSSH2_SESSION *session, ++ unsigned char **method, ++ size_t *method_len, ++ unsigned char **pubkeydata, ++ size_t *pubkeydata_len, ++ const char *privatekeydata, ++ size_t privatekeydata_len, ++ const char *passphrase); From 4ea698e30df06f370aa83204d5d4a67f7c7e88d6 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Tue, 12 Jul 2016 21:02:37 -0400 Subject: [PATCH 0439/1117] rename mbedtls patch --- deps/mbedtls.mk | 2 +- deps/patches/{mbedtls.patch => mbedtls-config.patch} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename deps/patches/{mbedtls.patch => mbedtls-config.patch} (100%) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 93121d9a59eb5..89987b9ce4cbb 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -37,7 +37,7 @@ $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt: $(SRCDIR)/srccache/$(MBEDTLS_S touch -c $@ $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt - -cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls.patch + -cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls-config.patch mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(MBEDTLS_OPTS) diff --git a/deps/patches/mbedtls.patch b/deps/patches/mbedtls-config.patch similarity index 100% rename from deps/patches/mbedtls.patch rename to deps/patches/mbedtls-config.patch From 68bfe7e18fa0e37386e580ec30462567a3ef1f3b Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Thu, 14 Jul 2016 11:17:32 -0400 Subject: [PATCH 0440/1117] use release tag for libssh2 --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/libssh2.version | 4 +- deps/patches/libssh2-mbedtls.patch | 529 +++++++++++++++++- 6 files changed, 530 insertions(+), 7 deletions(-) create mode 100644 deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/md5 create mode 100644 deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/sha512 delete mode 100644 deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/md5 delete mode 100644 deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/sha512 diff --git a/deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/md5 b/deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/md5 new file mode 100644 index 0000000000000..10e23f7fb56e2 --- /dev/null +++ b/deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/md5 @@ -0,0 +1 @@ +25c40973c9cdcd7919c0b74ca0d06dd1 diff --git a/deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/sha512 b/deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/sha512 new file mode 100644 index 0000000000000..5fdf997bca855 --- /dev/null +++ b/deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/sha512 @@ -0,0 +1 @@ +eea5e3638395cb5711f97b36adfa05a9c1cb24cf3b5639cb0a20d74855da025c2db2ff3eeb6bf37d49977e2d9fcf2b929e5c5d874c65754c280defaacb9f3eb8 diff --git a/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/md5 b/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/md5 deleted file mode 100644 index 8be6221cb0950..0000000000000 --- a/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -41eb49bcd2800bcf2cc12263be16cdfb diff --git a/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/sha512 b/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/sha512 deleted file mode 100644 index 35f599b6b6d17..0000000000000 --- a/deps/checksums/libssh2-7934c9ce2a029c43e3642a492d3b9e494d1542be.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -455a853076b94944a09965c57a97454989a0f6c87cd9927a75f3bce69ceff56cbe4a34b40deadcf70923434dd20381f6f4fae1564b41d3d8a970bee5f50efee0 diff --git a/deps/libssh2.version b/deps/libssh2.version index 5e0bbbfa8c85c..e661e4b436309 100644 --- a/deps/libssh2.version +++ b/deps/libssh2.version @@ -1,2 +1,2 @@ -LIBSSH2_BRANCH=master -LIBSSH2_SHA1=7934c9ce2a029c43e3642a492d3b9e494d1542be +LIBSSH2_BRANCH=libssh2-1.7.0 +LIBSSH2_SHA1=6d553a7bb94966466c4db0abf9a26660e786a275 diff --git a/deps/patches/libssh2-mbedtls.patch b/deps/patches/libssh2-mbedtls.patch index ffab8f7786685..8afe5c4b2ea4f 100644 --- a/deps/patches/libssh2-mbedtls.patch +++ b/deps/patches/libssh2-mbedtls.patch @@ -35,6 +35,47 @@ index 9d6c0be..8bc1292 100644 install: - mkdir bin +diff --git a/README b/README +index 39abc20..8a14856 100644 +--- a/README ++++ b/README +@@ -4,9 +4,9 @@ libssh2 - SSH2 library + libssh2 is a library implementing the SSH2 protocol, available under + the revised BSD license. + +-Web site: http://www.libssh2.org/ ++Web site: https://www.libssh2.org/ + +-Mailing list: http://cool.haxx.se/mailman/listinfo/libssh2-devel ++Mailing list: https://cool.haxx.se/mailman/listinfo/libssh2-devel + + License: see COPYING + +diff --git a/RELEASE-NOTES b/RELEASE-NOTES +index 2b22288..94525cb 100644 +--- a/RELEASE-NOTES ++++ b/RELEASE-NOTES +@@ -11,7 +11,7 @@ This release includes the following changes: + This release includes the following security advisory: + + o diffie_hellman_sha256: convert bytes to bits +- CVE-2016-0787: http://www.libssh2.org/adv_20160223.html ++ CVE-2016-0787: https://www.libssh2.org/adv_20160223.html + + This release includes the following bugfixes: + +@@ -49,8 +49,8 @@ advice from friends like these: + + Alexander Lamaison, Andreas Schneider, brian m. carlson, Daniel Stenberg, + David Byron, Jakob Egger, Kamil Dudka, Marc Hoersken, Mizunashi Mana, +- Patrick Monnerat, Paul Howarth, Salvador Fandino, Salvador Fandiño, +- Salvador Fandiño, Viktor Szakats, Will Cosgrove, +- (16 contributors) ++ Patrick Monnerat, Paul Howarth, Salvador Fandiño, Viktor Szakats, ++ Will Cosgrove ++ (14 contributors) + + Thanks! (and sorry if I forgot to mention someone) diff --git a/cmake/FindmbedTLS.cmake b/cmake/FindmbedTLS.cmake new file mode 100644 index 0000000..2f4adbc @@ -105,6 +146,184 @@ index 0000000..2f4adbc + MBEDX509_LIBRARY + MBEDCRYPTO_LIBRARY +) +diff --git a/docs/BINDINGS b/docs/BINDINGS +index b97758f..471f9be 100644 +--- a/docs/BINDINGS ++++ b/docs/BINDINGS +@@ -10,16 +10,16 @@ Cocoa/Objective-C + https://github.com/karelia/libssh2_sftp-Cocoa-wrapper + + Haskell +- FFI bindings - http://hackage.haskell.org/package/libssh2 ++ FFI bindings - https://hackage.haskell.org/package/libssh2 + + Perl +- Net::SSH2 - http://search.cpan.org/~rkitover/Net-SSH2-0.45/lib/Net/SSH2.pm ++ Net::SSH2 - https://metacpan.org/pod/Net::SSH2 + + PHP +- ssh2 - http://pecl.php.net/package/ssh2 ++ ssh2 - https://pecl.php.net/package/ssh2 + + Python +- pylibssh2 - http://www.wallix.org/pylibssh2-project/ ++ pylibssh2 - https://pypi.python.org/pypi/pylibssh2 + + Python-ctypes + +diff --git a/docs/INSTALL_AUTOTOOLS b/docs/INSTALL_AUTOTOOLS +index bc5a0eb..b469533 100644 +--- a/docs/INSTALL_AUTOTOOLS ++++ b/docs/INSTALL_AUTOTOOLS +@@ -284,7 +284,7 @@ Some ./configure options deserve additional comments: + * --with-libgcrypt-prefix=DIR + + libssh2 can use the Libgcrypt library +- (http://www.gnupg.org/) for cryptographic operations. ++ (https://www.gnupg.org/) for cryptographic operations. + Either Libgcrypt or OpenSSL is required. + + Configure will attempt to locate Libgcrypt +@@ -298,7 +298,7 @@ Some ./configure options deserve additional comments: + * --with-libssl-prefix=[DIR] + + libssh2 can use the OpenSSL library +- (http://www.openssl.org) for cryptographic operations. ++ (https://www.openssl.org) for cryptographic operations. + Either Libgcrypt or OpenSSL is required. + + Configure will attempt to locate OpenSSL in the +diff --git a/docs/INSTALL_CMAKE b/docs/INSTALL_CMAKE +index 7040370..d2d7169 100644 +--- a/docs/INSTALL_CMAKE ++++ b/docs/INSTALL_CMAKE +@@ -47,8 +47,8 @@ The following options are available: + * `CRYPTO_BACKEND=` + + Chooses a specific cryptography library to use for cryptographic +- operations. Can be `OpenSSL` (http://www.openssl.org), +- `Libgcrypt` (http://www.gnupg.org/), `WinCNG` (Windows Vista+) or ++ operations. Can be `OpenSSL` (https://www.openssl.org), ++ `Libgcrypt` (https://www.gnupg.org/), `WinCNG` (Windows Vista+) or + blank to use any library available. + + CMake will attempt to locate the libraries automatically. See [2] +@@ -161,14 +161,14 @@ builds your project: + Libssh2 + URL <libssh2 download location> + URL_HASH SHA1=<libssh2 archive SHA1> +- INSTALL_COMMAND "") ++ INSTALL_COMMAND "") + + ExternalProject_Add( + MyProject DEPENDS Libssh2 + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src + INSTALL_COMMAND "") + +-[1] http://www.cmake.org/cmake/resources/software.html +-[2] http://www.cmake.org/cmake/help/v3.0/manual/cmake-packages.7.html +-[3] http://www.cmake.org/cmake/help/v3.0/manual/cmake-packages.7.html#package-registry +-[4] http://www.kitware.com/media/html/BuildingExternalProjectsWithCMake2.8.html +\ No newline at end of file ++[1] https://www.cmake.org/cmake/resources/software.html ++[2] https://www.cmake.org/cmake/help/v3.0/manual/cmake-packages.7.html ++[3] https://www.cmake.org/cmake/help/v3.0/manual/cmake-packages.7.html#package-registry ++[4] http://www.kitware.com/media/html/BuildingExternalProjectsWithCMake2.8.html +diff --git a/docs/libssh2_sftp_get_channel.3 b/docs/libssh2_sftp_get_channel.3 +index f3d364a..d1d82bc 100644 +--- a/docs/libssh2_sftp_get_channel.3 ++++ b/docs/libssh2_sftp_get_channel.3 +@@ -1,21 +1,21 @@ +-.TH libssh2_sftp_get_channel 3 "9 Sep 2011" "libssh2 1.4.0" "libssh2 manual" +-.SH NAME +-libssh2_sftp_get_channel - return the channel of sftp +-.SH SYNOPSIS +-.nf +-#include <libssh2.h> +-#include <libssh2_sftp.h> +- +-.fi +-LIBSSH2_CHANNEL *libssh2_sftp_get_channel(LIBSSH2_SFTP *sftp); +-.SH DESCRIPTION +-\fIsftp\fP - SFTP instance as returned by +-.BR libssh2_sftp_init(3) +- +-Return the channel of the given sftp handle. +-.SH RETURN VALUE +-The channel of the SFTP instance or NULL if something was wrong. +-.SH AVAILABILITY +-Added in 1.4.0 +-.SH SEE ALSO +-.BR libssh2_sftp_init(3) ++.TH libssh2_sftp_get_channel 3 "9 Sep 2011" "libssh2 1.4.0" "libssh2 manual" ++.SH NAME ++libssh2_sftp_get_channel - return the channel of sftp ++.SH SYNOPSIS ++.nf ++#include <libssh2.h> ++#include <libssh2_sftp.h> ++ ++.fi ++LIBSSH2_CHANNEL *libssh2_sftp_get_channel(LIBSSH2_SFTP *sftp); ++.SH DESCRIPTION ++\fIsftp\fP - SFTP instance as returned by ++.BR libssh2_sftp_init(3) ++ ++Return the channel of the given sftp handle. ++.SH RETURN VALUE ++The channel of the SFTP instance or NULL if something was wrong. ++.SH AVAILABILITY ++Added in 1.4.0 ++.SH SEE ALSO ++.BR libssh2_sftp_init(3) +diff --git a/example/subsystem_netconf.c b/example/subsystem_netconf.c +index 96ef54a..82c4941 100644 +--- a/example/subsystem_netconf.c ++++ b/example/subsystem_netconf.c +@@ -250,7 +250,7 @@ int main(int argc, char *argv[]) + goto shutdown; + } + +- /* NETCONF: http://tools.ietf.org/html/draft-ietf-netconf-ssh-06 */ ++ /* NETCONF: https://tools.ietf.org/html/draft-ietf-netconf-ssh-06 */ + + fprintf(stderr, "Sending NETCONF client <hello>\n"); + snprintf(buf, sizeof(buf), +diff --git a/include/libssh2.h b/include/libssh2.h +index c157757..9ef9ff1 100644 +--- a/include/libssh2.h ++++ b/include/libssh2.h +@@ -202,7 +202,16 @@ typedef off_t libssh2_struct_stat_size; + #endif + + #ifndef LIBSSH2_STRUCT_STAT_SIZE_FORMAT +-# define LIBSSH2_STRUCT_STAT_SIZE_FORMAT "%zd" ++# ifdef __VMS ++/* We have to roll our own format here because %z is a C99-ism we don't have. */ ++# if __USE_OFF64_T || __USING_STD_STAT ++# define LIBSSH2_STRUCT_STAT_SIZE_FORMAT "%Ld" ++# else ++# define LIBSSH2_STRUCT_STAT_SIZE_FORMAT "%d" ++# endif ++# else ++# define LIBSSH2_STRUCT_STAT_SIZE_FORMAT "%zd" ++# endif + typedef struct stat libssh2_struct_stat; + typedef off_t libssh2_struct_stat_size; + #endif +diff --git a/libssh2.pc.in b/libssh2.pc.in +index 084918a..9663816 100644 +--- a/libssh2.pc.in ++++ b/libssh2.pc.in +@@ -8,7 +8,7 @@ libdir=@libdir@ + includedir=@includedir@ + + Name: libssh2 +-URL: http://www.libssh2.org/ ++URL: https://www.libssh2.org/ + Description: Library for SSH-based communication + Version: @LIBSSH2VER@ + Requires.private: @LIBSREQUIRED@ diff --git a/mbedtls/buildmbedtls b/mbedtls/buildmbedtls new file mode 100755 index 0000000..2385a8b @@ -162,8 +381,39 @@ index 0000000..1f774ec + //#define MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ + + /* ECP options */ +diff --git a/nw/GNUmakefile b/nw/GNUmakefile +index 0a1d0b3..8899fdd 100644 +--- a/nw/GNUmakefile ++++ b/nw/GNUmakefile +@@ -38,7 +38,7 @@ DEVLARC = $(DEVLDIR).zip + TARGET = libssh2 + VERSION = $(LIBSSH2_VERSION) + CPRIGHT = Copyright (c) $(LIBSSH2_COPYRIGHT_STR) +-WWWURL = http://www.libssh2.org/ ++WWWURL = https://www.libssh2.org/ + DESCR = libssh2 $(LIBSSH2_VERSION_STR) ($(LIBARCH)) - $(WWWURL) + MTSAFE = YES + STACK = 64000 +@@ -223,7 +223,7 @@ include ../Makefile.inc + OBJECTS := $(patsubst %.c,%.o,$(CSOURCES)) + ifeq ($(LIBARCH),CLIB) + # CLIB lacks of snprint() function - here's a replacement: +-# http://www.ijs.si/software/snprintf/ ++# https://www.ijs.si/software/snprintf/ + OBJECTS += snprintf.o + vpath %.c $(SNPRINTF) + endif +@@ -388,7 +388,7 @@ libssh2_config.h: GNUmakefile + @echo $(DL)** All your changes will be lost!!$(DL) >> $@ + @echo $(DL)*/$(DL) >> $@ + @echo $(DL)#define VERSION "$(LIBSSH2_VERSION_STR)"$(DL) >> $@ +- @echo $(DL)#define PACKAGE_BUGREPORT "http://sourceforge.net/projects/libssh2"$(DL) >> $@ ++ @echo $(DL)#define PACKAGE_BUGREPORT "https://github.com/libssh2/libssh2/issues"$(DL) >> $@ + ifeq ($(LIBARCH),CLIB) + @echo $(DL)#define OS "i586-pc-clib-NetWare"$(DL) >> $@ + @echo $(DL)#define NETDB_USE_INTERNET 1$(DL) >> $@ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index c29a2d8..987dfe0 100644 +index 1a76399..987dfe0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,7 +48,7 @@ set(CRYPTO_BACKEND @@ -175,7 +425,19 @@ index c29a2d8..987dfe0 100644 or empty to try any available") # If the crypto backend was given, rather than searching for the first -@@ -151,6 +151,21 @@ if(CRYPTO_BACKEND STREQUAL "WinCNG" OR NOT CRYPTO_BACKEND) +@@ -71,6 +71,11 @@ if(CRYPTO_BACKEND STREQUAL "OpenSSL" OR NOT CRYPTO_BACKEND) + list(APPEND PC_REQUIRES_PRIVATE libssl libcrypto) + + if (WIN32) ++ # Statically linking to OpenSSL requires crypt32 for some Windows APIs. ++ # This should really be handled by FindOpenSSL.cmake. ++ list(APPEND LIBRARIES crypt32) ++ list(APPEND PC_LIBS -lcrypt32) ++ + find_file(DLL_LIBEAY32 + NAMES libeay32.dll crypto.dll + HINTS ${_OPENSSL_ROOT_HINTS} PATHS ${_OPENSSL_ROOT_PATHS} +@@ -146,6 +151,21 @@ if(CRYPTO_BACKEND STREQUAL "WinCNG" OR NOT CRYPTO_BACKEND) endif() endif() @@ -197,7 +459,7 @@ index c29a2d8..987dfe0 100644 if(NOT CRYPTO_BACKEND) message(FATAL_ERROR "No suitable cryptography backend found.") endif() -@@ -209,6 +224,11 @@ target_include_directories(libssh2 +@@ -204,6 +224,11 @@ target_include_directories(libssh2 ## Options @@ -209,6 +471,19 @@ index c29a2d8..987dfe0 100644 add_feature_info("Shared library" BUILD_SHARED_LIBS "creating libssh2 as a shared library (.so/.dll)") +diff --git a/src/channel.c b/src/channel.c +index 32d914d..538a0ab 100644 +--- a/src/channel.c ++++ b/src/channel.c +@@ -270,7 +270,7 @@ _libssh2_channel_open(LIBSSH2_SESSION * session, const char *channel_type, + switch (reason_code) { + case SSH_OPEN_ADMINISTRATIVELY_PROHIBITED: + _libssh2_error(session, LIBSSH2_ERROR_CHANNEL_FAILURE, +- "Channel open failure (admininstratively prohibited)"); ++ "Channel open failure (administratively prohibited)"); + break; + case SSH_OPEN_CONNECT_FAILED: + _libssh2_error(session, LIBSSH2_ERROR_CHANNEL_FAILURE, diff --git a/src/crypto.h b/src/crypto.h index caad19f..aa997a3 100644 --- a/src/crypto.h @@ -224,6 +499,48 @@ index caad19f..aa997a3 100644 int _libssh2_rsa_new(libssh2_rsa_ctx ** rsa, const unsigned char *edata, unsigned long elen, +diff --git a/src/kex.c b/src/kex.c +index e89b36c..65b722f 100644 +--- a/src/kex.c ++++ b/src/kex.c +@@ -133,7 +133,7 @@ static int diffie_hellman_sha1(LIBSSH2_SESSION *session, + memset(&exchange_state->req_state, 0, sizeof(packet_require_state_t)); + + /* Generate x and e */ +- _libssh2_bn_rand(exchange_state->x, group_order, 0, -1); ++ _libssh2_bn_rand(exchange_state->x, group_order * 8 - 1, 0, -1); + _libssh2_bn_mod_exp(exchange_state->e, g, exchange_state->x, p, + exchange_state->ctx); + +diff --git a/src/libgcrypt.c b/src/libgcrypt.c +index e85aecd..366d007 100644 +--- a/src/libgcrypt.c ++++ b/src/libgcrypt.c +@@ -409,6 +409,9 @@ _libssh2_rsa_sha1_sign(LIBSSH2_SESSION * session, + } + + *signature = LIBSSH2_ALLOC(session, size); ++ if (!*signature) { ++ return -1; ++ } + memcpy(*signature, tmp, size); + *signature_len = size; + +diff --git a/src/libssh2_priv.h b/src/libssh2_priv.h +index ae691ad..b4296a2 100644 +--- a/src/libssh2_priv.h ++++ b/src/libssh2_priv.h +@@ -65,8 +65,8 @@ + consistent names of these fields. While arguable the best would to + change libssh2.h to use other names, that would break backwards + compatibility. For more information, see: +- http://www.mail-archive.com/libssh2-devel%40lists.sourceforge.net/msg00003.html +- http://www.mail-archive.com/libssh2-devel%40lists.sourceforge.net/msg00224.html ++ https://www.mail-archive.com/libssh2-devel%40lists.sourceforge.net/msg00003.html ++ https://www.mail-archive.com/libssh2-devel%40lists.sourceforge.net/msg00224.html + */ + #ifdef HAVE_POLL + # include <sys/poll.h> diff --git a/src/mbedtls.c b/src/mbedtls.c new file mode 100644 index 0000000..98bc549 @@ -1174,3 +1491,209 @@ index 0000000..f594575 + const char *privatekeydata, + size_t privatekeydata_len, + const char *passphrase); +diff --git a/src/sftp.c b/src/sftp.c +index c142713..7c44116 100644 +--- a/src/sftp.c ++++ b/src/sftp.c +@@ -1527,7 +1527,7 @@ static ssize_t sftp_read(LIBSSH2_SFTP_HANDLE * handle, char *buffer, + if (chunk->offset != filep->offset) { + /* This could happen if the server returns less bytes than + requested, which shouldn't happen for normal files. See: +- http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02 ++ https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02 + #section-6.4 + */ + return _libssh2_error(session, LIBSSH2_ERROR_SFTP_PROTOCOL, +diff --git a/src/wincng.c b/src/wincng.c +index 635b3e3..d3271b3 100755 +--- a/src/wincng.c ++++ b/src/wincng.c +@@ -861,7 +861,7 @@ _libssh2_wincng_rsa_new(libssh2_rsa_ctx **rsa, + memset(key, 0, keylen); + + +- /* http://msdn.microsoft.com/library/windows/desktop/aa375531.aspx */ ++ /* https://msdn.microsoft.com/library/windows/desktop/aa375531.aspx */ + rsakey = (BCRYPT_RSAKEY_BLOB *)key; + rsakey->BitLength = mlen * 8; + rsakey->cbPublicExp = elen; +@@ -1179,7 +1179,7 @@ _libssh2_wincng_dsa_new(libssh2_dsa_ctx **dsa, + memset(key, 0, keylen); + + +- /* http://msdn.microsoft.com/library/windows/desktop/aa833126.aspx */ ++ /* https://msdn.microsoft.com/library/windows/desktop/aa833126.aspx */ + dsakey = (BCRYPT_DSA_KEY_BLOB *)key; + dsakey->cbKey = length; + +@@ -1903,7 +1903,7 @@ _libssh2_wincng_bignum_mod_exp(_libssh2_bn *r, + return -1; + + +- /* http://msdn.microsoft.com/library/windows/desktop/aa375531.aspx */ ++ /* https://msdn.microsoft.com/library/windows/desktop/aa375531.aspx */ + rsakey = (BCRYPT_RSAKEY_BLOB *)key; + rsakey->Magic = BCRYPT_RSAPUBLIC_MAGIC; + rsakey->BitLength = m->length * 8; +diff --git a/vms/libssh2_config.h b/vms/libssh2_config.h +index 2eb09a8..b8f73e2 100644 +--- a/vms/libssh2_config.h ++++ b/vms/libssh2_config.h +@@ -14,6 +14,7 @@ typedef unsigned int socklen_t; /* missing in headers on VMS */ + /* Have's */ + + #define HAVE_UNISTD_H ++#define HAVE_STDLIB_H + #define HAVE_INTTYPES_H + #define HAVE_SYS_TIME_H + #define HAVE_SELECT +@@ -23,6 +24,8 @@ typedef unsigned int socklen_t; /* missing in headers on VMS */ + #define HAVE_NETINET_IN_H + #define HAVE_ARPA_INET_H + ++#define HAVE_GETTIMEOFDAY 1 ++ + #define POSIX_C_SOURCE + + /* Enable the possibility of using tracing */ +@@ -68,8 +71,11 @@ typedef unsigned int socklen_t; /* missing in headers on VMS */ + + #endif + ++/* Use OpenSSL */ ++#define LIBSSH2_OPENSSL 1 ++ + /* Compile in zlib support. We link against gnv$libzshr, as available +- * on encompasserve.com. ++ * from https://sourceforge.net/projects/vms-ports/files/. + */ + + #define LIBSSH2_HAVE_ZLIB +diff --git a/vms/libssh2_make_example.dcl b/vms/libssh2_make_example.dcl +index d8191f3..af4116d 100644 +--- a/vms/libssh2_make_example.dcl ++++ b/vms/libssh2_make_example.dcl +@@ -29,6 +29,7 @@ $ this = f$search("exadir:*.c;0") + $ if this .eqs. "" then goto EndLoop + $! + $ what = f$parse( this,,,"name") ++$ if what .eqs. "x11" then goto loop ! not portable yet + $ call Make + $! + $ goto Loop +@@ -84,7 +85,7 @@ $ define objdir 'objdir' + $ define exadir 'exadir' + $! + $ cc_include = "/include=([],[-.include])" +-$ cc_flags = "/name=shortened/show=all" ++$ cc_flags = "/name=shortened/show=all/define=(_USE_STD_STAT=1)" + $ link_opts = "objdir:libssh2_''thisid'.opt" + $! + $! +diff --git a/vms/libssh2_make_help.dcl b/vms/libssh2_make_help.dcl +index b28265d..b36512e 100644 +--- a/vms/libssh2_make_help.dcl ++++ b/vms/libssh2_make_help.dcl +@@ -11,23 +11,27 @@ $ man2help sys$input: libssh2.hlp -b 1 + + LIBSSH2 + +-OpenVMS port of the public domain libssh2 library, which +-provides an API to implement client SSH communciation. ++OpenVMS port of the libssh2 library, which provides an ++API to implement client SSH communication. + +-License information is available at the copying subtopic. ++License information is available at the Copying subtopic. + + $! + $ open/append mh libssh2.hlp + $ write mh helpversion + $ close mh + $! +-$ man2help -a [-]readme.; libssh2.hlp -b 2 +-$ man2help -a [-]authors.; libssh2.hlp -b 2 +-$ man2help -a [-]copying.; libssh2.hlp -b 2 +-$ man2help -a [-]news.; libssh2.hlp -b 2 +-$ man2help -a [-]release-notes.; libssh2.hlp -b 2 +-$ man2help -a [-]hacking.; libssh2.hlp -b 2 +-$ man2help -a [-]todo.; libssh2.hlp -b 2 ++$ man2help -a [-]README.; libssh2.hlp -b 2 ++$ man2help -a [-]COPYING.; libssh2.hlp -b 2 ++$ man2help -a [-]NEWS.; libssh2.hlp -b 2 ++$ man2help -a [-]RELEASE-NOTES.; libssh2.hlp -b 2 ++$ man2help -a [-.docs]AUTHORS.; libssh2.hlp -b 2 ++$ man2help -a [-.docs]BINDINGS.; libssh2.hlp -b 2 ++$ man2help -a [-.docs]HACKING.; libssh2.hlp -b 2 ++$ if f$search("[]HACKING_CRYPTO.") .nes. "" then delete []HACKING_CRYPTO.;* ++$ copy [-.docs]HACKING.CRYPTO; []HACKING_CRYPTO. ++$ man2help -a []HACKING_CRYPTO.; libssh2.hlp -b 2 ++$ man2help -a [-.docs]TODO.; libssh2.hlp -b 2 + $! + $ man2help -a sys$input: libssh2.hlp -b 2 + +diff --git a/vms/libssh2_make_kit.dcl b/vms/libssh2_make_kit.dcl +index dceca4f..6a14958 100644 +--- a/vms/libssh2_make_kit.dcl ++++ b/vms/libssh2_make_kit.dcl +@@ -194,7 +194,7 @@ $ write pt "=prompt JCB LIBSSH2 for OpenVMS" + $ write pt "" + $ write pt "libssh2 is an open source client side library that aims to implement" + $ write pt "the SSH protocol. This is the OpenVMS port of that library." +-$ write pt "Further information at http://www.libssh2.org." ++$ write pt "Further information at https://www.libssh2.org." + $ write pt "" + $ write pt "1 NEED_VMS83" + $ write pt "=prompt OpenVMS 8.3 or later is not installed on your system." +diff --git a/vms/libssh2_make_lib.dcl b/vms/libssh2_make_lib.dcl +index 6d8b13b..56d168f 100644 +--- a/vms/libssh2_make_lib.dcl ++++ b/vms/libssh2_make_lib.dcl +@@ -46,7 +46,8 @@ $! + $ define objdir 'objdir' + $ define srcdir 'srcdir' + $! +-$ cc_include = "/include=([],[-.include])" ++$ cc_include = "/include=([],[-.include],""/gnv$zlib_include"")" ++$ cc_define = "/DEFINE=(_USE_STD_STAT=1)" + $ link_opts = "objdir:libssh2_''thisid'.opt" + $! + $ pipe search [-.include]libssh2.h libssh2_version_major/nohead | (read sys$input l ; l = f$element(2," ",f$edit(l,"trim,compress")) ; - +@@ -137,7 +138,7 @@ $CaseLoop: + $! + $ if case .eq. 0 + $ then!camel case names +-$ cc_flags = "/names=(shortened,as_is)" ++$ cc_flags = "/names=(shortened,as_is)''cc_define'" + $ objlib = "libssh2_asis.olb" + $ endif + $! +@@ -149,7 +150,7 @@ $ rename [.cxx_repository]cxx$demangler_db.; *.lowercase + $ purge [.cxx_repository]cxx$demangler_db.lowercase + $ endif + $! +-$ cc_flags = "/names=(shortened)" ++$ cc_flags = "/names=(shortened)''cc_define'" + $ objlib = "libssh2_up.olb" + $ endif + $! +diff --git a/win32/libssh2_config.h b/win32/libssh2_config.h +index 3e3caf4..6ac2ef4 100644 +--- a/win32/libssh2_config.h ++++ b/win32/libssh2_config.h +@@ -24,6 +24,7 @@ + #define HAVE_SELECT + + #ifdef _MSC_VER ++#if _MSC_VER < 1900 + #define snprintf _snprintf + #if _MSC_VER < 1500 + #define vsnprintf _vsnprintf +@@ -31,6 +32,7 @@ + #define strdup _strdup + #define strncasecmp _strnicmp + #define strcasecmp _stricmp ++#endif + #else + #ifndef __MINGW32__ + #define strncasecmp strnicmp From 505a2a41a07e8f05952c0e5570376dc16250d6b7 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Thu, 14 Jul 2016 13:51:07 -0400 Subject: [PATCH 0441/1117] fix mbedtls build --- deps/mbedtls.mk | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 89987b9ce4cbb..7513d046c4136 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -7,12 +7,8 @@ else endif MBEDTLS_URL = https://tls.mbed.org/download/$(MBEDTLS_SRC).tgz -MBEDTLS_OBJ_SOURCE := $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedtls.$(SHLIB_EXT) \ - $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedx509.$(SHLIB_EXT) \ - $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedcrypto.$(SHLIB_EXT) -MBEDTLS_OBJ_TARGET := $(build_shlibdir)/libmbedtls.$(SHLIB_EXT) \ - $(build_shlibdir)/libmbedx509.$(SHLIB_EXT) \ - $(build_shlibdir)/libmbedcrypto.$(SHLIB_EXT) +MBEDTLS_OBJ_SOURCE := $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedcrypto.$(SHLIB_EXT) +MBEDTLS_OBJ_TARGET := $(build_shlibdir)/libmbedcrypto.$(SHLIB_EXT) MBEDTLS_OPTS := $(CMAKE_COMMON) -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release \ From 8aebd556191d2ad1199241a8f08467e380cc3fe6 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 14 Jul 2016 10:54:04 -0700 Subject: [PATCH 0442/1117] Replace outdated DEBUGGER.md with a reference to Gallium.jl [ci skip] --- DEBUGGER.md | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/DEBUGGER.md b/DEBUGGER.md index 29bcf8460876b..6caf8e7f69a77 100644 --- a/DEBUGGER.md +++ b/DEBUGGER.md @@ -1,34 +1,6 @@ The Julia Debugger ------------------ -In order to have a Julia debugger, we first need a version of LLDB -that understands Julia's JITed stack frames. To do so, build Julia -with instructions from https://github.com/Keno/Cxx.jl. Just the julia -parts are required. The Cxx part won't be necessary until the debugger -UI is released. +To install the Julia debugger, run `Pkg.add("Gallium")` and see documentation at -Use LLDB just as you would usually. - -This should at least get stack traces and the ability to step -through julia code. Please file any issues as bug reports. - -For local variables, you may get some, and you'll get more if you set -`DISABLE_OPT` in `src/options.h`. If you set the latter, it is useful -to collect a set of test cases that don't get local variables even -though they should. You will probably find a lot of functions don't -work, yet. - -OS X -==== - -* Enable FORCE_ELF in src/options. - -* Comment out `if (arch.GetTriple().getVendor() != llvm::Triple::Apple)` -in deps/llvm-svn/tools/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp - -* export `LLDB_DEBUGSERVER_PATH=/Applications/Xcode.app/Contents/SharedFrameworks/LLDB.framework/Resources/debugserver` - -References -========== - -The original post on julia-dev: https://groups.google.com/forum/#!topic/julia-dev/gcZ5dZJni5o +https://github.com/Keno/Gallium.jl#gallium From f5230bfab634d669ac5cc33dd7fd4fb0aa975984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1ll=20Haraldsson?= <Pall.Haraldsson@gmail.com> Date: Thu, 14 Jul 2016 19:05:14 +0000 Subject: [PATCH 0443/1117] Seems OS X 10.6 isn't really supported.. --- DISTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DISTRIBUTING.md b/DISTRIBUTING.md index da615a1546580..4f752bd957756 100644 --- a/DISTRIBUTING.md +++ b/DISTRIBUTING.md @@ -106,8 +106,7 @@ Note that if you want your `.app` to be able to run on OSX 10.6 Snow Leopard, you must pass `USE_SYSTEM_LIBUNWIND=1` as one of the make variables passed to both `make` processes. This disables the use of `libosxunwind`, a more modern libunwind that relies on OS features -available only in 10.7+. This is the reason why we offer [separate -downloads](http://julialang.org/downloads/) for OS X 10.6 and 10.7+. +available only in 10.7+. This is the reason why we support 10.7+ [while we did support 10.6 prior to Julia 0.3.0](http://julialang.org/downloads/platforms.html). Windows ------- From 7bcdab5544f0d0c7e0aa859e7699c96797fb5c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1ll=20Haraldsson?= <Pall.Haraldsson@gmail.com> Date: Thu, 14 Jul 2016 19:37:33 +0000 Subject: [PATCH 0444/1117] Update DISTRIBUTING.md --- DISTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DISTRIBUTING.md b/DISTRIBUTING.md index 4f752bd957756..9dd1dd972a133 100644 --- a/DISTRIBUTING.md +++ b/DISTRIBUTING.md @@ -106,7 +106,7 @@ Note that if you want your `.app` to be able to run on OSX 10.6 Snow Leopard, you must pass `USE_SYSTEM_LIBUNWIND=1` as one of the make variables passed to both `make` processes. This disables the use of `libosxunwind`, a more modern libunwind that relies on OS features -available only in 10.7+. This is the reason why we support 10.7+ [while we did support 10.6 prior to Julia 0.3.0](http://julialang.org/downloads/platforms.html). +available only in 10.7+. This is the reason why we support 10.7+ [while we did support 10.6 prior to Julia 0.3.0](http://julialang.org/downloads/platform.html). Windows ------- From b9f638115928b730360b7613d777d49500cb0439 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Fri, 8 Jul 2016 19:09:45 -0700 Subject: [PATCH 0445/1117] Rewrite setindex!(A::SparseMatrixCSC, v, i, j) such that it no longer expunges stored entries on zero assignment, throws a more explicit BoundsError when needed, and also for simplicity and clarity. Also add tests. --- base/sparse/sparsematrix.jl | 41 +++++++++++++++---------------------- test/sparsedir/sparse.jl | 9 ++++++++ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 3c0c5c47e5a92..d0866a65f6626 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -2483,32 +2483,25 @@ getindex{T<:Integer}(A::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVec function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, v, i::Integer, j::Integer) setindex!(A, convert(Tv, v), convert(Ti, i), convert(Ti, j)) end -function setindex!{Tv,Ti<:Integer}(A::SparseMatrixCSC{Tv,Ti}, v::Tv, i0::Ti, i1::Ti) - if !(1 <= i0 <= A.m && 1 <= i1 <= A.n); throw(BoundsError()); end - r1 = Int(A.colptr[i1]) - r2 = Int(A.colptr[i1+1]-1) - if v == 0 #either do nothing or delete entry if it exists - if r1 <= r2 - r1 = searchsortedfirst(A.rowval, i0, r1, r2, Forward) - if (r1 <= r2) && (A.rowval[r1] == i0) - deleteat!(A.rowval, r1) - deleteat!(A.nzval, r1) - @simd for j = (i1+1):(A.n+1) - @inbounds A.colptr[j] -= 1 - end - end - end +function setindex!{Tv,Ti<:Integer}(A::SparseMatrixCSC{Tv,Ti}, v::Tv, i::Ti, j::Ti) + if !((1 <= i <= A.m) & (1 <= j <= A.n)) + throw(BoundsError(A, (i,j))) + end + coljfirstk = Int(A.colptr[j]) + coljlastk = Int(A.colptr[j+1] - 1) + searchk = searchsortedfirst(A.rowval, i, coljfirstk, coljlastk, Base.Order.Forward) + if searchk <= coljlastk && A.rowval[searchk] == i + # Column j contains entry A[i,j]. Update and return + A.nzval[searchk] = v return A end - i = (r1 > r2) ? r1 : searchsortedfirst(A.rowval, i0, r1, r2, Forward) - - if (i <= r2) && (A.rowval[i] == i0) - A.nzval[i] = v - else - insert!(A.rowval, i, i0) - insert!(A.nzval, i, v) - @simd for j = (i1+1):(A.n+1) - @inbounds A.colptr[j] += 1 + # Column j does not contain entry A[i,j]. If v is nonzero, insert entry A[i,j] = v + # and return. If to the contrary v is zero, then simply return. + if v != 0 + insert!(A.rowval, searchk, i) + insert!(A.nzval, searchk, v) + @simd for m in (j + 1):(A.n + 1) + @inbounds A.colptr[m] += 1 end end return A diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 88835d4f274fa..47cccac054ccd 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -578,6 +578,14 @@ let a = spzeros(Int, 10, 10) @test countnz(a) == 19 @test a[:,2] == 2*sparse(ones(Int,10)) + # Zero-assignment behavior of setindex!(A, v, i, j) + a[1,3] = 0 + @test nnz(a) == 19 + @test countnz(a) == 18 + a[2,1] = 0 + @test nnz(a) == 19 + @test countnz(a) == 18 + a[1,:] = 1:10 @test a[1,:] == sparse([1:10;]) a[:,2] = 1:10 @@ -1397,6 +1405,7 @@ let x = UpperTriangular(A)*ones(n) @test UpperTriangular(A)\x ≈ ones(n) A[2,2] = 0 + dropzeros!(A) @test_throws LinAlg.SingularException LowerTriangular(A)\ones(n) @test_throws LinAlg.SingularException UpperTriangular(A)\ones(n) end From bf3456ad81ae0405ffa9f11c9717da83e7b2d14e Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 14 Jul 2016 14:57:34 -0400 Subject: [PATCH 0446/1117] disasm: handle non-function-local jumps more correctly if we could extract the name of the symbol stub, we would even have symbolic names for these now --- src/disasm.cpp | 53 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/src/disasm.cpp b/src/disasm.cpp index a96710526c627..433d5020f94a7 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -142,10 +142,12 @@ class SymbolTable { void createSymbols(); const char *lookupSymbolName(uint64_t addr, bool LocalOnly); MCSymbol *lookupSymbol(uint64_t addr); + StringRef getSymbolNameAt(uint64_t offset) const; + const char *lookupLocalPC(size_t addr); void setIP(uint64_t addr); uint64_t getIP() const; - StringRef getSymbolNameAt(uint64_t offset) const; }; + void SymbolTable::setIP(uint64_t addr) { ip = addr; @@ -154,6 +156,19 @@ uint64_t SymbolTable::getIP() const { return ip; } + +const char *SymbolTable::lookupLocalPC(size_t addr) { + jl_frame_t *frame = NULL; + jl_getFunctionInfo(&frame, + addr, + /*skipC*/0, + /*noInline*/1/* the entry pointer shouldn't have inlining */); + char *name = frame->func_name; // TODO: free me + free(frame->file_name); + free(frame); + return name; +} + StringRef SymbolTable::getSymbolNameAt(uint64_t offset) const { if (object == NULL) return StringRef(); @@ -218,34 +233,47 @@ void SymbolTable::insertAddress(uint64_t addr) // Create symbols for all addresses void SymbolTable::createSymbols() { + uint64_t Fptr = (uint64_t)MemObj.data(); + uint64_t Fsize = MemObj.size(); for (TableType::iterator isymb = Table.begin(), esymb = Table.end(); isymb != esymb; ++isymb) { - uint64_t addr = isymb->first - ip; std::ostringstream name; - name << "L" << addr; + uint64_t rel = isymb->first - ip; + uint64_t addr = isymb->first; + if (Fptr <= addr && addr < Fptr + Fsize) { + name << "L" << rel; + } + else { + const char *global = lookupLocalPC(addr); + if (!global) + continue; + name << global; + } + #ifdef LLVM37 MCSymbol *symb = Ctx.getOrCreateSymbol(StringRef(name.str())); assert(symb->isUndefined()); #else MCSymbol *symb = Ctx.GetOrCreateSymbol(StringRef(name.str())); - symb->setVariableValue(MCConstantExpr::Create(addr, Ctx)); + symb->setVariableValue(MCConstantExpr::Create(rel, Ctx)); #endif isymb->second = symb; } } + const char *SymbolTable::lookupSymbolName(uint64_t addr, bool LocalOnly) { TempName = std::string(); TableType::iterator Sym = Table.find(addr); - if (Sym != Table.end()) { - MCSymbol *symb = Table[addr]; - TempName = symb->getName().str(); + if (Sym != Table.end() && Sym->second) { + TempName = Sym->second->getName().str(); } else if (!LocalOnly) { TempName = getSymbolNameAt(addr + slide).str(); } return TempName.empty() ? NULL : TempName.c_str(); } + MCSymbol *SymbolTable::lookupSymbol(uint64_t addr) { if (!Table.count(addr)) return NULL; @@ -304,14 +332,7 @@ static int OpInfoLookup(void *DisInfo, uint64_t PC, uint64_t Offset, uint64_t Si case 8: { uint64_t val; std::memcpy(&val, bytes, 8); pointer = val; break; } default: return 0; // Cannot handle input address size } - jl_frame_t *frame = NULL; - jl_getFunctionInfo(&frame, - pointer, - /*skipC*/0, - /*noInline*/1/* the entry pointer shouldn't have inlining */); - char *name = frame->func_name; // TODO: free me - free(frame->file_name); - free(frame); + const char *name = SymTab->lookupLocalPC(pointer); if (!name) return 0; // Did not find symbolic information // Describe the symbol @@ -667,7 +688,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, case MCDisassembler::Success: if (pass == 0) { // Pass 0: Record all branch targets - if (MCIA && MCIA->isBranch(Inst)) { + if (MCIA && (MCIA->isBranch(Inst) || MCIA->isCall(Inst))) { uint64_t addr; #ifdef LLVM34 if (MCIA->evaluateBranch(Inst, Fptr+Index, insSize, addr)) From a7fcc622c0d622ad89f50ec630cacb2233481e67 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Thu, 14 Jul 2016 16:49:47 -0400 Subject: [PATCH 0447/1117] Add LLVM 3.8.1 patch list and checksums --- deps/checksums/cfe-3.8.1.src.tar.xz/md5 | 1 + deps/checksums/cfe-3.8.1.src.tar.xz/sha512 | 1 + .../compiler-rt-3.8.1.src.tar.xz/md5 | 1 + .../compiler-rt-3.8.1.src.tar.xz/sha512 | 1 + deps/checksums/llvm-3.8.1.src.tar.xz/md5 | 1 + deps/checksums/llvm-3.8.1.src.tar.xz/sha512 | 1 + deps/llvm.mk | 26 ++++++++++--------- 7 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 deps/checksums/cfe-3.8.1.src.tar.xz/md5 create mode 100644 deps/checksums/cfe-3.8.1.src.tar.xz/sha512 create mode 100644 deps/checksums/compiler-rt-3.8.1.src.tar.xz/md5 create mode 100644 deps/checksums/compiler-rt-3.8.1.src.tar.xz/sha512 create mode 100644 deps/checksums/llvm-3.8.1.src.tar.xz/md5 create mode 100644 deps/checksums/llvm-3.8.1.src.tar.xz/sha512 diff --git a/deps/checksums/cfe-3.8.1.src.tar.xz/md5 b/deps/checksums/cfe-3.8.1.src.tar.xz/md5 new file mode 100644 index 0000000000000..b190cebc87a5b --- /dev/null +++ b/deps/checksums/cfe-3.8.1.src.tar.xz/md5 @@ -0,0 +1 @@ +4ff2f8844a786edb0220f490f7896080 diff --git a/deps/checksums/cfe-3.8.1.src.tar.xz/sha512 b/deps/checksums/cfe-3.8.1.src.tar.xz/sha512 new file mode 100644 index 0000000000000..9c90c669cb03f --- /dev/null +++ b/deps/checksums/cfe-3.8.1.src.tar.xz/sha512 @@ -0,0 +1 @@ +72d23a410271b50f97371b13dd7a6c6c5c0a993e35df436ab716ece8521d83406aa3d4bf8fdecd8154139f39fd5e011e480d405225d8c3ff152d0a2ac4573e04 diff --git a/deps/checksums/compiler-rt-3.8.1.src.tar.xz/md5 b/deps/checksums/compiler-rt-3.8.1.src.tar.xz/md5 new file mode 100644 index 0000000000000..f46b850e33377 --- /dev/null +++ b/deps/checksums/compiler-rt-3.8.1.src.tar.xz/md5 @@ -0,0 +1 @@ +f140db073d2453f854fbe01cc46f3110 diff --git a/deps/checksums/compiler-rt-3.8.1.src.tar.xz/sha512 b/deps/checksums/compiler-rt-3.8.1.src.tar.xz/sha512 new file mode 100644 index 0000000000000..8f8875c9ec13b --- /dev/null +++ b/deps/checksums/compiler-rt-3.8.1.src.tar.xz/sha512 @@ -0,0 +1 @@ +bed3da5f8594c4bf71af406419fbeaf7ed5d8bf46adb305233a298271d34a9af1072bcb47d474ac19bb862cc7c7bc9e1d89bc1567133553f29480030cad1cf75 diff --git a/deps/checksums/llvm-3.8.1.src.tar.xz/md5 b/deps/checksums/llvm-3.8.1.src.tar.xz/md5 new file mode 100644 index 0000000000000..a5f1c598cdc59 --- /dev/null +++ b/deps/checksums/llvm-3.8.1.src.tar.xz/md5 @@ -0,0 +1 @@ +538467e6028bbc9259b1e6e015d25845 diff --git a/deps/checksums/llvm-3.8.1.src.tar.xz/sha512 b/deps/checksums/llvm-3.8.1.src.tar.xz/sha512 new file mode 100644 index 0000000000000..e9f501848b456 --- /dev/null +++ b/deps/checksums/llvm-3.8.1.src.tar.xz/sha512 @@ -0,0 +1 @@ +99bbb2cc5f337fd5edf1621f8028c8cb74011aa4af2531018dc05409b27f7b8d0c2f76a00115f677f7c013029d5d9f55d085a5b40433409aac4e9161d53bd366 diff --git a/deps/llvm.mk b/deps/llvm.mk index bf33bc4ca354b..4f050e7f7e744 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -431,23 +431,25 @@ $(eval $(call LLVM_PATCH,llvm-D14260)) $(eval $(call LLVM_PATCH,llvm-nodllalias)) $(eval $(call LLVM_PATCH,llvm-D21271-instcombine-tbaa-3.7)) $(eval $(call LLVM_PATCH,llvm-win64-reloc-dwarf)) -else ifeq ($(LLVM_VER),3.8.0) -$(eval $(call LLVM_PATCH,llvm-3.7.1_3)) +else ifeq ($(LLVM_VER_SHORT),3.8) +ifeq ($(LLVM_VER),3.8.0) +$(eval $(call LLVM_PATCH,llvm-D17326_unpack_load)) +endif +$(eval $(call LLVM_PATCH,llvm-3.7.1_3)) # Remove for 3.9 $(eval $(call LLVM_PATCH,llvm-D14260)) -$(eval $(call LLVM_PATCH,llvm-3.8.0_bindir)) -$(eval $(call LLVM_PATCH,llvm-3.8.0_winshlib)) -$(eval $(call LLVM_PATCH,llvm-nodllalias)) +$(eval $(call LLVM_PATCH,llvm-3.8.0_bindir)) # Remove for 3.9 +$(eval $(call LLVM_PATCH,llvm-3.8.0_winshlib)) # Remove for 3.9 +$(eval $(call LLVM_PATCH,llvm-nodllalias)) # Remove for 3.9 # Cygwin and openSUSE still use win32-threads mingw, https://llvm.org/bugs/show_bug.cgi?id=26365 $(eval $(call LLVM_PATCH,llvm-3.8.0_threads)) # fix replutil test on unix -$(eval $(call LLVM_PATCH,llvm-D17165-D18583)) +$(eval $(call LLVM_PATCH,llvm-D17165-D18583)) # Remove for 3.9 # Segfault for aggregate load -$(eval $(call LLVM_PATCH,llvm-D17326_unpack_load)) -$(eval $(call LLVM_PATCH,llvm-D17712)) -$(eval $(call LLVM_PATCH,llvm-PR26180)) -$(eval $(call LLVM_PATCH,llvm-PR27046)) -$(eval $(call LLVM_PATCH,llvm-3.8.0_ppc64_SUBFC8)) -$(eval $(call LLVM_PATCH,llvm-D21271-instcombine-tbaa-3.8)) +$(eval $(call LLVM_PATCH,llvm-D17712)) # Remove for 3.9 +$(eval $(call LLVM_PATCH,llvm-PR26180)) # Remove for 3.9 +$(eval $(call LLVM_PATCH,llvm-PR27046)) # Remove for 3.9 +$(eval $(call LLVM_PATCH,llvm-3.8.0_ppc64_SUBFC8)) # Remove for 3.9 +$(eval $(call LLVM_PATCH,llvm-D21271-instcombine-tbaa-3.8)) # Remove for 3.9 $(eval $(call LLVM_PATCH,llvm-win64-reloc-dwarf)) endif # LLVM_VER From d292d5af43bc4095934ead6f398e7c23ae07ac95 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Thu, 14 Jul 2016 16:59:50 -0500 Subject: [PATCH 0448/1117] Propagate compilecache flag for Pkg.test --- base/pkg/entry.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 16a20750019bc..120f6f114b0e0 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -700,8 +700,9 @@ function test!(pkg::AbstractString, try color = Base.have_color? "--color=yes" : "--color=no" codecov = coverage? ["--code-coverage=user", "--inline=no"] : ["--code-coverage=none"] + compilecache = "--compilecache=" * (Bool(Base.JLOptions().use_compilecache) ? "yes" : "no") julia_exe = Base.julia_cmd() - run(`$julia_exe --check-bounds=yes $codecov $color $test_path`) + run(`$julia_exe --check-bounds=yes $codecov $color $compilecache $test_path`) info("$pkg tests passed") catch err warnbanner(err, label="[ ERROR: $pkg ]") From 70e067c278861d315099907c538669466869d093 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Thu, 14 Jul 2016 19:25:23 -0400 Subject: [PATCH 0449/1117] Don't accidentally introduce constant uses from unowned functions --- src/jitlayers.h | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/jitlayers.h b/src/jitlayers.h index e9488124667d4..05f3a34faaf6b 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -92,14 +92,27 @@ static inline Function *function_proto(Function *F, Module *M = NULL) Function *NewF = Function::Create(F->getFunctionType(), Function::ExternalLinkage, F->getName(), M); - // FunctionType does not include any attributes. Copy them over manually - // as codegen may make decisions based on the presence of certain attributes - NewF->copyAttributesFrom(F); -#ifdef LLVM37 // Declarations are not allowed to have personality routines, but - // copyAttributesFrom sets them anyway, so clear them again manually - NewF->setPersonalityFn(nullptr); + // copyAttributesFrom sets them anyway. Temporarily unset the personlity + // routine from `F`, since copying it and then resetting is more expensive + // as well as introducing an extra use from this unowned function, which + // can cause crashes in the LLVMContext's global destructor. +#ifdef LLVM37 + llvm::Constant *OldPersonalityFn = nullptr; + if (F->hasPersonalityFn()) { + OldPersonalityFn = F->getPersonalityFn(); + F->setPersonalityFn(nullptr); + } +#endif + + // FunctionType does not include any attributes. Copy them over manually + // as codegen may make decisions based on the presence of certain attributes + NewF->copyAttributesFrom(F); + +#ifdef LLVM37 + if (OldPersonalityFn) + F->setPersonalityFn(OldPersonalityFn); #endif #ifdef LLVM35 From f53bb4f79ad6f28606ae98f365e4954f44893383 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 14 Jul 2016 20:35:09 -0700 Subject: [PATCH 0450/1117] typo fix /personlty/personality/ [ci skip] --- src/jitlayers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jitlayers.h b/src/jitlayers.h index 05f3a34faaf6b..c4e2fe0fa5a31 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -94,7 +94,7 @@ static inline Function *function_proto(Function *F, Module *M = NULL) F->getName(), M); // Declarations are not allowed to have personality routines, but - // copyAttributesFrom sets them anyway. Temporarily unset the personlity + // copyAttributesFrom sets them anyway. Temporarily unset the personality // routine from `F`, since copying it and then resetting is more expensive // as well as introducing an extra use from this unowned function, which // can cause crashes in the LLVMContext's global destructor. From fc8748b22cde05d810f2dc605a7ef4288efd0f11 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Thu, 14 Jul 2016 18:27:05 +0200 Subject: [PATCH 0451/1117] Move RST 'data' docstrings inline This moves all `.. data::` docstrings found in the RST docs into `base/` source files, so all docstrings found in the docs are now actual docstrings. This will help with the markdown manual conversion. Minor formatting changes include addition of some fullstops in several docstrings, and reformatting of the `StackTrace` docstring. --- base/c.jl | 118 ++++++++++++++++++++++++++++++++++ base/complex.jl | 5 ++ base/dates/types.jl | 63 ++++++++++++++++-- base/docs/basedocs.jl | 19 ++++++ base/float.jl | 31 +++++++++ base/irrationals.jl | 34 ++++++++++ base/linalg/uniformscaling.jl | 5 ++ base/stacktraces.jl | 35 +++++++++- doc/genstdlib.jl | 2 +- doc/stdlib/arrays.rst | 1 + doc/stdlib/base.rst | 14 ++-- doc/stdlib/c.rst | 90 ++++++++++++++++++-------- doc/stdlib/dates.rst | 16 ++++- doc/stdlib/libdl.rst | 2 + doc/stdlib/linalg.rst | 4 +- doc/stdlib/numbers.rst | 48 ++++++++++---- doc/stdlib/stacktraces.rst | 42 ++++++------ doc/stdlib/strings.rst | 1 + 18 files changed, 458 insertions(+), 72 deletions(-) diff --git a/base/c.jl b/base/c.jl index 8bd9b85179b24..28b9af1ca0a0a 100644 --- a/base/c.jl +++ b/base/c.jl @@ -6,15 +6,69 @@ import Core.Intrinsics: cglobal, box cfunction(f::Function, r, a) = ccall(:jl_function_ptr, Ptr{Void}, (Any, Any, Any), f, r, a) +""" + Ptr{T} + +A memory address referring to data of type `T`. However, there is no guarantee that the +memory is actually valid, or that it actually represents data of the specified type. +""" +Ptr + +""" + Ref{T} + +An object that safely references data of type `T`. This type is guaranteed to point to +valid, Julia-allocated memory of the correct type. The underlying data is protected from +freeing by the garbage collector as long as the `Ref` itself is referenced. + +When passed as a `ccall` argument (either as a `Ptr` or `Ref` type), a `Ref` object will be +converted to a native pointer to the data it references. + +There is no invalid (NULL) `Ref`. +""" +Ref + if ccall(:jl_is_char_signed, Ref{Bool}, ()) typealias Cchar Int8 else typealias Cchar UInt8 end +""" + Cchar + +Equivalent to the native `char` c-type. +""" +Cchar + +""" + Cuchar + +Equivalent to the native `unsigned char` c-type (`UInt8`). +""" typealias Cuchar UInt8 +""" + Cshort + +Equivalent to the native `signed short` c-type (`Int16`). +""" typealias Cshort Int16 +""" + Cushort + +Equivalent to the native `unsigned short` c-type (`UInt16`). +""" typealias Cushort UInt16 +""" + Cint + +Equivalent to the native `signed int` c-type (`Int32`). +""" typealias Cint Int32 +""" + Cuint + +Equivalent to the native `unsigned int` c-type (`UInt32`). +""" typealias Cuint UInt32 if is_windows() typealias Clong Int32 @@ -25,14 +79,78 @@ else typealias Culong UInt typealias Cwchar_t Int32 end +""" + Clong + +Equivalent to the native `signed long` c-type. +""" +Clong +""" + Culong + +Equivalent to the native `unsigned long` c-type. +""" +Culong +""" + Cwchar_t + +Equivalent to the native `wchar_t` c-type (`Int32`). +""" +Cwchar_t + +""" + Cptrdiff_t + +Equivalent to the native `ptrdiff_t` c-type (`Int`). +""" typealias Cptrdiff_t Int +""" + Csize_t + +Equivalent to the native `size_t` c-type (`UInt`). +""" typealias Csize_t UInt +""" + Cssize_t + +Equivalent to the native `ssize_t` c-type. +""" typealias Cssize_t Int +""" + Cintmax_t + +Equivalent to the native `intmax_t` c-type (`Int64`). +""" typealias Cintmax_t Int64 +""" + Cuintmax_t + +Equivalent to the native `uintmax_t` c-type (`UInt64`). +""" typealias Cuintmax_t UInt64 +""" + Clonglong + +Equivalent to the native `signed long long` c-type (`Int64`). +""" typealias Clonglong Int64 +""" + Culonglong + +Equivalent to the native `unsigned long long` c-type (`UInt64`). +""" typealias Culonglong UInt64 +""" + Cfloat + +Equivalent to the native `float` c-type (`Float32`). +""" typealias Cfloat Float32 +""" + Cdouble + +Equivalent to the native `double` c-type (`Float64`). +""" typealias Cdouble Float64 if !is_windows() diff --git a/base/complex.jl b/base/complex.jl index 669d9db150391..038a1d10df91f 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -7,6 +7,11 @@ end Complex(x::Real, y::Real) = Complex(promote(x,y)...) Complex(x::Real) = Complex(x, zero(x)) +""" + im + +The imaginary unit. +""" const im = Complex(false,true) typealias Complex128 Complex{Float64} diff --git a/base/dates/types.jl b/base/dates/types.jl index cbb9a1ed86f07..e8d9dcb14ae71 100644 --- a/base/dates/types.jl +++ b/base/dates/types.jl @@ -2,6 +2,19 @@ abstract AbstractTime +""" + Period + Year + Month + Week + Day + Hour + Minute + Second + Millisecond + +`Period` types represent discrete, human representations of time. +""" abstract Period <: AbstractTime abstract DatePeriod <: Period abstract TimePeriod <: Period @@ -19,10 +32,36 @@ for T in (:Hour,:Minute,:Second,:Millisecond) end end -# Instant types represent different monotonically increasing timelines +""" + Year(v) + Month(v) + Week(v) + Day(v) + Hour(v) + Minute(v) + Second(v) + Millisecond(v) + +Construct a `Period` type with the given `v` value. Input must be losslessly convertible +to an `Int64`. +""" +Period(v) + +""" + Instant + +`Instant` types represent integer-based, machine representations of time as continuous +timelines starting from an epoch. +""" abstract Instant <: AbstractTime -# UTInstant is based on UT seconds, or 1/86400th of a turn of the earth +""" + UTInstant{T} + +The `UTInstant` represents a machine timeline based on UT time (1 day = one revolution of +the earth). The `T` is a `Period` parameter that indicates the resolution or precision of +the instant. +""" immutable UTInstant{P<:Period} <: Instant periods::P end @@ -43,16 +82,30 @@ immutable ISOCalendar <: Calendar end abstract TimeZone immutable UTC <: TimeZone end -# TimeTypes wrap Instants to provide human representations of time +""" + TimeType + +`TimeType` types wrap `Instant` machine instances to provide human representations of the +machine instant. Both `DateTime` and `Date` are subtypes of `TimeType`. +""" abstract TimeType <: AbstractTime -# DateTime is a millisecond precision UTInstant interpreted by ISOCalendar +""" + DateTime + +`DateTime` wraps a `UTInstant{Millisecond}` and interprets it according to the proleptic +Gregorian calendar. +""" immutable DateTime <: TimeType instant::UTInstant{Millisecond} DateTime(instant::UTInstant{Millisecond}) = new(instant) end -# DateTime is a day precision UTInstant interpreted by ISOCalendar +""" + Date + +`Date` wraps a `UTInstant{Day}` and interprets it according to the proleptic Gregorian calendar. +""" immutable Date <: TimeType instant::UTInstant{Day} Date(instant::UTInstant{Day}) = new(instant) diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index dd027c01f8849..30a7114a4a783 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -657,6 +657,13 @@ kw"immutable" """ kw"@__LINE__" +""" + ans + +A variable referring to the last computed value, automatically set at the interactive prompt. +""" +kw"ans" + """ nothing @@ -673,4 +680,16 @@ generation specialization for that field. """ ANY +""" + DevNull + +Used in a stream redirect to discard all data written to it. Essentially equivalent to +/dev/null on Unix or NUL on Windows. Usage: + +```julia +run(pipeline(`cat test.txt`, DevNull)) +``` +""" +DevNull + end diff --git a/base/float.jl b/base/float.jl index 6d507f2d6b34b..f07ca5c68c194 100644 --- a/base/float.jl +++ b/base/float.jl @@ -2,13 +2,44 @@ ## floating point traits ## +""" + Inf16 + +Positive infinity of type `Float16`. +""" const Inf16 = box(Float16,unbox(UInt16,0x7c00)) +""" + NaN16 + +A not-a-number value of type `Float16`. +""" const NaN16 = box(Float16,unbox(UInt16,0x7e00)) +""" + Inf32 + +Positive infinity of type `Float32`. +""" const Inf32 = box(Float32,unbox(UInt32,0x7f800000)) +""" + NaN32 + +A not-a-number value of type `Float32`. +""" const NaN32 = box(Float32,unbox(UInt32,0x7fc00000)) const Inf64 = box(Float64,unbox(UInt64,0x7ff0000000000000)) const NaN64 = box(Float64,unbox(UInt64,0x7ff8000000000000)) + +""" + Inf + +Positive infinity of type `Float64`. +""" const Inf = Inf64 +""" + NaN + +A not-a-number value of type `Float64`. +""" const NaN = NaN64 ## conversions to floating-point ## diff --git a/base/irrationals.jl b/base/irrationals.jl index 0d05de33b6564..c68947a938b44 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -131,11 +131,45 @@ big(x::Irrational) = convert(BigFloat,x) @irrational φ 1.61803398874989484820 (1+sqrt(big(5)))/2 # aliases +""" + pi + π + +The constant pi. +""" const pi = π + +""" + e + eu + +The constant e. +""" const eu = e + +""" + γ + eulergamma + +Euler's constant. +""" const eulergamma = γ + +""" + φ + golden + +The golden ratio. +""" const golden = φ +""" + catalan + +Catalan's constant. +""" +catalan + # special behaviors # use exp for e^x or e.^x, as in diff --git a/base/linalg/uniformscaling.jl b/base/linalg/uniformscaling.jl index b262aed34072b..227e41ac5dc0c 100644 --- a/base/linalg/uniformscaling.jl +++ b/base/linalg/uniformscaling.jl @@ -7,6 +7,11 @@ immutable UniformScaling{T<:Number} λ::T end +""" + I + +An object of type `UniformScaling`, representing an identity matrix of any size. +""" const I = UniformScaling(1) eltype{T}(::Type{UniformScaling{T}}) = T diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 9be7b64ca9195..9fddd537351c3 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -11,7 +11,40 @@ export StackTrace, StackFrame, stacktrace, catch_stacktrace """ StackFrame -Stack information representing execution context. +Stack information representing execution context, with the following fields: + +- `func::Symbol` + + The name of the function containing the execution context. + +- `outer_linfo::Nullable{LambdaInfo}` + + The LambdaInfo containing the execution context (if it could be found). + +- `file::Symbol` + + The path to the file containing the execution context. + +- `line::Int` + + The line number in the file containing the execution context. + +- `inlined_file::Symbol` + + The path to the file containing the context for inlined code. + +- `inlined_line::Int` + + The line number in the file containing the context for inlined code. + +- `from_c::Bool` + + True if the code is from C. + +- `pointer::Int64` + + Representation of the pointer to the execution context as returned by `backtrace`. + """ immutable StackFrame # this type should be kept platform-agnostic so that profiles can be dumped on one machine and read on another "the name of the function containing the execution context" diff --git a/doc/genstdlib.jl b/doc/genstdlib.jl index ea9964735e438..2292d8b3ecfd5 100644 --- a/doc/genstdlib.jl +++ b/doc/genstdlib.jl @@ -3,7 +3,7 @@ module GenStdLib import Base.Docs: Binding, DocStr # Constants. -const DOCSTRING_DIRECTIVE = r"^(.. (function|type|variable):: ).*" +const DOCSTRING_DIRECTIVE = r"^(.. (function|type|variable|data):: ).*" # Types. diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 7b222dea1416c..b62866876cba1 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1080,3 +1080,4 @@ dense counterparts. The following functions are specific to sparse arrays. For additional (algorithmic) information, and for versions of these methods that forgo argument checking, see (unexported) parent methods :func:`Base.SparseArrays.unchecked_noalias_permute!` and :func:`Base.SparseArrays.unchecked_aliasing_permute!`\ . See also: :func:`Base.SparseArrays.permute` + diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 31d8e48433e1b..24e44f65b3963 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -228,8 +228,9 @@ Getting Around .. data:: ans - A variable referring to the last computed value, automatically set at the - interactive prompt. + .. Docstring generated from Julia source + + A variable referring to the last computed value, automatically set at the interactive prompt. All Objects ----------- @@ -758,8 +759,13 @@ System .. data:: DevNull - Used in a stream redirect to discard all data written to it. Essentially equivalent to /dev/null on Unix or NUL on Windows. - Usage: ``run(pipeline(`cat test.txt`, DevNull))`` + .. Docstring generated from Julia source + + Used in a stream redirect to discard all data written to it. Essentially equivalent to /dev/null on Unix or NUL on Windows. Usage: + + .. code-block:: julia + + run(pipeline(`cat test.txt`, DevNull)) .. function:: success(command) diff --git a/doc/stdlib/c.rst b/doc/stdlib/c.rst index 4c921310f23b1..b90f1fed69da7 100644 --- a/doc/stdlib/c.rst +++ b/doc/stdlib/c.rst @@ -161,93 +161,127 @@ .. data:: Ptr{T} - A memory address referring to data of type ``T``. - However, there is no guarantee that the memory is actually valid, - or that it actually represents data of the specified type. + .. Docstring generated from Julia source + + A memory address referring to data of type ``T``\ . However, there is no guarantee that the memory is actually valid, or that it actually represents data of the specified type. .. data:: Ref{T} - An object that safely references data of type ``T``. - This type is guaranteed to point to valid, Julia-allocated memory - of the correct type. The underlying data is protected from freeing by - the garbage collector as long as the ``Ref`` itself is referenced. + .. Docstring generated from Julia source - When passed as a ``ccall`` argument (either as a ``Ptr`` or ``Ref`` type), - a ``Ref`` object will be converted to a native pointer to the data it references. + An object that safely references data of type ``T``\ . This type is guaranteed to point to valid, Julia-allocated memory of the correct type. The underlying data is protected from freeing by the garbage collector as long as the ``Ref`` itself is referenced. - There is no invalid (NULL) ``Ref``. + When passed as a ``ccall`` argument (either as a ``Ptr`` or ``Ref`` type), a ``Ref`` object will be converted to a native pointer to the data it references. + + There is no invalid (NULL) ``Ref``\ . .. data:: Cchar - Equivalent to the native ``char`` c-type + .. Docstring generated from Julia source + + Equivalent to the native ``char`` c-type. .. data:: Cuchar - Equivalent to the native ``unsigned char`` c-type (UInt8) + .. Docstring generated from Julia source + + Equivalent to the native ``unsigned char`` c-type (``UInt8``\ ). .. data:: Cshort - Equivalent to the native ``signed short`` c-type (Int16) + .. Docstring generated from Julia source + + Equivalent to the native ``signed short`` c-type (``Int16``\ ). .. data:: Cushort - Equivalent to the native ``unsigned short`` c-type (UInt16) + .. Docstring generated from Julia source + + Equivalent to the native ``unsigned short`` c-type (``UInt16``\ ). .. data:: Cint - Equivalent to the native ``signed int`` c-type (Int32) + .. Docstring generated from Julia source + + Equivalent to the native ``signed int`` c-type (``Int32``\ ). .. data:: Cuint - Equivalent to the native ``unsigned int`` c-type (UInt32) + .. Docstring generated from Julia source + + Equivalent to the native ``unsigned int`` c-type (``UInt32``\ ). .. data:: Clong - Equivalent to the native ``signed long`` c-type + .. Docstring generated from Julia source + + Equivalent to the native ``signed long`` c-type. .. data:: Culong - Equivalent to the native ``unsigned long`` c-type + .. Docstring generated from Julia source + + Equivalent to the native ``unsigned long`` c-type. .. data:: Clonglong - Equivalent to the native ``signed long long`` c-type (Int64) + .. Docstring generated from Julia source + + Equivalent to the native ``signed long long`` c-type (``Int64``\ ). .. data:: Culonglong - Equivalent to the native ``unsigned long long`` c-type (UInt64) + .. Docstring generated from Julia source + + Equivalent to the native ``unsigned long long`` c-type (``UInt64``\ ). .. data:: Cintmax_t - Equivalent to the native ``intmax_t`` c-type (Int64) + .. Docstring generated from Julia source + + Equivalent to the native ``intmax_t`` c-type (``Int64``\ ). .. data:: Cuintmax_t - Equivalent to the native ``uintmax_t`` c-type (UInt64) + .. Docstring generated from Julia source + + Equivalent to the native ``uintmax_t`` c-type (``UInt64``\ ). .. data:: Csize_t - Equivalent to the native ``size_t`` c-type (UInt) + .. Docstring generated from Julia source + + Equivalent to the native ``size_t`` c-type (``UInt``\ ). .. data:: Cssize_t - Equivalent to the native ``ssize_t`` c-type + .. Docstring generated from Julia source + + Equivalent to the native ``ssize_t`` c-type. .. data:: Cptrdiff_t - Equivalent to the native ``ptrdiff_t`` c-type (Int) + .. Docstring generated from Julia source + + Equivalent to the native ``ptrdiff_t`` c-type (``Int``\ ). .. data:: Cwchar_t - Equivalent to the native ``wchar_t`` c-type (Int32) + .. Docstring generated from Julia source + + Equivalent to the native ``wchar_t`` c-type (``Int32``\ ). .. data:: Cfloat - Equivalent to the native ``float`` c-type (Float32) + .. Docstring generated from Julia source + + Equivalent to the native ``float`` c-type (``Float32``\ ). .. data:: Cdouble - Equivalent to the native ``double`` c-type (Float64) + .. Docstring generated from Julia source + + Equivalent to the native ``double`` c-type (``Float64``\ ). **************** LLVM Interface diff --git a/doc/stdlib/dates.rst b/doc/stdlib/dates.rst index 86ea8a182a593..78a2af3dd68e4 100644 --- a/doc/stdlib/dates.rst +++ b/doc/stdlib/dates.rst @@ -17,6 +17,8 @@ Dates and Time Types Second Millisecond + .. Docstring generated from Julia source + ``Period`` types represent discrete, human representations of time. .. type:: CompoundPeriod @@ -27,22 +29,32 @@ Dates and Time Types .. data:: Instant + .. Docstring generated from Julia source + ``Instant`` types represent integer-based, machine representations of time as continuous timelines starting from an epoch. .. data:: UTInstant{T} + .. Docstring generated from Julia source + The ``UTInstant`` represents a machine timeline based on UT time (1 day = one revolution of the earth). The ``T`` is a ``Period`` parameter that indicates the resolution or precision of the instant. .. data:: TimeType - ``TimeType`` types wrap ``Instant`` machine instances to provide human representations of the machine instant. Both ``DateTime`` and ``Date`` are subtypes of ``TimeType``. + .. Docstring generated from Julia source + + ``TimeType`` types wrap ``Instant`` machine instances to provide human representations of the machine instant. Both ``DateTime`` and ``Date`` are subtypes of ``TimeType``\ . .. data:: DateTime + .. Docstring generated from Julia source + ``DateTime`` wraps a ``UTInstant{Millisecond}`` and interprets it according to the proleptic Gregorian calendar. .. data:: Date + .. Docstring generated from Julia source + ``Date`` wraps a ``UTInstant{Day}`` and interprets it according to the proleptic Gregorian calendar. Dates Functions @@ -539,6 +551,8 @@ Periods Second(v) Millisecond(v) + .. Docstring generated from Julia source + Construct a ``Period`` type with the given ``v`` value. Input must be losslessly convertible to an ``Int64``\ . .. function:: CompoundPeriod(periods) -> CompoundPeriod diff --git a/doc/stdlib/libdl.rst b/doc/stdlib/libdl.rst index 1171a7fb60aa0..0d85f15bdc7fe 100644 --- a/doc/stdlib/libdl.rst +++ b/doc/stdlib/libdl.rst @@ -55,6 +55,8 @@ The names in :mod:`Base.Libdl` are not exported and need to be called e.g. as `` .. data:: dlext + .. Docstring generated from Julia source + File extension for dynamic libraries (e.g. dll, dylib, so) on the current platform. .. function:: find_library(names, locations) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index c671d355c2775..713d6c55912b6 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1894,7 +1894,9 @@ for ``Float64``, ``Float32``, ``Complex128``, and ``Complex64`` arrays. .. data:: I - An object of type ``UniformScaling``, representing an identity matrix of any size. + .. Docstring generated from Julia source + + An object of type ``UniformScaling``\ , representing an identity matrix of any size. LAPACK Functions ---------------- diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index d2048420317e6..0217ee5125257 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -170,54 +170,78 @@ General Number Functions and Constants .. data:: pi π - The constant pi + .. Docstring generated from Julia source + + The constant pi. .. data:: im - The imaginary unit + .. Docstring generated from Julia source + + The imaginary unit. .. data:: e eu - The constant e + .. Docstring generated from Julia source + + The constant e. .. data:: catalan - Catalan's constant + .. Docstring generated from Julia source + + Catalan's constant. .. data:: γ eulergamma - Euler's constant + .. Docstring generated from Julia source + + Euler's constant. .. data:: φ golden - The golden ratio + .. Docstring generated from Julia source + + The golden ratio. .. data:: Inf - Positive infinity of type ``Float64`` + .. Docstring generated from Julia source + + Positive infinity of type ``Float64``\ . .. data:: Inf32 - Positive infinity of type ``Float32`` + .. Docstring generated from Julia source + + Positive infinity of type ``Float32``\ . .. data:: Inf16 - Positive infinity of type ``Float16`` + .. Docstring generated from Julia source + + Positive infinity of type ``Float16``\ . .. data:: NaN - A not-a-number value of type ``Float64`` + .. Docstring generated from Julia source + + A not-a-number value of type ``Float64``\ . .. data:: NaN32 - A not-a-number value of type ``Float32`` + .. Docstring generated from Julia source + + A not-a-number value of type ``Float32``\ . .. data:: NaN16 - A not-a-number value of type ``Float16`` + .. Docstring generated from Julia source + + A not-a-number value of type ``Float16``\ . .. function:: issubnormal(f) -> Bool diff --git a/doc/stdlib/stacktraces.rst b/doc/stdlib/stacktraces.rst index 6b5de73730043..46a5f0b857d48 100644 --- a/doc/stdlib/stacktraces.rst +++ b/doc/stdlib/stacktraces.rst @@ -10,36 +10,40 @@ .. data:: StackFrame - Stack information representing execution context, with the following fields: + .. Docstring generated from Julia source + + Stack information representing execution context, with the following fields: + + * ``func::Symbol`` - ``func::Symbol`` - the name of the function containing the execution context + The name of the function containing the execution context. + * ``outer_linfo::Nullable{LambdaInfo}`` - ``outer_linfo::Nullable{LambdaInfo}`` - the LambdaInfo containing the execution context (if it could be found) + The LambdaInfo containing the execution context (if it could be found). + * ``file::Symbol`` - ``file::Symbol`` - the path to the file containing the execution context + The path to the file containing the execution context. + * ``line::Int`` - ``line::Int`` - the line number in the file containing the execution context + The line number in the file containing the execution context. + * ``inlined_file::Symbol`` - ``inlined_file::Symbol`` - the path to the file containing the context for inlined code + The path to the file containing the context for inlined code. + * ``inlined_line::Int`` - ``inlined_line::Int`` - the line number in the file containing the context for inlined code + The line number in the file containing the context for inlined code. + * ``from_c::Bool`` - ``from_c::Bool`` - true if the code is from C + True if the code is from C. + * ``pointer::Int64`` - ``pointer::Int64`` - representation of the pointer to the execution context as returned by ``backtrace`` + Representation of the pointer to the execution context as returned by ``backtrace``\ . .. data:: StackTrace - An alias for ``Vector{StackFrame}`` provided for convenience; returned by calls to - ``stacktrace`` and ``catch_stacktrace``. + .. Docstring generated from Julia source + + An alias for ``Vector{StackFrame}`` provided for convenience; returned by calls to ``stacktrace`` and ``catch_stacktrace``\ . .. function:: stacktrace([trace::Vector{Ptr{Void}},] [c_funcs::Bool=false]) -> StackTrace diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index 1e6cd2a46b5cb..ff929e55114c2 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -482,3 +482,4 @@ .. Docstring generated from Julia source General unescaping of traditional C and Unicode escape sequences. Reverse of :func:`escape_string`\ . See also :func:`unescape_string`\ . + From 58171d58b907728b69894ce2fd77f63a07c606b1 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Thu, 14 Jul 2016 22:04:05 +0200 Subject: [PATCH 0452/1117] Move `build_sysimg` docstring This moves the docs for `build_sysimg` inline in `contrib/build_sysimg.jl`. Adds an additional condition to the `build_sysimg.jl` file so that we can safely include the file in `genstdlib.jl` without a system image being built unintentionally. --- contrib/build_sysimg.jl | 17 ++++++++++++++++- doc/devdocs/sysimg.rst | 7 +++---- doc/genstdlib.jl | 4 ++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index 0d5ee14eba250..1d7b40d105cb2 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -12,6 +12,18 @@ function default_sysimg_path(debug=false) end end +""" + build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false) + +Rebuild the system image. Store it in `sysimg_path`, which defaults to a file named `sys.ji` +that sits in the same folder as `libjulia.{so,dylib}`, except on Windows where it defaults +to `JULIA_HOME/../lib/julia/sys.ji`. Use the cpu instruction set given by `cpu_target`. +Valid CPU targets are the same as for the `-C` option to `julia`, or the `-march` option to +`gcc`. Defaults to `native`, which means to use all CPU instructions available on the +current processor. Include the user image file given by `userimg_path`, which should contain +directives such as `using MyPackage` to include that package in the new system image. New +system image will not replace an older image unless `force` is set to true. +""" function build_sysimg(sysimg_path=nothing, cpu_target="native", userimg_path=nothing; force=false, debug=false) if sysimg_path == nothing sysimg_path = default_sysimg_path(debug) @@ -157,7 +169,10 @@ end # When running this file as a script, try to do so with default values. If arguments are passed # in, use them as the arguments to build_sysimg above -if !isinteractive() +# +# Also check whether we are running `genstdlib.jl`, in which case we don't want to build a +# system image and instead only need `build_sysimg`'s docstring to be available. +if !isdefined(Main, :GenStdLib) && !isinteractive() if length(ARGS) > 5 || ("--help" in ARGS || "-h" in ARGS) println("Usage: build_sysimg.jl <sysimg_path> <cpu_target> <usrimg_path.jl> [--force] [--debug] [--help]") println(" <sysimg_path> is an absolute, extensionless path to store the system image at") diff --git a/doc/devdocs/sysimg.rst b/doc/devdocs/sysimg.rst index 1e71ed0c51b7d..4ec0e8356546c 100644 --- a/doc/devdocs/sysimg.rst +++ b/doc/devdocs/sysimg.rst @@ -26,10 +26,9 @@ This will include a ``build_sysimg()`` function: .. function:: build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false) - Rebuild the system image. Store it in ``sysimg_path``, which defaults to a file named ``sys.ji`` that sits in the same folder as ``libjulia.{so,dylib}``, except on Windows where it defaults to ``JULIA_HOME/../lib/julia/sys.ji``. - Use the cpu instruction set given by ``cpu_target``. Valid CPU targets are the same as for the ``-C`` option to ``julia``, or the ``-march`` option to ``gcc``. Defaults to ``native``, which means to use all CPU instructions available on the current processor. - Include the user image file given by ``userimg_path``, which should contain directives such as ``using MyPackage`` to include that package in the new system image. - New system image will not replace an older image unless ``force`` is set to true. + .. Docstring generated from Julia source + + Rebuild the system image. Store it in ``sysimg_path``\ , which defaults to a file named ``sys.ji`` that sits in the same folder as ``libjulia.{so,dylib}``\ , except on Windows where it defaults to ``JULIA_HOME/../lib/julia/sys.ji``\ . Use the cpu instruction set given by ``cpu_target``\ . Valid CPU targets are the same as for the ``-C`` option to ``julia``\ , or the ``-march`` option to ``gcc``\ . Defaults to ``native``\ , which means to use all CPU instructions available on the current processor. Include the user image file given by ``userimg_path``\ , which should contain directives such as ``using MyPackage`` to include that package in the new system image. New system image will not replace an older image unless ``force`` is set to true. Note that this file can also be run as a script itself, with command line arguments taking the place of arguments passed to the ``build_sysimg`` function. For example, to build a system image in ``/tmp/sys.{so,dll,dylib}``, with the ``core2`` CPU instruction set, a user image of ``~/userimg.jl`` and ``force`` set to ``true``, one would execute: :: diff --git a/doc/genstdlib.jl b/doc/genstdlib.jl index 2292d8b3ecfd5..7471fc109f49b 100644 --- a/doc/genstdlib.jl +++ b/doc/genstdlib.jl @@ -200,4 +200,8 @@ validdocstr(other) = false end +# The docstring for `build_sysimg` is defined is this file and included within the +# `devdocs/sysimg.rst` file, so we include it here to make it visible to the docsystem. +include(joinpath("..", "contrib", "build_sysimg.jl")) + GenStdLib.translate(["manual", "stdlib", "devdocs"]) From 4d63beedf084edc225e8c7052f0eb919e723289c Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Thu, 14 Jul 2016 22:13:57 +0200 Subject: [PATCH 0453/1117] Remove 'variable' RST directives Replace with `.. data::` instead since they are both used for the same kind of docstrings. --- doc/genstdlib.jl | 2 +- doc/stdlib/base.rst | 2 +- doc/stdlib/constants.rst | 26 +++++++++++++------------- doc/stdlib/io-network.rst | 8 ++++---- doc/stdlib/libdl.rst | 18 +++++++++--------- doc/stdlib/math.rst | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/doc/genstdlib.jl b/doc/genstdlib.jl index 7471fc109f49b..9be91f56c5a43 100644 --- a/doc/genstdlib.jl +++ b/doc/genstdlib.jl @@ -3,7 +3,7 @@ module GenStdLib import Base.Docs: Binding, DocStr # Constants. -const DOCSTRING_DIRECTIVE = r"^(.. (function|type|variable|data):: ).*" +const DOCSTRING_DIRECTIVE = r"^(.. (function|type|data):: ).*" # Types. diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 24e44f65b3963..f961e60fa2270 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -966,7 +966,7 @@ System A singleton of this type provides a hash table interface to environment variables. -.. variable:: ENV +.. data:: ENV .. Docstring generated from Julia source diff --git a/doc/stdlib/constants.rst b/doc/stdlib/constants.rst index 3f32de5e8ccdb..e8530a79f0d41 100644 --- a/doc/stdlib/constants.rst +++ b/doc/stdlib/constants.rst @@ -3,79 +3,79 @@ Constants ========= -.. variable:: nothing +.. data:: nothing .. Docstring generated from Julia source The singleton instance of type ``Void``\ , used by convention when there is no value to return (as in a C ``void`` function). Can be converted to an empty ``Nullable`` value. -.. variable:: PROGRAM_FILE +.. data:: PROGRAM_FILE .. Docstring generated from Julia source A string containing the script name passed to Julia from the command line. Note that the script name remains unchanged from within included files. Alternatively see :data:`@__FILE__`\ . -.. variable:: ARGS +.. data:: ARGS .. Docstring generated from Julia source An array of the command line arguments passed to Julia, as strings. -.. variable:: C_NULL +.. data:: C_NULL .. Docstring generated from Julia source The C null pointer constant, sometimes used when calling external code. -.. variable:: VERSION +.. data:: VERSION .. Docstring generated from Julia source A ``VersionNumber`` object describing which version of Julia is in use. For details see :ref:`man-version-number-literals`\ . -.. variable:: LOAD_PATH +.. data:: LOAD_PATH .. Docstring generated from Julia source An array of paths (as strings) where the ``require`` function looks for code. -.. variable:: JULIA_HOME +.. data:: JULIA_HOME .. Docstring generated from Julia source A string containing the full path to the directory containing the ``julia`` executable. -.. variable:: ANY +.. data:: ANY .. Docstring generated from Julia source Equivalent to ``Any`` for dispatch purposes, but signals the compiler to skip code generation specialization for that field. -.. variable:: Sys.CPU_CORES +.. data:: Sys.CPU_CORES .. Docstring generated from Julia source The number of CPU cores in the system. -.. variable:: Sys.WORD_SIZE +.. data:: Sys.WORD_SIZE .. Docstring generated from Julia source Standard word size on the current machine, in bits. -.. variable:: Sys.KERNEL +.. data:: Sys.KERNEL .. Docstring generated from Julia source A symbol representing the name of the operating system, as returned by ``uname`` of the build configuration. -.. variable:: Sys.ARCH +.. data:: Sys.ARCH .. Docstring generated from Julia source A symbol representing the architecture of the build configuration. -.. variable:: Sys.MACHINE +.. data:: Sys.MACHINE .. Docstring generated from Julia source diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 936258c7ac119..edfea05d67835 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -7,19 +7,19 @@ General I/O ----------- -.. variable:: STDOUT +.. data:: STDOUT .. Docstring generated from Julia source Global variable referring to the standard out stream. -.. variable:: STDERR +.. data:: STDERR .. Docstring generated from Julia source Global variable referring to the standard error stream. -.. variable:: STDIN +.. data:: STDIN .. Docstring generated from Julia source @@ -1012,7 +1012,7 @@ Network I/O Converts the endianness of a value from that used by the Host to Little-endian. -.. variable:: ENDIAN_BOM +.. data:: ENDIAN_BOM .. Docstring generated from Julia source diff --git a/doc/stdlib/libdl.rst b/doc/stdlib/libdl.rst index 0d85f15bdc7fe..6da4dffbc60f7 100644 --- a/doc/stdlib/libdl.rst +++ b/doc/stdlib/libdl.rst @@ -22,14 +22,14 @@ The names in :mod:`Base.Libdl` are not exported and need to be called e.g. as `` Similar to :func:`dlopen`\ , except returns a ``NULL`` pointer instead of raising errors. -.. variable:: RTLD_DEEPBIND - RTLD_FIRST - RTLD_GLOBAL - RTLD_LAZY - RTLD_LOCAL - RTLD_NODELETE - RTLD_NOLOAD - RTLD_NOW +.. data:: RTLD_DEEPBIND + RTLD_FIRST + RTLD_GLOBAL + RTLD_LAZY + RTLD_LOCAL + RTLD_NODELETE + RTLD_NOLOAD + RTLD_NOW .. Docstring generated from Julia source @@ -65,7 +65,7 @@ The names in :mod:`Base.Libdl` are not exported and need to be called e.g. as `` Searches for the first library in ``names`` in the paths in the ``locations`` list, ``DL_LOAD_PATH``\ , or system library paths (in that order) which can successfully be dlopen'd. On success, the return value will be one of the names (potentially prefixed by one of the paths in locations). This string can be assigned to a ``global const`` and used as the library name in future ``ccall``\ 's. On failure, it returns the empty string. -.. variable:: DL_LOAD_PATH +.. data:: DL_LOAD_PATH .. Docstring generated from Julia source diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 5df626d3f6fd7..0b97384ccdf09 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -904,37 +904,37 @@ Mathematical Functions * :obj:`RoundUp` * :obj:`RoundDown` -.. variable:: RoundNearest +.. data:: RoundNearest .. Docstring generated from Julia source The default rounding mode. Rounds to the nearest integer, with ties (fractional values of 0.5) being rounded to the nearest even integer. -.. variable:: RoundNearestTiesAway +.. data:: RoundNearestTiesAway .. Docstring generated from Julia source Rounds to nearest integer, with ties rounded away from zero (C/C++ :func:`round` behaviour). -.. variable:: RoundNearestTiesUp +.. data:: RoundNearestTiesUp .. Docstring generated from Julia source Rounds to nearest integer, with ties rounded toward positive infinity (Java/JavaScript :func:`round` behaviour). -.. variable:: RoundToZero +.. data:: RoundToZero .. Docstring generated from Julia source :func:`round` using this rounding mode is an alias for :func:`trunc`\ . -.. variable:: RoundUp +.. data:: RoundUp .. Docstring generated from Julia source :func:`round` using this rounding mode is an alias for :func:`ceil`\ . -.. variable:: RoundDown +.. data:: RoundDown .. Docstring generated from Julia source From 17c4928bf1050567fd2930d6781fa41064a63efe Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 12 Jul 2016 18:42:15 -0700 Subject: [PATCH 0454/1117] Rework setindex!(A::SparseMatrixCSC, v, I, J) such that it no longer expunges stored entries on zero assignment. Also add tests. --- base/sparse/sparsematrix.jl | 129 +++++++++++++++--------------------- test/sparsedir/sparse.jl | 19 ++++++ 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index d0866a65f6626..76abeebb86f8f 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -2519,23 +2519,64 @@ setindex!(A::SparseMatrixCSC, x, ::Colon, ::Colon) = setindex!(A, x, 1:size(A, 1 setindex!(A::SparseMatrixCSC, x, ::Colon, j::Union{Integer, AbstractVector}) = setindex!(A, x, 1:size(A, 1), j) setindex!(A::SparseMatrixCSC, x, i::Union{Integer, AbstractVector}, ::Colon) = setindex!(A, x, i, 1:size(A, 2)) -setindex!{Tv,T<:Integer}(A::SparseMatrixCSC{Tv}, x::Number, I::AbstractVector{T}, J::AbstractVector{T}) = - (0 == x) ? spdelete!(A, I, J) : spset!(A, convert(Tv,x), I, J) - -function spset!{Tv,Ti<:Integer}(A::SparseMatrixCSC{Tv}, x::Tv, I::AbstractVector{Ti}, J::AbstractVector{Ti}) - !issorted(I) && (I = sort(I)) - !issorted(J) && (J = sort(J)) - - m, n = size(A) - lenI = length(I) - - if (!isempty(I) && (I[1] < 1 || I[end] > m)) || (!isempty(J) && (J[1] < 1 || J[end] > n)) +function setindex!{TvA,TiI<:Integer,TiJ<:Integer}(A::SparseMatrixCSC{TvA}, x::Number, + I::AbstractVector{TiI}, J::AbstractVector{TiJ}) + if isempty(I) || isempty(J); return A; end + if !issorted(I); I = sort(I); end + if !issorted(J); J = sort(J); end + if (I[1] < 1 || I[end] > A.m) || (J[1] < 1 || J[end] > A.n) throw(BoundsError(A, (I, J))) end - - if isempty(I) || isempty(J) - return A + if x == 0 + _spsetz_setindex!(A, I, J) + else + _spsetnz_setindex!(A, convert(TvA, x), I, J) + end +end +""" +Helper method for immediately preceding setindex! method. For all (i,j) such that i in I and +j in J, assigns zero to A[i,j] if A[i,j] is a presently-stored entry, and otherwise does nothing. +""" +function _spsetz_setindex!{TvA,TiI<:Integer,TiJ<:Integer}(A::SparseMatrixCSC{TvA}, + I::AbstractVector{TiI}, J::AbstractVector{TiJ}) + lengthI = length(I) + for j in J + coljAfirstk = A.colptr[j] + coljAlastk = A.colptr[j+1] - 1 + coljAfirstk > coljAlastk && continue + kA = coljAfirstk + kI = 1 + entrykArow = A.rowval[kA] + entrykIrow = I[kI] + while true + if entrykArow < entrykIrow + kA += 1 + kA > coljAlastk && break + entrykArow = A.rowval[kA] + elseif entrykArow > entrykIrow + kI += 1 + kI > lengthI && break + entrykIrow = I[kI] + else # entrykArow == entrykIrow + A.nzval[kA] = 0 + kA += 1 + kI += 1 + (kA > coljAlastk || kI > lengthI) && break + entrykArow = A.rowval[kA] + entrykIrow = I[kI] + end + end end +end +""" +Helper method for immediately preceding setindex! method. For all (i,j) such that i in I +and j in J, assigns x to A[i,j] if A[i,j] is a presently-stored entry, and allocates and +assigns x to A[i,j] if A[i,j] is not presently stored. +""" +function _spsetnz_setindex!{Tv,TiI<:Integer,TiJ<:Integer}(A::SparseMatrixCSC{Tv}, x::Tv, + I::AbstractVector{TiI}, J::AbstractVector{TiJ}) + m, n = size(A) + lenI = length(I) nnzA = nnz(A) + lenI * length(J) @@ -2638,66 +2679,6 @@ function spset!{Tv,Ti<:Integer}(A::SparseMatrixCSC{Tv}, x::Tv, I::AbstractVector return A end -function spdelete!{Tv,Ti<:Integer}(A::SparseMatrixCSC{Tv}, I::AbstractVector{Ti}, J::AbstractVector{Ti}) - m, n = size(A) - nnzA = nnz(A) - (nnzA == 0) && (return A) - - !issorted(I) && (I = sort(I)) - !issorted(J) && (J = sort(J)) - - if (!isempty(I) && (I[1] < 1 || I[end] > m)) || (!isempty(J) && (J[1] < 1 || J[end] > n)) - throw(BoundsError(A, (I, J))) - end - - if isempty(I) || isempty(J) - return A - end - - rowval = rowvalA = A.rowval - nzval = nzvalA = A.nzval - rowidx = 1 - ndel = 0 - @inbounds for col in 1:n - rrange = nzrange(A, col) - if ndel > 0 - A.colptr[col] = A.colptr[col] - ndel - end - - if isempty(rrange) || !(col in J) - nincl = length(rrange) - if(ndel > 0) && !isempty(rrange) - copy!(rowvalA, rowidx, rowval, rrange[1], nincl) - copy!(nzvalA, rowidx, nzval, rrange[1], nincl) - end - rowidx += nincl - else - for ridx in rrange - if rowval[ridx] in I - if ndel == 0 - rowval = copy(rowvalA) - nzval = copy(nzvalA) - end - ndel += 1 - else - if ndel > 0 - rowvalA[rowidx] = rowval[ridx] - nzvalA[rowidx] = nzval[ridx] - end - rowidx += 1 - end - end - end - end - - if ndel > 0 - A.colptr[n+1] = rowidx - deleteat!(rowvalA, rowidx:nnzA) - deleteat!(nzvalA, rowidx:nnzA) - end - return A -end - setindex!{Tv,Ti,T<:Integer}(A::SparseMatrixCSC{Tv,Ti}, S::Matrix, I::AbstractVector{T}, J::AbstractVector{T}) = setindex!(A, convert(SparseMatrixCSC{Tv,Ti}, S), I, J) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 47cccac054ccd..e186deafa197e 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -577,6 +577,7 @@ let a = spzeros(Int, 10, 10) a[:,2] = 2 @test countnz(a) == 19 @test a[:,2] == 2*sparse(ones(Int,10)) + b = copy(a) # Zero-assignment behavior of setindex!(A, v, i, j) a[1,3] = 0 @@ -586,6 +587,24 @@ let a = spzeros(Int, 10, 10) @test nnz(a) == 19 @test countnz(a) == 18 + # Zero-assignment behavior of setindex!(A, v, I, J) + a[1,:] = 0 + @test nnz(a) == 19 + @test countnz(a) == 9 + a[2,:] = 0 + @test nnz(a) == 19 + @test countnz(a) == 8 + a[:,1] = 0 + @test nnz(a) == 19 + @test countnz(a) == 8 + a[:,2] = 0 + @test nnz(a) == 19 + @test countnz(a) == 0 + a = copy(b) + a[:,:] = 0 + @test nnz(a) == 19 + @test countnz(a) == 0 + a[1,:] = 1:10 @test a[1,:] == sparse([1:10;]) a[:,2] = 1:10 From fe53e6b63865fc71d1c41f30930ccb2b4e7cbc65 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 12 Jul 2016 21:18:19 -0700 Subject: [PATCH 0455/1117] Revise setindex!(A::SparseMatrixCSC, B::SparseMatrixCSC, I, J) such that, when an assigning zero to entry A[i,j], if A[i,j] is not presently stored, the method does not introduce storage for A[i,j]. Also add tests. --- base/sparse/sparsematrix.jl | 32 ++++++++++++++++---------------- test/sparsedir/sparse.jl | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 76abeebb86f8f..5c48f161f18c2 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -2766,16 +2766,16 @@ function setindex!{Tv,Ti,T<:Integer}(A::SparseMatrixCSC{Tv,Ti}, B::SparseMatrixC rowA = rowvalS[ptrA] rowB = I[rowvalB[ptrB]] if rowA < rowB - if ~I_asgn[rowA] - rowvalA[ptrS] = rowA - nzvalA[ptrS] = nzvalS[ptrA] - ptrS += 1 - end + rowvalA[ptrS] = rowA + nzvalA[ptrS] = I_asgn[rowA] ? zero(Tv) : nzvalS[ptrA] + ptrS += 1 ptrA += 1 elseif rowB < rowA - rowvalA[ptrS] = rowB - nzvalA[ptrS] = nzvalB[ptrB] - ptrS += 1 + if nzvalB[ptrB] != zero(Tv) + rowvalA[ptrS] = rowB + nzvalA[ptrS] = nzvalB[ptrB] + ptrS += 1 + end ptrB += 1 else rowvalA[ptrS] = rowB @@ -2788,19 +2788,19 @@ function setindex!{Tv,Ti,T<:Integer}(A::SparseMatrixCSC{Tv,Ti}, B::SparseMatrixC while ptrA < stopA rowA = rowvalS[ptrA] - if ~I_asgn[rowA] - rowvalA[ptrS] = rowA - nzvalA[ptrS] = nzvalS[ptrA] - ptrS += 1 - end + rowvalA[ptrS] = rowA + nzvalA[ptrS] = I_asgn[rowA] ? zero(Tv) : nzvalS[ptrA] + ptrS += 1 ptrA += 1 end while ptrB < stopB rowB = I[rowvalB[ptrB]] - rowvalA[ptrS] = rowB - nzvalA[ptrS] = nzvalB[ptrB] - ptrS += 1 + if nzvalB[ptrB] != zero(Tv) + rowvalA[ptrS] = rowB + nzvalA[ptrS] = nzvalB[ptrB] + ptrS += 1 + end ptrB += 1 end diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index e186deafa197e..cbafc7929379b 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -605,6 +605,22 @@ let a = spzeros(Int, 10, 10) @test nnz(a) == 19 @test countnz(a) == 0 + # Zero-assignment behavior of setindex!(A, B::SparseMatrixCSC, I, J) + a = copy(b) + a[1:2,:] = spzeros(2, 10) + @test nnz(a) == 19 + @test countnz(a) == 8 + a[1:2,1:3] = sparse([1 0 1; 0 0 1]) + @test nnz(a) == 20 + @test countnz(a) == 11 + a = copy(b) + a[1:2,:] = let c = sparse(ones(2,10)); fill!(c.nzval, 0); c; end + @test nnz(a) == 19 + @test countnz(a) == 8 + a[1:2,1:3] = let c = sparse(ones(2,3)); c[1,2] = c[2,1] = c[2,2] = 0; c; end + @test nnz(a) == 20 + @test countnz(a) == 11 + a[1,:] = 1:10 @test a[1,:] == sparse([1:10;]) a[:,2] = 1:10 From 30e7d150480b6bce10c888a62bb7cd6eaede2271 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Wed, 13 Jul 2016 13:23:03 -0700 Subject: [PATCH 0456/1117] Revise setindex!(A::SparseMatrixCSC, x, I::AbstractMatrix{Bool}) such that it no longer expunges stored entries on zero assignment. Also add tests. --- base/sparse/sparsematrix.jl | 23 ++++++++++++----------- test/sparsedir/sparse.jl | 8 +++++++- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 5c48f161f18c2..d994a10fbcbb9 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -2836,7 +2836,7 @@ function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) colptrA = A.colptr; rowvalA = A.rowval; nzvalA = A.nzval colptrB = colptrA; rowvalB = rowvalA; nzvalB = nzvalA - nadd = ndel = 0 + nadd = 0 bidx = xidx = 1 r1 = r2 = 0 @@ -2852,7 +2852,7 @@ function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) if r1 <= r2 copylen = searchsortedfirst(rowvalA, row, r1, r2, Forward) - r1 if (copylen > 0) - if (nadd > 0) || (ndel > 0) + if (nadd > 0) copy!(rowvalB, bidx, rowvalA, r1, copylen) copy!(nzvalB, bidx, nzvalA, r1, copylen) end @@ -2861,13 +2861,17 @@ function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) end end - # 0: no change, 1: update, 2: delete, 3: add new - mode = ((r1 <= r2) && (rowvalA[r1] == row)) ? ((v == 0) ? 2 : 1) : ((v == 0) ? 0 : 3) + # 0: no change, 1: update, 2: add new + mode = ((r1 <= r2) && (rowvalA[r1] == row)) ? 1 : ((v == 0) ? 0 : 2) - if (mode > 1) && (nadd == 0) && (ndel == 0) + if (mode > 1) && (nadd == 0) # copy storage to take changes colptrA = copy(colptrB) memreq = (x == 0) ? 0 : n + # this x == 0 check and approach doesn't jive with use of v above + # and may not make sense generally, as scalar x == 0 probably + # means this section should never be called. also may not be generic. + # TODO: clean this up, maybe separate scalar and array X cases rowvalA = copy(rowvalB) nzvalA = copy(nzvalB) resize!(rowvalB, length(rowvalA)+memreq) @@ -2879,9 +2883,6 @@ function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) bidx += 1 r1 += 1 elseif mode == 2 - r1 += 1 - ndel += 1 - elseif mode == 3 rowvalB[bidx] = row nzvalB[bidx] = v bidx += 1 @@ -2891,7 +2892,7 @@ function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) end # if I[row, col] end # for row in 1:A.m - if ((nadd != 0) || (ndel != 0)) + if (nadd != 0) l = r2-r1+1 if l > 0 copy!(rowvalB, bidx, rowvalA, r1, l) @@ -2901,7 +2902,7 @@ function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) colptrB[col+1] = bidx if (xidx > n) && (length(colptrB) > (col+1)) - diff = nadd - ndel + diff = nadd colptrB[(col+2):end] = colptrA[(col+2):end] .+ diff r1 = colptrA[col+1] r2 = colptrA[end]-1 @@ -2918,7 +2919,7 @@ function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) (xidx > n) && break end # for col in 1:A.n - if (nadd != 0) || (ndel != 0) + if (nadd != 0) n = length(nzvalB) if n > (bidx-1) deleteat!(nzvalB, bidx:n) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index cbafc7929379b..33a00dbb1c10a 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -709,15 +709,21 @@ let S = sprand(50, 30, 0.5, x->round(Int,rand(x)*100)), I = sprand(Bool, 50, 30, sumS1 = sum(S) sumFI = sum(S[FI]) + nnzS1 = nnz(S) S[FI] = 0 - @test sum(S[FI]) == 0 sumS2 = sum(S) + cnzS2 = countnz(S) + @test sum(S[FI]) == 0 + @test nnz(S) == nnzS1 @test (sum(S) + sumFI) == sumS1 S[FI] = 10 + nnzS3 = nnz(S) @test sum(S) == sumS2 + 10*sum(FI) S[FI] = 0 @test sum(S) == sumS2 + @test nnz(S) == nnzS3 + @test countnz(S) == cnzS2 S[FI] = [1:sum(FI);] @test sum(S) == sumS2 + sum(1:sum(FI)) From c900fbd11dc8b6a0780c67a3be779eb7eed9f8d6 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Wed, 13 Jul 2016 13:38:45 -0700 Subject: [PATCH 0457/1117] Revise setindex!(A::SparseMatrixCSC, x, I::AbstractVector) such that it no longer expunges stored entries on zero assignment. Also add tests. --- base/sparse/sparsematrix.jl | 23 ++++++++++------------- test/sparsedir/sparse.jl | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index d994a10fbcbb9..d76aa544efa8a 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -2936,7 +2936,7 @@ function setindex!{Tv,Ti,T<:Real}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVecto colptrA = A.colptr; rowvalA = A.rowval; nzvalA = A.nzval; szA = size(A) colptrB = colptrA; rowvalB = rowvalA; nzvalB = nzvalA - nadd = ndel = 0 + nadd = 0 bidx = aidx = 1 S = issorted(I) ? (1:n) : sortperm(I) @@ -2962,8 +2962,8 @@ function setindex!{Tv,Ti,T<:Real}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVecto r2 = Int(colptrA[col+1] - 1) # copy from last position till current column - if (nadd > 0) || (ndel > 0) - colptrB[(lastcol+1):col] = colptrA[(lastcol+1):col] .+ (nadd - ndel) + if (nadd > 0) + colptrB[(lastcol+1):col] = colptrA[(lastcol+1):col] .+ nadd copylen = r1 - aidx if copylen > 0 copy!(rowvalB, bidx, rowvalA, aidx, copylen) @@ -2980,7 +2980,7 @@ function setindex!{Tv,Ti,T<:Real}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVecto if r1 <= r2 copylen = searchsortedfirst(rowvalA, row, r1, r2, Forward) - r1 if (copylen > 0) - if (nadd > 0) || (ndel > 0) + if (nadd > 0) copy!(rowvalB, bidx, rowvalA, r1, copylen) copy!(nzvalB, bidx, nzvalA, r1, copylen) end @@ -2990,13 +2990,14 @@ function setindex!{Tv,Ti,T<:Real}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVecto end end - # 0: no change, 1: update, 2: delete, 3: add new - mode = ((r1 <= r2) && (rowvalA[r1] == row)) ? ((v == 0) ? 2 : 1) : ((v == 0) ? 0 : 3) + # 0: no change, 1: update, 2: add new + mode = ((r1 <= r2) && (rowvalA[r1] == row)) ? 1 : ((v == 0) ? 0 : 2) - if (mode > 1) && (nadd == 0) && (ndel == 0) + if (mode > 1) && (nadd == 0) # copy storage to take changes colptrA = copy(colptrB) memreq = (x == 0) ? 0 : n + # see comment/TODO for same statement in preceding logical setindex! method rowvalA = copy(rowvalB) nzvalA = copy(nzvalB) resize!(rowvalB, length(rowvalA)+memreq) @@ -3009,10 +3010,6 @@ function setindex!{Tv,Ti,T<:Real}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVecto aidx += 1 r1 += 1 elseif mode == 2 - r1 += 1 - aidx += 1 - ndel += 1 - elseif mode == 3 rowvalB[bidx] = row nzvalB[bidx] = v bidx += 1 @@ -3021,8 +3018,8 @@ function setindex!{Tv,Ti,T<:Real}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVecto end # copy the rest - @inbounds if (nadd > 0) || (ndel > 0) - colptrB[(lastcol+1):end] = colptrA[(lastcol+1):end] .+ (nadd - ndel) + @inbounds if (nadd > 0) + colptrB[(lastcol+1):end] = colptrA[(lastcol+1):end] .+ nadd r1 = colptrA[end]-1 copylen = r1 - aidx + 1 if copylen > 0 diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 33a00dbb1c10a..47e94324ba2da 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -699,6 +699,20 @@ let A = speye(Int, 5), I=1:10, X=reshape([trues(10); falses(15)],5,5) @test A[I] == A[X] == [1,0,0,0,0,0,1,0,0,0] A[I] = [1:10;] @test A[I] == A[X] == collect(1:10) + A[I] = zeros(Int, 10) + @test nnz(A) == 13 + @test countnz(A) == 3 + @test A[I] == A[X] == zeros(Int, 10) + c = collect(11:20); c[1] = c[3] = 0 + A[I] = c + @test nnz(A) == 13 + @test countnz(A) == 11 + @test A[I] == A[X] == c + A = speye(Int, 5) + A[I] = c + @test nnz(A) == 12 + @test countnz(A) == 11 + @test A[I] == A[X] == c end let S = sprand(50, 30, 0.5, x->round(Int,rand(x)*100)), I = sprand(Bool, 50, 30, 0.2) From 78efe6c3040ed3cfc6efeda53afecea221ae8020 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Fri, 15 Jul 2016 12:32:46 -0700 Subject: [PATCH 0458/1117] Introduce rudimentary dropstored! methods for dropping stored entries from SparseMatrixCSCs. --- base/sparse/sparsematrix.jl | 103 ++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index d76aa544efa8a..ecacca3819e7c 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -3038,6 +3038,109 @@ function setindex!{Tv,Ti,T<:Real}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVecto A end +## dropstored! methods +""" + dropstored!(A::SparseMatrixCSC, i::Integer, j::Integer) + +Drop entry `A[i,j]` from `A` if `A[i,j]` is stored, and otherwise do nothing. +""" +function dropstored!(A::SparseMatrixCSC, i::Integer, j::Integer) + if !((1 <= i <= A.m) & (1 <= j <= A.n)) + throw(BoundsError(A, (i,j))) + end + coljfirstk = Int(A.colptr[j]) + coljlastk = Int(A.colptr[j+1] - 1) + searchk = searchsortedfirst(A.rowval, i, coljfirstk, coljlastk, Base.Order.Forward) + if searchk <= coljlastk && A.rowval[searchk] == i + # Entry A[i,j] is stored. Drop and return. + deleteat!(A.rowval, searchk) + deleteat!(A.nzval, searchk) + @simd for m in j:(A.n + 1) + @inbounds A.colptr[m] -= 1 + end + end + return A +end +""" + dropstored!(A::SparseMatrix, I::AbstractVector{<:Integer}, J::AbstractVector{<:Integer}) + +For each `(i,j)` where `i in I` and `j in J`, drop entry `A[i,j]` from `A` if `A[i,j]` is +stored and otherwise do nothing. Derivative forms: + + dropstored!(A::SparseMatrixCSC, i::Integer, J::AbstractVector{<:Integer}) + dropstored!(A::SparseMatrixCSC, I::AbstractVector{<:Integer}, j::Integer) +""" +function dropstored!{TiI<:Integer,TiJ<:Integer}(A::SparseMatrixCSC, + I::AbstractVector{TiI}, J::AbstractVector{TiJ}) + m, n = size(A) + nnzA = nnz(A) + (nnzA == 0) && (return A) + + !issorted(I) && (I = sort(I)) + !issorted(J) && (J = sort(J)) + + if (!isempty(I) && (I[1] < 1 || I[end] > m)) || (!isempty(J) && (J[1] < 1 || J[end] > n)) + throw(BoundsError(A, (I, J))) + end + + if isempty(I) || isempty(J) + return A + end + + rowval = rowvalA = A.rowval + nzval = nzvalA = A.nzval + rowidx = 1 + ndel = 0 + @inbounds for col in 1:n + rrange = nzrange(A, col) + if ndel > 0 + A.colptr[col] = A.colptr[col] - ndel + end + + if isempty(rrange) || !(col in J) + nincl = length(rrange) + if(ndel > 0) && !isempty(rrange) + copy!(rowvalA, rowidx, rowval, rrange[1], nincl) + copy!(nzvalA, rowidx, nzval, rrange[1], nincl) + end + rowidx += nincl + else + for ridx in rrange + if rowval[ridx] in I + if ndel == 0 + rowval = copy(rowvalA) + nzval = copy(nzvalA) + end + ndel += 1 + else + if ndel > 0 + rowvalA[rowidx] = rowval[ridx] + nzvalA[rowidx] = nzval[ridx] + end + rowidx += 1 + end + end + end + end + + if ndel > 0 + A.colptr[n+1] = rowidx + deleteat!(rowvalA, rowidx:nnzA) + deleteat!(nzvalA, rowidx:nnzA) + end + return A +end +dropstored!{T<:Integer}(A::SparseMatrixCSC, i::Integer, J::AbstractVector{T}) = dropstored!(A, [i], J) +dropstored!{T<:Integer}(A::SparseMatrixCSC, I::AbstractVector{T}, j::Integer) = dropstored!(A, I, [j]) +dropstored!(A::SparseMatrixCSC, ::Colon, j::Union{Integer,AbstractVector}) = dropstored!(A, 1:size(A,1), j) +dropstored!(A::SparseMatrixCSC, i::Union{Integer,AbstractVector}, ::Colon) = dropstored!(A, i, 1:size(A,2)) +dropstored!(A::SparseMatrixCSC, ::Colon, ::Colon) = dropstored!(A, 1:size(A,1), 1:size(A,2)) +dropstored!(A::SparseMatrixCSC, ::Colon) = dropstored!(A, :, :) +# TODO: Several of the preceding methods are optimization candidates. +# TODO: Implement linear indexing methods for dropstored! ? +# TODO: Implement logical indexing methods for dropstored! ? + + # Sparse concatenation function vcat(X::SparseMatrixCSC...) From 0e65c7a9831cc5d4062fed3ce4e0d871ba3238c6 Mon Sep 17 00:00:00 2001 From: Simon Kornblith <simon@simonster.com> Date: Fri, 8 Jul 2016 22:34:27 -0400 Subject: [PATCH 0459/1117] Fix #17105, @ inferred with kwargs Also fixes #9818 (incorrect @ code_typed with kwargs) since otherwise @ inferred with kwargs would sometimes be wrong. --- base/interactiveutil.jl | 18 +++++++++++------- base/test.jl | 19 +++++++++++++++---- test/test.jl | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 6ac54783f30e4..1760cda164318 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -277,13 +277,17 @@ typesof(args...) = Tuple{map(a->(isa(a,Type) ? Type{a} : typeof(a)), args)...} gen_call_with_extracted_types(fcn, ex0::Symbol) = Expr(:call, fcn, Meta.quot(ex0)) function gen_call_with_extracted_types(fcn, ex0) - if isa(ex0, Expr) && - (any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) || - ex0.head == :call) - # keyword args not used in dispatch, so just remove them - args = filter(a->!(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) - return Expr(:call, fcn, esc(args[1]), - Expr(:call, typesof, map(esc, args[2:end])...)) + if isa(ex0, Expr) + if any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) + # remove keyword args, but call the kwfunc + args = filter(a->!(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) + return :($(fcn)(Core.kwfunc($(esc(args[1]))), + Tuple{Vector{Any}, typeof($(esc(args[1]))), + $(typesof)($(map(esc, args[2:end])...)).parameters...})) + elseif ex0.head == :call + return Expr(:call, fcn, esc(ex0.args[1]), + Expr(:call, typesof, map(esc, ex0.args[2:end])...)) + end end exret = Expr(:none) is_macro = false diff --git a/base/test.jl b/base/test.jl index 2681d48ee573a..3776b3cc4d6b1 100644 --- a/base/test.jl +++ b/base/test.jl @@ -873,6 +873,7 @@ macro test_approx_eq(a, b) :(test_approx_eq($(esc(a)), $(esc(b)), $(string(a)), $(string(b)))) end +_args_and_call(args...; kwargs...) = (args[1:end-1], kwargs, args[end](args[1:end-1]...; kwargs...)) """ @inferred f(x) @@ -912,12 +913,22 @@ julia> @inferred max(1,2) ``` """ macro inferred(ex) - ex.head == :call || error("@inferred requires a call expression") + Meta.isexpr(ex, :call)|| error("@inferred requires a call expression") + args = gensym() + kwargs = gensym() + + # Need to only consturct a call with a kwargs if there are actually kwargs, since + # the inferred type depends on the presence/absence of kwargs + if any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex.args) + typecall = :($(ex.args[1])($(args)...; $(kwargs)...)) + else + typecall = :($(ex.args[1])($(args)...)) + end + Base.remove_linenums!(quote - vals = ($([esc(ex.args[i]) for i = 2:length(ex.args)]...),) - inftypes = Base.return_types($(esc(ex.args[1])), Base.typesof(vals...)) + $(esc(args)), $(esc(kwargs)), result = $(esc(Expr(:call, _args_and_call, ex.args[2:end]..., ex.args[1]))) + inftypes = $(Base.gen_call_with_extracted_types(Base.return_types, typecall)) @assert length(inftypes) == 1 - result = $(esc(ex.args[1]))(vals...) rettype = isa(result, Type) ? Type{result} : typeof(result) rettype == inftypes[1] || error("return type $rettype does not match inferred return type $(inftypes[1])") result diff --git a/test/test.jl b/test/test.jl index 0ef54e16b2f26..eb183681c82dd 100644 --- a/test/test.jl +++ b/test/test.jl @@ -278,6 +278,38 @@ for i in 1:6 @test typeof(tss[i].results[4].results[1]) == (iseven(i) ? Pass : Fail) end +# test @inferred +function uninferrable_function(i) + q = [1, "1"] + return q[i] +end + +@test_throws ErrorException @inferred(uninferrable_function(1)) +@test @inferred(identity(1)) == 1 + +# Ensure @inferred only evaluates the arguments once +inferred_test_global = 0 +function inferred_test_function() + global inferred_test_global + inferred_test_global += 1 + true +end +@test @inferred inferred_test_function() +@test inferred_test_global == 1 + # Issue #14928 # Make sure abstract error type works. @test_throws Exception error("") + +# Issue #17105 +# @inferred with kwargs +function inferrable_kwtest(x; y=1) + 2x +end +function uninferrable_kwtest(x; y=1) + 2x+y +end +@test @inferred(inferrable_kwtest(1)) == 2 +@test @inferred(inferrable_kwtest(1; y=1)) == 2 +@test @inferred(uninferrable_kwtest(1)) == 3 +@test_throws ErrorException @inferred(uninferrable_kwtest(1; y=2)) == 2 From e72e24f34cfe0ba4d9978d23d41f82b5cf574aef Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Fri, 15 Jul 2016 13:04:29 -0700 Subject: [PATCH 0460/1117] Introduce tests for dropstored!(A, i, j) and dropstored!(A, I, J). --- test/sparsedir/sparse.jl | 54 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 47e94324ba2da..3666e96e9bc89 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -755,6 +755,60 @@ let S = sprand(50, 30, 0.5, x->round(Int,rand(x)*100)) @test sum(S) == (sumS1 - sumS2 + sum(J)) end + +## dropstored! tests +let A = spzeros(Int, 10, 10) + # Introduce nonzeros in row and column two + A[1,:] = 1 + A[:,2] = 2 + @test nnz(A) == 19 + + # Test argument bounds checking for dropstored!(A, i, j) + @test_throws BoundsError Base.SparseArrays.dropstored!(A, 0, 1) + @test_throws BoundsError Base.SparseArrays.dropstored!(A, 1, 0) + @test_throws BoundsError Base.SparseArrays.dropstored!(A, 1, 11) + @test_throws BoundsError Base.SparseArrays.dropstored!(A, 11, 1) + + # Test argument bounds checking for dropstored!(A, I, J) + @test_throws BoundsError Base.SparseArrays.dropstored!(A, 0:1, 1:1) + @test_throws BoundsError Base.SparseArrays.dropstored!(A, 1:1, 0:1) + @test_throws BoundsError Base.SparseArrays.dropstored!(A, 10:11, 1:1) + @test_throws BoundsError Base.SparseArrays.dropstored!(A, 1:1, 10:11) + + # Test behavior of dropstored!(A, i, j) + # --> Test dropping a single stored entry + Base.SparseArrays.dropstored!(A, 1, 2) + @test nnz(A) == 18 + # --> Test dropping a single nonstored entry + Base.SparseArrays.dropstored!(A, 2, 1) + @test nnz(A) == 18 + + # Test behavior of dropstored!(A, I, J) and derivs. + # --> Test dropping a single row including stored and nonstored entries + Base.SparseArrays.dropstored!(A, 1, :) + @test nnz(A) == 9 + # --> Test dropping a single column including stored and nonstored entries + Base.SparseArrays.dropstored!(A, :, 2) + @test nnz(A) == 0 + # --> Introduce nonzeros in rows one and two and columns two and three + A[1:2,:] = 1 + A[:,2:3] = 2 + @test nnz(A) == 36 + # --> Test dropping multiple rows containing stored and nonstored entries + Base.SparseArrays.dropstored!(A, 1:3, :) + @test nnz(A) == 14 + # --> Test dropping multiple columns containing stored and nonstored entries + Base.SparseArrays.dropstored!(A, :, 2:4) + @test nnz(A) == 0 + # --> Introduce nonzeros in every other row + A[1:2:9, :] = 1 + @test nnz(A) == 50 + # --> Test dropping a block of the matrix towards the upper left + Base.SparseArrays.dropstored!(A, 2:5, 2:5) + @test nnz(A) == 42 +end + + #Issue 7507 @test (i7507=sparsevec(Dict{Int64, Float64}(), 10))==spzeros(10) From 48c44b19e38e8eb9847ec25df7963e7816ce2d13 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Fri, 15 Jul 2016 13:53:32 -0700 Subject: [PATCH 0461/1117] Revise setindex!(x::SparseVector, v, i) such that it no longer purges stored entries on zero assignment. Modify tests accordingly. --- base/sparse/sparsevector.jl | 7 +------ test/sparsedir/sparsevector.jl | 9 ++++++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index 4c303ff911cfa..b28c8ab2fe822 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -197,12 +197,7 @@ function setindex!{Tv,Ti<:Integer}(x::SparseVector{Tv,Ti}, v::Tv, i::Ti) m = length(nzind) k = searchsortedfirst(nzind, i) if 1 <= k <= m && nzind[k] == i # i found - if v == 0 - deleteat!(nzind, k) - deleteat!(nzval, k) - else - nzval[k] = v - end + nzval[k] = v else # i not found if v != 0 insert!(nzind, k, i) diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index ca94ed7974832..46919c2dd62dd 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -201,13 +201,16 @@ end let xc = copy(spv_x1) xc[5] = 0.0 - @test exact_equal(xc, SparseVector(8, [2, 6], [1.25, 3.5])) + @test exact_equal(xc, SparseVector(8, [2, 5, 6], [1.25, 0.0, 3.5])) xc[6] = 0.0 - @test exact_equal(xc, SparseVector(8, [2], [1.25])) + @test exact_equal(xc, SparseVector(8, [2, 5, 6], [1.25, 0.0, 0.0])) xc[2] = 0.0 - @test exact_equal(xc, SparseVector(8, Int[], Float64[])) + @test exact_equal(xc, SparseVector(8, [2, 5, 6], [0.0, 0.0, 0.0])) + + xc[1] = 0.0 + @test exact_equal(xc, SparseVector(8, [2, 5, 6], [0.0, 0.0, 0.0])) end From 31446cdb5e7b538c68c573bd81da0fc542034e78 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Fri, 15 Jul 2016 14:19:33 -0700 Subject: [PATCH 0462/1117] Introduce rudimentary dropstored! method for dropping entries from SparseVectors. Also introduce associated tests. --- base/sparse/sparsevector.jl | 22 ++++++++++++++++++++++ test/sparsedir/sparsevector.jl | 12 ++++++++++++ 2 files changed, 34 insertions(+) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index b28c8ab2fe822..b07360a707c93 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -211,6 +211,28 @@ setindex!{Tv, Ti<:Integer}(x::SparseVector{Tv,Ti}, v, i::Integer) = setindex!(x, convert(Tv, v), convert(Ti, i)) +### dropstored! +""" + dropstored!(x::SparseVector, i::Integer) + +Drop entry `x[i]` from `x` if `x[i]` is stored and otherwise do nothing. +""" +function dropstored!(x::SparseVector, i::Integer) + if !(1 <= i <= x.n) + throw(BoundsError(x, i)) + end + searchk = searchsortedfirst(x.nzind, i) + if searchk <= length(x.nzind) && x.nzind[searchk] == i + # Entry x[i] is stored. Drop and return. + deleteat!(x.nzind, searchk) + deleteat!(x.nzval, searchk) + end + return x +end +# TODO: Implement linear collection indexing methods for dropstored! ? +# TODO: Implement logical indexing methods for dropstored! ? + + ### Conversion # convert SparseMatrixCSC to SparseVector diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index 46919c2dd62dd..62a7649cc1e4d 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -213,6 +213,18 @@ let xc = copy(spv_x1) @test exact_equal(xc, SparseVector(8, [2, 5, 6], [0.0, 0.0, 0.0])) end +## dropstored! tests +let x = SparseVector(10, [2, 7, 9], [2.0, 7.0, 9.0]) + # Test argument bounds checking for dropstored!(x, i) + @test_throws BoundsError Base.SparseArrays.dropstored!(x, 0) + @test_throws BoundsError Base.SparseArrays.dropstored!(x, 11) + # Test behavior of dropstored!(x, i) + # --> Test dropping a single stored entry + @test Base.SparseArrays.dropstored!(x, 2) == SparseVector(10, [7, 9], [7.0, 9.0]) + # --> Test dropping a single nonstored entry + @test Base.SparseArrays.dropstored!(x, 5) == SparseVector(10, [7, 9], [7.0, 9.0]) +end + ### Array manipulation From 4e2149d9f46d29ef3c445982c510734efe7cfc9c Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 14 Jul 2016 14:24:04 -0700 Subject: [PATCH 0463/1117] Note in NEWS.md that setindex! for sparse matrices and vectors no longer purges allocated entries on zero assignment and that dropstored! now exists to drop stored entries. --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.md b/NEWS.md index b96a6f1f6568d..df92f944d367d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -196,6 +196,10 @@ Library improvements * All `sparse` methods now retain provided numerical zeros as structural nonzeros; to drop numerical zeros, use `dropzeros!` ([#14798], [#15242]). + * `setindex!` methods for sparse matrices and vectors no longer purge allocated entries + on zero assignment. To drop stored entries from sparse matrices and vectors, use + `Base.SparseArrays.dropstored!` ([#17404]). + * New `foreach` function for calling a function on every element of a collection when the results are not needed ([#13774]). @@ -343,3 +347,4 @@ Deprecated or removed [#17323]: https://github.com/JuliaLang/julia/issues/17323 [#17374]: https://github.com/JuliaLang/julia/issues/17374 [#17402]: https://github.com/JuliaLang/julia/issues/17402 +[#17404]: https://github.com/JuliaLang/julia/issues/17404 From e7657d991fd4a863647eda3eaa7e99f83423d577 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 15 Jul 2016 15:31:41 -0700 Subject: [PATCH 0464/1117] typo fix /consturct/construct/ [ci skip] --- base/test.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/test.jl b/base/test.jl index 3776b3cc4d6b1..13a3566c1e9b3 100644 --- a/base/test.jl +++ b/base/test.jl @@ -917,7 +917,7 @@ macro inferred(ex) args = gensym() kwargs = gensym() - # Need to only consturct a call with a kwargs if there are actually kwargs, since + # Need to only construct a call with a kwargs if there are actually kwargs, since # the inferred type depends on the presence/absence of kwargs if any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex.args) typecall = :($(ex.args[1])($(args)...; $(kwargs)...)) From 5f5edddeee617d9b1c7d62b995ab1928cdaaee62 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 23 Jun 2016 12:12:16 -0700 Subject: [PATCH 0465/1117] Replace full(X) calls with convert(Array, X) throughout base/sparse. --- base/sparse/linalg.jl | 10 ++++---- base/sparse/sparsematrix.jl | 48 ++++++++++++++++++------------------- base/sparse/umfpack.jl | 2 +- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/base/sparse/linalg.jl b/base/sparse/linalg.jl index a80e5f4a6cb51..d32f62977ecd9 100644 --- a/base/sparse/linalg.jl +++ b/base/sparse/linalg.jl @@ -292,8 +292,8 @@ end A_ldiv_B!{T,Ti}(L::LowerTriangular{T,SparseMatrixCSC{T,Ti}}, B::StridedVecOrMat) = fwdTriSolve!(L.data, B) A_ldiv_B!{T,Ti}(U::UpperTriangular{T,SparseMatrixCSC{T,Ti}}, B::StridedVecOrMat) = bwdTriSolve!(U.data, B) -(\){T,Ti}(L::LowerTriangular{T,SparseMatrixCSC{T,Ti}}, B::SparseMatrixCSC) = A_ldiv_B!(L, full(B)) -(\){T,Ti}(U::UpperTriangular{T,SparseMatrixCSC{T,Ti}}, B::SparseMatrixCSC) = A_ldiv_B!(U, full(B)) +(\){T,Ti}(L::LowerTriangular{T,SparseMatrixCSC{T,Ti}}, B::SparseMatrixCSC) = A_ldiv_B!(L, convert(Array, B)) +(\){T,Ti}(U::UpperTriangular{T,SparseMatrixCSC{T,Ti}}, B::SparseMatrixCSC) = A_ldiv_B!(U, convert(Array, B)) ## triu, tril @@ -504,7 +504,7 @@ function norm(A::SparseMatrixCSC,p::Real=2) return float(real(zero(eltype(A)))) elseif m == 1 || n == 1 # TODO: compute more efficiently using A.nzval directly - return norm(full(A), p) + return norm(convert(Array, A), p) else Tnorm = typeof(float(real(zero(eltype(A))))) Tsum = promote_type(Float64,Tnorm) @@ -519,7 +519,7 @@ function norm(A::SparseMatrixCSC,p::Real=2) end return convert(Tnorm, nA) elseif p==2 - throw(ArgumentError("2-norm not yet implemented for sparse matrices. Try norm(full(A)) or norm(A, p) where p=1 or Inf.")) + throw(ArgumentError("2-norm not yet implemented for sparse matrices. Try norm(convert(Array, A)) or norm(A, p) where p=1 or Inf.")) elseif p==Inf rowSum = zeros(Tsum,m) for i=1:length(A.nzval) @@ -544,7 +544,7 @@ function cond(A::SparseMatrixCSC, p::Real=2) normA = norm(A, Inf) return normA * normAinv elseif p == 2 - throw(ArgumentError("2-norm condition number is not implemented for sparse matrices, try cond(full(A), 2) instead")) + throw(ArgumentError("2-norm condition number is not implemented for sparse matrices, try cond(convert(Array, A), 2) instead")) else throw(ArgumentError("second argument must be either 1 or Inf, got $p")) end diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index ecacca3819e7c..629e96a4a5e3a 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1682,39 +1682,39 @@ for op in (+, -, min, max, &, |, $) end end # macro -(.+)(A::SparseMatrixCSC, B::Number) = full(A) .+ B -( +)(A::SparseMatrixCSC, B::Array ) = full(A) + B -(.+)(A::Number, B::SparseMatrixCSC) = A .+ full(B) -( +)(A::Array , B::SparseMatrixCSC) = A + full(B) +(.+)(A::SparseMatrixCSC, B::Number) = convert(Array, A) .+ B +( +)(A::SparseMatrixCSC, B::Array ) = convert(Array, A) + B +(.+)(A::Number, B::SparseMatrixCSC) = A .+ convert(Array, B) +( +)(A::Array , B::SparseMatrixCSC) = A + convert(Array, B) -(.-)(A::SparseMatrixCSC, B::Number) = full(A) .- B -( -)(A::SparseMatrixCSC, B::Array ) = full(A) - B -(.-)(A::Number, B::SparseMatrixCSC) = A .- full(B) -( -)(A::Array , B::SparseMatrixCSC) = A - full(B) +(.-)(A::SparseMatrixCSC, B::Number) = convert(Array, A) .- B +( -)(A::SparseMatrixCSC, B::Array ) = convert(Array, A) - B +(.-)(A::Number, B::SparseMatrixCSC) = A .- convert(Array, B) +( -)(A::Array , B::SparseMatrixCSC) = A - convert(Array, B) (.*)(A::AbstractArray, B::AbstractArray) = broadcast_zpreserving(*, A, B) (.*)(A::SparseMatrixCSC, B::Number) = SparseMatrixCSC(A.m, A.n, copy(A.colptr), copy(A.rowval), A.nzval .* B) (.*)(A::Number, B::SparseMatrixCSC) = SparseMatrixCSC(B.m, B.n, copy(B.colptr), copy(B.rowval), A .* B.nzval) (./)(A::SparseMatrixCSC, B::Number) = SparseMatrixCSC(A.m, A.n, copy(A.colptr), copy(A.rowval), A.nzval ./ B) -(./)(A::Number, B::SparseMatrixCSC) = (./)(A, full(B)) -(./)(A::SparseMatrixCSC, B::Array) = (./)(full(A), B) -(./)(A::Array, B::SparseMatrixCSC) = (./)(A, full(B)) -(./)(A::SparseMatrixCSC, B::SparseMatrixCSC) = (./)(full(A), full(B)) +(./)(A::Number, B::SparseMatrixCSC) = (./)(A, convert(Array, B)) +(./)(A::SparseMatrixCSC, B::Array) = (./)(convert(Array, A), B) +(./)(A::Array, B::SparseMatrixCSC) = (./)(A, convert(Array, B)) +(./)(A::SparseMatrixCSC, B::SparseMatrixCSC) = (./)(convert(Array, A), convert(Array, B)) -(.\)(A::SparseMatrixCSC, B::Number) = (.\)(full(A), B) +(.\)(A::SparseMatrixCSC, B::Number) = (.\)(convert(Array, A), B) (.\)(A::Number, B::SparseMatrixCSC) = SparseMatrixCSC(B.m, B.n, copy(B.colptr), copy(B.rowval), A .\ B.nzval ) -(.\)(A::SparseMatrixCSC, B::Array) = (.\)(full(A), B) -(.\)(A::Array, B::SparseMatrixCSC) = (.\)(A, full(B)) -(.\)(A::SparseMatrixCSC, B::SparseMatrixCSC) = (.\)(full(A), full(B)) +(.\)(A::SparseMatrixCSC, B::Array) = (.\)(convert(Array, A), B) +(.\)(A::Array, B::SparseMatrixCSC) = (.\)(A, convert(Array, B)) +(.\)(A::SparseMatrixCSC, B::SparseMatrixCSC) = (.\)(convert(Array, A), convert(Array, B)) (.^)(A::SparseMatrixCSC, B::Number) = B==0 ? sparse(ones(typeof(one(eltype(A)).^B), A.m, A.n)) : SparseMatrixCSC(A.m, A.n, copy(A.colptr), copy(A.rowval), A.nzval .^ B) (.^)(::Irrational{:e}, B::SparseMatrixCSC) = exp(B) -(.^)(A::Number, B::SparseMatrixCSC) = (.^)(A, full(B)) -(.^)(A::SparseMatrixCSC, B::Array) = (.^)(full(A), B) -(.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, full(B)) +(.^)(A::Number, B::SparseMatrixCSC) = (.^)(A, convert(Array, B)) +(.^)(A::SparseMatrixCSC, B::Array) = (.^)(convert(Array, A), B) +(.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, convert(Array, B)) .+{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(+, spzeros(promote_op(+, Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) @@ -2822,11 +2822,11 @@ setindex!(A::SparseMatrixCSC, x::Matrix, I::AbstractVector{Bool}, J::AbstractVec setindex!{T<:Integer}(A::SparseMatrixCSC, x::Matrix, I::AbstractVector{T}, J::AbstractVector{Bool}) = setindex!(A, sparse(x), I, find(J)) setindex!{T<:Integer}(A::SparseMatrixCSC, x::Matrix, I::AbstractVector{Bool}, J::AbstractVector{T}) = setindex!(A, sparse(x), find(I),J) -setindex!(A::Matrix, x::SparseMatrixCSC, I::Integer, J::AbstractVector{Bool}) = setindex!(A, full(x), I, find(J)) -setindex!(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::Integer) = setindex!(A, full(x), find(I), J) -setindex!(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{Bool}) = setindex!(A, full(x), find(I), find(J)) -setindex!{T<:Integer}(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{T}, J::AbstractVector{Bool}) = setindex!(A, full(x), I, find(J)) -setindex!{T<:Integer}(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{T}) = setindex!(A, full(x), find(I), J) +setindex!(A::Matrix, x::SparseMatrixCSC, I::Integer, J::AbstractVector{Bool}) = setindex!(A, convert(Array, x), I, find(J)) +setindex!(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::Integer) = setindex!(A, convert(Array, x), find(I), J) +setindex!(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{Bool}) = setindex!(A, convert(Array, x), find(I), find(J)) +setindex!{T<:Integer}(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{T}, J::AbstractVector{Bool}) = setindex!(A, convert(Array, x), I, find(J)) +setindex!{T<:Integer}(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{T}) = setindex!(A, convert(Array, x), find(I), J) setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVector{Bool}) = throw(BoundsError()) function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) diff --git a/base/sparse/umfpack.jl b/base/sparse/umfpack.jl index 3b426bb9f234d..0120fd2d4fbcd 100644 --- a/base/sparse/umfpack.jl +++ b/base/sparse/umfpack.jl @@ -154,7 +154,7 @@ lufact{Tv<:Union{Complex32,Complex64}, Ti<:UMFITypes}(A::SparseMatrixCSC{Tv,Ti}) lufact{T<:AbstractFloat}(A::Union{SparseMatrixCSC{T},SparseMatrixCSC{Complex{T}}}) = throw(ArgumentError(string("matrix type ", typeof(A), "not supported. ", "Try lufact(convert(SparseMatrixCSC{Float64/Complex128,Int}, A)) for ", - "sparse floating point LU using UMFPACK or lufact(full(A)) for generic ", + "sparse floating point LU using UMFPACK or lufact(convert(Array, A)) for generic ", "dense LU."))) lufact(A::SparseMatrixCSC) = lufact(float(A)) lufact(A::SparseMatrixCSC, pivot::Type{Val{false}}) = lufact(A) From dd3d4bf16fe74fe1f34b7dea062d5cfd9bddb61a Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 23 Jun 2016 12:27:17 -0700 Subject: [PATCH 0466/1117] Replace full(X) calls with convert(Array, X) throughout base/linalg. --- base/linalg/bidiag.jl | 16 ++++++++-------- base/linalg/dense.jl | 18 +++++++++--------- base/linalg/diagonal.jl | 2 +- base/linalg/hessenberg.jl | 4 ++-- base/linalg/lq.jl | 7 ++++--- base/linalg/lu.jl | 4 ++-- base/linalg/qr.jl | 7 +++++-- base/linalg/special.jl | 20 ++++++++++---------- base/linalg/symmetric.jl | 4 ++-- base/linalg/triangular.jl | 14 +++++++------- base/linalg/tridiag.jl | 4 ++-- base/linalg/uniformscaling.jl | 4 ++-- 12 files changed, 54 insertions(+), 50 deletions(-) diff --git a/base/linalg/bidiag.jl b/base/linalg/bidiag.jl index 15913677df72a..1eaff9c3ff533 100644 --- a/base/linalg/bidiag.jl +++ b/base/linalg/bidiag.jl @@ -18,7 +18,7 @@ end Constructs an upper (`isupper=true`) or lower (`isupper=false`) bidiagonal matrix using the given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` and provides efficient specialized linear solvers, but may be converted into a regular -matrix with [`full`](:func:`full`). `ev`'s length must be one less than the length of `dv`. +matrix with [`convert(Array, _)`](:func:`convert`). `ev`'s length must be one less than the length of `dv`. **Example** @@ -38,7 +38,7 @@ Bidiagonal(dv::AbstractVector, ev::AbstractVector) = throw(ArgumentError("did yo Constructs an upper (`uplo='U'`) or lower (`uplo='L'`) bidiagonal matrix using the given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` and provides efficient specialized linear solvers, but may be converted into a regular -matrix with [`full`](:func:`full`). `ev`'s length must be one less than the length of `dv`. +matrix with [`convert(Array, _)`](:func:`convert`). `ev`'s length must be one less than the length of `dv`. **Example** @@ -302,7 +302,7 @@ end function A_mul_B_td!(C::AbstractMatrix, A::BiTriSym, B::BiTriSym) check_A_mul_B!_sizes(C, A, B) n = size(A,1) - n <= 3 && return A_mul_B!(C, full(A), full(B)) + n <= 3 && return A_mul_B!(C, convert(Array, A), convert(Array, B)) fill!(C, zero(eltype(C))) Al = diag(A, -1) Ad = diag(A, 0) @@ -361,7 +361,7 @@ function A_mul_B_td!(C::AbstractVecOrMat, A::BiTriSym, B::AbstractVecOrMat) if size(C,2) != nB throw(DimensionMismatch("A has second dimension $nA, B has $(size(B,2)), C has $(size(C,2)) but all must match")) end - nA <= 3 && return A_mul_B!(C, full(A), full(B)) + nA <= 3 && return A_mul_B!(C, convert(Array, A), convert(Array, B)) l = diag(A, -1) d = diag(A, 0) u = diag(A, 1) @@ -382,7 +382,7 @@ end function A_mul_B_td!(C::AbstractMatrix, A::AbstractMatrix, B::BiTriSym) check_A_mul_B!_sizes(C, A, B) n = size(A,1) - n <= 3 && return A_mul_B!(C, full(A), full(B)) + n <= 3 && return A_mul_B!(C, convert(Array, A), convert(Array, B)) m = size(B,2) Bl = diag(B, -1) Bd = diag(B, 0) @@ -412,12 +412,12 @@ end SpecialMatrix = Union{Bidiagonal, SymTridiagonal, Tridiagonal} # to avoid ambiguity warning, but shouldn't be necessary -*(A::AbstractTriangular, B::SpecialMatrix) = full(A) * full(B) -*(A::SpecialMatrix, B::SpecialMatrix) = full(A) * full(B) +*(A::AbstractTriangular, B::SpecialMatrix) = convert(Array, A) * convert(Array, B) +*(A::SpecialMatrix, B::SpecialMatrix) = convert(Array, A) * convert(Array, B) #Generic multiplication for func in (:*, :Ac_mul_B, :A_mul_Bc, :/, :A_rdiv_Bc) - @eval ($func){T}(A::Bidiagonal{T}, B::AbstractVector{T}) = ($func)(full(A), B) + @eval ($func){T}(A::Bidiagonal{T}, B::AbstractVector{T}) = ($func)(convert(Array, A), B) end #Linear solvers diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index 2a68cc7473c6d..ffb5930e600f5 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -206,7 +206,7 @@ expm(x::Number) = exp(x) function expm!{T<:BlasFloat}(A::StridedMatrix{T}) n = checksquare(A) if ishermitian(A) - return full(expm(Hermitian(A))) + return convert(Array, expm(Hermitian(A))) end ilo, ihi, scale = LAPACK.gebal!('B', A) # modifies A nA = norm(A, 1) @@ -321,13 +321,13 @@ function logm{T}(A::StridedMatrix{T}) return full(logm(Symmetric(A))) end if ishermitian(A) - return full(logm(Hermitian(A))) + return convert(Array, logm(Hermitian(A))) end # Use Schur decomposition n = checksquare(A) if istriu(A) - retmat = full(logm(UpperTriangular(complex(A)))) + retmat = convert(Array, logm(UpperTriangular(complex(A)))) d = diag(A) else S,Q,d = schur(complex(A)) @@ -358,27 +358,27 @@ logm(a::Complex) = log(a) function sqrtm{T<:Real}(A::StridedMatrix{T}) if issymmetric(A) - return full(sqrtm(Symmetric(A))) + return convert(Array, sqrtm(Symmetric(A))) end n = checksquare(A) if istriu(A) - return full(sqrtm(UpperTriangular(A))) + return convert(Array, sqrtm(UpperTriangular(A))) else SchurF = schurfact(complex(A)) - R = full(sqrtm(UpperTriangular(SchurF[:T]))) + R = convert(Array, sqrtm(UpperTriangular(SchurF[:T]))) return SchurF[:vectors] * R * SchurF[:vectors]' end end function sqrtm{T<:Complex}(A::StridedMatrix{T}) if ishermitian(A) - return full(sqrtm(Hermitian(A))) + return convert(Array, sqrtm(Hermitian(A))) end n = checksquare(A) if istriu(A) - return full(sqrtm(UpperTriangular(A))) + return convert(Array, sqrtm(UpperTriangular(A))) else SchurF = schurfact(A) - R = full(sqrtm(UpperTriangular(SchurF[:T]))) + R = convert(Array, sqrtm(UpperTriangular(SchurF[:T]))) return SchurF[:vectors] * R * SchurF[:vectors]' end end diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 0d377c4b9308f..d26f4739482c2 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -314,7 +314,7 @@ svdvals(D::Diagonal) = [svdvals(v) for v in D.diag] function svd{T<:Number}(D::Diagonal{T}) S = abs(D.diag) piv = sortperm(S, rev = true) - U = full(Diagonal(D.diag ./ S)) + U = convert(Array, Diagonal(D.diag ./ S)) Up = hcat([U[:,i] for i = 1:length(D.diag)][piv]...) V = eye(D) Vp = hcat([V[:,i] for i = 1:length(D.diag)][piv]...) diff --git a/base/linalg/hessenberg.jl b/base/linalg/hessenberg.jl index 83088cda465eb..2b9f2466b7984 100644 --- a/base/linalg/hessenberg.jl +++ b/base/linalg/hessenberg.jl @@ -23,7 +23,7 @@ end Compute the Hessenberg decomposition of `A` and return a `Hessenberg` object. If `F` is the factorization object, the unitary matrix can be accessed with `F[:Q]` and the Hessenberg matrix with `F[:H]`. When `Q` is extracted, the resulting type is the `HessenbergQ` object, -and may be converted to a regular matrix with [`full`](:func:`full`). +and may be converted to a regular matrix with [`convert(Array, _)`](:func:`convert`). """ hessfact @@ -56,7 +56,7 @@ end convert{T<:BlasFloat}(::Type{Matrix}, A::HessenbergQ{T}) = LAPACK.orghr!(1, size(A.factors, 1), copy(A.factors), A.τ) convert(::Type{Array}, A::HessenbergQ) = convert(Matrix, A) full(A::HessenbergQ) = convert(Array, A) -convert(::Type{AbstractMatrix}, F::Hessenberg) = (fq = full(F[:Q]); (fq * F[:H]) * fq') +convert(::Type{AbstractMatrix}, F::Hessenberg) = (fq = convert(Array, F[:Q]); (fq * F[:H]) * fq') convert(::Type{AbstractArray}, F::Hessenberg) = convert(AbstractMatrix, F) convert(::Type{Matrix}, F::Hessenberg) = convert(Array, convert(AbstractArray, F)) convert(::Type{Array}, F::Hessenberg) = convert(Matrix, F) diff --git a/base/linalg/lq.jl b/base/linalg/lq.jl index cb10bb81bac5d..a0806370b1f1b 100644 --- a/base/linalg/lq.jl +++ b/base/linalg/lq.jl @@ -42,8 +42,9 @@ zeros if the full `Q` is requested. """ function lq(A::Union{Number, AbstractMatrix}; thin::Bool=true) F = lqfact(A) - F[:L], full(F[:Q], thin=thin) + F[:L], thin ? convert(Array, F[:Q]) : thickQ(F[:Q]) end +thickQ{T}(Q::LQPackedQ{T}) = A_mul_B!(Q, eye(T, size(Q.factors,2), size(Q.factors,1))) copy(A::LQ) = LQ(copy(A.factors), copy(A.τ)) @@ -112,8 +113,8 @@ size(A::LQPackedQ) = size(A.factors) ## Multiplication by LQ A_mul_B!{T<:BlasFloat}(A::LQ{T}, B::StridedVecOrMat{T}) = A[:L]*LAPACK.ormlq!('L','N',A.factors,A.τ,B) -A_mul_B!{T<:BlasFloat}(A::LQ{T}, B::QR{T}) = A[:L]*LAPACK.ormlq!('L','N',A.factors,A.τ,full(B)) -A_mul_B!{T<:BlasFloat}(A::QR{T}, B::LQ{T}) = A_mul_B!(zeros(full(A)), full(A), full(B)) +A_mul_B!{T<:BlasFloat}(A::LQ{T}, B::QR{T}) = A[:L]*LAPACK.ormlq!('L','N',A.factors,A.τ,convert(Array, B)) +A_mul_B!{T<:BlasFloat}(A::QR{T}, B::LQ{T}) = A_mul_B!(zeros(convert(Array, A)), convert(Array, A), convert(Array, B)) function *{TA,TB}(A::LQ{TA},B::StridedVecOrMat{TB}) TAB = promote_type(TA, TB) A_mul_B!(convert(Factorization{TAB},A), copy_oftype(B, TAB)) diff --git a/base/linalg/lu.jl b/base/linalg/lu.jl index 5201decd46fc6..49ebea091c2f8 100644 --- a/base/linalg/lu.jl +++ b/base/linalg/lu.jl @@ -308,7 +308,7 @@ factorize(A::Tridiagonal) = lufact(A) function getindex{T}(F::Base.LinAlg.LU{T,Tridiagonal{T}}, d::Symbol) m, n = size(F) if d == :L - L = full(Bidiagonal(ones(T, n), F.factors.dl, false)) + L = convert(Array, Bidiagonal(ones(T, n), F.factors.dl, false)) for i = 2:n tmp = L[F.ipiv[i], 1:i - 1] L[F.ipiv[i], 1:i - 1] = L[i, 1:i - 1] @@ -316,7 +316,7 @@ function getindex{T}(F::Base.LinAlg.LU{T,Tridiagonal{T}}, d::Symbol) end return L elseif d == :U - U = full(Bidiagonal(F.factors.d, F.factors.du, true)) + U = convert(Array, Bidiagonal(F.factors.d, F.factors.du, true)) for i = 1:n - 2 U[i,i + 2] = F.factors.du2[i] end diff --git a/base/linalg/qr.jl b/base/linalg/qr.jl index ba1797bc6105e..82009e25252c2 100644 --- a/base/linalg/qr.jl +++ b/base/linalg/qr.jl @@ -174,13 +174,14 @@ qr(A::Union{Number, AbstractMatrix}, pivot::Union{Type{Val{false}}, Type{Val{tru _qr(A, pivot, thin=thin) function _qr(A::Union{Number, AbstractMatrix}, ::Type{Val{false}}; thin::Bool=true) F = qrfact(A, Val{false}) - full(getq(F), thin=thin), F[:R]::Matrix{eltype(F)} + (thin ? convert(Array, getq(F)) : thickQ(getq(F))), F[:R]::Matrix{eltype(F)} end function _qr(A::Union{Number, AbstractMatrix}, ::Type{Val{true}}; thin::Bool=true) F = qrfact(A, Val{true}) - full(getq(F), thin=thin), F[:R]::Matrix{eltype(F)}, F[:p]::Vector{BlasInt} + (thin ? convert(Array, getq(F)) : thickQ(getq(F))), F[:R]::Matrix{eltype(F)}, F[:p]::Vector{BlasInt} end + """ qr(v::AbstractVector) @@ -325,6 +326,8 @@ function full{T}(A::Union{QRPackedQ{T},QRCompactWYQ{T}}; thin::Bool = true) end end +thickQ{T}(Q::Union{QRPackedQ{T},QRCompactWYQ{T}}) = A_mul_B!(Q, eye(T, size(Q.factors, 1))) + size(A::Union{QR,QRCompactWY,QRPivoted}, dim::Integer) = size(A.factors, dim) size(A::Union{QR,QRCompactWY,QRPivoted}) = size(A.factors) size(A::Union{QRPackedQ,QRCompactWYQ}, dim::Integer) = 0 < dim ? (dim <= 2 ? size(A.factors, 1) : 1) : throw(BoundsError()) diff --git a/base/linalg/special.jl b/base/linalg/special.jl index f70d67130807b..c8d86d559b689 100644 --- a/base/linalg/special.jl +++ b/base/linalg/special.jl @@ -6,21 +6,21 @@ convert{T}(::Type{Bidiagonal}, A::Diagonal{T})=Bidiagonal(A.diag, zeros(T, size(A.diag,1)-1), true) convert{T}(::Type{SymTridiagonal}, A::Diagonal{T})=SymTridiagonal(A.diag, zeros(T, size(A.diag,1)-1)) convert{T}(::Type{Tridiagonal}, A::Diagonal{T})=Tridiagonal(zeros(T, size(A.diag,1)-1), A.diag, zeros(T, size(A.diag,1)-1)) -convert(::Type{LowerTriangular}, A::Bidiagonal) = !A.isupper ? LowerTriangular(full(A)) : throw(ArgumentError("Bidiagonal matrix must have lower off diagonal to be converted to LowerTriangular")) -convert(::Type{UpperTriangular}, A::Bidiagonal) = A.isupper ? UpperTriangular(full(A)) : throw(ArgumentError("Bidiagonal matrix must have upper off diagonal to be converted to UpperTriangular")) +convert(::Type{LowerTriangular}, A::Bidiagonal) = !A.isupper ? LowerTriangular(convert(Array, A)) : throw(ArgumentError("Bidiagonal matrix must have lower off diagonal to be converted to LowerTriangular")) +convert(::Type{UpperTriangular}, A::Bidiagonal) = A.isupper ? UpperTriangular(convert(Array, A)) : throw(ArgumentError("Bidiagonal matrix must have upper off diagonal to be converted to UpperTriangular")) function convert(::Type{UnitUpperTriangular}, A::Diagonal) if !all(A.diag .== one(eltype(A))) throw(ArgumentError("matrix cannot be represented as UnitUpperTriangular")) end - UnitUpperTriangular(full(A)) + UnitUpperTriangular(convert(Array, A)) end function convert(::Type{UnitLowerTriangular}, A::Diagonal) if !all(A.diag .== one(eltype(A))) throw(ArgumentError("matrix cannot be represented as UnitLowerTriangular")) end - UnitLowerTriangular(full(A)) + UnitLowerTriangular(convert(Array, A)) end function convert(::Type{Diagonal}, A::Union{Bidiagonal, SymTridiagonal}) @@ -72,14 +72,14 @@ function convert(::Type{Tridiagonal}, A::SymTridiagonal) end function convert(::Type{Diagonal}, A::AbstractTriangular) - if full(A) != diagm(diag(A)) + if convert(Array, A) != diagm(diag(A)) throw(ArgumentError("matrix cannot be represented as Diagonal")) end Diagonal(diag(A)) end function convert(::Type{Bidiagonal}, A::AbstractTriangular) - fA = full(A) + fA = convert(Array, A) if fA == diagm(diag(A)) + diagm(diag(fA, 1), 1) return Bidiagonal(diag(A), diag(fA,1), true) elseif fA == diagm(diag(A)) + diagm(diag(fA, -1), -1) @@ -92,7 +92,7 @@ end convert(::Type{SymTridiagonal}, A::AbstractTriangular) = convert(SymTridiagonal, convert(Tridiagonal, A)) function convert(::Type{Tridiagonal}, A::AbstractTriangular) - fA = full(A) + fA = convert(Array, A) if fA == diagm(diag(A)) + diagm(diag(fA, 1), 1) + diagm(diag(fA, -1), -1) return Tridiagonal(diag(fA, -1), diag(A), diag(fA,1)) else @@ -154,12 +154,12 @@ for op in (:+, :-) end for matrixtype in (:SymTridiagonal,:Tridiagonal,:Bidiagonal,:Matrix) @eval begin - ($op)(A::AbstractTriangular, B::($matrixtype)) = ($op)(full(A), B) - ($op)(A::($matrixtype), B::AbstractTriangular) = ($op)(A, full(B)) + ($op)(A::AbstractTriangular, B::($matrixtype)) = ($op)(convert(Array, A), B) + ($op)(A::($matrixtype), B::AbstractTriangular) = ($op)(A, convert(Array, B)) end end end A_mul_Bc!(A::AbstractTriangular, B::QRCompactWYQ) = A_mul_Bc!(full!(A),B) A_mul_Bc!(A::AbstractTriangular, B::QRPackedQ) = A_mul_Bc!(full!(A),B) -A_mul_Bc(A::AbstractTriangular, B::Union{QRCompactWYQ,QRPackedQ}) = A_mul_Bc(full(A), B) +A_mul_Bc(A::AbstractTriangular, B::Union{QRCompactWYQ,QRPackedQ}) = A_mul_Bc(convert(Array, A), B) diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index 40898a9aa2789..02a0904c33cc0 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -174,8 +174,8 @@ A_mul_B!{T<:BlasFloat,S<:StridedMatrix}(C::StridedMatrix{T}, A::StridedMatrix{T} A_mul_B!{T<:BlasComplex,S<:StridedMatrix}(C::StridedMatrix{T}, A::Hermitian{T,S}, B::StridedMatrix{T}) = BLAS.hemm!('L', A.uplo, one(T), A.data, B, zero(T), C) A_mul_B!{T<:BlasComplex,S<:StridedMatrix}(C::StridedMatrix{T}, A::StridedMatrix{T}, B::Hermitian{T,S}) = BLAS.hemm!('R', B.uplo, one(T), B.data, A, zero(T), C) -*(A::HermOrSym, B::HermOrSym) = full(A)*full(B) -*(A::StridedMatrix, B::HermOrSym) = A*full(B) +*(A::HermOrSym, B::HermOrSym) = convert(Array, A)*convert(Array, B) +*(A::StridedMatrix, B::HermOrSym) = A*convert(Array, B) bkfact(A::HermOrSym) = bkfact(A.data, Symbol(A.uplo), issymmetric(A)) factorize(A::HermOrSym) = bkfact(A) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 5dd80ac8f8cd5..37b39d132da91 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -362,7 +362,7 @@ scale!(c::Number, A::Union{UpperTriangular,LowerTriangular}) = scale!(A,c) +(A::UnitLowerTriangular, B::LowerTriangular) = LowerTriangular(tril(A.data, -1) + B.data + I) +(A::UnitUpperTriangular, B::UnitUpperTriangular) = UpperTriangular(triu(A.data, 1) + triu(B.data, 1) + 2I) +(A::UnitLowerTriangular, B::UnitLowerTriangular) = LowerTriangular(tril(A.data, -1) + tril(B.data, -1) + 2I) -+(A::AbstractTriangular, B::AbstractTriangular) = full(A) + full(B) ++(A::AbstractTriangular, B::AbstractTriangular) = convert(Array, A) + convert(Array, B) -(A::UpperTriangular, B::UpperTriangular) = UpperTriangular(A.data - B.data) -(A::LowerTriangular, B::LowerTriangular) = LowerTriangular(A.data - B.data) @@ -372,15 +372,15 @@ scale!(c::Number, A::Union{UpperTriangular,LowerTriangular}) = scale!(A,c) -(A::UnitLowerTriangular, B::LowerTriangular) = LowerTriangular(tril(A.data, -1) - B.data + I) -(A::UnitUpperTriangular, B::UnitUpperTriangular) = UpperTriangular(triu(A.data, 1) - triu(B.data, 1)) -(A::UnitLowerTriangular, B::UnitLowerTriangular) = LowerTriangular(tril(A.data, -1) - tril(B.data, -1)) --(A::AbstractTriangular, B::AbstractTriangular) = full(A) - full(B) +-(A::AbstractTriangular, B::AbstractTriangular) = convert(Array, A) - convert(Array, B) ###################### # BlasFloat routines # ###################### A_mul_B!(A::Tridiagonal, B::AbstractTriangular) = A*full!(B) -A_mul_B!(C::AbstractMatrix, A::AbstractTriangular, B::Tridiagonal) = A_mul_B!(C, full(A), B) -A_mul_B!(C::AbstractMatrix, A::Tridiagonal, B::AbstractTriangular) = A_mul_B!(C, A, full(B)) +A_mul_B!(C::AbstractMatrix, A::AbstractTriangular, B::Tridiagonal) = A_mul_B!(C, convert(Array, A), B) +A_mul_B!(C::AbstractMatrix, A::Tridiagonal, B::AbstractTriangular) = A_mul_B!(C, A, convert(Array, B)) A_mul_B!(C::AbstractVector, A::AbstractTriangular, B::AbstractVector) = A_mul_B!(A, copy!(C, B)) A_mul_B!(C::AbstractMatrix, A::AbstractTriangular, B::AbstractVecOrMat) = A_mul_B!(A, copy!(C, B)) A_mul_B!(C::AbstractVecOrMat, A::AbstractTriangular, B::AbstractVecOrMat) = A_mul_B!(A, copy!(C, B)) @@ -437,7 +437,7 @@ for (t, uploc, isunitc) in ((:LowerTriangular, 'L', 'N'), elseif p == Inf return inv(LAPACK.trcon!('I', $uploc, $isunitc, A.data)) else #use fallback - return cond(full(A), p) + return cond(convert(Array, A), p) end end end @@ -1321,7 +1321,7 @@ end ## Some Triangular-Triangular cases. We might want to write taylored methods for these cases, but I'm not sure it is worth it. for t in (UpperTriangular, UnitUpperTriangular, LowerTriangular, UnitLowerTriangular) @eval begin - (*)(A::Tridiagonal, B::$t) = A_mul_B!(full(A), B) + (*)(A::Tridiagonal, B::$t) = A_mul_B!(convert(Array, A), B) end end @@ -1987,7 +1987,7 @@ eigfact(A::AbstractTriangular) = Eigen(eigvals(A), eigvecs(A)) #Generic singular systems for func in (:svd, :svdfact, :svdfact!, :svdvals) @eval begin - ($func)(A::AbstractTriangular) = ($func)(full(A)) + ($func)(A::AbstractTriangular) = ($func)(convert(Array, A)) end end diff --git a/base/linalg/tridiag.jl b/base/linalg/tridiag.jl index aee5e05483456..0e79204b5069e 100644 --- a/base/linalg/tridiag.jl +++ b/base/linalg/tridiag.jl @@ -19,7 +19,7 @@ end Construct a symmetric tridiagonal matrix from the diagonal and first sub/super-diagonal, respectively. The result is of type `SymTridiagonal` and provides efficient specialized -eigensolvers, but may be converted into a regular matrix with [`full`](:func:`full`). +eigensolvers, but may be converted into a regular matrix with [`convert(Array, _)`](:func:`convert`). """ SymTridiagonal{T}(dv::Vector{T}, ev::Vector{T}) = SymTridiagonal{T}(dv, ev) @@ -327,7 +327,7 @@ end Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type `Tridiagonal` and provides efficient specialized linear -solvers, but may be converted into a regular matrix with [`full`](:func:`full`). +solvers, but may be converted into a regular matrix with [`convert(Array, _)`](:func:`convert`). The lengths of `dl` and `du` must be one less than the length of `d`. """ # Basic constructor takes in three dense vectors of same type diff --git a/base/linalg/uniformscaling.jl b/base/linalg/uniformscaling.jl index 227e41ac5dc0c..db26156fef0a2 100644 --- a/base/linalg/uniformscaling.jl +++ b/base/linalg/uniformscaling.jl @@ -61,7 +61,7 @@ for (t1, t2) in ((:UnitUpperTriangular, :UpperTriangular), end function (-)(J::UniformScaling, UL::Union{UpperTriangular,UnitUpperTriangular}) - ULnew = similar(full(UL), promote_type(eltype(J), eltype(UL))) + ULnew = similar(convert(Array, UL), promote_type(eltype(J), eltype(UL))) n = size(ULnew, 1) ULold = UL.data for j = 1:n @@ -77,7 +77,7 @@ function (-)(J::UniformScaling, UL::Union{UpperTriangular,UnitUpperTriangular}) return UpperTriangular(ULnew) end function (-)(J::UniformScaling, UL::Union{LowerTriangular,UnitLowerTriangular}) - ULnew = similar(full(UL), promote_type(eltype(J), eltype(UL))) + ULnew = similar(convert(Array, UL), promote_type(eltype(J), eltype(UL))) n = size(ULnew, 1) ULold = UL.data for j = 1:n From 30ad6de71e0ae1fca508bb7f031cf3ab7948956d Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 23 Jun 2016 12:36:43 -0700 Subject: [PATCH 0467/1117] Replace full(X) calls with convert(Array, X) elsewhere in base/ (outside base/sparse and base/linalg), apart from the no-op fallback for `AbstractArray`s in abstractarray.jl. --- base/abstractarray.jl | 10 +++++----- base/test.jl | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 3e82aeb2245fa..0497852f798ff 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -896,7 +896,7 @@ function typed_vcat{T}(::Type{T}, V::AbstractVector...) for Vk in V n += length(Vk) end - a = similar(full(V[1]), T, n) + a = similar(convert(Array, V[1]), T, n) pos = 1 for k=1:length(V) Vk = V[k] @@ -924,7 +924,7 @@ function typed_hcat{T}(::Type{T}, A::AbstractVecOrMat...) nd = ndims(Aj) ncols += (nd==2 ? size(Aj,2) : 1) end - B = similar(full(A[1]), T, nrows, ncols) + B = similar(convert(Array, A[1]), T, nrows, ncols) pos = 1 if dense for k=1:nargs @@ -956,7 +956,7 @@ function typed_vcat{T}(::Type{T}, A::AbstractMatrix...) throw(ArgumentError("number of columns of each array must match (got $(map(x->size(x,2), A)))")) end end - B = similar(full(A[1]), T, nrows, ncols) + B = similar(convert(Array, A[1]), T, nrows, ncols) pos = 1 for k=1:nargs Ak = A[k] @@ -1003,7 +1003,7 @@ function cat_t(catdims, typeC::Type, X...) end end - C = similar(isa(X[1],AbstractArray) ? full(X[1]) : [X[1]], typeC, tuple(dimsC...)) + C = similar(isa(X[1],AbstractArray) ? convert(Array, X[1]) : [X[1]], typeC, tuple(dimsC...)) if length(catdims)>1 fill!(C,0) end @@ -1075,7 +1075,7 @@ function typed_hvcat{T}(::Type{T}, rows::Tuple{Vararg{Int}}, as::AbstractMatrix. a += rows[i] end - out = similar(full(as[1]), T, nr, nc) + out = similar(convert(Array, as[1]), T, nr, nc) a = 1 r = 1 diff --git a/base/test.jl b/base/test.jl index 13a3566c1e9b3..e82bb3a7b8ec6 100644 --- a/base/test.jl +++ b/base/test.jl @@ -817,7 +817,7 @@ end approx_full(x::AbstractArray) = x approx_full(x::Number) = x -approx_full(x) = full(x) +approx_full(x) = convert(Array, x) function test_approx_eq(va, vb, Eps, astr, bstr) va = approx_full(va) From 46b9780025302c619039da760e907c68014e3a38 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 23 Jun 2016 13:47:13 -0700 Subject: [PATCH 0468/1117] Migrate full(X) to convert(Array, X) in tests in test/sparsedir. --- test/sparsedir/cholmod.jl | 16 +- test/sparsedir/sparse.jl | 340 ++++++++++++++++----------------- test/sparsedir/sparsevector.jl | 130 ++++++------- test/sparsedir/spqr.jl | 4 +- test/sparsedir/umfpack.jl | 2 +- 5 files changed, 246 insertions(+), 246 deletions(-) diff --git a/test/sparsedir/cholmod.jl b/test/sparsedir/cholmod.jl index f7b4461911a70..f42afcf6d252b 100644 --- a/test/sparsedir/cholmod.jl +++ b/test/sparsedir/cholmod.jl @@ -356,8 +356,8 @@ for elty in (Float64, Complex{Float64}) # Factor @test_throws ArgumentError cholfact(A1) - @test_throws Base.LinAlg.PosDefException cholfact(A1 + A1' - 2eigmax(full(A1 + A1'))I) - @test_throws Base.LinAlg.PosDefException cholfact(A1 + A1', shift=-2eigmax(full(A1 + A1'))) + @test_throws Base.LinAlg.PosDefException cholfact(A1 + A1' - 2eigmax(convert(Array, A1 + A1'))I) + @test_throws Base.LinAlg.PosDefException cholfact(A1 + A1', shift=-2eigmax(convert(Array, A1 + A1'))) @test_throws ArgumentError ldltfact(A1 + A1' - 2real(A1[1,1])I) @test_throws ArgumentError ldltfact(A1 + A1', shift=-2real(A1[1,1])) @test_throws ArgumentError cholfact(A1) @@ -372,19 +372,19 @@ for elty in (Float64, Complex{Float64}) @test F\CHOLMOD.Sparse(sparse(ones(elty, 5))) ≈ A1pd\ones(5) @test_throws DimensionMismatch F\CHOLMOD.Dense(ones(elty, 4)) @test_throws DimensionMismatch F\CHOLMOD.Sparse(sparse(ones(elty, 4))) - @test F'\ones(elty, 5) ≈ full(A1pd)'\ones(5) - @test F'\sparse(ones(elty, 5)) ≈ full(A1pd)'\ones(5) - @test logdet(F) ≈ logdet(full(A1pd)) + @test F'\ones(elty, 5) ≈ convert(Array, A1pd)'\ones(5) + @test F'\sparse(ones(elty, 5)) ≈ convert(Array, A1pd)'\ones(5) + @test logdet(F) ≈ logdet(convert(Array, A1pd)) @test det(F) == exp(logdet(F)) let # to test supernodal, we must use a larger matrix Ftmp = sprandn(100,100,0.1) Ftmp = Ftmp'Ftmp + I - @test logdet(cholfact(Ftmp)) ≈ logdet(full(Ftmp)) + @test logdet(cholfact(Ftmp)) ≈ logdet(convert(Array, Ftmp)) end - @test logdet(ldltfact(A1pd)) ≈ logdet(full(A1pd)) + @test logdet(ldltfact(A1pd)) ≈ logdet(convert(Array, A1pd)) @test isposdef(A1pd) @test !isposdef(A1) - @test !isposdef(A1 + A1' |> t -> t - 2eigmax(full(t))*I) + @test !isposdef(A1 + A1' |> t -> t - 2eigmax(convert(Array, t))*I) if elty <: Real @test CHOLMOD.issymmetric(Sparse(A1pd, 0)) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 3666e96e9bc89..e70122b5851a2 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -7,7 +7,7 @@ using Base.Test @test Base.SparseArrays.indtype(sparse(ones(Int8,2),ones(Int8,2),rand(2))) == Int8 # check sparse matrix construction -@test isequal(full(sparse(complex(ones(5,5),ones(5,5)))), complex(ones(5,5),ones(5,5))) +@test isequal(convert(Array, sparse(complex(ones(5,5),ones(5,5)))), complex(ones(5,5),ones(5,5))) @test_throws ArgumentError sparse([1,2,3], [1,2], [1,2,3], 3, 3) @test_throws ArgumentError sparse([1,2,3], [1,2,3], [1,2], 3, 3) @test_throws ArgumentError sparse([1,2,3], [1,2,3], [1,2,3], 0, 1) @@ -22,8 +22,8 @@ do33 = ones(3) @test isequal(se33 * se33, se33) # check sparse binary op -@test all(full(se33 + convert(SparseMatrixCSC{Float32,Int32}, se33)) == 2*eye(3)) -@test all(full(se33 * convert(SparseMatrixCSC{Float32,Int32}, se33)) == eye(3)) +@test all(convert(Array, se33 + convert(SparseMatrixCSC{Float32,Int32}, se33)) == 2*eye(3)) +@test all(convert(Array, se33 * convert(SparseMatrixCSC{Float32,Int32}, se33)) == eye(3)) # check horiz concatenation @test all([se33 se33] == sparse([1, 2, 3, 1, 2, 3], [1, 2, 3, 4, 5, 6], ones(6))) @@ -52,7 +52,7 @@ se33_i32 = speye(Int32, 3, 3) # check mixed sparse-dense concatenation sz33 = spzeros(3, 3) de33 = eye(3) -@test all([se33 de33; sz33 se33] == full([se33 se33; sz33 se33 ])) +@test all([se33 de33; sz33 se33] == convert(Array, [se33 se33; sz33 se33 ])) # check splicing + concatenation on # random instances, with nested vcat @@ -66,9 +66,9 @@ end a116 = copy(reshape(1:16, 4, 4)) s116 = sparse(a116) p = [4, 1, 2, 3, 2] -@test full(s116[p,:]) == a116[p,:] -@test full(s116[:,p]) == a116[:,p] -@test full(s116[p,p]) == a116[p,p] +@test convert(Array, s116[p,:]) == a116[p,:] +@test convert(Array, s116[:,p]) == a116[:,p] +@test convert(Array, s116[p,p]) == a116[p,p] # sparse assign p = [4, 1, 3] @@ -97,16 +97,16 @@ end for i = 1:5 a = sprand(10, 5, 0.5) b = rand(5) - @test maximum(abs(a*b - full(a)*b)) < 100*eps() + @test maximum(abs(a*b - convert(Array, a)*b)) < 100*eps() end # sparse matrix * BitArray A = sprand(5,5,0.2) B = trues(5) -@test A*B ≈ full(A)*B +@test A*B ≈ convert(Array, A)*B B = trues(5,5) -@test A*B ≈ full(A)*B -@test B*A ≈ B*full(A) +@test A*B ≈ convert(Array, A)*B +@test B*A ≈ B*convert(Array, A) # complex matrix-vector multiplication and left-division if Base.USE_GPL_LIBS @@ -117,91 +117,91 @@ for i = 1:5 d = randn(5) + im*randn(5) α = rand(Complex128) β = rand(Complex128) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(A_mul_B!(similar(b), a, b) - full(a)*b)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. - @test (maximum(abs(A_mul_B!(similar(c), a, c) - full(a)*c)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) - @test (maximum(abs((a'*c + d) - (full(a)'*c + d))) < 1000*eps()) - @test (maximum(abs((α*a.'*c + β*d) - (α*full(a).'*c + β*d))) < 1000*eps()) - @test (maximum(abs((a.'*c + d) - (full(a).'*c + d))) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(A_mul_B!(similar(b), a, b) - convert(Array, a)*b)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. + @test (maximum(abs(A_mul_B!(similar(c), a, c) - convert(Array, a)*c)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs((a'*c + d) - (convert(Array, a)'*c + d))) < 1000*eps()) + @test (maximum(abs((α*a.'*c + β*d) - (α*convert(Array, a).'*c + β*d))) < 1000*eps()) + @test (maximum(abs((a.'*c + d) - (convert(Array, a).'*c + d))) < 1000*eps()) c = randn(6) + im*randn(6) @test_throws DimensionMismatch α*a.'*c + β*c @test_throws DimensionMismatch α*a.'*ones(5) + β*c a = speye(5) + 0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2) b = randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) a = speye(5) + tril(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) a = speye(5) + tril(0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2)) b = randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2)) b = randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) a = spdiagm(randn(5)) + im*spdiagm(randn(5)) b = randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) + @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) + @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) end end @@ -209,24 +209,24 @@ end for i = 1:5 a = sprand(10, 5, 0.7) b = sprand(5, 15, 0.3) - @test maximum(abs(a*b - full(a)*full(b))) < 100*eps() - @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:sortcols) - full(a)*full(b))) < 100*eps() - @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:doubletranspose) - full(a)*full(b))) < 100*eps() - @test full(kron(a,b)) == kron(full(a), full(b)) - @test full(kron(full(a),b)) == kron(full(a), full(b)) - @test full(kron(a,full(b))) == kron(full(a), full(b)) + @test maximum(abs(a*b - convert(Array, a)*convert(Array, b))) < 100*eps() + @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:sortcols) - convert(Array, a)*convert(Array, b))) < 100*eps() + @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:doubletranspose) - convert(Array, a)*convert(Array, b))) < 100*eps() + @test convert(Array, kron(a,b)) == kron(convert(Array, a), convert(Array, b)) + @test convert(Array, kron(convert(Array, a),b)) == kron(convert(Array, a), convert(Array, b)) + @test convert(Array, kron(a,convert(Array, b))) == kron(convert(Array, a), convert(Array, b)) c = sparse(rand(Float32,5,5)) d = sparse(rand(Float64,5,5)) - @test full(kron(c,d)) == kron(full(c),full(d)) + @test convert(Array, kron(c,d)) == kron(convert(Array, c),convert(Array, d)) f = Diagonal(rand(5)) - @test full(a*f) == full(a)*f - @test full(f*b) == f*full(b) + @test convert(Array, a*f) == convert(Array, a)*f + @test convert(Array, f*b) == f*convert(Array, b) end # scale and scale! sA = sprandn(3, 7, 0.5) sC = similar(sA) -dA = full(sA) +dA = convert(Array, sA) b = randn(7) @test dA * Diagonal(b) == sA * Diagonal(b) @test dA * Diagonal(b) == scale!(sC, sA, b) @@ -286,7 +286,7 @@ end # conj cA = sprandn(5,5,0.2) + im*sprandn(5,5,0.2) -@test full(conj(cA)) == conj(full(cA)) +@test convert(Array, conj(cA)) == conj(convert(Array, cA)) # Test SparseMatrixCSC [c]transpose[!] and permute[!] methods let smalldim = 5, largedim = 10, nzprob = 0.4 @@ -326,19 +326,19 @@ let smalldim = 5, largedim = 10, nzprob = 0.4 A = sprand(m, n, nzprob) At = transpose(A) # transpose[!] - fullAt = transpose(full(A)) + fullAt = transpose(convert(Array, A)) @test transpose(A) == fullAt @test transpose!(similar(At), A) == fullAt # ctranspose[!] C = A + im*A/2 - fullCh = ctranspose(full(C)) + fullCh = ctranspose(convert(Array, C)) @test ctranspose(C) == fullCh @test ctranspose!(similar(sparse(fullCh)), C) == fullCh # permute[!] p = randperm(m) q = randperm(n) - fullPAQ = full(A)[p,q] - @test permute(A, p, q) == sparse(full(A[p,q])) + fullPAQ = convert(Array, A)[p,q] + @test permute(A, p, q) == sparse(convert(Array, A[p,q])) @test permute!(similar(A), A, p, q) == fullPAQ @test permute!(similar(A), A, p, q, similar(At)) == fullPAQ @test permute!(copy(A), p, q) == fullPAQ @@ -349,19 +349,19 @@ end # transpose of SubArrays A = view(sprandn(10, 10, 0.3), 1:4, 1:4) -@test transpose(full(A)) == full(transpose(A)) -@test ctranspose(full(A)) == full(ctranspose(A)) +@test transpose(convert(Array, A)) == convert(Array, transpose(A)) +@test ctranspose(convert(Array, A)) == convert(Array, ctranspose(A)) # exp A = sprandn(5,5,0.2) -@test e.^A ≈ e.^full(A) +@test e.^A ≈ e.^convert(Array, A) # reductions pA = sparse(rand(3, 7)) for arr in (se33, sA, pA) for f in (sum, prod, minimum, maximum, var) - farr = full(arr) + farr = convert(Array, arr) @test f(arr) ≈ f(farr) @test f(arr, 1) ≈ f(farr, 1) @test f(arr, 2) ≈ f(farr, 2) @@ -399,7 +399,7 @@ for f in (sum, prod, minimum, maximum, var) end # spdiagm -@test full(spdiagm((ones(2), ones(2)), (0, -1), 3, 3)) == +@test convert(Array, spdiagm((ones(2), ones(2)), (0, -1), 3, 3)) == [1.0 0.0 0.0; 1.0 1.0 0.0; 0.0 1.0 0.0] # issue #4986, reinterpret @@ -417,7 +417,7 @@ K,J,V = findnz(SparseMatrixCSC(2,1,[1,3],[1,2],[1.0,0.0])) # https://groups.google.com/d/msg/julia-users/Yq4dh8NOWBQ/GU57L90FZ3EJ A = speye(Bool, 5) -@test find(A) == find(x -> x == true, A) == find(full(A)) +@test find(A) == find(x -> x == true, A) == find(convert(Array, A)) # issue #5824 @@ -463,16 +463,16 @@ end # Unary functions a = sprand(5,15, 0.5) -afull = full(a) +afull = convert(Array, a) for op in (:sin, :cos, :tan, :ceil, :floor, :abs, :abs2) @eval begin - @test ($op)(afull) == full($(op)(a)) + @test ($op)(afull) == convert(Array, $(op)(a)) end end for op in (:ceil, :floor) @eval begin - @test ($op)(Int,afull) == full($(op)(Int,a)) + @test ($op)(Int,afull) == convert(Array, $(op)(Int,a)) end end @@ -499,42 +499,42 @@ for (aa116, ss116) in [(a116, s116), (ad116, sd116)] @test ss116[:,:] == copy(ss116) # range indexing - @test full(ss116[i,:]) == aa116[i,:] - @test full(ss116[:,j]) == aa116[:,j] - @test full(ss116[i,1:2:end]) == aa116[i,1:2:end] - @test full(ss116[1:2:end,j]) == aa116[1:2:end,j] - @test full(ss116[i,end:-2:1]) == aa116[i,end:-2:1] - @test full(ss116[end:-2:1,j]) == aa116[end:-2:1,j] + @test convert(Array, ss116[i,:]) == aa116[i,:] + @test convert(Array, ss116[:,j]) == aa116[:,j] + @test convert(Array, ss116[i,1:2:end]) == aa116[i,1:2:end] + @test convert(Array, ss116[1:2:end,j]) == aa116[1:2:end,j] + @test convert(Array, ss116[i,end:-2:1]) == aa116[i,end:-2:1] + @test convert(Array, ss116[end:-2:1,j]) == aa116[end:-2:1,j] # float-range indexing is not supported # sorted vector indexing - @test full(ss116[i,[3:2:end-3;]]) == aa116[i,[3:2:end-3;]] - @test full(ss116[[3:2:end-3;],j]) == aa116[[3:2:end-3;],j] - @test full(ss116[i,[end-3:-2:1;]]) == aa116[i,[end-3:-2:1;]] - @test full(ss116[[end-3:-2:1;],j]) == aa116[[end-3:-2:1;],j] + @test convert(Array, ss116[i,[3:2:end-3;]]) == aa116[i,[3:2:end-3;]] + @test convert(Array, ss116[[3:2:end-3;],j]) == aa116[[3:2:end-3;],j] + @test convert(Array, ss116[i,[end-3:-2:1;]]) == aa116[i,[end-3:-2:1;]] + @test convert(Array, ss116[[end-3:-2:1;],j]) == aa116[[end-3:-2:1;],j] # unsorted vector indexing with repetition p = [4, 1, 2, 3, 2, 6] - @test full(ss116[p,:]) == aa116[p,:] - @test full(ss116[:,p]) == aa116[:,p] - @test full(ss116[p,p]) == aa116[p,p] + @test convert(Array, ss116[p,:]) == aa116[p,:] + @test convert(Array, ss116[:,p]) == aa116[:,p] + @test convert(Array, ss116[p,p]) == aa116[p,p] # bool indexing li = bitrand(size(aa116,1)) lj = bitrand(size(aa116,2)) - @test full(ss116[li,j]) == aa116[li,j] - @test full(ss116[li,:]) == aa116[li,:] - @test full(ss116[i,lj]) == aa116[i,lj] - @test full(ss116[:,lj]) == aa116[:,lj] - @test full(ss116[li,lj]) == aa116[li,lj] + @test convert(Array, ss116[li,j]) == aa116[li,j] + @test convert(Array, ss116[li,:]) == aa116[li,:] + @test convert(Array, ss116[i,lj]) == aa116[i,lj] + @test convert(Array, ss116[:,lj]) == aa116[:,lj] + @test convert(Array, ss116[li,lj]) == aa116[li,lj] # empty indices for empty in (1:0, Int[]) - @test full(ss116[empty,:]) == aa116[empty,:] - @test full(ss116[:,empty]) == aa116[:,empty] - @test full(ss116[empty,lj]) == aa116[empty,lj] - @test full(ss116[li,empty]) == aa116[li,empty] - @test full(ss116[empty,empty]) == aa116[empty,empty] + @test convert(Array, ss116[empty,:]) == aa116[empty,:] + @test convert(Array, ss116[:,empty]) == aa116[:,empty] + @test convert(Array, ss116[empty,lj]) == aa116[empty,lj] + @test convert(Array, ss116[li,empty]) == aa116[li,empty] + @test convert(Array, ss116[empty,empty]) == aa116[empty,empty] end # out of bounds indexing @@ -564,7 +564,7 @@ S1290 = SparseMatrixCSC(3, 3, UInt8[1,1,1,1], UInt8[], Int64[]) S1290[end] = 3 @test S1290[end] == (S1290[1] + S1290[2,2]) @test 6 == sum(diag(S1290)) - @test full(S1290)[[3,1],1] == full(S1290[[3,1],1]) + @test convert(Array, S1290)[[3,1],1] == convert(Array, S1290[[3,1],1]) # end @@ -716,8 +716,8 @@ let A = speye(Int, 5), I=1:10, X=reshape([trues(10); falses(15)],5,5) end let S = sprand(50, 30, 0.5, x->round(Int,rand(x)*100)), I = sprand(Bool, 50, 30, 0.2) - FS = full(S) - FI = full(I) + FS = convert(Array, S) + FI = convert(Array, I) @test sparse(FS[FI]) == S[I] == S[FI] @test sum(S[FI]) + sum(S[!FI]) == sum(S) @@ -841,7 +841,7 @@ let A = sprand(5,5,0.5,(n)->rand(Float64,n)), ACPY = copy(A) end # indmax, indmin, findmax, findmin -let S = sprand(100,80, 0.5), A = full(S) +let S = sprand(100,80, 0.5), A = convert(Array, S) @test indmax(S) == indmax(A) @test indmin(S) == indmin(A) @test findmin(S) == findmin(A) @@ -851,7 +851,7 @@ let S = sprand(100,80, 0.5), A = full(S) end end -let S = spzeros(10,8), A = full(S) +let S = spzeros(10,8), A = convert(Array, S) @test indmax(S) == indmax(A) == 1 @test indmin(S) == indmin(A) == 1 end @@ -1032,7 +1032,7 @@ end # issue #9917 @test sparse([]') == reshape(sparse([]), 1, 0) -@test full(sparse([])) == zeros(0) +@test convert(Array, sparse([])) == zeros(0) @test_throws BoundsError sparse([])[1] @test_throws BoundsError sparse([])[1] = 1 x = speye(100) @@ -1066,22 +1066,22 @@ end # test sparse constructors from special matrices T = Tridiagonal(randn(4),randn(5),randn(4)) S = sparse(T) -@test norm(full(T) - full(S)) == 0.0 +@test norm(convert(Array, T) - convert(Array, S)) == 0.0 T = SymTridiagonal(randn(5),rand(4)) S = sparse(T) -@test norm(full(T) - full(S)) == 0.0 +@test norm(convert(Array, T) - convert(Array, S)) == 0.0 B = Bidiagonal(randn(5),randn(4),true) S = sparse(B) -@test norm(full(B) - full(S)) == 0.0 +@test norm(convert(Array, B) - convert(Array, S)) == 0.0 B = Bidiagonal(randn(5),randn(4),false) S = sparse(B) -@test norm(full(B) - full(S)) == 0.0 +@test norm(convert(Array, B) - convert(Array, S)) == 0.0 # promotion in spdiagm @test spdiagm(([1,2],[3.5],[4+5im]), (0,1,-1), 2,2) == [1 3.5; 4+5im 2] #Test broadcasting of sparse matrixes -let A = sprand(10,10,0.3), B = sprand(10,10,0.3), CF = rand(10,10), AF = full(A), BF = full(B), C = sparse(CF) +let A = sprand(10,10,0.3), B = sprand(10,10,0.3), CF = rand(10,10), AF = convert(Array, A), BF = convert(Array, B), C = sparse(CF) @test A .* B == AF .* BF @test A[1,:] .* B == AF[1,:] .* BF @test A[:,1] .* B == AF[:,1] .* BF @@ -1173,18 +1173,18 @@ A = speye(5) @test size(similar(A,Complex128,Int8)) == (5,5) @test typeof(similar(A,Complex128,Int8)) == SparseMatrixCSC{Complex128,Int8} @test similar(A,Complex128,(6,6)) == spzeros(Complex128,6,6) -@test convert(Matrix,A) == full(A) +@test convert(Matrix,A) == convert(Array, A) # test float A = sprand(Bool, 5,5,0.0) @test eltype(float(A)) == Float64 # issue #11658 A = sprand(Bool, 5,5,0.2) -@test float(A) == float(full(A)) +@test float(A) == float(convert(Array, A)) # test sparsevec A = sparse(ones(5,5)) -@test all(full(sparsevec(A)) .== ones(25)) -@test all(full(sparsevec([1:5;],1)) .== ones(5)) +@test all(convert(Array, sparsevec(A)) .== ones(25)) +@test all(convert(Array, sparsevec([1:5;],1)) .== ones(5)) @test_throws ArgumentError sparsevec([1:5;], [1:4;]) #test sparse @@ -1262,11 +1262,11 @@ end # triu/tril A = sprand(5,5,0.2) -AF = full(A) -@test full(triu(A,1)) == triu(AF,1) -@test full(tril(A,1)) == tril(AF,1) -@test full(triu!(copy(A), 2)) == triu(AF,2) -@test full(tril!(copy(A), 2)) == tril(AF,2) +AF = convert(Array, A) +@test convert(Array, triu(A,1)) == triu(AF,1) +@test convert(Array, tril(A,1)) == tril(AF,1) +@test convert(Array, triu!(copy(A), 2)) == triu(AF,2) +@test convert(Array, tril!(copy(A), 2)) == tril(AF,2) @test_throws BoundsError tril(A,6) @test_throws BoundsError tril(A,-6) @test_throws BoundsError triu(A,6) @@ -1374,10 +1374,10 @@ nonzeros(A1)[2:5]=0 # UniformScaling A = sprandn(10,10,0.5) -@test A + I == full(A) + I -@test I + A == I + full(A) -@test A - I == full(A) - I -@test I - A == I - full(A) +@test A + I == convert(Array, A) + I +@test I + A == I + convert(Array, A) +@test A - I == convert(Array, A) - I +@test I - A == I - convert(Array, A) # Test error path if triplet vectors are not all the same length (#12177) @test_throws ArgumentError sparse([1,2,3], [1,2], [1,2,3], 3, 3) @@ -1405,15 +1405,15 @@ end Ac = sprandn(10,10,.1) + im* sprandn(10,10,.1) Ar = sprandn(10,10,.1) Ai = ceil(Int,Ar*100) -@test norm(Ac,1) ≈ norm(full(Ac),1) -@test norm(Ac,Inf) ≈ norm(full(Ac),Inf) -@test vecnorm(Ac) ≈ vecnorm(full(Ac)) -@test norm(Ar,1) ≈ norm(full(Ar),1) -@test norm(Ar,Inf) ≈ norm(full(Ar),Inf) -@test vecnorm(Ar) ≈ vecnorm(full(Ar)) -@test norm(Ai,1) ≈ norm(full(Ai),1) -@test norm(Ai,Inf) ≈ norm(full(Ai),Inf) -@test vecnorm(Ai) ≈ vecnorm(full(Ai)) +@test norm(Ac,1) ≈ norm(convert(Array, Ac),1) +@test norm(Ac,Inf) ≈ norm(convert(Array, Ac),Inf) +@test vecnorm(Ac) ≈ vecnorm(convert(Array, Ac)) +@test norm(Ar,1) ≈ norm(convert(Array, Ar),1) +@test norm(Ar,Inf) ≈ norm(convert(Array, Ar),Inf) +@test vecnorm(Ar) ≈ vecnorm(convert(Array, Ar)) +@test norm(Ai,1) ≈ norm(convert(Array, Ai),1) +@test norm(Ai,Inf) ≈ norm(convert(Array, Ai),Inf) +@test vecnorm(Ai) ≈ vecnorm(convert(Array, Ai)) # test sparse matrix cond A = sparse(reshape([1.0],1,1)) @@ -1422,10 +1422,10 @@ Ar = sprandn(20,20,.5) @test cond(A,1) == 1.0 # For a discussion of the tolerance, see #14778 if Base.USE_GPL_LIBS - @test 0.99 <= cond(Ar, 1) \ norm(Ar, 1) * norm(inv(full(Ar)), 1) < 3 - @test 0.99 <= cond(Ac, 1) \ norm(Ac, 1) * norm(inv(full(Ac)), 1) < 3 - @test 0.99 <= cond(Ar, Inf) \ norm(Ar, Inf) * norm(inv(full(Ar)), Inf) < 3 - @test 0.99 <= cond(Ac, Inf) \ norm(Ac, Inf) * norm(inv(full(Ac)), Inf) < 3 + @test 0.99 <= cond(Ar, 1) \ norm(Ar, 1) * norm(inv(convert(Array, Ar)), 1) < 3 + @test 0.99 <= cond(Ac, 1) \ norm(Ac, 1) * norm(inv(convert(Array, Ac)), 1) < 3 + @test 0.99 <= cond(Ar, Inf) \ norm(Ar, Inf) * norm(inv(convert(Array, Ar)), Inf) < 3 + @test 0.99 <= cond(Ac, Inf) \ norm(Ac, Inf) * norm(inv(convert(Array, Ac)), Inf) < 3 end @test_throws ArgumentError cond(A,2) @test_throws ArgumentError cond(A,3) @@ -1441,9 +1441,9 @@ Aci = ceil(Int64,100*sprand(20,20,.5))+ im*ceil(Int64,sprand(20,20,.5)) Ar = sprandn(20,20,.5) Ari = ceil(Int64,100*Ar) if Base.USE_GPL_LIBS - @test_approx_eq_eps Base.SparseArrays.normestinv(Ac,3) norm(inv(full(Ac)),1) 1e-4 - @test_approx_eq_eps Base.SparseArrays.normestinv(Aci,3) norm(inv(full(Aci)),1) 1e-4 - @test_approx_eq_eps Base.SparseArrays.normestinv(Ar) norm(inv(full(Ar)),1) 1e-4 + @test_approx_eq_eps Base.SparseArrays.normestinv(Ac,3) norm(inv(convert(Array, Ac)),1) 1e-4 + @test_approx_eq_eps Base.SparseArrays.normestinv(Aci,3) norm(inv(convert(Array, Aci)),1) 1e-4 + @test_approx_eq_eps Base.SparseArrays.normestinv(Ar) norm(inv(convert(Array, Ar)),1) 1e-4 @test_throws ArgumentError Base.SparseArrays.normestinv(Ac,0) @test_throws ArgumentError Base.SparseArrays.normestinv(Ac,21) end @@ -1474,32 +1474,32 @@ let @test typeof(min(A13024, B13024)) == SparseMatrixCSC{Bool,Int} for op in (+, -, &, |, $, max, min) - @test op(A13024, B13024) == op(full(A13024), full(B13024)) + @test op(A13024, B13024) == op(convert(Array, A13024), convert(Array, B13024)) end end let A = 2. * speye(5,5) - @test full(spones(A)) == eye(full(A)) + @test convert(Array, spones(A)) == eye(convert(Array, A)) end let A = spdiagm(rand(5)) + sprandn(5,5,0.2) + im*sprandn(5,5,0.2) A = A + A' - @test !Base.USE_GPL_LIBS || abs(det(factorize(Hermitian(A)))) ≈ abs(det(factorize(full(A)))) + @test !Base.USE_GPL_LIBS || abs(det(factorize(Hermitian(A)))) ≈ abs(det(factorize(convert(Array, A)))) A = spdiagm(rand(5)) + sprandn(5,5,0.2) + im*sprandn(5,5,0.2) A = A*A' - @test !Base.USE_GPL_LIBS || abs(det(factorize(Hermitian(A)))) ≈ abs(det(factorize(full(A)))) + @test !Base.USE_GPL_LIBS || abs(det(factorize(Hermitian(A)))) ≈ abs(det(factorize(convert(Array, A)))) A = spdiagm(rand(5)) + sprandn(5,5,0.2) A = A + A.' - @test !Base.USE_GPL_LIBS || abs(det(factorize(Symmetric(A)))) ≈ abs(det(factorize(full(A)))) + @test !Base.USE_GPL_LIBS || abs(det(factorize(Symmetric(A)))) ≈ abs(det(factorize(convert(Array, A)))) A = spdiagm(rand(5)) + sprandn(5,5,0.2) A = A*A.' - @test !Base.USE_GPL_LIBS || abs(det(factorize(Symmetric(A)))) ≈ abs(det(factorize(full(A)))) + @test !Base.USE_GPL_LIBS || abs(det(factorize(Symmetric(A)))) ≈ abs(det(factorize(convert(Array, A)))) @test factorize(triu(A)) == triu(A) @test isa(factorize(triu(A)), UpperTriangular{Float64, SparseMatrixCSC{Float64, Int}}) @test factorize(tril(A)) == tril(A) @test isa(factorize(tril(A)), LowerTriangular{Float64, SparseMatrixCSC{Float64, Int}}) - @test !Base.USE_GPL_LIBS || factorize(A[:,1:4])\ones(size(A,1)) ≈ full(A[:,1:4])\ones(size(A,1)) + @test !Base.USE_GPL_LIBS || factorize(A[:,1:4])\ones(size(A,1)) ≈ convert(Array, A[:,1:4])\ones(size(A,1)) @test_throws ErrorException chol(A) @test_throws ErrorException lu(A) @test_throws ErrorException eig(A) @@ -1532,12 +1532,12 @@ let @test issparse(LinAlg.UnitLowerTriangular(m)) @test issparse(UpperTriangular(m)) @test issparse(LinAlg.UnitUpperTriangular(m)) - @test issparse(Symmetric(full(m))) == false - @test issparse(Hermitian(full(m))) == false - @test issparse(LowerTriangular(full(m))) == false - @test issparse(LinAlg.UnitLowerTriangular(full(m))) == false - @test issparse(UpperTriangular(full(m))) == false - @test issparse(LinAlg.UnitUpperTriangular(full(m))) == false + @test issparse(Symmetric(convert(Array, m))) == false + @test issparse(Hermitian(convert(Array, m))) == false + @test issparse(LowerTriangular(convert(Array, m))) == false + @test issparse(LinAlg.UnitLowerTriangular(convert(Array, m))) == false + @test issparse(UpperTriangular(convert(Array, m))) == false + @test issparse(LinAlg.UnitUpperTriangular(convert(Array, m))) == false end let diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index 62a7649cc1e4d..2e05de46b80b7 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -26,11 +26,11 @@ let x = spv_x1 @test nonzeros(x) == [1.25, -0.75, 3.5] end -# full +# convert(Array, _) for (x, xf) in [(spv_x1, x1_full)] - @test isa(full(x), Vector{Float64}) - @test full(x) == xf + @test isa(convert(Array, x), Vector{Float64}) + @test convert(Array, x) == xf end ### Show @@ -173,7 +173,7 @@ let x = sprand(100, 0.5) r = x[I] @test isa(r, SparseVector{Float64,Int}) @test all(nonzeros(r) .!= 0.0) - @test full(r) == full(x)[I] + @test convert(Array, r) == convert(Array, x)[I] end # setindex @@ -369,11 +369,11 @@ let x = spv_x1, xf = x1_full xm = convert(SparseMatrixCSC, x) @test isa(xm, SparseMatrixCSC{Float64,Int}) - @test full(xm) == reshape(xf, 8, 1) + @test convert(Array, xm) == reshape(xf, 8, 1) xm = convert(SparseMatrixCSC{Float32}, x) @test isa(xm, SparseMatrixCSC{Float32,Int}) - @test full(xm) == reshape(convert(Vector{Float32}, xf), 8, 1) + @test convert(Array, xm) == reshape(convert(Vector{Float32}, xf), 8, 1) end @@ -393,22 +393,22 @@ let m = 80, n = 100 @test nnz(H) == tnnz Hr = zeros(m, n) for j = 1:n - Hr[:,j] = full(A[j]) + Hr[:,j] = convert(Array, A[j]) end - @test full(H) == Hr + @test convert(Array, H) == Hr V = vcat(A...) @test isa(V, SparseVector{Float64,Int}) @test length(V) == m * n Vr = vec(Hr) - @test full(V) == Vr + @test convert(Array, V) == Vr end ## sparsemat: combinations with sparse matrix let S = sprand(4, 8, 0.5) - Sf = full(S) + Sf = convert(Array, S) @assert isa(Sf, Matrix{Float64}) # get a single column @@ -416,84 +416,84 @@ let S = sprand(4, 8, 0.5) col = S[:, j] @test isa(col, SparseVector{Float64,Int}) @test length(col) == size(S,1) - @test full(col) == Sf[:,j] + @test convert(Array, col) == Sf[:,j] end # Get a reshaped vector v = S[:] @test isa(v, SparseVector{Float64,Int}) @test length(v) == length(S) - @test full(v) == Sf[:] + @test convert(Array, v) == Sf[:] # Get a linear subset for i=0:length(S) v = S[1:i] @test isa(v, SparseVector{Float64,Int}) @test length(v) == i - @test full(v) == Sf[1:i] + @test convert(Array, v) == Sf[1:i] end for i=1:length(S)+1 v = S[i:end] @test isa(v, SparseVector{Float64,Int}) @test length(v) == length(S) - i + 1 - @test full(v) == Sf[i:end] + @test convert(Array, v) == Sf[i:end] end for i=0:div(length(S),2) v = S[1+i:end-i] @test isa(v, SparseVector{Float64,Int}) @test length(v) == length(S) - 2i - @test full(v) == Sf[1+i:end-i] + @test convert(Array, v) == Sf[1+i:end-i] end end let r = [1,10], S = sparse(r, r, r) - Sf = full(S) + Sf = convert(Array, S) @assert isa(Sf, Matrix{Int}) inds = [1,1,1,1,1,1] v = S[inds] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(inds) - @test full(v) == Sf[inds] + @test convert(Array, v) == Sf[inds] inds = [2,2,2,2,2,2] v = S[inds] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(inds) - @test full(v) == Sf[inds] + @test convert(Array, v) == Sf[inds] # get a single column for j = 1:size(S,2) col = S[:, j] @test isa(col, SparseVector{Int,Int}) @test length(col) == size(S,1) - @test full(col) == Sf[:,j] + @test convert(Array, col) == Sf[:,j] end # Get a reshaped vector v = S[:] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(S) - @test full(v) == Sf[:] + @test convert(Array, v) == Sf[:] # Get a linear subset for i=0:length(S) v = S[1:i] @test isa(v, SparseVector{Int,Int}) @test length(v) == i - @test full(v) == Sf[1:i] + @test convert(Array, v) == Sf[1:i] end for i=1:length(S)+1 v = S[i:end] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(S) - i + 1 - @test full(v) == Sf[i:end] + @test convert(Array, v) == Sf[i:end] end for i=0:div(length(S),2) v = S[1+i:end-i] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(S) - 2i - @test full(v) == Sf[1+i:end-i] + @test convert(Array, v) == Sf[1+i:end-i] end end @@ -502,10 +502,10 @@ end ### Data rnd_x0 = sprand(50, 0.6) -rnd_x0f = full(rnd_x0) +rnd_x0f = convert(Array, rnd_x0) rnd_x1 = sprand(50, 0.7) * 4.0 -rnd_x1f = full(rnd_x1) +rnd_x1f = convert(Array, rnd_x1) spv_x1 = SparseVector(8, [2, 5, 6], [1.25, -0.75, 3.5]) spv_x2 = SparseVector(8, [1, 2, 6, 7], [3.25, 4.0, -5.5, -6.0]) @@ -532,10 +532,10 @@ let x = spv_x1, x2 = x2 = spv_x2 @test exact_equal(x - x2, xb) @test exact_equal(x2 - x, -xb) - @test full(x) + x2 == full(xa) - @test full(x) - x2 == full(xb) - @test x + full(x2) == full(xa) - @test x - full(x2) == full(xb) + @test convert(Array, x) + x2 == convert(Array, xa) + @test convert(Array, x) - x2 == convert(Array, xb) + @test x + convert(Array, x2) == convert(Array, xa) + @test x - convert(Array, x2) == convert(Array, xb) # multiplies xm = SparseVector(8, [2, 6], [5.0, -19.25]) @@ -543,8 +543,8 @@ let x = spv_x1, x2 = x2 = spv_x2 @test exact_equal(x .* x2, xm) @test exact_equal(x2 .* x, xm) - @test full(x) .* x2 == full(xm) - @test x .* full(x2) == full(xm) + @test convert(Array, x) .* x2 == convert(Array, xm) + @test x .* convert(Array, x2) == convert(Array, xm) # max & min @test exact_equal(max(x, x), x) @@ -584,7 +584,7 @@ function check_nz2z_z2z{T}(f::Function, x::SparseVector{T}, xf::Vector{T}) isa(r, AbstractSparseVector) || error("$f(x) is not a sparse vector.") eltype(r) == R || error("$f(x) results in eltype = $(eltype(r)), expect $R") all(r.nzval .!= 0) || error("$f(x) contains zeros in nzval.") - full(r) == f(xf) || error("Incorrect results found in $f(x).") + convert(Array, r) == f(xf) || error("Incorrect results found in $f(x).") end for f in [floor, ceil, trunc, round] @@ -671,14 +671,14 @@ end ### BLAS Level-1 let x = sprand(16, 0.5), x2 = sprand(16, 0.4) - xf = full(x) - xf2 = full(x2) + xf = convert(Array, x) + xf2 = convert(Array, x2) # axpy! for c in [1.0, -1.0, 2.0, -2.0] - y = full(x) + y = convert(Array, x) @test is(Base.axpy!(c, x2, y), y) - @test y == full(x2 * c + x) + @test y == convert(Array, x2 * c + x) end # scale @@ -704,15 +704,15 @@ let x = sprand(16, 0.5), x2 = sprand(16, 0.4) @test dot(x2, x2) == sumabs2(x2) @test dot(x, x2) ≈ dv @test dot(x2, x) ≈ dv - @test dot(full(x), x2) ≈ dv - @test dot(x, full(x2)) ≈ dv + @test dot(convert(Array, x), x2) ≈ dv + @test dot(x, convert(Array, x2)) ≈ dv end end let x = complex(sprand(32, 0.6), sprand(32, 0.6)), y = complex(sprand(32, 0.6), sprand(32, 0.6)) - xf = full(x)::Vector{Complex128} - yf = full(y)::Vector{Complex128} + xf = convert(Array, x)::Vector{Complex128} + yf = convert(Array, y)::Vector{Complex128} @test dot(x, x) ≈ dot(xf, xf) @test dot(x, y) ≈ dot(xf, yf) end @@ -723,7 +723,7 @@ end ## dense A * sparse x -> dense y let A = randn(9, 16), x = sprand(16, 0.7) - xf = full(x) + xf = convert(Array, x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) rr = α*A*xf + β*y @@ -736,7 +736,7 @@ let A = randn(9, 16), x = sprand(16, 0.7) end let A = randn(16, 9), x = sprand(16, 0.7) - xf = full(x) + xf = convert(Array, x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) rr = α*A'xf + β*y @@ -751,8 +751,8 @@ end ## sparse A * sparse x -> dense y let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7) - Af = full(A) - xf = full(x) + Af = convert(Array, A) + xf = convert(Array, x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) rr = α*Af*xf + β*y @@ -765,8 +765,8 @@ let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7) end let A = sprandn(16, 9, 0.5), x = sprand(16, 0.7) - Af = full(A) - xf = full(x) + Af = convert(Array, A) + xf = convert(Array, x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) rr = α*Af'xf + β*y @@ -781,9 +781,9 @@ end let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), x = complex(sprandn(8, 0.6), sprandn(8, 0.6)), x2 = complex(sprandn(7, 0.75), sprandn(7, 0.75)) - Af = full(A) - xf = full(x) - x2f = full(x2) + Af = convert(Array, A) + xf = convert(Array, x) + x2f = convert(Array, x2) @test SparseArrays.densemv(A, x; trans='N') ≈ Af * xf @test SparseArrays.densemv(A, x2; trans='T') ≈ Af.' * x2f @test SparseArrays.densemv(A, x2; trans='C') ≈ Af'x2f @@ -792,39 +792,39 @@ end ## sparse A * sparse x -> sparse y let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7), x2 = sprand(9, 0.7) - Af = full(A) - xf = full(x) - x2f = full(x2) + Af = convert(Array, A) + xf = convert(Array, x) + x2f = convert(Array, x2) y = A*x @test isa(y, SparseVector{Float64,Int}) @test all(nonzeros(y) .!= 0.0) - @test full(y) ≈ Af * xf + @test convert(Array, y) ≈ Af * xf y = At_mul_B(A, x2) @test isa(y, SparseVector{Float64,Int}) @test all(nonzeros(y) .!= 0.0) - @test full(y) ≈ Af'x2f + @test convert(Array, y) ≈ Af'x2f end let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), x = complex(sprandn(8, 0.6), sprandn(8, 0.6)), x2 = complex(sprandn(7, 0.75), sprandn(7, 0.75)) - Af = full(A) - xf = full(x) - x2f = full(x2) + Af = convert(Array, A) + xf = convert(Array, x) + x2f = convert(Array, x2) y = A*x @test isa(y, SparseVector{Complex128,Int}) - @test full(y) ≈ Af * xf + @test convert(Array, y) ≈ Af * xf y = At_mul_B(A, x2) @test isa(y, SparseVector{Complex128,Int}) - @test full(y) ≈ Af.' * x2f + @test convert(Array, y) ≈ Af.' * x2f y = Ac_mul_B(A, x2) @test isa(y, SparseVector{Complex128,Int}) - @test full(y) ≈ Af'x2f + @test convert(Array, y) ≈ Af'x2f end # left-division operations involving triangular matrices and sparse vectors (#14005) @@ -965,7 +965,7 @@ end sv = sparse(1:10) sm = convert(SparseMatrixCSC, sv) sv[1] = 0 -@test full(sm)[2:end] == collect(2:10) +@test convert(Array, sm)[2:end] == collect(2:10) # Ensure that sparsevec with all-zero values returns an array of zeros @test sparsevec([1,2,3],[0,0,0]) == [0,0,0] @@ -991,8 +991,8 @@ s14013 = sparse([10.0 0.0 30.0; 0.0 1.0 0.0]) a14013 = [10.0 0.0 30.0; 0.0 1.0 0.0] @test s14013 == a14013 @test vec(s14013) == s14013[:] == a14013[:] -@test full(s14013)[1,:] == s14013[1,:] == a14013[1,:] == [10.0, 0.0, 30.0] -@test full(s14013)[2,:] == s14013[2,:] == a14013[2,:] == [0.0, 1.0, 0.0] +@test convert(Array, s14013)[1,:] == s14013[1,:] == a14013[1,:] == [10.0, 0.0, 30.0] +@test convert(Array, s14013)[2,:] == s14013[2,:] == a14013[2,:] == [0.0, 1.0, 0.0] # Issue 14046 s14046 = sprand(5, 1.0) @@ -1024,9 +1024,9 @@ for Tv in [Float32, Float64, Int64, Int32, Complex128] sparr = Sp(arr) fillval = rand(Tv) fill!(sparr, fillval) - @test full(sparr) == fillval * ones(arr) + @test convert(Array, sparr) == fillval * ones(arr) fill!(sparr, 0) - @test full(sparr) == zeros(arr) + @test convert(Array, sparr) == zeros(arr) end end end diff --git a/test/sparsedir/spqr.jl b/test/sparsedir/spqr.jl index 5411dd503f7a7..ef15ea35db0b8 100644 --- a/test/sparsedir/spqr.jl +++ b/test/sparsedir/spqr.jl @@ -25,8 +25,8 @@ for eltyA in (Float64, Complex{Float64}) end @inferred A\B - @test A\B[:,1] ≈ full(A)\B[:,1] - @test A\B ≈ full(A)\B + @test A\B[:,1] ≈ convert(Array, A)\B[:,1] + @test A\B ≈ convert(Array, A)\B @test_throws DimensionMismatch A\B[1:m-1,:] @test A[1:9,:]*(A[1:9,:]\ones(eltyB, 9)) ≈ ones(9) # Underdetermined system diff --git a/test/sparsedir/umfpack.jl b/test/sparsedir/umfpack.jl index 6db636a20353e..9c609ab7969c4 100644 --- a/test/sparsedir/umfpack.jl +++ b/test/sparsedir/umfpack.jl @@ -23,7 +23,7 @@ for Tv in (Float64, Complex128) L,U,p,q,Rs = lua[:(:)] @test (Diagonal(Rs) * A)[p,q] ≈ L * U - det(lua) ≈ det(full(A)) + det(lua) ≈ det(convert(Array, A)) b = [8., 45., -3., 3., 19.] x = lua\b From 15ddf79948db0b1c7bb962a8f37811a3b618d221 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 23 Jun 2016 14:13:35 -0700 Subject: [PATCH 0469/1117] Migrate full(X) to convert(Array, X) in tests in test/linalg. --- test/linalg/arnoldi.jl | 4 +- test/linalg/bidiag.jl | 24 ++--- test/linalg/cholesky.jl | 22 ++--- test/linalg/dense.jl | 4 +- test/linalg/diagonal.jl | 32 +++---- test/linalg/hessenberg.jl | 4 +- test/linalg/lapack.jl | 2 +- test/linalg/lq.jl | 28 +++--- test/linalg/lu.jl | 22 ++--- test/linalg/matmul.jl | 2 +- test/linalg/qr.jl | 50 +++++----- test/linalg/schur.jl | 2 +- test/linalg/special.jl | 66 ++++++------- test/linalg/svd.jl | 2 +- test/linalg/symmetric.jl | 8 +- test/linalg/triangular.jl | 170 +++++++++++++++++----------------- test/linalg/tridiag.jl | 45 +++++---- test/linalg/uniformscaling.jl | 32 +++---- 18 files changed, 259 insertions(+), 260 deletions(-) diff --git a/test/linalg/arnoldi.jl b/test/linalg/arnoldi.jl index 81dc9217972ef..5c781a062e20b 100644 --- a/test/linalg/arnoldi.jl +++ b/test/linalg/arnoldi.jl @@ -155,7 +155,7 @@ debug && println("real svds") let # svds test A = sparse([1, 1, 2, 3, 4], [2, 1, 1, 3, 1], [2.0, -1.0, 6.1, 7.0, 1.5]) S1 = svds(A, nsv = 2) - S2 = svd(full(A)) + S2 = svd(convert(Array, A)) ## singular values match: @test S1[1][:S] ≈ S2[2][1:2] @@ -196,7 +196,7 @@ debug && println("complex svds") let # complex svds test A = sparse([1, 1, 2, 3, 4], [2, 1, 1, 3, 1], exp(im*[2.0:2:10;])) S1 = svds(A, nsv = 2) - S2 = svd(full(A)) + S2 = svd(convert(Array, A)) ## singular values match: @test S1[1][:S] ≈ S2[2][1:2] diff --git a/test/linalg/bidiag.jl b/test/linalg/bidiag.jl index d4d7f397f555f..f96677b4038eb 100644 --- a/test/linalg/bidiag.jl +++ b/test/linalg/bidiag.jl @@ -62,12 +62,12 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test size(T, 1) == size(T, 2) == n @test size(T) == (n, n) - @test full(T) == diagm(dv) + diagm(ev, isupper?1:-1) - @test Bidiagonal(full(T), isupper) == T + @test convert(Array, T) == diagm(dv) + diagm(ev, isupper?1:-1) + @test Bidiagonal(convert(Array, T), isupper) == T @test big(T) == T - @test full(abs(T)) == abs(diagm(dv)) + abs(diagm(ev, isupper?1:-1)) - @test full(real(T)) == real(diagm(dv)) + real(diagm(ev, isupper?1:-1)) - @test full(imag(T)) == imag(diagm(dv)) + imag(diagm(ev, isupper?1:-1)) + @test convert(Array, abs(T)) == abs(diagm(dv)) + abs(diagm(ev, isupper?1:-1)) + @test convert(Array, real(T)) == real(diagm(dv)) + real(diagm(ev, isupper?1:-1)) + @test convert(Array, imag(T)) == imag(diagm(dv)) + imag(diagm(ev, isupper?1:-1)) z = zeros(elty, n) debug && println("Idempotent tests") @@ -113,7 +113,7 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) b += im*convert(Matrix{elty}, rand(1:10, n, 2)) end end - Tfull = full(T) + Tfull = convert(Array, T) condT = cond(map(Complex128,Tfull)) promty = typeof((zero(relty)*zero(relty) + zero(relty)*zero(relty))/one(relty)) if relty != BigFloat @@ -184,7 +184,7 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) debug && println("Singular systems") if (elty <: BlasReal) - @test full(svdfact(T)) ≈ full(svdfact!(copy(Tfull))) + @test convert(Array, svdfact(T)) ≈ convert(Array, svdfact!(copy(Tfull))) @test svdvals(Tfull) ≈ svdvals(T) u1, d1, v1 = svd(Tfull) u2, d2, v2 = svd(T) @@ -206,9 +206,9 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) dv = convert(Vector{elty}, relty <: AbstractFloat ? randn(n) : rand(1:10, n)) ev = convert(Vector{elty}, relty <: AbstractFloat ? randn(n-1) : rand(1:10, n-1)) T2 = Bidiagonal(dv, ev, isupper2) - Tfull2 = full(T2) + Tfull2 = convert(Array, T2) for op in (+, -, *) - @test full(op(T, T2)) ≈ op(Tfull, Tfull2) + @test convert(Array, op(T, T2)) ≈ op(Tfull, Tfull2) end end @@ -234,7 +234,7 @@ A = Bidiagonal(ones(Float32,10),ones(Float32,9),true) B = rand(Float64,10,10) C = Tridiagonal(rand(Float64,9),rand(Float64,10),rand(Float64,9)) @test promote_rule(Matrix{Float64}, Bidiagonal{Float64}) == Matrix{Float64} -@test promote(B,A) == (B,convert(Matrix{Float64},full(A))) +@test promote(B,A) == (B,convert(Matrix{Float64},convert(Array, A))) @test promote(C,A) == (C,Tridiagonal(zeros(Float64,9),convert(Vector{Float64},A.dv),convert(Vector{Float64},A.ev))) import Base.LinAlg: fillslots!, UnitLowerTriangular @@ -271,14 +271,14 @@ let #fill! b = Bidiagonal(randn(1,1), true) st = SymTridiagonal(randn(1,1)) for x in (b, st) - @test full(fill!(x, val)) == fill!(full(x), val) + @test convert(Array, fill!(x, val)) == fill!(convert(Array, x), val) end b = Bidiagonal(randn(2,2), true) st = SymTridiagonal(randn(3), randn(2)) t = Tridiagonal(randn(3,3)) for x in (b, t, st) @test_throws ArgumentError fill!(x, val) - @test full(fill!(x, 0)) == fill!(full(x), 0) + @test convert(Array, fill!(x, 0)) == fill!(convert(Array, x), 0) end end end diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index 00e4696b4f12b..29fafb69e6bd6 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -46,7 +46,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) for i=1:n, j=1:n @test E[i,j] <= (n+1)ε/(1-(n+1)ε)*real(sqrt(apd[i,i]*apd[j,j])) end - E = abs(apd - full(capd)) + E = abs(apd - convert(Array, capd)) for i=1:n, j=1:n @test E[i,j] <= (n+1)ε/(1-(n+1)ε)*real(sqrt(apd[i,i]*apd[j,j])) end @@ -65,13 +65,13 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) end # test chol of 2x2 Strang matrix - S = convert(AbstractMatrix{eltya},full(SymTridiagonal([2,2],[-1]))) + S = convert(AbstractMatrix{eltya},convert(Array, SymTridiagonal([2,2],[-1]))) U = Bidiagonal([2,sqrt(eltya(3))],[-1],true) / sqrt(eltya(2)) - @test full(chol(S)) ≈ full(U) + @test convert(Array, chol(S)) ≈ convert(Array, U) #lower Cholesky factor lapd = cholfact(apd, :L) - @test full(lapd) ≈ apd + @test convert(Array, lapd) ≈ apd l = lapd[:L] @test l*l' ≈ apd @test triu(capd.factors) ≈ lapd[:U] @@ -95,12 +95,12 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) if isreal(apd) @test apd*inv(cpapd) ≈ eye(n) end - @test full(cpapd) ≈ apd + @test convert(Array, cpapd) ≈ apd #getindex @test_throws KeyError cpapd[:Z] @test size(cpapd) == size(apd) - @test full(copy(cpapd)) ≈ apd + @test convert(Array, copy(cpapd)) ≈ apd @test det(cpapd) ≈ det(apd) @test cpapd[:P]*cpapd[:L]*cpapd[:U]*cpapd[:P]' ≈ apd end @@ -151,9 +151,9 @@ begin # Cholesky factor of Matrix with non-commutative elements, here 2x2-matrices X = Matrix{Float64}[0.1*rand(2,2) for i in 1:3, j = 1:3] - L = full(Base.LinAlg._chol!(X*X', LowerTriangular)) - U = full(Base.LinAlg._chol!(X*X', UpperTriangular)) - XX = full(X*X') + L = convert(Array, Base.LinAlg._chol!(X*X', LowerTriangular)) + U = convert(Array, Base.LinAlg._chol!(X*X', UpperTriangular)) + XX = convert(Array, X*X') @test sum(sum(norm, L*L' - XX)) < eps() @test sum(sum(norm, U'*U - XX)) < eps() @@ -167,8 +167,8 @@ for elty in (Float32, Float64, Complex{Float32}, Complex{Float64}) A = randn(5,5) end A = convert(Matrix{elty}, A'A) - @test full(cholfact(A)[:L]) ≈ full(invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{LowerTriangular}}, copy(A), LowerTriangular)) - @test full(cholfact(A)[:U]) ≈ full(invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{UpperTriangular}}, copy(A), UpperTriangular)) + @test convert(Array, cholfact(A)[:L]) ≈ convert(Array, invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{LowerTriangular}}, copy(A), LowerTriangular)) + @test convert(Array, cholfact(A)[:U]) ≈ convert(Array, invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{UpperTriangular}}, copy(A), UpperTriangular)) end # Test up- and downdates diff --git a/test/linalg/dense.jl b/test/linalg/dense.jl index 3f1c7a3c45995..8f640a864d57e 100644 --- a/test/linalg/dense.jl +++ b/test/linalg/dense.jl @@ -120,12 +120,12 @@ debug && println("Factorize") @test factorize(A) == Bidiagonal(d,e,true) if eltya <: Real A = diagm(d) + diagm(e,1) + diagm(e,-1) - @test full(factorize(A)) ≈ full(factorize(SymTridiagonal(d,e))) + @test convert(Array, factorize(A)) ≈ convert(Array, factorize(SymTridiagonal(d,e))) A = diagm(d) + diagm(e,1) + diagm(e,-1) + diagm(f,2) + diagm(f,-2) @test inv(factorize(A)) ≈ inv(factorize(Symmetric(A))) end A = diagm(d) + diagm(e,1) + diagm(e2,-1) - @test full(factorize(A)) ≈ full(factorize(Tridiagonal(e2,d,e))) + @test convert(Array, factorize(A)) ≈ convert(Array, factorize(Tridiagonal(e2,d,e))) A = diagm(d) + diagm(e,1) + diagm(f,2) @test factorize(A) == UpperTriangular(A) end # for eltya diff --git a/test/linalg/diagonal.jl b/test/linalg/diagonal.jl index be856257cef9d..22597c55f4f32 100644 --- a/test/linalg/diagonal.jl +++ b/test/linalg/diagonal.jl @@ -27,9 +27,9 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test typeof(convert(Diagonal{Complex64},D)) == Diagonal{Complex64} @test typeof(convert(AbstractMatrix{Complex64},D)) == Diagonal{Complex64} - @test full(real(D)) == real(DM) - @test full(abs(D)) == abs(DM) - @test full(imag(D)) == imag(DM) + @test convert(Array, real(D)) == real(DM) + @test convert(Array, abs(D)) == abs(DM) + @test convert(Array, imag(D)) == imag(DM) @test parent(D) == d @test diag(D) == d @@ -79,12 +79,12 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test_throws SingularException A_ldiv_B!(Diagonal(zeros(relty,n)),copy(v)) b = rand(elty,n,n) b = sparse(b) - @test A_ldiv_B!(D,copy(b)) ≈ full(D)\full(b) + @test A_ldiv_B!(D,copy(b)) ≈ convert(Array, D)\convert(Array, b) @test_throws SingularException A_ldiv_B!(Diagonal(zeros(elty,n)),copy(b)) b = view(rand(elty,n),collect(1:n)) b2 = copy(b) c = A_ldiv_B!(D,b) - d = full(D)\b2 + d = convert(Array, D)\b2 for i in 1:n @test c[i] ≈ d[i] end @@ -101,23 +101,23 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) D2 = Diagonal(d) DM2= diagm(d) for op in (+, -, *) - @test full(op(D, D2)) ≈ op(DM, DM2) + @test convert(Array, op(D, D2)) ≈ op(DM, DM2) end # binary ops with plain numbers a = rand() - @test full(a*D) ≈ a*DM - @test full(D*a) ≈ DM*a - @test full(D/a) ≈ DM/a + @test convert(Array, a*D) ≈ a*DM + @test convert(Array, D*a) ≈ DM*a + @test convert(Array, D/a) ≈ DM/a if relty <: BlasFloat b = rand(elty,n,n) b = sparse(b) - @test A_mul_B!(copy(D), copy(b)) ≈ full(D)*full(b) - @test At_mul_B!(copy(D), copy(b)) ≈ full(D).'*full(b) - @test Ac_mul_B!(copy(D), copy(b)) ≈ full(D)'*full(b) + @test A_mul_B!(copy(D), copy(b)) ≈ convert(Array, D)*convert(Array, b) + @test At_mul_B!(copy(D), copy(b)) ≈ convert(Array, D).'*convert(Array, b) + @test Ac_mul_B!(copy(D), copy(b)) ≈ convert(Array, D)'*convert(Array, b) end - @test U.'*D ≈ U.'*full(D) - @test U'*D ≈ U'*full(D) + @test U.'*D ≈ U.'*convert(Array, D) + @test U'*D ≈ U'*convert(Array, D) #division of two Diagonals @test D/D2 ≈ Diagonal(D.diag./D2.diag) @@ -155,7 +155,7 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) debug && println("conj and transpose") @test transpose(D) == D if elty <: BlasComplex - @test full(conj(D)) ≈ conj(DM) + @test convert(Array, conj(D)) ≈ conj(DM) @test ctranspose(D) == conj(D) end @@ -233,7 +233,7 @@ end # inv for d in (randn(n), [1, 2, 3], [1im, 2im, 3im]) D = Diagonal(d) - @test inv(D) ≈ inv(full(D)) + @test inv(D) ≈ inv(convert(Array, D)) end @test_throws SingularException inv(Diagonal(zeros(n))) @test_throws SingularException inv(Diagonal([0, 1, 2])) diff --git a/test/linalg/hessenberg.jl b/test/linalg/hessenberg.jl index 8ea3b290056bc..163f05570ac4b 100644 --- a/test/linalg/hessenberg.jl +++ b/test/linalg/hessenberg.jl @@ -26,11 +26,11 @@ debug && println("\ntype of a: ", eltya, " type of b: ", eltyb, "\n") @test size(H[:Q], 2) == size(A, 2) @test size(H[:Q]) == size(A) @test_throws KeyError H[:Z] - @test full(H) ≈ A + @test convert(Array, H) ≈ A @test (H[:Q] * H[:H]) * H[:Q]' ≈ A @test (H[:Q]' *A) * H[:Q] ≈ H[:H] #getindex for HessenbergQ - @test H[:Q][1,1] ≈ full(H[:Q])[1,1] + @test H[:Q][1,1] ≈ convert(Array, H[:Q])[1,1] end end end diff --git a/test/linalg/lapack.jl b/test/linalg/lapack.jl index 51a47eace3a4a..42a0b6f6c4554 100644 --- a/test/linalg/lapack.jl +++ b/test/linalg/lapack.jl @@ -46,7 +46,7 @@ let # gebrd, bdsqr & throw for bdsdc d, e = convert(Vector{elty}, randn(n)), convert(Vector{elty}, randn(n - 1)) U, Vt, C = eye(elty, n), eye(elty, n), eye(elty, n) s, _ = LAPACK.bdsqr!('U', copy(d), copy(e), Vt, U, C) - @test full(Bidiagonal(d, e, true)) ≈ U*Diagonal(s)*Vt + @test convert(Array, Bidiagonal(d, e, true)) ≈ U*Diagonal(s)*Vt @test_throws ArgumentError LAPACK.bdsqr!('A', d, e, Vt, U, C) @test_throws DimensionMismatch LAPACK.bdsqr!('U', d, [e; 1], Vt, U, C) diff --git a/test/linalg/lq.jl b/test/linalg/lq.jl index e898dbcfdaed8..aaaa960a8ef20 100644 --- a/test/linalg/lq.jl +++ b/test/linalg/lq.jl @@ -52,25 +52,25 @@ debug && println("LQ decomposition") @test size(lqa[:Q],3) == 1 @test Base.LinAlg.getq(lqa) == lqa[:Q] @test_throws KeyError lqa[:Z] - @test full(lqa') ≈ a' + @test convert(Array, lqa') ≈ a' @test lqa * lqa' ≈ a * a' @test lqa' * lqa ≈ a' * a - @test q*full(q, thin = false)' ≈ eye(eltya,n) + @test q*Base.LinAlg.thickQ(q)' ≈ eye(eltya,n) @test l*q ≈ a - @test full(lqa) ≈ a - @test full(copy(lqa)) ≈ a + @test convert(Array, lqa) ≈ a + @test convert(Array, copy(lqa)) ≈ a @test_approx_eq_eps a*(lqa\b) b 3000ε @test_approx_eq_eps lqa*b qra[:Q]*qra[:R]*b 3000ε - @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*full(q, thin=false) eye(n) 5000ε + @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*Base.LinAlg.thickQ(q) eye(n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) end - @test_approx_eq_eps q*b full(q, thin=false)*b 100ε - @test_approx_eq_eps q.'*b full(q, thin=false).'*b 100ε - @test_approx_eq_eps q'*b full(q, thin=false)'*b 100ε - @test_approx_eq_eps a*q a*full(q, thin=false) 100ε - @test_approx_eq_eps a*q.' a*full(q, thin=false).' 100ε - @test_approx_eq_eps a*q' a*full(q, thin=false)' 100ε + @test_approx_eq_eps q*b Base.LinAlg.thickQ(q)*b 100ε + @test_approx_eq_eps q.'*b Base.LinAlg.thickQ(q).'*b 100ε + @test_approx_eq_eps q'*b Base.LinAlg.thickQ(q)'*b 100ε + @test_approx_eq_eps a*q a*Base.LinAlg.thickQ(q) 100ε + @test_approx_eq_eps a*q.' a*Base.LinAlg.thickQ(q).' 100ε + @test_approx_eq_eps a*q' a*Base.LinAlg.thickQ(q)' 100ε @test_throws DimensionMismatch q*b[1:n1 + 1] @test_throws DimensionMismatch Ac_mul_B(q,ones(eltya,n+2,n+2)) @test_throws DimensionMismatch ones(eltyb,n+2,n+2)*q @@ -80,10 +80,10 @@ debug && println("LQ decomposition") debug && println("Matmul with LQ factorizations") lqa = lqfact(a[:,1:n1]) l,q = lqa[:L], lqa[:Q] - @test full(q)*full(q)' ≈ eye(eltya,n1) - @test (full(q,thin=false)'*full(q,thin=false))[1:n1,:] ≈ eye(eltya,n1,n) + @test convert(Array, q)*convert(Array, q)' ≈ eye(eltya,n1) + @test (Base.LinAlg.thickQ(q)'*Base.LinAlg.thickQ(q))[1:n1,:] ≈ eye(eltya,n1,n) @test_throws DimensionMismatch A_mul_B!(eye(eltya,n+1),q) - @test Ac_mul_B!(q,full(q)) ≈ eye(eltya,n1) + @test Ac_mul_B!(q,convert(Array, q)) ≈ eye(eltya,n1) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) end diff --git a/test/linalg/lu.jl b/test/linalg/lu.jl index ca5831029d204..a0e291c0d4c9d 100644 --- a/test/linalg/lu.jl +++ b/test/linalg/lu.jl @@ -36,7 +36,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) if eltya <: BlasFloat num = rand(eltya) @test lu(num) == (one(eltya),num,1) - @test full(lufact(num)) ≈ eltya[num] + @test convert(Array, lufact(num)) ≈ eltya[num] end for eltyb in (Float32, Float64, Complex64, Complex128, Int) b = eltyb == Int ? rand(1:5, n, 2) : convert(Matrix{eltyb}, eltyb <: Complex ? complex(breal, bimg) : breal) @@ -68,7 +68,7 @@ debug && println("(Automatic) Square LU decomposition") @test norm(a'*(lua'\a') - a', 1) < ε*κ*n^2 @test norm(a*(lua\c) - c, 1) < ε*κ*n # c is a vector @test norm(a'*(lua'\c) - c, 1) < ε*κ*n # c is a vector - @test full(lua) ≈ a + @test convert(Array, lua) ≈ a if eltya <: Real && eltyb <: Real @test norm(a.'*(lua.'\b) - b,1) < ε*κ*n*2 # Two because the right hand side has two columns @test norm(a.'*(lua.'\c) - c,1) < ε*κ*n @@ -81,13 +81,13 @@ debug && println("(Automatic) Square LU decomposition") end debug && println("Tridiagonal LU") - κd = cond(full(d),1) + κd = cond(convert(Array, d),1) lud = lufact(d) @test lufact(lud) == lud @test_throws KeyError lud[:Z] - @test lud[:L]*lud[:U] ≈ lud[:P]*full(d) - @test lud[:L]*lud[:U] ≈ full(d)[lud[:p],:] - @test full(lud) ≈ d + @test lud[:L]*lud[:U] ≈ lud[:P]*convert(Array, d) + @test lud[:L]*lud[:U] ≈ convert(Array, d)[lud[:p],:] + @test convert(Array, lud) ≈ d f = zeros(eltyb, n+1) @test_throws DimensionMismatch lud\f @test_throws DimensionMismatch lud.'\f @@ -102,17 +102,17 @@ debug && println("Tridiagonal LU") @test norm(d*(lud\b) - b, 1) < ε*κd*n*2 # Two because the right hand side has two columns if eltya <: Real - @test norm((lud.'\b) - full(d.')\b, 1) < ε*κd*n*2 # Two because the right hand side has two columns + @test norm((lud.'\b) - convert(Array, d.')\b, 1) < ε*κd*n*2 # Two because the right hand side has two columns end if eltya <: Complex - @test norm((lud'\b) - full(d')\b, 1) < ε*κd*n*2 # Two because the right hand side has two columns + @test norm((lud'\b) - convert(Array, d')\b, 1) < ε*κd*n*2 # Two because the right hand side has two columns end end end if eltya <: BlasFloat && eltyb <: BlasFloat e = rand(eltyb,n,n) @test norm(e/lud - e/d,1) < ε*κ*n^2 - @test norm((lud.'\e') - full(d.')\e',1) < ε*κd*n^2 + @test norm((lud.'\e') - convert(Array, d.')\e',1) < ε*κd*n^2 #test singular du = rand(eltya,n-1) dl = rand(eltya,n-1) @@ -136,11 +136,11 @@ end # test conversion routine a = Tridiagonal(rand(9),rand(10),rand(9)) -fa = full(a) +fa = convert(Array, a) falu = lufact(fa) alu = lufact(a) falu = convert(typeof(falu),alu) -@test full(alu) == fa +@test convert(Array, alu) == fa # Test rational matrices ## Integrate in general tests when more linear algebra is implemented in julia diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 44bb9a6afa85b..b8f72bca058c4 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -336,7 +336,7 @@ A = [RootInt(3) RootInt(5)] function test_mul(C, A, B) A_mul_B!(C, A, B) - @test full(A) * full(B) ≈ C + @test convert(Array, A) * convert(Array, B) ≈ C @test A*B ≈ C end diff --git a/test/linalg/qr.jl b/test/linalg/qr.jl index 8df1097b926d6..78e64b7744acb 100644 --- a/test/linalg/qr.jl +++ b/test/linalg/qr.jl @@ -46,15 +46,15 @@ debug && println("QR decomposition (without pivoting)") @inferred qr(a) q, r = qra[:Q], qra[:R] @test_throws KeyError qra[:Z] - @test q'*full(q, thin=false) ≈ eye(n) - @test q*full(q, thin=false)' ≈ eye(n) - @test q'*eye(n)' ≈ full(q, thin=false)' - @test full(q, thin=false)'q ≈ eye(n) - @test eye(n)'q' ≈ full(q, thin=false)' + @test q'*Base.LinAlg.thickQ(q) ≈ eye(n) + @test q*Base.LinAlg.thickQ(q)' ≈ eye(n) + @test q'*eye(n)' ≈ Base.LinAlg.thickQ(q)' + @test Base.LinAlg.thickQ(q)'q ≈ eye(n) + @test eye(n)'q' ≈ Base.LinAlg.thickQ(q)' @test q*r ≈ a @test_approx_eq_eps a*(qra\b) b 3000ε - @test full(qra) ≈ a - @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*full(q, thin=false) eye(n) 5000ε + @test convert(Array, qra) ≈ a + @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*Base.LinAlg.thickQ(q) eye(n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) ac = copy(a) @@ -66,14 +66,14 @@ debug && println("Thin QR decomposition (without pivoting)") @inferred qr(a[:,1:n1], Val{false}) q,r = qra[:Q], qra[:R] @test_throws KeyError qra[:Z] - @test q'*full(q, thin=false) ≈ eye(n) - @test q'*full(q) ≈ eye(n,n1) + @test q'*Base.LinAlg.thickQ(q) ≈ eye(n) + @test q'*convert(Array, q) ≈ eye(n,n1) @test q*r ≈ a[:,1:n1] - @test_approx_eq_eps q*b[1:n1] full(q)*b[1:n1] 100ε - @test_approx_eq_eps q*b full(q, thin=false)*b 100ε + @test_approx_eq_eps q*b[1:n1] convert(Array, q)*b[1:n1] 100ε + @test_approx_eq_eps q*b Base.LinAlg.thickQ(q)*b 100ε @test_throws DimensionMismatch q*b[1:n1 + 1] @test_throws DimensionMismatch b[1:n1 + 1]*q' - @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*full(q, thin=false) eye(n1,n) 5000ε + @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*Base.LinAlg.thickQ(q) eye(n1,n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) end @@ -86,14 +86,14 @@ debug && println("(Automatic) Fat (pivoted) QR decomposition") q,r = qrpa[:Q], qrpa[:R] @test_throws KeyError qrpa[:Z] p = qrpa[:p] - @test q'*full(q, thin=false) ≈ eye(n1) - @test q*full(q, thin=false)' ≈ eye(n1) - @test (UpperTriangular(eye(eltya,size(q,2)))*q')*full(q, thin=false) ≈ eye(n1) + @test q'*Base.LinAlg.thickQ(q) ≈ eye(n1) + @test q*Base.LinAlg.thickQ(q)' ≈ eye(n1) + @test (UpperTriangular(eye(eltya,size(q,2)))*q')*Base.LinAlg.thickQ(q) ≈ eye(n1) @test q*r ≈ (isa(qrpa,QRPivoted) ? a[1:n1,p] : a[1:n1,:]) @test q*r[:,invperm(p)] ≈ a[1:n1,:] @test q*r*qrpa[:P].' ≈ a[1:n1,:] @test_approx_eq_eps a[1:n1,:]*(qrpa\b[1:n1]) b[1:n1] 5000ε - @test full(qrpa) ≈ a[1:5,:] + @test convert(Array, qrpa) ≈ a[1:5,:] @test_throws DimensionMismatch q*b[1:n1+1] @test_throws DimensionMismatch b[1:n1+1]*q' if eltya != Int @@ -105,14 +105,14 @@ debug && println("(Automatic) Thin (pivoted) QR decomposition") q,r = qrpa[:Q], qrpa[:R] @test_throws KeyError qrpa[:Z] p = qrpa[:p] - @test q'*full(q, thin=false) ≈ eye(n) - @test q*full(q, thin=false)' ≈ eye(n) + @test q'*Base.LinAlg.thickQ(q) ≈ eye(n) + @test q*Base.LinAlg.thickQ(q)' ≈ eye(n) @test q*r ≈ a[:,p] @test q*r[:,invperm(p)] ≈ a[:,1:n1] - @test full(qrpa) ≈ a[:,1:5] + @test convert(Array, qrpa) ≈ a[:,1:5] @test_throws DimensionMismatch q*b[1:n1+1] @test_throws DimensionMismatch b[1:n1+1]*q' - @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*full(q, thin=false) eye(n1,n) 5000ε + @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*Base.LinAlg.thickQ(q) eye(n1,n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) end @@ -123,9 +123,9 @@ debug && println("Matmul with QR factorizations") if eltya != Int qrpa = factorize(a[:,1:n1]) q, r = qrpa[:Q], qrpa[:R] - @test A_mul_B!(full(q, thin=false)',q) ≈ eye(n) + @test A_mul_B!(Base.LinAlg.thickQ(q)',q) ≈ eye(n) @test_throws DimensionMismatch A_mul_B!(eye(eltya,n+1),q) - @test A_mul_Bc!(full(q, thin=false),q) ≈ eye(n) + @test A_mul_Bc!(Base.LinAlg.thickQ(q),q) ≈ eye(n) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) @test_throws DimensionMismatch Base.LinAlg.A_mul_B!(q,zeros(eltya,n1+1)) @@ -133,9 +133,9 @@ debug && println("Matmul with QR factorizations") qra = qrfact(a[:,1:n1], Val{false}) q, r = qra[:Q], qra[:R] - @test A_mul_B!(full(q, thin=false)',q) ≈ eye(n) + @test A_mul_B!(Base.LinAlg.thickQ(q)',q) ≈ eye(n) @test_throws DimensionMismatch A_mul_B!(eye(eltya,n+1),q) - @test A_mul_Bc!(full(q, thin=false),q) ≈ eye(n) + @test A_mul_Bc!(Base.LinAlg.thickQ(q),q) ≈ eye(n) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) @test_throws DimensionMismatch q * eye(Int8,n+4) @@ -154,7 +154,7 @@ end # Issue 7304 let A = [-√.5 -√.5; -√.5 √.5] - Q = full(qrfact(A)[:Q]) + Q = convert(Array, qrfact(A)[:Q]) @test vecnorm(A-Q) < eps() end diff --git a/test/linalg/schur.jl b/test/linalg/schur.jl index 5370afd3f82f5..2f195f087808a 100644 --- a/test/linalg/schur.jl +++ b/test/linalg/schur.jl @@ -38,7 +38,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) @test sort(real(f[:values])) ≈ sort(real(d)) @test sort(imag(f[:values])) ≈ sort(imag(d)) @test istriu(f[:Schur]) || eltype(a)<:Real - @test full(f) ≈ a + @test convert(Array, f) ≈ a @test_throws KeyError f[:A] debug && println("Reorder Schur") diff --git a/test/linalg/special.jl b/test/linalg/special.jl index 89ddde5ee4b00..2ccaf724c9dd5 100644 --- a/test/linalg/special.jl +++ b/test/linalg/special.jl @@ -11,11 +11,11 @@ let a=[1.0:n;] A=Diagonal(a) for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Tridiagonal, LowerTriangular, UpperTriangular, Matrix] debug && println("newtype is $(newtype)") - @test full(convert(newtype, A)) == full(A) + @test convert(Array, convert(newtype, A)) == convert(Array, A) end for newtype in [Base.LinAlg.UnitUpperTriangular, Base.LinAlg.UnitLowerTriangular] @test_throws ArgumentError convert(newtype, A) - @test full(convert(newtype, Diagonal(ones(n)))) == eye(n) + @test convert(Array, convert(newtype, Diagonal(ones(n)))) == eye(n) end for isupper in (true, false) @@ -23,56 +23,56 @@ let a=[1.0:n;] A=Bidiagonal(a, [1.0:n-1;], isupper) for newtype in [Bidiagonal, Tridiagonal, isupper ? UpperTriangular : LowerTriangular, Matrix] debug && println("newtype is $(newtype)") - @test full(convert(newtype, A)) == full(A) - @test full(newtype(A)) == full(A) + @test convert(Array, convert(newtype, A)) == convert(Array, A) + @test convert(Array, newtype(A)) == convert(Array, A) end @test_throws ArgumentError convert(SymTridiagonal, A) A=Bidiagonal(a, zeros(n-1), isupper) #morally Diagonal for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Tridiagonal, isupper ? UpperTriangular : LowerTriangular, Matrix] debug && println("newtype is $(newtype)") - @test full(convert(newtype, A)) == full(A) - @test full(newtype(A)) == full(A) + @test convert(Array, convert(newtype, A)) == convert(Array, A) + @test convert(Array, newtype(A)) == convert(Array, A) end end A = SymTridiagonal(a, [1.0:n-1;]) for newtype in [Tridiagonal, Matrix] - @test full(convert(newtype, A)) == full(A) + @test convert(Array, convert(newtype, A)) == convert(Array, A) end for newtype in [Diagonal, Bidiagonal] @test_throws ArgumentError convert(newtype,A) end A = SymTridiagonal(a, zeros(n-1)) - @test full(convert(Bidiagonal,A)) == full(A) + @test convert(Array, convert(Bidiagonal,A)) == convert(Array, A) A = Tridiagonal(zeros(n-1), [1.0:n;], zeros(n-1)) #morally Diagonal for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Matrix] - @test full(convert(newtype, A)) == full(A) + @test convert(Array, convert(newtype, A)) == convert(Array, A) end A = Tridiagonal(ones(n-1), [1.0:n;], ones(n-1)) #not morally Diagonal for newtype in [SymTridiagonal, Matrix] - @test full(convert(newtype, A)) == full(A) + @test convert(Array, convert(newtype, A)) == convert(Array, A) end for newtype in [Diagonal, Bidiagonal] @test_throws ArgumentError convert(newtype,A) end A = Tridiagonal(zeros(n-1), [1.0:n;], ones(n-1)) #not morally Diagonal - @test full(convert(Bidiagonal, A)) == full(A) + @test convert(Array, convert(Bidiagonal, A)) == convert(Array, A) A = UpperTriangular(Tridiagonal(zeros(n-1), [1.0:n;], ones(n-1))) - @test full(convert(Bidiagonal, A)) == full(A) + @test convert(Array, convert(Bidiagonal, A)) == convert(Array, A) A = Tridiagonal(ones(n-1), [1.0:n;], zeros(n-1)) #not morally Diagonal - @test full(convert(Bidiagonal, A)) == full(A) + @test convert(Array, convert(Bidiagonal, A)) == convert(Array, A) A = LowerTriangular(Tridiagonal(ones(n-1), [1.0:n;], zeros(n-1))) - @test full(convert(Bidiagonal, A)) == full(A) + @test convert(Array, convert(Bidiagonal, A)) == convert(Array, A) @test_throws ArgumentError convert(SymTridiagonal,A) - A = LowerTriangular(full(Diagonal(a))) #morally Diagonal + A = LowerTriangular(convert(Array, Diagonal(a))) #morally Diagonal for newtype in [Diagonal, Bidiagonal, SymTridiagonal, LowerTriangular, Matrix] - @test full(convert(newtype, A)) == full(A) + @test convert(Array, convert(newtype, A)) == convert(Array, A) end - A = UpperTriangular(full(Diagonal(a))) #morally Diagonal + A = UpperTriangular(convert(Array, Diagonal(a))) #morally Diagonal for newtype in [Diagonal, Bidiagonal, SymTridiagonal, UpperTriangular, Matrix] - @test full(convert(newtype, A)) == full(A) + @test convert(Array, convert(newtype, A)) == convert(Array, A) end A = UpperTriangular(triu(rand(n,n))) for newtype in [Diagonal, Bidiagonal, Tridiagonal, SymTridiagonal] @@ -80,7 +80,7 @@ let a=[1.0:n;] end A = Diagonal(a) for newtype in [UpperTriangular, LowerTriangular] - @test full(convert(newtype,A)) == full(A) + @test convert(Array, convert(newtype,A)) == convert(Array, A) end end @@ -92,26 +92,26 @@ let a=[1.0:n;] for type2 in Spectypes B = convert(type1,A) C = convert(type2,A) - @test full(B + C) ≈ full(A + A) - @test full(B - C) ≈ full(A - A) + @test convert(Array, B + C) ≈ convert(Array, A + A) + @test convert(Array, B - C) ≈ convert(Array, A - A) end end B = SymTridiagonal(a, ones(n-1)) for Spectype in [Diagonal, Bidiagonal, Tridiagonal, Matrix] - @test full(B + convert(Spectype,A)) ≈ full(B + A) - @test full(convert(Spectype,A) + B) ≈ full(B + A) - @test full(B - convert(Spectype,A)) ≈ full(B - A) - @test full(convert(Spectype,A) - B) ≈ full(A - B) + @test convert(Array, B + convert(Spectype,A)) ≈ convert(Array, B + A) + @test convert(Array, convert(Spectype,A) + B) ≈ convert(Array, B + A) + @test convert(Array, B - convert(Spectype,A)) ≈ convert(Array, B - A) + @test convert(Array, convert(Spectype,A) - B) ≈ convert(Array, A - B) end C = rand(n,n) for TriType in [Base.LinAlg.UnitLowerTriangular, Base.LinAlg.UnitUpperTriangular, UpperTriangular, LowerTriangular] D = TriType(C) for Spectype in [Diagonal, Bidiagonal, Tridiagonal, Matrix] - @test full(D + convert(Spectype,A)) ≈ full(D + A) - @test full(convert(Spectype,A) + D) ≈ full(A + D) - @test full(D - convert(Spectype,A)) ≈ full(D - A) - @test full(convert(Spectype,A) - D) ≈ full(A - D) + @test convert(Array, D + convert(Spectype,A)) ≈ convert(Array, D + A) + @test convert(Array, convert(Spectype,A) + D) ≈ convert(Array, A + D) + @test convert(Array, D - convert(Spectype,A)) ≈ convert(Array, D - A) + @test convert(Array, convert(Spectype,A) - D) ≈ convert(Array, A - D) end end end @@ -122,9 +122,9 @@ for typ in [UpperTriangular,LowerTriangular,Base.LinAlg.UnitUpperTriangular,Base atri = typ(a) b = rand(n,n) qrb = qrfact(b,Val{true}) - @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ full(atri) * qrb[:Q]' - @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ full(atri) * qrb[:Q]' + @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ convert(Array, atri) * qrb[:Q]' + @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ convert(Array, atri) * qrb[:Q]' qrb = qrfact(b,Val{false}) - @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ full(atri) * qrb[:Q]' - @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ full(atri) * qrb[:Q]' + @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ convert(Array, atri) * qrb[:Q]' + @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ convert(Array, atri) * qrb[:Q]' end diff --git a/test/linalg/svd.jl b/test/linalg/svd.jl index d98a63c1d7625..f76ca1c3eea41 100644 --- a/test/linalg/svd.jl +++ b/test/linalg/svd.jl @@ -39,7 +39,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) usv = svdfact(a) @test usv[:S] === svdvals(usv) @test usv[:U] * (Diagonal(usv[:S]) * usv[:Vt]) ≈ a - @test full(usv) ≈ a + @test convert(Array, usv) ≈ a @test usv[:Vt]' ≈ usv[:V] @test_throws KeyError usv[:Z] b = rand(eltya,n) diff --git a/test/linalg/symmetric.jl b/test/linalg/symmetric.jl index 3a33105b5372f..e7a8e12f8c96f 100644 --- a/test/linalg/symmetric.jl +++ b/test/linalg/symmetric.jl @@ -56,8 +56,8 @@ let n=10 @test isa(similar(Symmetric(asym), Int, (3,2)), Matrix{Int}) @test isa(similar(Hermitian(asym), Int, (3,2)), Matrix{Int}) - # full - @test asym == full(Hermitian(asym)) + # convert(Array, _) + @test asym == convert(Array, Hermitian(asym)) # parent @@ -111,7 +111,7 @@ let n=10 eig(Hermitian(asym), d[1] - 1, (d[2] + d[3])/2) # same result, but checks that method works @test eigvals(Hermitian(asym), 1:2) ≈ d[1:2] @test eigvals(Hermitian(asym), d[1] - 1, (d[2] + d[3])/2) ≈ d[1:2] - @test full(eigfact(asym)) ≈ asym + @test convert(Array, eigfact(asym)) ≈ asym # relation to svdvals @test sum(sort(abs(eigvals(Hermitian(asym))))) == sum(sort(svdvals(Hermitian(asym)))) @@ -213,7 +213,7 @@ end #Issue #7933 let A7933 = [1 2; 3 4] B7933 = copy(A7933) - C7933 = full(Symmetric(A7933)) + C7933 = convert(Array, Symmetric(A7933)) @test A7933 == B7933 end diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index 4c570babc80e9..e9a794d90000a 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -11,7 +11,7 @@ srand(123) debug && println("Test basic type functionality") @test_throws DimensionMismatch LowerTriangular(randn(5, 4)) -@test LowerTriangular(randn(3, 3)) |> t -> [size(t, i) for i = 1:3] == [size(full(t), i) for i = 1:3] +@test LowerTriangular(randn(3, 3)) |> t -> [size(t, i) for i = 1:3] == [size(convert(Array, t), i) for i = 1:3] # The following test block tries to call all methods in base/linalg/triangular.jl in order for a combination of input element types. Keep the ordering when adding code. for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloat}, Int) @@ -29,13 +29,13 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa # Convert @test convert(AbstractMatrix{elty1}, A1) == A1 - @test convert(Matrix, A1) == full(A1) + @test convert(Matrix, A1) == convert(Array, A1) # full! - @test full!(copy(A1)) == full(A1) + @test full!(copy(A1)) == convert(Array, A1) # fill! - @test full!(fill!(copy(A1), 1)) == full(t1(ones(size(A1)...))) + @test full!(fill!(copy(A1), 1)) == convert(Array, t1(ones(size(A1)...))) # similar @test isa(similar(A1), t1) @@ -48,14 +48,14 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa # getindex ## Linear indexing for i = 1:length(A1) - @test A1[i] == full(A1)[i] + @test A1[i] == convert(Array, A1)[i] end @test isa(A1[2:4,1], Vector) ## Cartesian indexing for i = 1:size(A1, 1) for j = 1:size(A1, 2) - @test A1[i,j] == full(A1)[i,j] + @test A1[i,j] == convert(Array, A1)[i,j] end end @@ -101,8 +101,8 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa #tril/triu if uplo1 == :L @test tril(A1,0) == A1 - @test tril(A1,-1) == LowerTriangular(tril(full(A1),-1)) - @test tril(A1,1) == t1(tril(tril(full(A1),1))) + @test tril(A1,-1) == LowerTriangular(tril(convert(Array, A1),-1)) + @test tril(A1,1) == t1(tril(tril(convert(Array, A1),1))) @test_throws ArgumentError tril!(A1,n+1) @test triu(A1,0) == t1(diagm(diag(A1))) @test triu(A1,-1) == t1(tril(triu(A1.data,-1))) @@ -110,8 +110,8 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test_throws ArgumentError triu!(A1,n+1) else @test triu(A1,0) == A1 - @test triu(A1,1) == UpperTriangular(triu(full(A1),1)) - @test triu(A1,-1) == t1(triu(triu(full(A1),-1))) + @test triu(A1,1) == UpperTriangular(triu(convert(Array, A1),1)) + @test triu(A1,-1) == t1(triu(triu(convert(Array, A1),-1))) @test_throws ArgumentError triu!(A1,n+1) @test tril(A1,0) == t1(diagm(diag(A1))) @test tril(A1,1) == t1(triu(tril(A1.data,1))) @@ -125,11 +125,11 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa # [c]transpose[!] (test views as well, see issue #14317) let vrange = 1:n-1, viewA1 = t1(view(A1.data, vrange, vrange)) # transpose - @test full(A1.') == full(A1).' - @test full(viewA1.') == full(viewA1).' + @test convert(Array, A1.') == convert(Array, A1).' + @test convert(Array, viewA1.') == convert(Array, viewA1).' # ctranspose - @test full(A1') == full(A1)' - @test full(viewA1') == full(viewA1)' + @test convert(Array, A1') == convert(Array, A1)' + @test convert(Array, viewA1') == convert(Array, viewA1)' # transpose! @test transpose!(copy(A1)) == A1.' @test transpose!(t1(view(copy(A1).data, vrange, vrange))) == viewA1.' @@ -139,21 +139,21 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa end # diag - @test diag(A1) == diag(full(A1)) + @test diag(A1) == diag(convert(Array, A1)) # real - @test full(real(A1)) == real(full(A1)) - @test full(imag(A1)) == imag(full(A1)) - @test full(abs(A1)) == abs(full(A1)) + @test convert(Array, real(A1)) == real(convert(Array, A1)) + @test convert(Array, imag(A1)) == imag(convert(Array, A1)) + @test convert(Array, abs(A1)) == abs(convert(Array, A1)) # Unary operations - @test full(-A1) == -full(A1) + @test convert(Array, -A1) == -convert(Array, A1) # copy and copy! (test views as well, see issue #14317) let vrange = 1:n-1, viewA1 = t1(view(A1.data, vrange, vrange)) # copy - @test copy(A1) == copy(full(A1)) - @test copy(viewA1) == copy(full(viewA1)) + @test copy(A1) == copy(convert(Array, A1)) + @test copy(viewA1) == copy(convert(Array, viewA1)) # copy! B = similar(A1) copy!(B, A1) @@ -171,7 +171,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa #expm/logm if (elty1 == Float64 || elty1 == Complex128) && (t1 == UpperTriangular || t1 == LowerTriangular) - @test expm(full(logm(A1))) ≈ full(A1) + @test expm(convert(Array, logm(A1))) ≈ convert(Array, A1) end # scale @@ -209,23 +209,23 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa end # Binary operations - @test A1*0.5 == full(A1)*0.5 - @test 0.5*A1 == 0.5*full(A1) - @test A1/0.5 == full(A1)/0.5 - @test 0.5\A1 == 0.5\full(A1) + @test A1*0.5 == convert(Array, A1)*0.5 + @test 0.5*A1 == 0.5*convert(Array, A1) + @test A1/0.5 == convert(Array, A1)/0.5 + @test 0.5\A1 == 0.5\convert(Array, A1) # inversion - @test inv(A1) ≈ inv(lufact(full(A1))) - inv(full(A1)) # issue #11298 + @test inv(A1) ≈ inv(lufact(convert(Array, A1))) + inv(convert(Array, A1)) # issue #11298 @test isa(inv(A1), t1) # make sure the call to LAPACK works right if elty1 <: BlasFloat - @test Base.LinAlg.inv!(copy(A1)) ≈ inv(lufact(full(A1))) + @test Base.LinAlg.inv!(copy(A1)) ≈ inv(lufact(convert(Array, A1))) end # Determinant - @test_approx_eq_eps det(A1) det(lufact(full(A1))) sqrt(eps(real(float(one(elty1)))))*n*n - @test_approx_eq_eps logdet(A1) logdet(lufact(full(A1))) sqrt(eps(real(float(one(elty1)))))*n*n + @test_approx_eq_eps det(A1) det(lufact(convert(Array, A1))) sqrt(eps(real(float(one(elty1)))))*n*n + @test_approx_eq_eps logdet(A1) logdet(lufact(convert(Array, A1))) sqrt(eps(real(float(one(elty1)))))*n*n # Matrix square root @test sqrtm(A1) |> t -> t*t ≈ A1 @@ -237,7 +237,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa if !(elty1 in (BigFloat, Complex{BigFloat})) # Not handled yet vals, vecs = eig(A1) if (t1 == UpperTriangular || t1 == LowerTriangular) && elty1 != Int # Cannot really handle degenerate eigen space and Int matrices will probably have repeated eigenvalues. - @test_approx_eq_eps vecs*diagm(vals)/vecs full(A1) sqrt(eps(float(real(one(vals[1])))))*(norm(A1, Inf)*n)^2 + @test_approx_eq_eps vecs*diagm(vals)/vecs convert(Array, A1) sqrt(eps(float(real(one(vals[1])))))*(norm(A1, Inf)*n)^2 end end @@ -246,7 +246,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa for p in (1.0, Inf) @test_approx_eq_eps cond(A1, p) cond(A1, p) (cond(A1, p) + cond(A1, p)) end - @test cond(A1,2) == cond(full(A1),2) + @test cond(A1,2) == cond(convert(Array, A1),2) end if !(elty1 in (BigFloat, Complex{BigFloat})) # Not implemented yet @@ -275,19 +275,19 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa end # Binary operations - @test full(A1 + A2) == full(A1) + full(A2) - @test full(A1 - A2) == full(A1) - full(A2) + @test convert(Array, A1 + A2) == convert(Array, A1) + convert(Array, A2) + @test convert(Array, A1 - A2) == convert(Array, A1) - convert(Array, A2) # Triangular-Triangualar multiplication and division - @test full(A1*A2) ≈ full(A1)*full(A2) - @test full(A1.'A2) ≈ full(A1).'full(A2) - @test full(A1'A2) ≈ full(A1)'full(A2) - @test full(A1*A2.') ≈ full(A1)*full(A2).' - @test full(A1*A2') ≈ full(A1)*full(A2)' - @test full(A1.'A2.') ≈ full(A1).'full(A2).' - @test full(A1'A2') ≈ full(A1)'full(A2)' - @test full(A1/A2) ≈ full(A1)/full(A2) - @test full(A1\A2) ≈ full(A1)\full(A2) + @test convert(Array, A1*A2) ≈ convert(Array, A1)*convert(Array, A2) + @test convert(Array, A1.'A2) ≈ convert(Array, A1).'convert(Array, A2) + @test convert(Array, A1'A2) ≈ convert(Array, A1)'convert(Array, A2) + @test convert(Array, A1*A2.') ≈ convert(Array, A1)*convert(Array, A2).' + @test convert(Array, A1*A2') ≈ convert(Array, A1)*convert(Array, A2)' + @test convert(Array, A1.'A2.') ≈ convert(Array, A1).'convert(Array, A2).' + @test convert(Array, A1'A2') ≈ convert(Array, A1)'convert(Array, A2)' + @test convert(Array, A1/A2) ≈ convert(Array, A1)/convert(Array, A2) + @test convert(Array, A1\A2) ≈ convert(Array, A1)\convert(Array, A2) @test_throws DimensionMismatch eye(n+1)/A2 @test_throws DimensionMismatch eye(n+1)/A2.' @test_throws DimensionMismatch eye(n+1)/A2' @@ -308,29 +308,29 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa if !(eltyB in (BigFloat, Complex{BigFloat})) # rand does not support BigFloat and Complex{BigFloat} as of Dec 2015 Tri = Tridiagonal(rand(eltyB,n-1),rand(eltyB,n),rand(eltyB,n-1)) - @test Base.LinAlg.A_mul_B!(Tri,copy(A1)) ≈ Tri*full(A1) + @test Base.LinAlg.A_mul_B!(Tri,copy(A1)) ≈ Tri*convert(Array, A1) end # Triangular-dense Matrix/vector multiplication - @test A1*B[:,1] ≈ full(A1)*B[:,1] - @test A1*B ≈ full(A1)*B - @test A1.'B[:,1] ≈ full(A1).'B[:,1] - @test A1'B[:,1] ≈ full(A1)'B[:,1] - @test A1.'B ≈ full(A1).'B - @test A1'B ≈ full(A1)'B - @test A1*B.' ≈ full(A1)*B.' - @test A1*B' ≈ full(A1)*B' - @test B*A1 ≈ B*full(A1) - @test B[:,1].'A1 ≈ B[:,1].'full(A1) - @test B[:,1]'A1 ≈ B[:,1]'full(A1) - @test B.'A1 ≈ B.'full(A1) - @test B'A1 ≈ B'full(A1) - @test B*A1.' ≈ B*full(A1).' - @test B*A1' ≈ B*full(A1)' - @test B[:,1].'A1.' ≈ B[:,1].'full(A1).' - @test B[:,1]'A1' ≈ B[:,1]'full(A1)' - @test B.'A1.' ≈ B.'full(A1).' - @test B'A1' ≈ B'full(A1)' + @test A1*B[:,1] ≈ convert(Array, A1)*B[:,1] + @test A1*B ≈ convert(Array, A1)*B + @test A1.'B[:,1] ≈ convert(Array, A1).'B[:,1] + @test A1'B[:,1] ≈ convert(Array, A1)'B[:,1] + @test A1.'B ≈ convert(Array, A1).'B + @test A1'B ≈ convert(Array, A1)'B + @test A1*B.' ≈ convert(Array, A1)*B.' + @test A1*B' ≈ convert(Array, A1)*B' + @test B*A1 ≈ B*convert(Array, A1) + @test B[:,1].'A1 ≈ B[:,1].'convert(Array, A1) + @test B[:,1]'A1 ≈ B[:,1]'convert(Array, A1) + @test B.'A1 ≈ B.'convert(Array, A1) + @test B'A1 ≈ B'convert(Array, A1) + @test B*A1.' ≈ B*convert(Array, A1).' + @test B*A1' ≈ B*convert(Array, A1)' + @test B[:,1].'A1.' ≈ B[:,1].'convert(Array, A1).' + @test B[:,1]'A1' ≈ B[:,1]'convert(Array, A1)' + @test B.'A1.' ≈ B.'convert(Array, A1).' + @test B'A1' ≈ B'convert(Array, A1)' if eltyB == elty1 @test A_mul_B!(zeros(B),A1,B) ≈ A1*B @@ -346,29 +346,29 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test_throws DimensionMismatch Base.LinAlg.A_mul_Bt!(ones(eltyB,n+1,n+1),A1) # ... and division - @test A1\B[:,1] ≈ full(A1)\B[:,1] - @test A1\B ≈ full(A1)\B - @test A1.'\B[:,1] ≈ full(A1).'\B[:,1] - @test A1'\B[:,1] ≈ full(A1)'\B[:,1] - @test A1.'\B ≈ full(A1).'\B - @test A1'\B ≈ full(A1)'\B - @test A1\B.' ≈ full(A1)\B.' - @test A1\B' ≈ full(A1)\B' - @test A1.'\B.' ≈ full(A1).'\B.' - @test A1'\B' ≈ full(A1)'\B' + @test A1\B[:,1] ≈ convert(Array, A1)\B[:,1] + @test A1\B ≈ convert(Array, A1)\B + @test A1.'\B[:,1] ≈ convert(Array, A1).'\B[:,1] + @test A1'\B[:,1] ≈ convert(Array, A1)'\B[:,1] + @test A1.'\B ≈ convert(Array, A1).'\B + @test A1'\B ≈ convert(Array, A1)'\B + @test A1\B.' ≈ convert(Array, A1)\B.' + @test A1\B' ≈ convert(Array, A1)\B' + @test A1.'\B.' ≈ convert(Array, A1).'\B.' + @test A1'\B' ≈ convert(Array, A1)'\B' @test_throws DimensionMismatch A1\ones(elty1,n+2) @test_throws DimensionMismatch A1'\ones(elty1,n+2) @test_throws DimensionMismatch A1.'\ones(elty1,n+2) if t1 == UpperTriangular || t1 == LowerTriangular @test_throws Base.LinAlg.SingularException naivesub!(t1(zeros(elty1,n,n)),ones(eltyB,n)) end - @test B/A1 ≈ B/full(A1) - @test B/A1.' ≈ B/full(A1).' - @test B/A1' ≈ B/full(A1)' - @test B.'/A1 ≈ B.'/full(A1) - @test B'/A1 ≈ B'/full(A1) - @test B.'/A1.' ≈ B.'/full(A1).' - @test B'/A1' ≈ B'/full(A1)' + @test B/A1 ≈ B/convert(Array, A1) + @test B/A1.' ≈ B/convert(Array, A1).' + @test B/A1' ≈ B/convert(Array, A1)' + @test B.'/A1 ≈ B.'/convert(Array, A1) + @test B'/A1 ≈ B'/convert(Array, A1) + @test B.'/A1.' ≈ B.'/convert(Array, A1).' + @test B'/A1' ≈ B'/convert(Array, A1)' # Error bounds !(elty1 in (BigFloat, Complex{BigFloat})) && !(eltyB in (BigFloat, Complex{BigFloat})) && errorbounds(A1, A1\B, B) @@ -403,8 +403,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) debug && println("Solve upper triangular system") Atri = UpperTriangular(lufact(A)[:U]) |> t -> eltya <: Complex && eltyb <: Real ? real(t) : t # Here the triangular matrix can't be too badly conditioned - b = convert(Matrix{eltyb}, eltya <: Complex ? full(Atri)*ones(n, 2) : full(Atri)*ones(n, 2)) - x = full(Atri) \ b + b = convert(Matrix{eltyb}, eltya <: Complex ? convert(Array, Atri)*ones(n, 2) : convert(Array, Atri)*ones(n, 2)) + x = convert(Array, Atri) \ b debug && println("Test error estimates") if eltya != BigFloat && eltyb != BigFloat @@ -431,8 +431,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) debug && println("Solve lower triangular system") Atri = UpperTriangular(lufact(A)[:U]) |> t -> eltya <: Complex && eltyb <: Real ? real(t) : t # Here the triangular matrix can't be too badly conditioned - b = convert(Matrix{eltyb}, eltya <: Complex ? full(Atri)*ones(n, 2) : full(Atri)*ones(n, 2)) - x = full(Atri)\b + b = convert(Matrix{eltyb}, eltya <: Complex ? convert(Array, Atri)*ones(n, 2) : convert(Array, Atri)*ones(n, 2)) + x = convert(Array, Atri)\b debug && println("Test error estimates") if eltya != BigFloat && eltyb != BigFloat diff --git a/test/linalg/tridiag.jl b/test/linalg/tridiag.jl index 53fb78a416fb4..7436882ff2fbe 100644 --- a/test/linalg/tridiag.jl +++ b/test/linalg/tridiag.jl @@ -42,7 +42,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) F[i,i+1] = du[i] F[i+1,i] = dl[i] end - @test full(T) == F + @test convert(Array, T) == F # elementary operations on tridiagonals @test conj(T) == Tridiagonal(conj(dl), conj(d), conj(du)) @@ -62,7 +62,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) @test Tridiagonal(dl, d, du) + Tridiagonal(du, d, dl) == SymTridiagonal(2d, dl+du) @test SymTridiagonal(d, dl) + Tridiagonal(dl, d, du) == Tridiagonal(dl + dl, d+d, dl+du) @test convert(SymTridiagonal,Tridiagonal(Ts)) == Ts - @test full(convert(SymTridiagonal{Complex64},Tridiagonal(Ts))) == convert(Matrix{Complex64},full(Ts)) + @test convert(Array, convert(SymTridiagonal{Complex64},Tridiagonal(Ts))) == convert(Matrix{Complex64},convert(Array, Ts)) if elty == Int vv = rand(1:100, n) BB = rand(1:100, n, 2) @@ -101,7 +101,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) # symmetric tridiagonal if elty <: Real Ts = SymTridiagonal(d, dl) - Fs = full(Ts) + Fs = convert(Array, Ts) Tldlt = factorize(Ts) @test_throws DimensionMismatch Tldlt\rand(elty,n+1) @test size(Tldlt) == size(Ts) @@ -119,7 +119,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) invFsv = Fs\vv x = Ts\vv @test x ≈ invFsv - @test full(full(Tldlt)) ≈ Fs + @test convert(Array, convert(Array, Tldlt)) ≈ Fs end # similar @@ -237,7 +237,7 @@ let n = 12 #Size of matrix problem to test @test_throws ArgumentError SymTridiagonal(rand(n,n)) A = SymTridiagonal(a, b) - fA = map(elty <: Complex ? Complex128 : Float64, full(A)) + fA = map(elty <: Complex ? Complex128 : Float64, convert(Array, A)) debug && println("getindex") @test_throws BoundsError A[n+1,1] @@ -289,10 +289,10 @@ let n = 12 #Size of matrix problem to test @test B - A == A - B debug && println("Multiplication with strided vector") - @test A*ones(n) ≈ full(A)*ones(n) + @test A*ones(n) ≈ convert(Array, A)*ones(n) debug && println("Multiplication with strided matrix") - @test A*ones(n, 2) ≈ full(A)*ones(n, 2) + @test A*ones(n, 2) ≈ convert(Array, A)*ones(n, 2) debug && println("Eigensystems") if elty <: Real @@ -312,13 +312,13 @@ let n = 12 #Size of matrix problem to test debug && println("stegr! call with index range") F = eigfact(SymTridiagonal(a, b),1:2) - fF = eigfact(Symmetric(full(SymTridiagonal(a, b))),1:2) + fF = eigfact(Symmetric(convert(Array, SymTridiagonal(a, b))),1:2) Test.test_approx_eq_modphase(F[:vectors], fF[:vectors]) @test F[:values] ≈ fF[:values] debug && println("stegr! call with value range") F = eigfact(SymTridiagonal(a, b),0.0,1.0) - fF = eigfact(Symmetric(full(SymTridiagonal(a, b))),0.0,1.0) + fF = eigfact(Symmetric(convert(Array, SymTridiagonal(a, b))),0.0,1.0) Test.test_approx_eq_modphase(F[:vectors], fF[:vectors]) @test F[:values] ≈ fF[:values] end @@ -332,15 +332,15 @@ let n = 12 #Size of matrix problem to test end B = SymTridiagonal(a, b) - fB = map(elty <: Complex ? Complex128 : Float64, full(B)) + fB = map(elty <: Complex ? Complex128 : Float64, convert(Array, B)) for op in (+, -, *) - @test full(op(A, B)) ≈ op(fA, fB) + @test convert(Array, op(A, B)) ≈ op(fA, fB) end α = rand(elty) - @test full(α*A) ≈ α*full(A) - @test full(A*α) ≈ full(A)*α - @test full(A/α) ≈ full(A)/α + @test convert(Array, α*A) ≈ α*convert(Array, A) + @test convert(Array, A*α) ≈ convert(Array, A)*α + @test convert(Array, A/α) ≈ convert(Array, A)/α debug && println("A_mul_B!") @test_throws DimensionMismatch A_mul_B!(zeros(elty,n,n),B,ones(elty,n+1,n)) @@ -363,7 +363,7 @@ let n = 12 #Size of matrix problem to test @test_throws ArgumentError Tridiagonal(a,a,a) A = Tridiagonal(a, b, c) - fA = map(elty <: Complex ? Complex128 : Float64, full(A)) + fA = map(elty <: Complex ? Complex128 : Float64, convert(Array, A)) debug && println("Similar, size, and copy!") B = similar(A) @@ -411,22 +411,21 @@ let n = 12 #Size of matrix problem to test end debug && println("Multiplication with strided vector") - @test A*ones(n) ≈ full(A)*ones(n) + @test A*ones(n) ≈ convert(Array, A)*ones(n) debug && println("Multiplication with strided matrix") - @test A*ones(n, 2) ≈ full(A)*ones(n, 2) - + @test A*ones(n, 2) ≈ convert(Array, A)*ones(n, 2) B = Tridiagonal(a, b, c) - fB = map(elty <: Complex ? Complex128 : Float64, full(B)) + fB = map(elty <: Complex ? Complex128 : Float64, convert(Array, B)) for op in (+, -, *) - @test full(op(A, B)) ≈ op(fA, fB) + @test convert(Array, op(A, B)) ≈ op(fA, fB) end α = rand(elty) - @test full(α*A) ≈ α*full(A) - @test full(A*α) ≈ full(A)*α - @test full(A/α) ≈ full(A)/α + @test convert(Array, α*A) ≈ α*convert(Array, A) + @test convert(Array, A*α) ≈ convert(Array, A)*α + @test convert(Array, A/α) ≈ convert(Array, A)/α @test_throws ArgumentError convert(SymTridiagonal{elty},A) diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index 4d1c3e20d0ad8..a07a38056489a 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -76,10 +76,10 @@ let AA = randn(2, 2) else T = LowerTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == full(T) + J - @test J + T == J + full(T) - @test T - J == full(T) - J - @test J - T == J - full(T) + @test T + J == convert(Array, T) + J + @test J + T == J + convert(Array, T) + @test T - J == convert(Array, T) - J + @test J - T == J - convert(Array, T) @test T\I == inv(T) if atype == "Array" @@ -87,10 +87,10 @@ let AA = randn(2, 2) else T = LinAlg.UnitLowerTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == full(T) + J - @test J + T == J + full(T) - @test T - J == full(T) - J - @test J - T == J - full(T) + @test T + J == convert(Array, T) + J + @test J + T == J + convert(Array, T) + @test T - J == convert(Array, T) - J + @test J - T == J - convert(Array, T) @test T\I == inv(T) if atype == "Array" @@ -98,10 +98,10 @@ let AA = randn(2, 2) else T = UpperTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == full(T) + J - @test J + T == J + full(T) - @test T - J == full(T) - J - @test J - T == J - full(T) + @test T + J == convert(Array, T) + J + @test J + T == J + convert(Array, T) + @test T - J == convert(Array, T) - J + @test J - T == J - convert(Array, T) @test T\I == inv(T) if atype == "Array" @@ -109,10 +109,10 @@ let AA = randn(2, 2) else T = LinAlg.UnitUpperTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == full(T) + J - @test J + T == J + full(T) - @test T - J == full(T) - J - @test J - T == J - full(T) + @test T + J == convert(Array, T) + J + @test J + T == J + convert(Array, T) + @test T - J == convert(Array, T) - J + @test J - T == J - convert(Array, T) @test T\I == inv(T) @test I\A == A From 930d29237bfb186e11dc0ae1079ab18e0142aab5 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 23 Jun 2016 14:28:23 -0700 Subject: [PATCH 0470/1117] Migrate full(X) to convert(Array, X) in tests outside of test/sparsedir and test/linalg. Migrate `full` to `convert` in some documentation. --- doc/manual/arrays.rst | 2 +- doc/stdlib/linalg.rst | 12 ++++++------ test/hashing.jl | 2 +- test/perf/threads/stockcorr/pstockcorr.jl | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 44100b50b9094..f4244b1fad44f 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -831,7 +831,7 @@ reference. +----------------------------------------+----------------------------------+--------------------------------------------+ | :func:`speye(n) <speye>` | :func:`eye(n) <eye>` | Creates a *n*-by-*n* identity matrix. | +----------------------------------------+----------------------------------+--------------------------------------------+ -| :func:`full(S) <full>` | :func:`sparse(A) <sparse>` | Interconverts between dense | +| :func:`convert(Array, S) <convert>` | :func:`sparse(A) <sparse>` | Interconverts between dense | | | | and sparse formats. | +----------------------------------------+----------------------------------+--------------------------------------------+ | :func:`sprand(m,n,d) <sprand>` | :func:`rand(m,n) <rand>` | Creates a *m*-by-*n* random matrix (of | diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 713d6c55912b6..dc9ea8a81f0bf 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -110,7 +110,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . + Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`convert`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . **Example** @@ -125,7 +125,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Constructs an upper (``uplo='U'``\ ) or lower (``uplo='L'``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . + Constructs an upper (``uplo='U'``\ ) or lower (``uplo='L'``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`convert`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . **Example** @@ -154,13 +154,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Construct a symmetric tridiagonal matrix from the diagonal and first sub/super-diagonal, respectively. The result is of type ``SymTridiagonal`` and provides efficient specialized eigensolvers, but may be converted into a regular matrix with :func:`full`\ . + Construct a symmetric tridiagonal matrix from the diagonal and first sub/super-diagonal, respectively. The result is of type ``SymTridiagonal`` and provides efficient specialized eigensolvers, but may be converted into a regular matrix with :func:`convert`\ . .. function:: Tridiagonal(dl, d, du) .. Docstring generated from Julia source - Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . + Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`convert`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . .. function:: Symmetric(A, uplo=:U) @@ -658,7 +658,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the Hessenberg decomposition of ``A`` and return a ``Hessenberg`` object. If ``F`` is the factorization object, the unitary matrix can be accessed with ``F[:Q]`` and the Hessenberg matrix with ``F[:H]``\ . When ``Q`` is extracted, the resulting type is the ``HessenbergQ`` object, and may be converted to a regular matrix with :func:`full`\ . + Compute the Hessenberg decomposition of ``A`` and return a ``Hessenberg`` object. If ``F`` is the factorization object, the unitary matrix can be accessed with ``F[:Q]`` and the Hessenberg matrix with ``F[:H]``\ . When ``Q`` is extracted, the resulting type is the ``HessenbergQ`` object, and may be converted to a regular matrix with :func:`convert`\ . .. function:: hessfact!(A) @@ -974,7 +974,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . + Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`convert`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . .. function:: rank(M) diff --git a/test/hashing.jl b/test/hashing.jl index 1919226f41d2a..b143db1b3c387 100644 --- a/test/hashing.jl +++ b/test/hashing.jl @@ -89,7 +89,7 @@ end x = sprand(10, 10, 0.5) x[1] = 1 x.nzval[1] = 0 -@test hash(x) == hash(full(x)) +@test hash(x) == hash(convert(Array, x)) let a = QuoteNode(1), b = QuoteNode(1.0) @test (hash(a)==hash(b)) == (a==b) diff --git a/test/perf/threads/stockcorr/pstockcorr.jl b/test/perf/threads/stockcorr/pstockcorr.jl index 076feb7cc58b2..93d4bcf0864c2 100644 --- a/test/perf/threads/stockcorr/pstockcorr.jl +++ b/test/perf/threads/stockcorr/pstockcorr.jl @@ -78,7 +78,7 @@ function pstockcorr(n) SimulPriceB[1,:] = CurrentPrice[2] ## Generating the paths of stock prices by Geometric Brownian Motion - const UpperTriangle = full(chol(Corr)) # UpperTriangle Matrix by Cholesky decomposition + const UpperTriangle = convert(Array, chol(Corr)) # UpperTriangle Matrix by Cholesky decomposition # Optimization: pre-allocate these for performance # NOTE: the new GC will hopefully fix this, but currently GC time From 6765c0df10d5e53b7d7125783bbaf26c54748f0c Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 9 Jul 2016 13:06:23 -0400 Subject: [PATCH 0471/1117] Remove unnecessary GC root on the return path Fixes #15457 --- src/ccall.cpp | 6 +++++- src/codegen.cpp | 8 ++++++++ src/llvm-gcroot.cpp | 50 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index c6eb11fe434c5..4455312eafe1d 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -821,7 +821,7 @@ class FunctionMover : public ValueMaterializer // llvmcall(ir, (rettypes...), (argtypes...), args...) static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) { - JL_NARGSV(llvmcall, 3) + JL_NARGSV(llvmcall, 3); jl_value_t *rt = NULL, *at = NULL, *ir = NULL, *decl = NULL; jl_svec_t *stt = NULL; JL_GC_PUSH5(&ir, &rt, &at, &stt, &decl); @@ -1009,6 +1009,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c f->setLinkage(GlobalValue::LinkOnceODRLinkage); // the actual call + builder.CreateCall(prepare_call(gcroot_flush_func)); CallInst *inst = builder.CreateCall(f, ArrayRef<Value*>(&argvals[0], nargt)); if (isString) ctx->to_inline.push_back(inst); @@ -1440,6 +1441,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(!isVa); assert(nargt == 0); JL_GC_POP(); + builder.CreateCall(prepare_call(gcroot_flush_func)); emit_signal_fence(); builder.CreateLoad(ctx->signalPage, true); emit_signal_fence(); @@ -1471,6 +1473,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(!isVa); assert(nargt == 0); JL_GC_POP(); + builder.CreateCall(prepare_call(gcroot_flush_func)); Value *pdefer_sig = emit_defer_signal(ctx); Value *defer_sig = builder.CreateLoad(pdefer_sig); defer_sig = builder.CreateAdd(defer_sig, @@ -1486,6 +1489,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(!isVa); assert(nargt == 0); JL_GC_POP(); + builder.CreateCall(prepare_call(gcroot_flush_func)); Value *pdefer_sig = emit_defer_signal(ctx); Value *defer_sig = builder.CreateLoad(pdefer_sig); emit_signal_fence(); diff --git a/src/codegen.cpp b/src/codegen.cpp index af6c6ea34a3a4..c83e02d6e6d10 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -391,6 +391,7 @@ static Function *gcroot_func; static Function *gcstore_func; static Function *gckill_func; static Function *jlcall_frame_func; +static Function *gcroot_flush_func; static std::vector<Type *> two_pvalue_llvmt; static std::vector<Type *> three_pvalue_llvmt; @@ -3386,6 +3387,7 @@ static void finalize_gc_frame(Function *F) M->getOrInsertFunction(gckill_func->getName(), gckill_func->getFunctionType()); M->getOrInsertFunction(gcstore_func->getName(), gcstore_func->getFunctionType()); M->getOrInsertFunction(jlcall_frame_func->getName(), jlcall_frame_func->getFunctionType()); + M->getOrInsertFunction(gcroot_flush_func->getName(), gcroot_flush_func->getFunctionType()); Function *jl_get_ptls_states = M->getFunction("jl_get_ptls_states"); CallInst *ptlsStates = NULL; @@ -3456,6 +3458,7 @@ void finalize_gc_frame(Module *m) m->getFunction("julia.gc_root_kill")->eraseFromParent(); m->getFunction("julia.gc_store")->eraseFromParent(); m->getFunction("julia.jlcall_frame_decl")->eraseFromParent(); + m->getFunction("julia.gcroot_flush")->eraseFromParent(); } static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_tupletype_t *argt, @@ -5595,6 +5598,11 @@ static void init_julia_llvm_env(Module *m) "julia.jlcall_frame_decl", m); add_named_global(jlcall_frame_func, (void*)NULL, /*dllimport*/false); + gcroot_flush_func = Function::Create(FunctionType::get(T_void, false), + Function::ExternalLinkage, + "julia.gcroot_flush", m); + add_named_global(gcroot_flush_func, (void*)NULL, /*dllimport*/false); + // set up optimization passes #ifdef LLVM37 // No DataLayout pass needed anymore. diff --git a/src/llvm-gcroot.cpp b/src/llvm-gcroot.cpp index 3bc65b08b364b..b8b11f475906b 100644 --- a/src/llvm-gcroot.cpp +++ b/src/llvm-gcroot.cpp @@ -53,6 +53,7 @@ class JuliaGCAllocator { gckill_func(M.getFunction("julia.gc_root_kill")), gc_store_func(M.getFunction("julia.gc_store")), jlcall_frame_func(M.getFunction("julia.jlcall_frame_decl")), + gcroot_flush_func(M.getFunction("julia.gcroot_flush")), tbaa_gcframe(tbaa) { /* Algorithm sketch: @@ -82,7 +83,8 @@ Function *const gcroot_func; Function *const gckill_func; Function *const gc_store_func; Function *const jlcall_frame_func; -MDNode *tbaa_gcframe; +Function *const gcroot_flush_func; +MDNode *const tbaa_gcframe; typedef std::pair<CallInst*, unsigned> frame_register; class liveness { @@ -461,11 +463,54 @@ unsigned find_space_for(CallInst *callInst, return n; } +void rearrangeRoots() +{ + for (auto BB = F.begin(), E(F.end()); BB != E; BB++) { + auto terminst = BB->getTerminator(); + if (!isa<ReturnInst>(terminst) && !isa<UnreachableInst>(terminst)) + continue; + SmallVector<StoreInst*, 16> toRemove; + for (auto I = BB->rbegin(), E(BB->rend()); I != E; ++I) { + // Only handle the simplest case for now, give up if there's a call + // or load from the GC frame. + // (Assume we don't have loads that can alias GC frame + // unless the source address is a `julia.gc_root_decl`) + Instruction *inst = &*I; + if (isa<CallInst>(inst)) + break; + if (LoadInst *loadInst = dyn_cast<LoadInst>(inst)) { + CallInst *loadAddr = + dyn_cast<CallInst>(loadInst->getPointerOperand()); + if (loadAddr && loadAddr->getCalledFunction() == gcroot_func) + break; + continue; + } + if (StoreInst *storeInst = dyn_cast<StoreInst>(inst)) { + CallInst *storeAddr = + dyn_cast<CallInst>(storeInst->getPointerOperand()); + if (storeAddr && storeAddr->getCalledFunction() == gcroot_func) + toRemove.push_back(storeInst); + continue; + } + } + for (auto inst: toRemove) { + CallInst *decl = cast<CallInst>(inst->getPointerOperand()); + inst->eraseFromParent(); + // TODO removing unused slot should probably be handled later + // when we allocate the frame + if (decl->use_empty()) { + decl->eraseFromParent(); + } + } + } +} + public: void allocate_frame() { Instruction *last_gcframe_inst = gcframe; collapseRedundantRoots(); + rearrangeRoots(); /* # initialize the kill BasicBlock of all jlcall-frames * bb-uses : map<BB, map< pair<inst, arg-offset>, assign|live|kill > > @@ -665,7 +710,8 @@ void allocate_frame() ++i; // delete the now unused gckill information if (CallInst* callInst = dyn_cast<CallInst>(inst)) { - if (callInst->getCalledFunction() == gckill_func) { + Value *callee = callInst->getCalledFunction(); + if (callee == gckill_func || callee == gcroot_flush_func) { callInst->eraseFromParent(); } } From 7dae005fd6fb50eb3d8fad871718b3c312f7827c Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Fri, 15 Jul 2016 21:19:39 -0400 Subject: [PATCH 0472/1117] fix libgit2 & libssh2 build --- deps/libgit2.mk | 6 +++--- deps/libssh2.mk | 2 +- deps/patches/libgit2-ssh.patch | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index c343f14ae0bff..06683918bb86d 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -24,11 +24,11 @@ LIBGIT2_OPTS += -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_ endif $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(LIBSSH2_OBJ_TARGET) -ifeq ($(OS),WINNT) - -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p0 -f < $(SRCDIR)/patches/libgit2-ssh.patch -else +ifeq ($(OS),Linux) -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch endif + -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p0 -f < $(SRCDIR)/patches/libgit2-ssh.patch + mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 92deec707b34f..c32d72643b72d 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -10,7 +10,7 @@ LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT) LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \ -DCMAKE_PREFIX_PATH=$(build_prefix) \ -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \ - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib ifeq ($(OS),WINNT) LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF diff --git a/deps/patches/libgit2-ssh.patch b/deps/patches/libgit2-ssh.patch index 4c7081e37e2bb..2c21194b2f749 100644 --- a/deps/patches/libgit2-ssh.patch +++ b/deps/patches/libgit2-ssh.patch @@ -8,7 +8,7 @@ + FIND_PACKAGE(Libssh2) + GET_TARGET_PROPERTY(LIBSSH2_INCLUDE_DIRS Libssh2::libssh2 INTERFACE_INCLUDE_DIRECTORIES) + GET_TARGET_PROPERTY(LIBSSH2_LOCATION Libssh2::libssh2 IMPORTED_LOCATION_RELEASE) -+ GET_FILENAME_COMPONENT(LIBSSH2_LIBRARY_DIRS ${LIBSSH2_LOCATION} DIRECTORY) ++ GET_FILENAME_COMPONENT(LIBSSH2_LIBRARY_DIRS ${LIBSSH2_LOCATION} PATH) + SET(LIBSSH2_LIBRARIES "-lssh2") ENDIF() -IF (LIBSSH2_FOUND) From 8598556205604ed900ce08c1a5c38e3cfdf4318b Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 15 Jul 2016 21:16:20 -0700 Subject: [PATCH 0473/1117] Minor grammar fix in a comment and revert some unnecessary whitespace additions to make the pr diff smaller --- Makefile | 1 - base/libgit2/libgit2.jl | 3 +-- deps/libgit2.mk | 1 - deps/mbedtls.mk | 4 ++-- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 23e9e26919068..7a5b562896b47 100644 --- a/Makefile +++ b/Makefile @@ -364,7 +364,6 @@ ifeq ($(OS),Linux) -$(INSTALL_M) $(build_libdir)/libssl*.so* $(DESTDIR)$(private_libdir) -$(INSTALL_M) $(build_libdir)/libcrypto*.so* $(DESTDIR)$(private_libdir) endif - endif ifeq ($(USE_SYSTEM_LIBUV),0) diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index b8d5e78bde40f..5294cb8fb9b74 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -511,7 +511,6 @@ function set_ssl_cert_locations(cert_loc) ccall((:git_libgit2_opts, :libgit2), Cint, (Cint, Cstring, Cstring), Cint(Consts.SET_SSL_CERT_LOCATIONS), cert_file, cert_dir) - end function __init__() @@ -522,7 +521,7 @@ function __init__() end # Look for OpenSSL env variable for CA bundle (linux only) - # windows and macOS are use their security backends + # windows and macOS use the OS native security backends @static if is_linux() cert_loc = if "SSL_CERT_DIR" in keys(ENV) ENV["SSL_CERT_DIR"] diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 06683918bb86d..97f3c278964c3 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -28,7 +28,6 @@ ifeq ($(OS),Linux) -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch endif -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p0 -f < $(SRCDIR)/patches/libgit2-ssh.patch - mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 7513d046c4136..9020857f6b5ed 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -1,9 +1,9 @@ ## mbedtls ifeq ($(USE_GPL_LIBS), 1) - MBEDTLS_SRC = mbedtls-$(MBEDTLS_VER)-gpl +MBEDTLS_SRC = mbedtls-$(MBEDTLS_VER)-gpl else - MBEDTLS_SRC = mbedtls-$(MBEDTLS_VER)-apache +MBEDTLS_SRC = mbedtls-$(MBEDTLS_VER)-apache endif MBEDTLS_URL = https://tls.mbed.org/download/$(MBEDTLS_SRC).tgz From 095d475cf7e577cd5408a778d8670f470be4f212 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 15 Jul 2016 21:43:24 -0700 Subject: [PATCH 0474/1117] Remove many unrelated changes from libssh2-mbedtls.patch --- deps/patches/libssh2-mbedtls.patch | 590 ----------------------------- 1 file changed, 590 deletions(-) diff --git a/deps/patches/libssh2-mbedtls.patch b/deps/patches/libssh2-mbedtls.patch index 8afe5c4b2ea4f..2311d1e210b49 100644 --- a/deps/patches/libssh2-mbedtls.patch +++ b/deps/patches/libssh2-mbedtls.patch @@ -1,81 +1,3 @@ -diff --git a/.travis.yml b/.travis.yml -index 9d6c0be..8bc1292 100644 ---- a/.travis.yml -+++ b/.travis.yml -@@ -49,6 +49,10 @@ env: - - ADDRESS_SIZE=64 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF - - ADDRESS_SIZE=64 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON - - ADDRESS_SIZE=64 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=ON -+ - ADDRESS_SIZE=64 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=OFF -+ - ADDRESS_SIZE=64 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF -+ - ADDRESS_SIZE=64 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON -+ - ADDRESS_SIZE=64 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=ON - - ADDRESS_SIZE=32 CRYPTO_BACKEND=OpenSSL BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=OFF - - ADDRESS_SIZE=32 CRYPTO_BACKEND=OpenSSL BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF - - ADDRESS_SIZE=32 CRYPTO_BACKEND=OpenSSL BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON -@@ -57,6 +61,11 @@ env: - - ADDRESS_SIZE=32 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF - - ADDRESS_SIZE=32 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON - - ADDRESS_SIZE=32 CRYPTO_BACKEND=Libgcrypt BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=ON -+ - ADDRESS_SIZE=32 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=OFF -+ - ADDRESS_SIZE=32 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=OFF -+ - ADDRESS_SIZE=32 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=OFF ENABLE_ZLIB_COMPRESSION=ON -+ - ADDRESS_SIZE=32 CRYPTO_BACKEND=mbedTLS BUILD_SHARED_LIBS=ON ENABLE_ZLIB_COMPRESSION=ON -+ - - before_install: - - sudo add-apt-repository --yes ppa:kalakris/cmake -@@ -66,6 +75,9 @@ before_install: - - if [ $ADDRESS_SIZE = '32' ]; then sudo apt-get install -y linux-libc-dev linux-libc-dev:i386; fi - - if [ $ADDRESS_SIZE = '32' ]; then sudo apt-get install -y gcc-multilib libgcrypt11-dev:i386 libssl-dev:i386 zlib1g-dev:i386; fi - - if [ $ADDRESS_SIZE = '32' ]; then export TOOLCHAIN_OPTION="-DCMAKE_TOOLCHAIN_FILE=../cmake/Toolchain-Linux-32.cmake"; fi -+ - if [ $CRYPTO_BACKEND = 'mbedTLS' ]; then mbedtls/buildmbedtls; fi -+ - if [ $CRYPTO_BACKEND = 'mbedTLS' ]; then export TOOLCHAIN_OPTION="-DCMAKE_PREFIX_PATH=../mbedtls/install"; fi -+ - if [ $CRYPTO_BACKEND = 'mbedTLS' ]; then export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/mbedtls/install/lib; fi - - install: - - mkdir bin -diff --git a/README b/README -index 39abc20..8a14856 100644 ---- a/README -+++ b/README -@@ -4,9 +4,9 @@ libssh2 - SSH2 library - libssh2 is a library implementing the SSH2 protocol, available under - the revised BSD license. - --Web site: http://www.libssh2.org/ -+Web site: https://www.libssh2.org/ - --Mailing list: http://cool.haxx.se/mailman/listinfo/libssh2-devel -+Mailing list: https://cool.haxx.se/mailman/listinfo/libssh2-devel - - License: see COPYING - -diff --git a/RELEASE-NOTES b/RELEASE-NOTES -index 2b22288..94525cb 100644 ---- a/RELEASE-NOTES -+++ b/RELEASE-NOTES -@@ -11,7 +11,7 @@ This release includes the following changes: - This release includes the following security advisory: - - o diffie_hellman_sha256: convert bytes to bits -- CVE-2016-0787: http://www.libssh2.org/adv_20160223.html -+ CVE-2016-0787: https://www.libssh2.org/adv_20160223.html - - This release includes the following bugfixes: - -@@ -49,8 +49,8 @@ advice from friends like these: - - Alexander Lamaison, Andreas Schneider, brian m. carlson, Daniel Stenberg, - David Byron, Jakob Egger, Kamil Dudka, Marc Hoersken, Mizunashi Mana, -- Patrick Monnerat, Paul Howarth, Salvador Fandino, Salvador Fandiño, -- Salvador Fandiño, Viktor Szakats, Will Cosgrove, -- (16 contributors) -+ Patrick Monnerat, Paul Howarth, Salvador Fandiño, Viktor Szakats, -+ Will Cosgrove -+ (14 contributors) - - Thanks! (and sorry if I forgot to mention someone) diff --git a/cmake/FindmbedTLS.cmake b/cmake/FindmbedTLS.cmake new file mode 100644 index 0000000..2f4adbc @@ -146,272 +68,6 @@ index 0000000..2f4adbc + MBEDX509_LIBRARY + MBEDCRYPTO_LIBRARY +) -diff --git a/docs/BINDINGS b/docs/BINDINGS -index b97758f..471f9be 100644 ---- a/docs/BINDINGS -+++ b/docs/BINDINGS -@@ -10,16 +10,16 @@ Cocoa/Objective-C - https://github.com/karelia/libssh2_sftp-Cocoa-wrapper - - Haskell -- FFI bindings - http://hackage.haskell.org/package/libssh2 -+ FFI bindings - https://hackage.haskell.org/package/libssh2 - - Perl -- Net::SSH2 - http://search.cpan.org/~rkitover/Net-SSH2-0.45/lib/Net/SSH2.pm -+ Net::SSH2 - https://metacpan.org/pod/Net::SSH2 - - PHP -- ssh2 - http://pecl.php.net/package/ssh2 -+ ssh2 - https://pecl.php.net/package/ssh2 - - Python -- pylibssh2 - http://www.wallix.org/pylibssh2-project/ -+ pylibssh2 - https://pypi.python.org/pypi/pylibssh2 - - Python-ctypes - -diff --git a/docs/INSTALL_AUTOTOOLS b/docs/INSTALL_AUTOTOOLS -index bc5a0eb..b469533 100644 ---- a/docs/INSTALL_AUTOTOOLS -+++ b/docs/INSTALL_AUTOTOOLS -@@ -284,7 +284,7 @@ Some ./configure options deserve additional comments: - * --with-libgcrypt-prefix=DIR - - libssh2 can use the Libgcrypt library -- (http://www.gnupg.org/) for cryptographic operations. -+ (https://www.gnupg.org/) for cryptographic operations. - Either Libgcrypt or OpenSSL is required. - - Configure will attempt to locate Libgcrypt -@@ -298,7 +298,7 @@ Some ./configure options deserve additional comments: - * --with-libssl-prefix=[DIR] - - libssh2 can use the OpenSSL library -- (http://www.openssl.org) for cryptographic operations. -+ (https://www.openssl.org) for cryptographic operations. - Either Libgcrypt or OpenSSL is required. - - Configure will attempt to locate OpenSSL in the -diff --git a/docs/INSTALL_CMAKE b/docs/INSTALL_CMAKE -index 7040370..d2d7169 100644 ---- a/docs/INSTALL_CMAKE -+++ b/docs/INSTALL_CMAKE -@@ -47,8 +47,8 @@ The following options are available: - * `CRYPTO_BACKEND=` - - Chooses a specific cryptography library to use for cryptographic -- operations. Can be `OpenSSL` (http://www.openssl.org), -- `Libgcrypt` (http://www.gnupg.org/), `WinCNG` (Windows Vista+) or -+ operations. Can be `OpenSSL` (https://www.openssl.org), -+ `Libgcrypt` (https://www.gnupg.org/), `WinCNG` (Windows Vista+) or - blank to use any library available. - - CMake will attempt to locate the libraries automatically. See [2] -@@ -161,14 +161,14 @@ builds your project: - Libssh2 - URL <libssh2 download location> - URL_HASH SHA1=<libssh2 archive SHA1> -- INSTALL_COMMAND "") -+ INSTALL_COMMAND "") - - ExternalProject_Add( - MyProject DEPENDS Libssh2 - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src - INSTALL_COMMAND "") - --[1] http://www.cmake.org/cmake/resources/software.html --[2] http://www.cmake.org/cmake/help/v3.0/manual/cmake-packages.7.html --[3] http://www.cmake.org/cmake/help/v3.0/manual/cmake-packages.7.html#package-registry --[4] http://www.kitware.com/media/html/BuildingExternalProjectsWithCMake2.8.html -\ No newline at end of file -+[1] https://www.cmake.org/cmake/resources/software.html -+[2] https://www.cmake.org/cmake/help/v3.0/manual/cmake-packages.7.html -+[3] https://www.cmake.org/cmake/help/v3.0/manual/cmake-packages.7.html#package-registry -+[4] http://www.kitware.com/media/html/BuildingExternalProjectsWithCMake2.8.html -diff --git a/docs/libssh2_sftp_get_channel.3 b/docs/libssh2_sftp_get_channel.3 -index f3d364a..d1d82bc 100644 ---- a/docs/libssh2_sftp_get_channel.3 -+++ b/docs/libssh2_sftp_get_channel.3 -@@ -1,21 +1,21 @@ --.TH libssh2_sftp_get_channel 3 "9 Sep 2011" "libssh2 1.4.0" "libssh2 manual" --.SH NAME --libssh2_sftp_get_channel - return the channel of sftp --.SH SYNOPSIS --.nf --#include <libssh2.h> --#include <libssh2_sftp.h> -- --.fi --LIBSSH2_CHANNEL *libssh2_sftp_get_channel(LIBSSH2_SFTP *sftp); --.SH DESCRIPTION --\fIsftp\fP - SFTP instance as returned by --.BR libssh2_sftp_init(3) -- --Return the channel of the given sftp handle. --.SH RETURN VALUE --The channel of the SFTP instance or NULL if something was wrong. --.SH AVAILABILITY --Added in 1.4.0 --.SH SEE ALSO --.BR libssh2_sftp_init(3) -+.TH libssh2_sftp_get_channel 3 "9 Sep 2011" "libssh2 1.4.0" "libssh2 manual" -+.SH NAME -+libssh2_sftp_get_channel - return the channel of sftp -+.SH SYNOPSIS -+.nf -+#include <libssh2.h> -+#include <libssh2_sftp.h> -+ -+.fi -+LIBSSH2_CHANNEL *libssh2_sftp_get_channel(LIBSSH2_SFTP *sftp); -+.SH DESCRIPTION -+\fIsftp\fP - SFTP instance as returned by -+.BR libssh2_sftp_init(3) -+ -+Return the channel of the given sftp handle. -+.SH RETURN VALUE -+The channel of the SFTP instance or NULL if something was wrong. -+.SH AVAILABILITY -+Added in 1.4.0 -+.SH SEE ALSO -+.BR libssh2_sftp_init(3) -diff --git a/example/subsystem_netconf.c b/example/subsystem_netconf.c -index 96ef54a..82c4941 100644 ---- a/example/subsystem_netconf.c -+++ b/example/subsystem_netconf.c -@@ -250,7 +250,7 @@ int main(int argc, char *argv[]) - goto shutdown; - } - -- /* NETCONF: http://tools.ietf.org/html/draft-ietf-netconf-ssh-06 */ -+ /* NETCONF: https://tools.ietf.org/html/draft-ietf-netconf-ssh-06 */ - - fprintf(stderr, "Sending NETCONF client <hello>\n"); - snprintf(buf, sizeof(buf), -diff --git a/include/libssh2.h b/include/libssh2.h -index c157757..9ef9ff1 100644 ---- a/include/libssh2.h -+++ b/include/libssh2.h -@@ -202,7 +202,16 @@ typedef off_t libssh2_struct_stat_size; - #endif - - #ifndef LIBSSH2_STRUCT_STAT_SIZE_FORMAT --# define LIBSSH2_STRUCT_STAT_SIZE_FORMAT "%zd" -+# ifdef __VMS -+/* We have to roll our own format here because %z is a C99-ism we don't have. */ -+# if __USE_OFF64_T || __USING_STD_STAT -+# define LIBSSH2_STRUCT_STAT_SIZE_FORMAT "%Ld" -+# else -+# define LIBSSH2_STRUCT_STAT_SIZE_FORMAT "%d" -+# endif -+# else -+# define LIBSSH2_STRUCT_STAT_SIZE_FORMAT "%zd" -+# endif - typedef struct stat libssh2_struct_stat; - typedef off_t libssh2_struct_stat_size; - #endif -diff --git a/libssh2.pc.in b/libssh2.pc.in -index 084918a..9663816 100644 ---- a/libssh2.pc.in -+++ b/libssh2.pc.in -@@ -8,7 +8,7 @@ libdir=@libdir@ - includedir=@includedir@ - - Name: libssh2 --URL: http://www.libssh2.org/ -+URL: https://www.libssh2.org/ - Description: Library for SSH-based communication - Version: @LIBSSH2VER@ - Requires.private: @LIBSREQUIRED@ -diff --git a/mbedtls/buildmbedtls b/mbedtls/buildmbedtls -new file mode 100755 -index 0000000..2385a8b ---- /dev/null -+++ b/mbedtls/buildmbedtls -@@ -0,0 +1,18 @@ -+#!/bin/sh -+set -ex -+ -+MBEDTLS_SRC_ARCH=mbedtls-2.2.1 -+ -+# download mbedtls sources -+cd mbedtls -+wget https://github.com/ARMmbed/mbedtls/archive/$MBEDTLS_SRC_ARCH.tar.gz -+tar -xzf $MBEDTLS_SRC_ARCH.tar.gz -+ -+# patch mbedtls configuration -+cd mbedtls-$MBEDTLS_SRC_ARCH -+patch -p0 < ../mbedtls.patch -+ -+# build library -+cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ -+ -DCMAKE_INSTALL_PREFIX:PATH=../install -+make install -diff --git a/mbedtls/mbedtls.patch b/mbedtls/mbedtls.patch -new file mode 100644 -index 0000000..1f774ec ---- /dev/null -+++ b/mbedtls/mbedtls.patch -@@ -0,0 +1,27 @@ -+diff --git include/mbedtls/config.h include/mbedtls/config.h -+index 0efee04..787cd8c 100644 -+--- include/mbedtls/config.h -++++ include/mbedtls/config.h -+@@ -2434,19 +2434,19 @@ -+ -+ /* MPI / BIGNUM options */ -+ //#define MBEDTLS_MPI_WINDOW_SIZE 6 /**< Maximum windows size used. */ -+-//#define MBEDTLS_MPI_MAX_SIZE 1024 /**< Maximum number of bytes for usable MPIs. */ -++#define MBEDTLS_MPI_MAX_SIZE 2048 /**< Maximum number of bytes for usable MPIs. */ -+ -+ /* CTR_DRBG options */ -+ //#define MBEDTLS_CTR_DRBG_ENTROPY_LEN 48 /**< Amount of entropy used per seed by default (48 with SHA-512, 32 with SHA-256) */ -+ //#define MBEDTLS_CTR_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ -+ //#define MBEDTLS_CTR_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ -+-//#define MBEDTLS_CTR_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -++#define MBEDTLS_CTR_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ -+ //#define MBEDTLS_CTR_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ -+ -+ /* HMAC_DRBG options */ -+ //#define MBEDTLS_HMAC_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ -+ //#define MBEDTLS_HMAC_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ -+-//#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -++#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ -+ //#define MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ -+ -+ /* ECP options */ -diff --git a/nw/GNUmakefile b/nw/GNUmakefile -index 0a1d0b3..8899fdd 100644 ---- a/nw/GNUmakefile -+++ b/nw/GNUmakefile -@@ -38,7 +38,7 @@ DEVLARC = $(DEVLDIR).zip - TARGET = libssh2 - VERSION = $(LIBSSH2_VERSION) - CPRIGHT = Copyright (c) $(LIBSSH2_COPYRIGHT_STR) --WWWURL = http://www.libssh2.org/ -+WWWURL = https://www.libssh2.org/ - DESCR = libssh2 $(LIBSSH2_VERSION_STR) ($(LIBARCH)) - $(WWWURL) - MTSAFE = YES - STACK = 64000 -@@ -223,7 +223,7 @@ include ../Makefile.inc - OBJECTS := $(patsubst %.c,%.o,$(CSOURCES)) - ifeq ($(LIBARCH),CLIB) - # CLIB lacks of snprint() function - here's a replacement: --# http://www.ijs.si/software/snprintf/ -+# https://www.ijs.si/software/snprintf/ - OBJECTS += snprintf.o - vpath %.c $(SNPRINTF) - endif -@@ -388,7 +388,7 @@ libssh2_config.h: GNUmakefile - @echo $(DL)** All your changes will be lost!!$(DL) >> $@ - @echo $(DL)*/$(DL) >> $@ - @echo $(DL)#define VERSION "$(LIBSSH2_VERSION_STR)"$(DL) >> $@ -- @echo $(DL)#define PACKAGE_BUGREPORT "http://sourceforge.net/projects/libssh2"$(DL) >> $@ -+ @echo $(DL)#define PACKAGE_BUGREPORT "https://github.com/libssh2/libssh2/issues"$(DL) >> $@ - ifeq ($(LIBARCH),CLIB) - @echo $(DL)#define OS "i586-pc-clib-NetWare"$(DL) >> $@ - @echo $(DL)#define NETDB_USE_INTERNET 1$(DL) >> $@ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a76399..987dfe0 100644 --- a/src/CMakeLists.txt @@ -459,31 +115,6 @@ index 1a76399..987dfe0 100644 if(NOT CRYPTO_BACKEND) message(FATAL_ERROR "No suitable cryptography backend found.") endif() -@@ -204,6 +224,11 @@ target_include_directories(libssh2 - - ## Options - -+option(CLEAR_MEMORY "Enable clearing of memory before being freed" OFF) -+if(CLEAR_MEMORY) -+ add_definitions(-DLIBSSH2_CLEAR_MEMORY) -+endif(CLEAR_MEMORY) -+ - add_feature_info("Shared library" BUILD_SHARED_LIBS - "creating libssh2 as a shared library (.so/.dll)") - -diff --git a/src/channel.c b/src/channel.c -index 32d914d..538a0ab 100644 ---- a/src/channel.c -+++ b/src/channel.c -@@ -270,7 +270,7 @@ _libssh2_channel_open(LIBSSH2_SESSION * session, const char *channel_type, - switch (reason_code) { - case SSH_OPEN_ADMINISTRATIVELY_PROHIBITED: - _libssh2_error(session, LIBSSH2_ERROR_CHANNEL_FAILURE, -- "Channel open failure (admininstratively prohibited)"); -+ "Channel open failure (administratively prohibited)"); - break; - case SSH_OPEN_CONNECT_FAILED: - _libssh2_error(session, LIBSSH2_ERROR_CHANNEL_FAILURE, diff --git a/src/crypto.h b/src/crypto.h index caad19f..aa997a3 100644 --- a/src/crypto.h @@ -526,21 +157,6 @@ index e85aecd..366d007 100644 memcpy(*signature, tmp, size); *signature_len = size; -diff --git a/src/libssh2_priv.h b/src/libssh2_priv.h -index ae691ad..b4296a2 100644 ---- a/src/libssh2_priv.h -+++ b/src/libssh2_priv.h -@@ -65,8 +65,8 @@ - consistent names of these fields. While arguable the best would to - change libssh2.h to use other names, that would break backwards - compatibility. For more information, see: -- http://www.mail-archive.com/libssh2-devel%40lists.sourceforge.net/msg00003.html -- http://www.mail-archive.com/libssh2-devel%40lists.sourceforge.net/msg00224.html -+ https://www.mail-archive.com/libssh2-devel%40lists.sourceforge.net/msg00003.html -+ https://www.mail-archive.com/libssh2-devel%40lists.sourceforge.net/msg00224.html - */ - #ifdef HAVE_POLL - # include <sys/poll.h> diff --git a/src/mbedtls.c b/src/mbedtls.c new file mode 100644 index 0000000..98bc549 @@ -1491,209 +1107,3 @@ index 0000000..f594575 + const char *privatekeydata, + size_t privatekeydata_len, + const char *passphrase); -diff --git a/src/sftp.c b/src/sftp.c -index c142713..7c44116 100644 ---- a/src/sftp.c -+++ b/src/sftp.c -@@ -1527,7 +1527,7 @@ static ssize_t sftp_read(LIBSSH2_SFTP_HANDLE * handle, char *buffer, - if (chunk->offset != filep->offset) { - /* This could happen if the server returns less bytes than - requested, which shouldn't happen for normal files. See: -- http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02 -+ https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02 - #section-6.4 - */ - return _libssh2_error(session, LIBSSH2_ERROR_SFTP_PROTOCOL, -diff --git a/src/wincng.c b/src/wincng.c -index 635b3e3..d3271b3 100755 ---- a/src/wincng.c -+++ b/src/wincng.c -@@ -861,7 +861,7 @@ _libssh2_wincng_rsa_new(libssh2_rsa_ctx **rsa, - memset(key, 0, keylen); - - -- /* http://msdn.microsoft.com/library/windows/desktop/aa375531.aspx */ -+ /* https://msdn.microsoft.com/library/windows/desktop/aa375531.aspx */ - rsakey = (BCRYPT_RSAKEY_BLOB *)key; - rsakey->BitLength = mlen * 8; - rsakey->cbPublicExp = elen; -@@ -1179,7 +1179,7 @@ _libssh2_wincng_dsa_new(libssh2_dsa_ctx **dsa, - memset(key, 0, keylen); - - -- /* http://msdn.microsoft.com/library/windows/desktop/aa833126.aspx */ -+ /* https://msdn.microsoft.com/library/windows/desktop/aa833126.aspx */ - dsakey = (BCRYPT_DSA_KEY_BLOB *)key; - dsakey->cbKey = length; - -@@ -1903,7 +1903,7 @@ _libssh2_wincng_bignum_mod_exp(_libssh2_bn *r, - return -1; - - -- /* http://msdn.microsoft.com/library/windows/desktop/aa375531.aspx */ -+ /* https://msdn.microsoft.com/library/windows/desktop/aa375531.aspx */ - rsakey = (BCRYPT_RSAKEY_BLOB *)key; - rsakey->Magic = BCRYPT_RSAPUBLIC_MAGIC; - rsakey->BitLength = m->length * 8; -diff --git a/vms/libssh2_config.h b/vms/libssh2_config.h -index 2eb09a8..b8f73e2 100644 ---- a/vms/libssh2_config.h -+++ b/vms/libssh2_config.h -@@ -14,6 +14,7 @@ typedef unsigned int socklen_t; /* missing in headers on VMS */ - /* Have's */ - - #define HAVE_UNISTD_H -+#define HAVE_STDLIB_H - #define HAVE_INTTYPES_H - #define HAVE_SYS_TIME_H - #define HAVE_SELECT -@@ -23,6 +24,8 @@ typedef unsigned int socklen_t; /* missing in headers on VMS */ - #define HAVE_NETINET_IN_H - #define HAVE_ARPA_INET_H - -+#define HAVE_GETTIMEOFDAY 1 -+ - #define POSIX_C_SOURCE - - /* Enable the possibility of using tracing */ -@@ -68,8 +71,11 @@ typedef unsigned int socklen_t; /* missing in headers on VMS */ - - #endif - -+/* Use OpenSSL */ -+#define LIBSSH2_OPENSSL 1 -+ - /* Compile in zlib support. We link against gnv$libzshr, as available -- * on encompasserve.com. -+ * from https://sourceforge.net/projects/vms-ports/files/. - */ - - #define LIBSSH2_HAVE_ZLIB -diff --git a/vms/libssh2_make_example.dcl b/vms/libssh2_make_example.dcl -index d8191f3..af4116d 100644 ---- a/vms/libssh2_make_example.dcl -+++ b/vms/libssh2_make_example.dcl -@@ -29,6 +29,7 @@ $ this = f$search("exadir:*.c;0") - $ if this .eqs. "" then goto EndLoop - $! - $ what = f$parse( this,,,"name") -+$ if what .eqs. "x11" then goto loop ! not portable yet - $ call Make - $! - $ goto Loop -@@ -84,7 +85,7 @@ $ define objdir 'objdir' - $ define exadir 'exadir' - $! - $ cc_include = "/include=([],[-.include])" --$ cc_flags = "/name=shortened/show=all" -+$ cc_flags = "/name=shortened/show=all/define=(_USE_STD_STAT=1)" - $ link_opts = "objdir:libssh2_''thisid'.opt" - $! - $! -diff --git a/vms/libssh2_make_help.dcl b/vms/libssh2_make_help.dcl -index b28265d..b36512e 100644 ---- a/vms/libssh2_make_help.dcl -+++ b/vms/libssh2_make_help.dcl -@@ -11,23 +11,27 @@ $ man2help sys$input: libssh2.hlp -b 1 - - LIBSSH2 - --OpenVMS port of the public domain libssh2 library, which --provides an API to implement client SSH communciation. -+OpenVMS port of the libssh2 library, which provides an -+API to implement client SSH communication. - --License information is available at the copying subtopic. -+License information is available at the Copying subtopic. - - $! - $ open/append mh libssh2.hlp - $ write mh helpversion - $ close mh - $! --$ man2help -a [-]readme.; libssh2.hlp -b 2 --$ man2help -a [-]authors.; libssh2.hlp -b 2 --$ man2help -a [-]copying.; libssh2.hlp -b 2 --$ man2help -a [-]news.; libssh2.hlp -b 2 --$ man2help -a [-]release-notes.; libssh2.hlp -b 2 --$ man2help -a [-]hacking.; libssh2.hlp -b 2 --$ man2help -a [-]todo.; libssh2.hlp -b 2 -+$ man2help -a [-]README.; libssh2.hlp -b 2 -+$ man2help -a [-]COPYING.; libssh2.hlp -b 2 -+$ man2help -a [-]NEWS.; libssh2.hlp -b 2 -+$ man2help -a [-]RELEASE-NOTES.; libssh2.hlp -b 2 -+$ man2help -a [-.docs]AUTHORS.; libssh2.hlp -b 2 -+$ man2help -a [-.docs]BINDINGS.; libssh2.hlp -b 2 -+$ man2help -a [-.docs]HACKING.; libssh2.hlp -b 2 -+$ if f$search("[]HACKING_CRYPTO.") .nes. "" then delete []HACKING_CRYPTO.;* -+$ copy [-.docs]HACKING.CRYPTO; []HACKING_CRYPTO. -+$ man2help -a []HACKING_CRYPTO.; libssh2.hlp -b 2 -+$ man2help -a [-.docs]TODO.; libssh2.hlp -b 2 - $! - $ man2help -a sys$input: libssh2.hlp -b 2 - -diff --git a/vms/libssh2_make_kit.dcl b/vms/libssh2_make_kit.dcl -index dceca4f..6a14958 100644 ---- a/vms/libssh2_make_kit.dcl -+++ b/vms/libssh2_make_kit.dcl -@@ -194,7 +194,7 @@ $ write pt "=prompt JCB LIBSSH2 for OpenVMS" - $ write pt "" - $ write pt "libssh2 is an open source client side library that aims to implement" - $ write pt "the SSH protocol. This is the OpenVMS port of that library." --$ write pt "Further information at http://www.libssh2.org." -+$ write pt "Further information at https://www.libssh2.org." - $ write pt "" - $ write pt "1 NEED_VMS83" - $ write pt "=prompt OpenVMS 8.3 or later is not installed on your system." -diff --git a/vms/libssh2_make_lib.dcl b/vms/libssh2_make_lib.dcl -index 6d8b13b..56d168f 100644 ---- a/vms/libssh2_make_lib.dcl -+++ b/vms/libssh2_make_lib.dcl -@@ -46,7 +46,8 @@ $! - $ define objdir 'objdir' - $ define srcdir 'srcdir' - $! --$ cc_include = "/include=([],[-.include])" -+$ cc_include = "/include=([],[-.include],""/gnv$zlib_include"")" -+$ cc_define = "/DEFINE=(_USE_STD_STAT=1)" - $ link_opts = "objdir:libssh2_''thisid'.opt" - $! - $ pipe search [-.include]libssh2.h libssh2_version_major/nohead | (read sys$input l ; l = f$element(2," ",f$edit(l,"trim,compress")) ; - -@@ -137,7 +138,7 @@ $CaseLoop: - $! - $ if case .eq. 0 - $ then!camel case names --$ cc_flags = "/names=(shortened,as_is)" -+$ cc_flags = "/names=(shortened,as_is)''cc_define'" - $ objlib = "libssh2_asis.olb" - $ endif - $! -@@ -149,7 +150,7 @@ $ rename [.cxx_repository]cxx$demangler_db.; *.lowercase - $ purge [.cxx_repository]cxx$demangler_db.lowercase - $ endif - $! --$ cc_flags = "/names=(shortened)" -+$ cc_flags = "/names=(shortened)''cc_define'" - $ objlib = "libssh2_up.olb" - $ endif - $! -diff --git a/win32/libssh2_config.h b/win32/libssh2_config.h -index 3e3caf4..6ac2ef4 100644 ---- a/win32/libssh2_config.h -+++ b/win32/libssh2_config.h -@@ -24,6 +24,7 @@ - #define HAVE_SELECT - - #ifdef _MSC_VER -+#if _MSC_VER < 1900 - #define snprintf _snprintf - #if _MSC_VER < 1500 - #define vsnprintf _vsnprintf -@@ -31,6 +32,7 @@ - #define strdup _strdup - #define strncasecmp _strnicmp - #define strcasecmp _stricmp -+#endif - #else - #ifndef __MINGW32__ - #define strncasecmp strnicmp From 3a6213adfe992b73cf6df3bf73c761748ebba6d5 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 15 Jul 2016 23:34:47 -0700 Subject: [PATCH 0475/1117] Remove libgit2-custom-tls.patch for now not being used yet in this PR --- deps/patches/libgit2-custom-tls.patch | 65 --------------------------- 1 file changed, 65 deletions(-) delete mode 100644 deps/patches/libgit2-custom-tls.patch diff --git a/deps/patches/libgit2-custom-tls.patch b/deps/patches/libgit2-custom-tls.patch deleted file mode 100644 index cb06fe3e512a6..0000000000000 --- a/deps/patches/libgit2-custom-tls.patch +++ /dev/null @@ -1,65 +0,0 @@ -diff --git a/src/settings.c b/src/settings.c -index 00a3ef0..21430bc 100644 ---- a/src/settings.c -+++ b/src/settings.c -@@ -29,9 +29,7 @@ int git_libgit2_features(void) - #ifdef GIT_THREADS - | GIT_FEATURE_THREADS - #endif --#if defined(GIT_OPENSSL) || defined(GIT_WINHTTP) || defined(GIT_SECURE_TRANSPORT) - | GIT_FEATURE_HTTPS --#endif - #if defined(GIT_SSH) - | GIT_FEATURE_SSH - #endif -diff --git a/src/transport.c b/src/transport.c -index 327052f..32f8464 100644 ---- a/src/transport.c -+++ b/src/transport.c -@@ -29,9 +29,7 @@ static transport_definition local_transport_definition = { "file://", git_transp - static transport_definition transports[] = { - { "git://", git_transport_smart, &git_subtransport_definition }, - { "http://", git_transport_smart, &http_subtransport_definition }, --#if defined(GIT_OPENSSL) || defined(GIT_WINHTTP) || defined(GIT_SECURE_TRANSPORT) - { "https://", git_transport_smart, &http_subtransport_definition }, --#endif - { "file://", git_transport_local, NULL }, - #ifdef GIT_SSH - { "ssh://", git_transport_smart, &ssh_subtransport_definition }, -diff --git a/src/transports/http.c b/src/transports/http.c -index 4fbbfbb..30520a0 100644 ---- a/src/transports/http.c -+++ b/src/transports/http.c -@@ -620,7 +620,6 @@ static int http_connect(http_subtransport *t) - - error = git_stream_connect(t->io); - --#if defined(GIT_OPENSSL) || defined(GIT_SECURE_TRANSPORT) || defined(GIT_CURL) - if ((!error || error == GIT_ECERTIFICATE) && t->owner->certificate_check_cb != NULL && - git_stream_is_encrypted(t->io)) { - git_cert *cert; -@@ -640,7 +639,7 @@ static int http_connect(http_subtransport *t) - return error; - } - } --#endif -+ - if (error < 0) - return error; - -diff --git a/tests/core/features.c b/tests/core/features.c -index 85cddfe..cf5e190 100644 ---- a/tests/core/features.c -+++ b/tests/core/features.c -@@ -17,11 +17,7 @@ void test_core_features__0(void) - cl_assert((caps & GIT_FEATURE_THREADS) == 0); - #endif - --#if defined(GIT_OPENSSL) || defined(GIT_WINHTTP) || defined(GIT_SECURE_TRANSPORT) - cl_assert((caps & GIT_FEATURE_HTTPS) != 0); --#else -- cl_assert((caps & GIT_FEATURE_HTTPS) == 0); --#endif - - #if defined(GIT_SSH) - cl_assert((caps & GIT_FEATURE_SSH) != 0); From 24c656f001296d59148ea790e7adc76d0b93c8f7 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 15 Jul 2016 23:35:45 -0700 Subject: [PATCH 0476/1117] Minor makefile cleanups make cleaning and installation a bit more uniform Move CMAKE_PREFIX_PATH to CMAKE_COMMON --- deps/Makefile | 6 +++--- deps/libgit2.mk | 4 ++-- deps/libssh2.mk | 5 ++--- deps/mbedtls.mk | 9 ++++----- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index fc337f49dfbb7..f8044f7149955 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -20,9 +20,9 @@ include $(SRCDIR)/llvm-ver.make # autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv # custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind # entirely custom: virtualenv -# CMake libs: libgit2 +# CMake libs: libgit2 libssh2 mbedtls # -# downloaded from git: llvm-svn, libuv, libopenlibm, utf8proc, openspecfun, libgit2 +# downloaded from git: llvm-svn, libuv, libopenlibm, utf8proc, openspecfun, libgit2, libssh2 # # there are rules in this file with the . replaced by a % # this is some magic Makefile trick that tells make @@ -46,7 +46,7 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXX CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS) CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS) -CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) +CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) ifneq ($(VERBOSE), 0) CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON endif diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 97f3c278964c3..f677aad788837 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -7,7 +7,7 @@ $(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,build/libgit2.$(SHLIB_ LIBGIT2_OBJ_SOURCE := $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/libgit2.$(SHLIB_EXT) LIBGIT2_OBJ_TARGET := $(build_shlibdir)/libgit2.$(SHLIB_EXT) -LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DTHREADSAFE=ON -DCMAKE_PREFIX_PATH=$(build_prefix) +LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DTHREADSAFE=ON ifeq ($(OS),WINNT) LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON ifneq ($(ARCH),x86_64) @@ -61,7 +61,7 @@ endif touch -c $@ clean-libgit2: - -rm -rf $(BUILDDIR)/$(LIBGIT2_SRC_DIR) + -$(MAKE) -C $(BUILDDIR)/$(LIBGIT2_SRC_DIR) clean -rm -f $(LIBGIT2_OBJ_TARGET) get-libgit2: $(LIBGIT2_SRC_FILE) diff --git a/deps/libssh2.mk b/deps/libssh2.mk index c32d72643b72d..7306e0dd7afa8 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -8,7 +8,6 @@ LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT) LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT) LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \ - -DCMAKE_PREFIX_PATH=$(build_prefix) \ -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib @@ -39,7 +38,7 @@ endif echo 1 > $@ $(LIBSSH2_OBJ_TARGET): $(LIBSSH2_OBJ_SOURCE) | $(build_shlibdir) -ifeq ($(OS),WINNT) +ifeq ($(BUILD_OS),WINNT) $(MAKE) -C $(BUILDDIR)/$(LIBSSH2_SRC_DIR) install else $(call make-install,$(LIBSSH2_SRC_DIR),) @@ -47,7 +46,7 @@ endif touch -c $(LIBSSH2_OBJ_TARGET) clean-libssh2: - -rm -rf $(BUILDDIR)/$(LIBSSH2_SRC_DIR) + -$(MAKE) -C $(BUILDDIR)/$(LIBSSH2_SRC_DIR) clean -rm -f $(LIBSSH2_OBJ_TARGET) get-libssh2: $(LIBSSH2_SRC_FILE) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 9020857f6b5ed..0d74657122128 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -58,14 +58,13 @@ endif touch -c $(MBEDTLS_OBJ_TARGET) clean-mbedtls: - -rm -rf $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) + -$(MAKE) -C $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) clean -rm -f $(MBEDTLS_OBJ_TARGET) distclean-mbedtls: - -rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz - -rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC) - -rm -rf $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) - + -rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz \ + $(SRCDIR)/srccache/$(MBEDTLS_SRC) \ + $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) get-mbedtls: $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz configure-mbedtls: $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile From 9a061b9f03499f35d3458ffc1d3107072ac249f5 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 15 Jul 2016 23:14:30 -0700 Subject: [PATCH 0477/1117] Add libssh2 and mbedtls to LICENSE and README add 1.7 and 2.2 as minimum libssh2 and mbedtls versions, respectively --- LICENSE.md | 6 ++++-- README.md | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 067fd16490f7c..fd312fcee06f3 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -50,7 +50,6 @@ Julia's standard library uses the following external libraries, which have their own licenses: - [ARPACK](http://www.caam.rice.edu/software/ARPACK/RiceBSD.txt#LICENSE) [BSD-3] -- [ATLAS](http://math-atlas.sourceforge.net/faq.html#license) [BSD-3] - [DSFMT](http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/LICENSE.txt) [BSD-3] - [OPENLIBM](https://github.com/JuliaLang/openlibm/blob/master/LICENSE.md) [MIT, BSD-2, ISC] - [OPENSPECFUN](https://github.com/JuliaLang/openspecfun) [MIT, public domain] @@ -58,6 +57,8 @@ their own licenses: - [FFTW](http://fftw.org/doc/License-and-Copyright.html) [GPL2+] - [GMP](http://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+] - [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception] +- [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3] +- [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0] - [MPFR](http://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+] - [OPENBLAS](https://raw.github.com/xianyi/OpenBLAS/master/LICENSE) [BSD-3] - [LAPACK](http://netlib.org/lapack/LICENSE.txt) [BSD-3] @@ -86,4 +87,5 @@ Julia bundles the following external programs and libraries on some platforms: - [LIBEXPAT](http://expat.cvs.sourceforge.net/viewvc/expat/expat/README) - [OPENSSL](https://github.com/openssl/openssl/blob/master/LICENSE) -On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). +On some platforms, distributions of Julia contain SSL certificate authority certificates, +released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). diff --git a/README.md b/README.md index 3c07cf02d49e8..09e65ff030d82 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,9 @@ Julia uses the following external libraries, which are automatically downloaded - **[PCRE]** (>= 10.00) — Perl-compatible regular expressions library. - **[GMP]** (>= 5.0) — GNU multiple precision arithmetic library, needed for `BigInt` support. - **[MPFR]** (>= 3.0) — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support. -- **[libgit2]** (>= 0.21) — Git linkable library, used by Julia's package manager +- **[libgit2]** (>= 0.24) — Git linkable library, used by Julia's package manager +- **[libssh2]** (>= 1.7) — library for SSH transport, used by libgit2 for packages with SSH remotes +- **[mbedtls]** (>= 2.2) — library used for cryptography and transport layer security, used by libssh2 - **[utf8proc]** (>= 2.0) — a library for processing UTF-8 encoded Unicode strings - **[libosxunwind]** — clone of [libunwind], a library that determines the call-chain of a program @@ -321,6 +323,8 @@ For a longer overview of Julia's dependencies, see these [slides](https://github [libosxunwind]: https://github.com/JuliaLang/libosxunwind [libunwind]: http://www.nongnu.org/libunwind [openssl]: https://www.openssl.org +[libssh2]: https://www.libssh2.org +[mbedtls]: https://tls.mbed.org/ <a name="System-Provided-Libraries"> ### System Provided Libraries From 7386c178a8ea9a175f4f486fb1993a7f8cf112d9 Mon Sep 17 00:00:00 2001 From: Lyndon White <lyndon.white@research.uwa.edu.au> Date: Sat, 16 Jul 2016 18:52:46 +1000 Subject: [PATCH 0478/1117] =Marked float bit definitions as pure, so they will inline. --- base/float.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/float.jl b/base/float.jl index f07ca5c68c194..20b7e9a2513ca 100644 --- a/base/float.jl +++ b/base/float.jl @@ -519,9 +519,9 @@ exponent_one(::Type{Float32}) = 0x3f80_0000 exponent_half(::Type{Float32}) = 0x3f00_0000 significand_mask(::Type{Float32}) = 0x007f_ffff -significand_bits{T<:AbstractFloat}(::Type{T}) = trailing_ones(significand_mask(T)) -exponent_bits{T<:AbstractFloat}(::Type{T}) = sizeof(T)*8 - significand_bits(T) - 1 -exponent_bias{T<:AbstractFloat}(::Type{T}) = Int(exponent_one(T) >> significand_bits(T)) +@pure significand_bits{T<:AbstractFloat}(::Type{T}) = trailing_ones(significand_mask(T)) +@pure exponent_bits{T<:AbstractFloat}(::Type{T}) = sizeof(T)*8 - significand_bits(T) - 1 +@pure exponent_bias{T<:AbstractFloat}(::Type{T}) = Int(exponent_one(T) >> significand_bits(T)) ## Array operations on floating point numbers ## From a1dbcf6587f84dcaf9a999ed51d7e2f928b52b80 Mon Sep 17 00:00:00 2001 From: Lyndon White <lyndon.white@research.uwa.edu.au> Date: Sat, 16 Jul 2016 19:09:53 +1000 Subject: [PATCH 0479/1117] =tests for exp2(int) --- test/math.jl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/math.jl b/test/math.jl index 33902962b3999..a896c57d1bea7 100644 --- a/test/math.jl +++ b/test/math.jl @@ -192,6 +192,25 @@ end @test exp2(Float16(2.)) ≈ exp2(2.) @test log(e) == 1 +@testset "check exp2(::Int) matches exp2(::Float)" begin + for ii in -2048:2048 + expected = exp2(float(ii)) + @test(exp2(Int16(ii)) == expected) + @test(exp2(Int32(ii)) == expected) + @test(exp2(Int64(ii)) == expected) + @test(exp2(Int128(ii)) == expected) + if ii>=0 + @test(exp2(UInt16(ii)) == expected) + @test(exp2(UInt32(ii)) == expected) + @test(exp2(UInt64(ii)) == expected) + @test(exp2(UInt128(ii)) == expected) + end + end + + @test(exp2(false) == exp2(float(false))) + @test(exp2(true) == exp2(float(true))) +end + for T in (Int, Float64, BigFloat) @test deg2rad(T(180)) ≈ 1pi @test deg2rad(T[45, 60]) ≈ [pi/T(4), pi/T(3)] From b4b542bd95c03213aadf340ad36533dcc92bafb2 Mon Sep 17 00:00:00 2001 From: Lyndon White <lyndon.white@research.uwa.edu.au> Date: Sat, 16 Jul 2016 19:29:31 +1000 Subject: [PATCH 0480/1117] =Fast exp2(::Int) definition added --- base/math.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/base/math.jl b/base/math.jl index 66b9717e80efc..efe734554b766 100644 --- a/base/math.jl +++ b/base/math.jl @@ -124,6 +124,23 @@ for f in (:sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :expm1) @eval ($f)(x::AbstractFloat) = error("not implemented for ", typeof(x)) end +#functions with special cases for integer arguements +@inline function exp2(x::Integer) + if x > 1023 + Inf64 + elseif x < -1074 + Float64(0.0) + elseif x <= -1023 + #Result will be a subnormal number + reinterpret(Float64, Int64(1) << (x + 1074)) + else + #If x is a Int128, and is outside the range of Int64, then it is not -123<x<=1023 + #We will cast everything to Int64 to avoid errors incase of Int128 + reinterpret(Float64, (exponent_bias(Float64) + Int64(x)) << significand_bits(Float64)) + end +end + + # TODO: GNU libc has exp10 as an extension; should openlibm? exp10(x::Float64) = 10.0^x exp10(x::Float32) = 10.0f0^x From 021bae7059e47b97b0685747f6c177d521648aff Mon Sep 17 00:00:00 2001 From: Lyndon White <lyndon.white@research.uwa.edu.au> Date: Sat, 16 Jul 2016 20:03:58 +1000 Subject: [PATCH 0481/1117] =No Testsets --- test/math.jl | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/test/math.jl b/test/math.jl index a896c57d1bea7..75e6978bdabd1 100644 --- a/test/math.jl +++ b/test/math.jl @@ -192,25 +192,24 @@ end @test exp2(Float16(2.)) ≈ exp2(2.) @test log(e) == 1 -@testset "check exp2(::Int) matches exp2(::Float)" begin - for ii in -2048:2048 - expected = exp2(float(ii)) - @test(exp2(Int16(ii)) == expected) - @test(exp2(Int32(ii)) == expected) - @test(exp2(Int64(ii)) == expected) - @test(exp2(Int128(ii)) == expected) - if ii>=0 - @test(exp2(UInt16(ii)) == expected) - @test(exp2(UInt32(ii)) == expected) - @test(exp2(UInt64(ii)) == expected) - @test(exp2(UInt128(ii)) == expected) - end +# check exp2(::Integer) matches exp2(::Float)" +for ii in -2048:2048 + expected = exp2(float(ii)) + @test(exp2(Int16(ii)) == expected) + @test(exp2(Int32(ii)) == expected) + @test(exp2(Int64(ii)) == expected) + @test(exp2(Int128(ii)) == expected) + if ii>=0 + @test(exp2(UInt16(ii)) == expected) + @test(exp2(UInt32(ii)) == expected) + @test(exp2(UInt64(ii)) == expected) + @test(exp2(UInt128(ii)) == expected) end - - @test(exp2(false) == exp2(float(false))) - @test(exp2(true) == exp2(float(true))) end +@test(exp2(false) == exp2(float(false))) +@test(exp2(true) == exp2(float(true))) + for T in (Int, Float64, BigFloat) @test deg2rad(T(180)) ≈ 1pi @test deg2rad(T[45, 60]) ≈ [pi/T(4), pi/T(3)] From b6d6258faa1d32d47d1305724cadd98ea46d9015 Mon Sep 17 00:00:00 2001 From: Lyndon White <lyndon.white@research.uwa.edu.au> Date: Sat, 16 Jul 2016 20:08:38 +1000 Subject: [PATCH 0482/1117] =retab --- test/math.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/math.jl b/test/math.jl index 75e6978bdabd1..fc3f94b2b90a5 100644 --- a/test/math.jl +++ b/test/math.jl @@ -194,17 +194,17 @@ end # check exp2(::Integer) matches exp2(::Float)" for ii in -2048:2048 - expected = exp2(float(ii)) - @test(exp2(Int16(ii)) == expected) - @test(exp2(Int32(ii)) == expected) - @test(exp2(Int64(ii)) == expected) - @test(exp2(Int128(ii)) == expected) - if ii>=0 - @test(exp2(UInt16(ii)) == expected) - @test(exp2(UInt32(ii)) == expected) - @test(exp2(UInt64(ii)) == expected) - @test(exp2(UInt128(ii)) == expected) - end + expected = exp2(float(ii)) + @test(exp2(Int16(ii)) == expected) + @test(exp2(Int32(ii)) == expected) + @test(exp2(Int64(ii)) == expected) + @test(exp2(Int128(ii)) == expected) + if ii>=0 + @test(exp2(UInt16(ii)) == expected) + @test(exp2(UInt32(ii)) == expected) + @test(exp2(UInt64(ii)) == expected) + @test(exp2(UInt128(ii)) == expected) + end end @test(exp2(false) == exp2(float(false))) From 2626da1d919e0aa76f8d71b35807f59dbcf0433a Mon Sep 17 00:00:00 2001 From: Lyndon White <lyndon.white@research.uwa.edu.au> Date: Sat, 16 Jul 2016 20:27:59 +1000 Subject: [PATCH 0483/1117] =Retab. --- base/math.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/base/math.jl b/base/math.jl index efe734554b766..ec0f226b2a9a5 100644 --- a/base/math.jl +++ b/base/math.jl @@ -126,7 +126,7 @@ end #functions with special cases for integer arguements @inline function exp2(x::Integer) - if x > 1023 + if x > 1023 Inf64 elseif x < -1074 Float64(0.0) @@ -140,7 +140,6 @@ end end end - # TODO: GNU libc has exp10 as an extension; should openlibm? exp10(x::Float64) = 10.0^x exp10(x::Float32) = 10.0f0^x From b7b7e7ae9d33ceeb2825b5b7a0b1182bba52de33 Mon Sep 17 00:00:00 2001 From: Lyndon White <lyndon.white@research.uwa.edu.au> Date: Sat, 16 Jul 2016 20:42:11 +1000 Subject: [PATCH 0484/1117] =typos in comments --- base/math.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/math.jl b/base/math.jl index ec0f226b2a9a5..8997fdbc35333 100644 --- a/base/math.jl +++ b/base/math.jl @@ -124,18 +124,18 @@ for f in (:sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :expm1) @eval ($f)(x::AbstractFloat) = error("not implemented for ", typeof(x)) end -#functions with special cases for integer arguements +# functions with special cases for integer arguements @inline function exp2(x::Integer) if x > 1023 Inf64 elseif x < -1074 Float64(0.0) elseif x <= -1023 - #Result will be a subnormal number + # Result will be a subnormal number reinterpret(Float64, Int64(1) << (x + 1074)) else - #If x is a Int128, and is outside the range of Int64, then it is not -123<x<=1023 - #We will cast everything to Int64 to avoid errors incase of Int128 + # If x is a Int128, and is outside the range of Int64, then it is not -1023<x<=1023 + # We will cast everything to Int64 to avoid errors in case of Int128 reinterpret(Float64, (exponent_bias(Float64) + Int64(x)) << significand_bits(Float64)) end end From 24fde490e941ab9cebc8782aa77652b79d51e491 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Sat, 16 Jul 2016 11:57:56 +0100 Subject: [PATCH 0485/1117] call utf8proc_category rather than utf8proc_get_property --- src/flisp/julia_extensions.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/flisp/julia_extensions.c b/src/flisp/julia_extensions.c index 61746cc39c0ba..f43fbd7479ec2 100644 --- a/src/flisp/julia_extensions.c +++ b/src/flisp/julia_extensions.c @@ -51,7 +51,7 @@ value_t fl_skipws(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) return skipped; } -static int is_wc_cat_id_start(uint32_t wc, utf8proc_propval_t cat) +static int is_wc_cat_id_start(uint32_t wc, utf8proc_category_t cat) { return (cat == UTF8PROC_CATEGORY_LU || cat == UTF8PROC_CATEGORY_LL || cat == UTF8PROC_CATEGORY_LT || cat == UTF8PROC_CATEGORY_LM || @@ -110,8 +110,7 @@ JL_DLLEXPORT int jl_id_start_char(uint32_t wc) return 1; if (wc < 0xA1 || wc > 0x10ffff) return 0; - const utf8proc_property_t *prop = utf8proc_get_property(wc); - return is_wc_cat_id_start(wc, prop->category); + return is_wc_cat_id_start(wc, utf8proc_category((utf8proc_int32_t) wc)); } JL_DLLEXPORT int jl_id_char(uint32_t wc) @@ -121,8 +120,7 @@ JL_DLLEXPORT int jl_id_char(uint32_t wc) return 1; if (wc < 0xA1 || wc > 0x10ffff) return 0; - const utf8proc_property_t *prop = utf8proc_get_property(wc); - utf8proc_propval_t cat = prop->category; + utf8proc_category_t cat = utf8proc_category((utf8proc_int32_t) wc); if (is_wc_cat_id_start(wc, cat)) return 1; if (cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_MC || cat == UTF8PROC_CATEGORY_ND || cat == UTF8PROC_CATEGORY_PC || From 1b359e78baf3fbefe7ff717ac7cb637989a99a48 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Fri, 15 Jul 2016 20:46:50 -0400 Subject: [PATCH 0486/1117] Fix Dict key completion for qualified names. --- base/REPLCompletions.jl | 22 +++++--- test/replcompletions.jl | 118 +++++++++++++++++++++------------------- 2 files changed, 76 insertions(+), 64 deletions(-) diff --git a/base/REPLCompletions.jl b/base/REPLCompletions.jl index 473ab5ad28750..ff87762b6c1fb 100644 --- a/base/REPLCompletions.jl +++ b/base/REPLCompletions.jl @@ -397,14 +397,20 @@ function dict_identifier_key(str,tag) frange, end_of_indentifier = find_start_brace(str_close, c_start='[', c_end=']') isempty(frange) && return (nothing, nothing, nothing) - identifier = Symbol(str[frange[1]:end_of_indentifier]) - isdefined(Main,identifier) || return (nothing, nothing, nothing) + obj = Main + for name in split(str[frange[1]:end_of_indentifier], '.') + Base.isidentifier(name) || return (nothing, nothing, nothing) + sym = Symbol(name) + isdefined(obj, sym) || return (nothing, nothing, nothing) + obj = getfield(obj, sym) + # Avoid `isdefined(::Array, ::Symbol)` + isa(obj, Array) && return (nothing, nothing, nothing) + end begin_of_key = findnext(x->!in(x,whitespace_chars), str, end_of_indentifier+2) - begin_of_key==0 && return (identifier, nothing, nothing) + begin_of_key==0 && return (true, nothing, nothing) partial_key = str[begin_of_key:end] - main_id = getfield(Main,identifier) - typeof(main_id) <: Associative && length(main_id)<1e6 || return (identifier, nothing, nothing) - main_id, partial_key, begin_of_key + (isa(obj, Associative) && length(obj) < 1e6) || return (true, nothing, nothing) + return (obj, partial_key, begin_of_key) end function completions(string, pos) @@ -416,8 +422,8 @@ function completions(string, pos) # if completing a key in a Dict identifier, partial_key, loc = dict_identifier_key(partial,inc_tag) - if identifier != nothing - if partial_key != nothing + if identifier !== nothing + if partial_key !== nothing matches = [] for key in keys(identifier) rkey = repr(key) diff --git a/test/replcompletions.jl b/test/replcompletions.jl index 8abd0c35a01f1..89b59ed7855db 100644 --- a/test/replcompletions.jl +++ b/test/replcompletions.jl @@ -51,7 +51,11 @@ module CompletionFoo const tuple = (1, 2) test_y_array=[CompletionFoo.Test_y(rand()) for i in 1:10] + test_dict = Dict("abc"=>1, "abcd"=>10, :bar=>2, :bar2=>9, Base=>3, + contains=>4, `ls`=>5, 66=>7, 67=>8, ("q",3)=>11, + "α"=>12, :α=>13) end +test_repl_comp_dict = CompletionFoo.test_dict function temp_pkg_dir(fn::Function) # Used in tests below to setup and teardown a sandboxed package directory @@ -663,59 +667,61 @@ c, r, res = test_complete(s) @test isempty(c) # test Dicts -test_dict = Dict("abc"=>1, "abcd"=>10, :bar=>2, :bar2=>9, Base=>3, contains=>4, `ls`=>5, - 66=>7, 67=>8, ("q",3)=>11, "α"=>12, :α=>13) -s="test_dict[\"ab" -c,r = test_complete(s) -@test c == Any["\"abc\"","\"abcd\""] -s="test_dict[\"abcd" -c,r = test_complete(s) -@test c == Any["\"abcd\"]"] -s="test_dict[ \"abcd" # leading whitespace -c,r = test_complete(s) -@test c == Any["\"abcd\"]"] -s="test_dict[\"abcd]" # trailing close bracket -c,r = completions(s,endof(s)-1) -@test c == Any["\"abcd\""] -s="test_dict[:b" -c,r = test_complete(s) -@test c == Any[":bar",":bar2"] -s="test_dict[:bar2" -c,r = test_complete(s) -@test c == Any[":bar2]"] -s="test_dict[Ba" -c,r = test_complete(s) -@test c == Any["Base]"] -s="test_dict[co" -c,r = test_complete(s) -@test c == Any["contains]"] -s="test_dict[`l" -c,r = test_complete(s) -@test c == Any["`ls`]"] -s="test_dict[6" -c,r = test_complete(s) -@test c == Any["66","67"] -s="test_dict[66" -c,r = test_complete(s) -@test c == Any["66]"] -s="test_dict[(" -c,r = test_complete(s) -@test c == Any["(\"q\",3)]"] -s="test_dict[\"\\alp" -c,r = test_complete(s) -@test c == String["\\alpha"] -s="test_dict[\"\\alpha" -c,r = test_complete(s) -@test c == String["α"] -s="test_dict[\"α" -c,r = test_complete(s) -@test c == Any["\"α\"]"] -s="test_dict[:\\alp" -c,r = test_complete(s) -@test c == String["\\alpha"] -s="test_dict[:\\alpha" -c,r = test_complete(s) -@test c == String["α"] -s="test_dict[:α" -c,r = test_complete(s) -@test c == Any[":α]"] +function test_dict_completion(dict_name) + s = "$dict_name[\"ab" + c, r = test_complete(s) + @test c == Any["\"abc\"", "\"abcd\""] + s = "$dict_name[\"abcd" + c, r = test_complete(s) + @test c == Any["\"abcd\"]"] + s = "$dict_name[ \"abcd" # leading whitespace + c, r = test_complete(s) + @test c == Any["\"abcd\"]"] + s = "$dict_name[\"abcd]" # trailing close bracket + c, r = completions(s, endof(s) - 1) + @test c == Any["\"abcd\""] + s = "$dict_name[:b" + c, r = test_complete(s) + @test c == Any[":bar", ":bar2"] + s = "$dict_name[:bar2" + c, r = test_complete(s) + @test c == Any[":bar2]"] + s = "$dict_name[Ba" + c, r = test_complete(s) + @test c == Any["Base]"] + s = "$dict_name[co" + c, r = test_complete(s) + @test c == Any["contains]"] + s = "$dict_name[`l" + c, r = test_complete(s) + @test c == Any["`ls`]"] + s = "$dict_name[6" + c, r = test_complete(s) + @test c == Any["66", "67"] + s = "$dict_name[66" + c, r = test_complete(s) + @test c == Any["66]"] + s = "$dict_name[(" + c, r = test_complete(s) + @test c == Any["(\"q\",3)]"] + s = "$dict_name[\"\\alp" + c, r = test_complete(s) + @test c == String["\\alpha"] + s = "$dict_name[\"\\alpha" + c, r = test_complete(s) + @test c == String["α"] + s = "$dict_name[\"α" + c, r = test_complete(s) + @test c == Any["\"α\"]"] + s = "$dict_name[:\\alp" + c, r = test_complete(s) + @test c == String["\\alpha"] + s = "$dict_name[:\\alpha" + c, r = test_complete(s) + @test c == String["α"] + s = "$dict_name[:α" + c, r = test_complete(s) + @test c == Any[":α]"] +end +test_dict_completion("CompletionFoo.test_dict") +test_dict_completion("test_repl_comp_dict") From 9e1abe1bed1ef216eae3ea613222153cb1a6a3b3 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 16 Jul 2016 10:21:50 -0400 Subject: [PATCH 0487/1117] Fix showerror for keyword arguments error when the function is not in Main or Base --- base/reflection.jl | 2 +- base/replutil.jl | 2 +- test/replutil.jl | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index 36b8ad34a2314..c81471b0801ef 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -160,7 +160,7 @@ function _subtypes(m::Module, x::DataType, sts=Set(), visited=Set()) push!(visited, m) for s in names(m,true) if isdefined(m,s) - t = eval(m,s) + t = getfield(m, s) if isa(t, DataType) && t.name.name == s && supertype(t).name == x.name ti = typeintersect(t, x) ti != Bottom && push!(sts, ti) diff --git a/base/replutil.jl b/base/replutil.jl index edbc2863e53a9..478b8cfe4c3e8 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -267,7 +267,7 @@ function showerror(io::IO, ex::MethodError) name = ft.name.mt.name f_is_function = false kwargs = Any[] - if startswith(string(ft.name), "#kw#") + if startswith(string(ft.name.name), "#kw#") f = ex.args[2] ft = typeof(f) name = ft.name.mt.name diff --git a/test/replutil.jl b/test/replutil.jl index c0a34b29df5e9..e44ef80ddee1b 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -95,14 +95,31 @@ m_error = try method_c6(1, x=1) catch e; e; end showerror(buf, m_error) error_out1 = takebuf_string(buf) +module TestKWError +method_c6_in_module(; x=1) = x +method_c6_in_module(a; y=1) = y +end +m_error = try TestKWError.method_c6_in_module(y=1) catch e; e; end +showerror(buf, m_error) +error_out2 = takebuf_string(buf) +m_error = try TestKWError.method_c6_in_module(1, x=1) catch e; e; end +showerror(buf, m_error) +error_out3 = takebuf_string(buf) + if Base.have_color @test contains(error_out, "method_c6(; x)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") @test contains(error_out, "method_c6(\e[1m\e[31m::Any\e[0m; y)") @test contains(error_out1, "method_c6(::Any; y)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") + @test contains(error_out2, "method_c6_in_module(; x)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") + @test contains(error_out2, "method_c6_in_module(\e[1m\e[31m::Any\e[0m; y)") + @test contains(error_out3, "method_c6_in_module(::Any; y)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") else @test contains(error_out, "method_c6(; x) got an unsupported keyword argument \"y\"") @test contains(error_out, "method_c6(!Matched::Any; y)") @test contains(error_out1, "method_c6(::Any; y) got an unsupported keyword argument \"x\"") + @test contains(error_out2, "method_c6_in_module(; x) got an unsupported keyword argument \"y\"") + @test contains(error_out2, "method_c6_in_module(!Matched::Any; y)") + @test contains(error_out3, "method_c6_in_module(::Any; y) got an unsupported keyword argument \"x\"") end method_c7(a, b; kargs...) = a From ad6330074bede3322738911b47195ce890e619f1 Mon Sep 17 00:00:00 2001 From: Lyndon White <lyndon.white@research.uwa.edu.au> Date: Sun, 17 Jul 2016 03:09:15 +1000 Subject: [PATCH 0488/1117] =BigInt math returns BigFloat --- base/math.jl | 4 ++-- test/bigint.jl | 19 +++++++++++++++++++ test/math.jl | 4 +--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/base/math.jl b/base/math.jl index 8997fdbc35333..a26f9ac853863 100644 --- a/base/math.jl +++ b/base/math.jl @@ -124,8 +124,8 @@ for f in (:sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :expm1) @eval ($f)(x::AbstractFloat) = error("not implemented for ", typeof(x)) end -# functions with special cases for integer arguements -@inline function exp2(x::Integer) +# functions with special cases for integer arguments +@inline function exp2(x::Base.BitInteger) if x > 1023 Inf64 elseif x < -1074 diff --git a/test/bigint.jl b/test/bigint.jl index 53cb5a9ccc7e2..9fd05362dfcb6 100644 --- a/test/bigint.jl +++ b/test/bigint.jl @@ -311,3 +311,22 @@ ndigits(rand(big(-999:999)), big(2)^rand(2:999)) @test big(5)^true == big(5) @test big(5)^false == one(BigInt) + + +# operations that when applied to Int64 give Float64, should give BigFloat +@test typeof(exp(a)) == BigFloat +@test typeof(exp2(a)) == BigFloat +@test typeof(exp10(a)) == BigFloat +@test typeof(expm1(a)) == BigFloat +@test typeof(erf(a)) == BigFloat +@test typeof(erfc(a)) == BigFloat +@test typeof(cosh(a)) == BigFloat +@test typeof(sinh(a)) == BigFloat +@test typeof(tanh(a)) == BigFloat +@test typeof(sech(a)) == BigFloat +@test typeof(csch(a)) == BigFloat +@test typeof(coth(a)) == BigFloat +@test typeof(cbrt(a)) == BigFloat +@test typeof(tan(a)) == BigFloat +@test typeof(cos(a)) == BigFloat +@test typeof(sin(a)) == BigFloat diff --git a/test/math.jl b/test/math.jl index fc3f94b2b90a5..a46c31702d078 100644 --- a/test/math.jl +++ b/test/math.jl @@ -199,7 +199,7 @@ for ii in -2048:2048 @test(exp2(Int32(ii)) == expected) @test(exp2(Int64(ii)) == expected) @test(exp2(Int128(ii)) == expected) - if ii>=0 + if ii >= 0 @test(exp2(UInt16(ii)) == expected) @test(exp2(UInt32(ii)) == expected) @test(exp2(UInt64(ii)) == expected) @@ -207,8 +207,6 @@ for ii in -2048:2048 end end -@test(exp2(false) == exp2(float(false))) -@test(exp2(true) == exp2(float(true))) for T in (Int, Float64, BigFloat) @test deg2rad(T(180)) ≈ 1pi From 21b745502f4680df00f4ea2f4f9dd5659aa386ed Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 16 Jul 2016 20:58:28 -0700 Subject: [PATCH 0489/1117] save time on appveyor by using libgit2 and deps from nightly rather than recompiling it and libssh2 and mbedtls every time now that the nightlies include it --- contrib/windows/msys_build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 2c31c77473dff..41258a21f414d 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -173,9 +173,9 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user # libuv since its static lib is no longer included in the binaries # openlibm since we need it as a static library to work properly # utf8proc since its headers are not in the binary download -echo 'override STAGE1_DEPS = libuv mbedtls' >> Make.user -echo 'override STAGE2_DEPS = utf8proc libssh2' >> Make.user -echo 'override STAGE3_DEPS = libgit2' >> Make.user +echo 'override STAGE1_DEPS = libuv' >> Make.user +echo 'override STAGE2_DEPS = utf8proc' >> Make.user +echo 'override STAGE3_DEPS = ' >> Make.user if [ -n "$USEMSVC" ]; then # Openlibm doesn't build well with MSVC right now From 7b3457e9d0c109d725a4bca82ede9bd49d48da09 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 16 Jul 2016 21:23:07 -0700 Subject: [PATCH 0490/1117] temporarily distclean libgit2 and deps on travis until we can fix caching / patch application [av skip] --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 94edf04dbd460..d5f64bf36e2bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,6 +95,7 @@ before_install: script: - make -C moreutils mispipe - make $BUILDOPTS -C base version_git.jl.phony + - make $BUILDOPTS NO_GIT=1 -C deps distclean-libgit2 distclean-libssh2 distclean-mbedtls - moreutils/mispipe "make $BUILDOPTS NO_GIT=1 -C deps" bar > deps.log || cat deps.log - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia install | moreutils/ts -s "%.s" - make $BUILDOPTS NO_GIT=1 build-stats From 318853af2f2a066f62a0dcba6b9f22eb7f6be349 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Thu, 14 Jul 2016 22:03:43 -0400 Subject: [PATCH 0491/1117] Avoid that cor(x,x) != 1 by trading a sqrt for two / and use clamp to avoid that cor(x,y) > 1. Make sure that the loop vectorizes Convert corzm into corm to avoid allocation of tempraries. --- base/statistics.jl | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/base/statistics.jl b/base/statistics.jl index 764a8fa70a394..c7516f750ee00 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -368,25 +368,6 @@ function corzm(x::AbstractMatrix, vardim::Int=1) c = unscaled_covzm(x, vardim) return cov2cor!(c, sqrt!(diag(c))) end -function corzm(x::AbstractVector, y::AbstractVector) - n = length(x) - length(y) == n || throw(DimensionMismatch("inconsistent lengths")) - x1 = x[1] - y1 = y[1] - xx = abs2(x1) - yy = abs2(y1) - xy = x1 * conj(y1) - i = 1 - while i < n - i += 1 - @inbounds xi = x[i] - @inbounds yi = y[i] - xx += abs2(xi) - yy += abs2(yi) - xy += xi * conj(yi) - end - return xy / (sqrt(xx) * sqrt(yy)) -end corzm(x::AbstractVector, y::AbstractMatrix, vardim::Int=1) = cov2cor!(unscaled_covzm(x, y, vardim), sqrt(sumabs2(x)), sqrt!(sumabs2(y, vardim))) corzm(x::AbstractMatrix, y::AbstractVector, vardim::Int=1) = @@ -398,7 +379,27 @@ corzm(x::AbstractMatrix, y::AbstractMatrix, vardim::Int=1) = corm{T}(x::AbstractVector{T}, xmean) = one(real(T)) corm(x::AbstractMatrix, xmean, vardim::Int=1) = corzm(x .- xmean, vardim) -corm(x::AbstractVector, xmean, y::AbstractVector, ymean) = corzm(x .- xmean, y .- ymean) +function corm(x::AbstractVector, mx::Number, y::AbstractVector, my::Number) + n = length(x) + length(y) == n || throw(DimensionMismatch("inconsistent lengths")) + n > 0 || throw(ArgumentError("correlation only defined for non-empty vectors")) + + @inbounds begin + # Initialize the accumulators + xx = zero(sqrt(x[1] * x[1])) + yy = zero(sqrt(y[1] * y[1])) + xy = zero(xx * yy) + + @simd for i = 1:n + xi = x[i] - mx + yi = y[i] - my + xx += abs2(xi) + yy += abs2(yi) + xy += xi * yi' + end + end + return clamp(xy / max(xx, yy) / sqrt(min(xx, yy) / max(xx, yy)), -1, 1) +end corm(x::AbstractVecOrMat, xmean, y::AbstractVecOrMat, ymean, vardim::Int=1) = corzm(x .- xmean, y .- ymean, vardim) From d90f035ec594d81be7fdd3312f955957915b08f4 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 16 Jul 2016 07:38:41 -0400 Subject: [PATCH 0492/1117] Fix arguments evaluation order in callsite devitalization. Fix #17449 --- base/inference.jl | 8 +++++--- test/core.jl | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index bb11810f68621..c5c497419b3fb 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2360,13 +2360,15 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference local linfo_var = add_slot!(enclosing, LambdaInfo, false) local ex = copy(e) local stmts = [] - for i = 1:length(atypes); local i + local arg_hoisted = false + for i = length(atypes):-1:1; local i local ti = atypes[i] - if isa(ti, Union) + if arg_hoisted || isa(ti, Union) aei = ex.args[i] if !effect_free(aei, sv, false) + arg_hoisted = true newvar = newvar!(sv, ti) - push!(stmts, Expr(:(=), newvar, aei)) + insert!(stmts, 1, Expr(:(=), newvar, aei)) ex.args[i] = newvar end end diff --git a/test/core.jl b/test/core.jl index d98db8c25826f..1e153aba58154 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4462,3 +4462,20 @@ end f17147(::Tuple) = 1 f17147{N}(::Vararg{Tuple,N}) = 2 @test f17147((), ()) == 2 + +# issue #17449, argument evaluation order +@noinline f17449(x, y) = nothing +@noinline function g17449(r) + r[] = :g + return 1 +end +@noinline function k17449(r, v) + r[] = :k + return v ? 1 : 1.0 +end +function h17449(v) + r = Ref(:h) + f17449(g17449(r), k17449(r, v)) + return r[] +end +@test h17449(true) === :k From ef37ad141e948eb033ea29666a726b8ca67bd7be Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Sun, 17 Jul 2016 17:57:03 -0400 Subject: [PATCH 0493/1117] Add appropriate install_name_tool invocations to libgit2+dependencies Fixes #17460 --- deps/libgit2.mk | 3 +-- deps/libssh2.mk | 2 +- deps/mbedtls.mk | 9 +++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index f677aad788837..6cf21f6137494 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -19,8 +19,6 @@ else LIBGIT2_OPTS += -DBUILD_CLAR=OFF -DDLLTOOL=`which $(CROSS_COMPILE)dlltool` LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY endif -else -LIBGIT2_OPTS += -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE endif $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(LIBSSH2_OBJ_TARGET) @@ -49,6 +47,7 @@ ifeq ($(BUILD_OS),WINNT) else $(call make-install,$(LIBGIT2_SRC_DIR),) endif + $(INSTALL_NAME_CMD)libgit2.$(SHLIB_EXT) $@ ifeq ($(OS),Linux) @# If we're on linux, copy over libssl and libcrypto for libgit2 -LIBGIT_LIBS=$$(ldd "$@" | tail -n +2 | awk '{print $$(NF-1)}'); \ diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 7306e0dd7afa8..e9f002a140439 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -8,7 +8,6 @@ LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT) LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT) LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \ - -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib ifeq ($(OS),WINNT) @@ -43,6 +42,7 @@ ifeq ($(BUILD_OS),WINNT) else $(call make-install,$(LIBSSH2_SRC_DIR),) endif + $(INSTALL_NAME_CMD)libssh2.$(SHLIB_EXT) $@ touch -c $(LIBSSH2_OBJ_TARGET) clean-libssh2: diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 0d74657122128..af3f90e3f34e2 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -11,8 +11,7 @@ MBEDTLS_OBJ_SOURCE := $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedcrypto.$ MBEDTLS_OBJ_TARGET := $(build_shlibdir)/libmbedcrypto.$(SHLIB_EXT) MBEDTLS_OPTS := $(CMAKE_COMMON) -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ - -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE + -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release ifeq ($(OS),WINNT) MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=OFF @@ -55,6 +54,12 @@ ifeq ($(OS), WINNT) else $(call make-install,mbedtls-$(MBEDTLS_VER),) endif + $(INSTALL_NAME_CMD)libmbedx509.$(SHLIB_EXT) $(build_shlibdir)/libmbedx509.$(SHLIB_EXT) + $(INSTALL_NAME_CMD)libmbedtls.$(SHLIB_EXT) $(build_shlibdir)/libmbedtls.$(SHLIB_EXT) + $(INSTALL_NAME_CHANGE_CMD) libmbedx509.0.dylib @rpath/libmbedx509.$(SHLIB_EXT) $(build_shlibdir)/libmbedtls.$(SHLIB_EXT) + $(INSTALL_NAME_CHANGE_CMD) libmbedcrypto.0.dylib @rpath/libmbedcrypto.$(SHLIB_EXT) $(build_shlibdir)/libmbedtls.$(SHLIB_EXT) + $(INSTALL_NAME_CHANGE_CMD) libmbedcrypto.0.dylib @rpath/libmbedcrypto.$(SHLIB_EXT) $(build_shlibdir)/libmbedx509.$(SHLIB_EXT) + $(INSTALL_NAME_CMD)libmbedcrypto.$(SHLIB_EXT) $@ touch -c $(MBEDTLS_OBJ_TARGET) clean-mbedtls: From 6db58c7933f12dba462e6e6b7cff4ebbddbff1ff Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sun, 17 Jul 2016 18:03:31 -0400 Subject: [PATCH 0494/1117] Fix coverage for the first statement in the function Fixes #17442 --- src/codegen.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index c83e02d6e6d10..a794ab72b7a7a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4643,6 +4643,8 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func if (ctx.debug_enabled) // set initial line number builder.SetCurrentDebugLocation(topdebugloc); + if (do_coverage) + coverageVisitLine(ctx.file, toplineno); bool prevlabel = false; int lno = -1; int prevlno = -1; From 8a753e521a6469e9f26e54142eed4d7630b9c782 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Sun, 17 Jul 2016 18:53:31 -0400 Subject: [PATCH 0495/1117] Correctly set RPATH on Linux --- deps/libgit2.mk | 4 ++++ deps/libssh2.mk | 4 ++++ deps/mbedtls.mk | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 6cf21f6137494..0f5a3bd10e7d6 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -21,6 +21,10 @@ LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MO endif endif +ifeq ($(OS),Linux) +LIBGIT2_OPTS += -DCMAKE_INSTALL_RPATH="\$$ORIGIN" +endif + $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(LIBSSH2_OBJ_TARGET) ifeq ($(OS),Linux) -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch diff --git a/deps/libssh2.mk b/deps/libssh2.mk index e9f002a140439..e322b6bbe17ce 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -19,6 +19,10 @@ else LIBSSH2_OPTS += -DCRYPTO_BACKEND=mbedTLS -DENABLE_ZLIB_COMPRESSION=ON endif +ifeq ($(OS),Linux) +LIBSSH2_OPTS += -DCMAKE_INSTALL_RPATH="\$$ORIGIN" +endif + $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(MBEDTLS_OBJ_TARGET) -cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-mbedtls.patch mkdir -p $(dir $@) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index af3f90e3f34e2..eb6934f49901b 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -22,6 +22,10 @@ else MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=ON endif +ifeq ($(OS),Linux) +MBEDTLS_OPTS += -DCMAKE_INSTALL_RPATH="\$$ORIGIN" +endif + $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ $(MBEDTLS_URL) From 6e6adb778944e2f8f9652d1316e383e02d39a43a Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Sat, 16 Jul 2016 17:14:03 -0400 Subject: [PATCH 0496/1117] Include lib/julia in libjulia RPATH --- Make.inc | 6 ++++-- src/Makefile | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Make.inc b/Make.inc index f13ad5e4d75f4..c8d5beaa09532 100644 --- a/Make.inc +++ b/Make.inc @@ -842,11 +842,13 @@ ifeq ($(OS), WINNT) RPATH := RPATH_ORIGIN := else ifeq ($(OS), Darwin) - RPATH := -Wl,-rpath,'@executable_path/$(build_private_libdir_rel)' -Wl,-rpath,'@executable_path/$(build_libdir_rel)' + RPATH := -Wl,-rpath,'@executable_path/$(build_libdir_rel)' RPATH_ORIGIN := -Wl,-rpath,'@loader_path/' + RPATH_LIB := -Wl,-rpath,'@loader_path/' -Wl,-rpath,'@loader_path/julia/' else - RPATH := -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)' -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-z,origin + RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-z,origin RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin + RPATH_LIB := -Wl,-rpath,'$$ORIGIN' -Wl,-rpath,'$$ORIGIN/julia' -Wl,-z,origin endif # --whole-archive diff --git a/src/Makefile b/src/Makefile index 8e783ef471cc1..e1c18c8eeda04 100644 --- a/src/Makefile +++ b/src/Makefile @@ -213,7 +213,7 @@ else endif $(build_shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT): $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV) - @$(call PRINT_LINK, $(CXXLD) $(CXXFLAGS) $(CXXLDFLAGS) $(DEBUGFLAGS) $(DOBJS) $(RPATH_ORIGIN) -o $@ $(LDFLAGS) $(JLIBLDFLAGS) $(DEBUG_LIBS) $(SONAME_DEBUG)) + @$(call PRINT_LINK, $(CXXLD) $(CXXFLAGS) $(CXXLDFLAGS) $(DEBUGFLAGS) $(DOBJS) $(RPATH_LIB) -o $@ $(LDFLAGS) $(JLIBLDFLAGS) $(DEBUG_LIBS) $(SONAME_DEBUG)) $(INSTALL_NAME_CMD)libjulia-debug.$(SHLIB_EXT) $@ ifneq ($(OS), WINNT) @ln -sf libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia-debug.$(JL_MAJOR_SHLIB_EXT) @@ -226,7 +226,7 @@ $(BUILDDIR)/libjulia-debug.a: $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/ libjulia-debug: $(build_shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT): $(SRCDIR)/julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV) - @$(call PRINT_LINK, $(CXXLD) $(CXXFLAGS) $(CXXLDFLAGS) $(SHIPFLAGS) $(OBJS) $(RPATH_ORIGIN) -o $@ $(LDFLAGS) $(JLIBLDFLAGS) $(RELEASE_LIBS) $(SONAME)) + @$(call PRINT_LINK, $(CXXLD) $(CXXFLAGS) $(CXXLDFLAGS) $(SHIPFLAGS) $(OBJS) $(RPATH_LIB) -o $@ $(LDFLAGS) $(JLIBLDFLAGS) $(RELEASE_LIBS) $(SONAME)) $(INSTALL_NAME_CMD)libjulia.$(SHLIB_EXT) $@ ifneq ($(OS), WINNT) @ln -sf libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia.$(JL_MAJOR_SHLIB_EXT) From a65993080a3d1b08fc2447515b1c21f026353249 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 18 Jul 2016 02:58:25 -0400 Subject: [PATCH 0497/1117] Work around a cmake bug that was causing unnecessary reconfiguration see https://cmake.org/Bug/view.php?id=14366, the cache was getting deleted because it thought CMAKE_C_COMPILER was changing, causing issues with missing destdir paths when doing the install --- deps/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index f8044f7149955..08c3f96fc503e 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -50,7 +50,8 @@ CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH= ifneq ($(VERBOSE), 0) CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON endif -CMAKE_COMMON += -DCMAKE_C_COMPILER="$(CC_BASE)" +# The call to which here is to work around https://cmake.org/Bug/view.php?id=14366 +CMAKE_COMMON += -DCMAKE_C_COMPILER="$$(which $(CC_BASE))" ifneq ($(strip $(CMAKE_CC_ARG)),) CMAKE_COMMON += -DCMAKE_C_COMPILER_ARG1="$(CMAKE_CC_ARG)" endif @@ -62,7 +63,7 @@ endif ifeq ($(OS),WINNT) CMAKE_COMMON += -DCMAKE_SYSTEM_NAME=Windows ifneq ($(BUILD_OS),WINNT) -CMAKE_COMMON += -DCMAKE_RC_COMPILER=`which $(CROSS_COMPILE)windres` +CMAKE_COMMON += -DCMAKE_RC_COMPILER="$$(which $(CROSS_COMPILE)windres)" endif endif From 49446dff805c12c7901f872ccdf2ed4282594c56 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 18 Jul 2016 03:02:15 -0400 Subject: [PATCH 0498/1117] Use a .patch-applied file to make rebuilds more robust with libgit2, libssh2, and mbedtls - the dependencies are order-only '|' on CMakeLists.txt since applying the patches changes that file --- deps/libgit2.mk | 13 ++++++++++--- deps/libssh2.mk | 7 +++++-- deps/mbedtls.mk | 7 +++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 0f5a3bd10e7d6..75d27f195a376 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -25,11 +25,18 @@ ifeq ($(OS),Linux) LIBGIT2_OPTS += -DCMAKE_INSTALL_RPATH="\$$ORIGIN" endif -$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(LIBSSH2_OBJ_TARGET) +$(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied: | $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p0 -f < $(SRCDIR)/patches/libgit2-ssh.patch + echo 1 > $@ + +$(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied | $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch + echo 1 > $@ + ifeq ($(OS),Linux) - -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied endif - -cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p0 -f < $(SRCDIR)/patches/libgit2-ssh.patch +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(LIBSSH2_OBJ_TARGET) mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) diff --git a/deps/libssh2.mk b/deps/libssh2.mk index e322b6bbe17ce..41b1c82cd2eb2 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -23,8 +23,11 @@ ifeq ($(OS),Linux) LIBSSH2_OPTS += -DCMAKE_INSTALL_RPATH="\$$ORIGIN" endif -$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(MBEDTLS_OBJ_TARGET) - -cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-mbedtls.patch +$(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied: | $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-mbedtls.patch + echo 1 > $@ + +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied $(MBEDTLS_OBJ_TARGET) mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBSSH2_OPTS) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index eb6934f49901b..79c8513a44579 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -35,8 +35,11 @@ $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt: $(SRCDIR)/srccache/$(MBEDTLS_S $(TAR) -C $(dir $@) --strip-components 1 -xf $< touch -c $@ -$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt - -cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls-config.patch +$(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied: | $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls-config.patch + echo 1 > $@ + +$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(MBEDTLS_OPTS) From ba5dab1b2e95ba6a51a767f8a5c43e70f0db2587 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 18 Jul 2016 06:50:02 -0500 Subject: [PATCH 0499/1117] Short-circuit some calls to flatten for better --inline=no performance Ref https://github.com/JuliaLang/julia/commit/c68a2cd9919cdf4a23243b313670ad8dc41a9c27 --- base/multidimensional.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 199ef18cf5afc..9637892bd2877 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -25,7 +25,10 @@ CartesianIndex{N}(index::NTuple{N,Integer}) = CartesianIndex{N}(index) (::Type{CartesianIndex{N}}){N}() = CartesianIndex{N}(()) # Un-nest passed CartesianIndexes CartesianIndex(index::Union{Integer, CartesianIndex}...) = CartesianIndex(flatten(index)) -Base.@pure flatten(I) = (_flatten(I...)...,) +Base.@pure flatten(I::Tuple{}) = I +Base.@pure flatten(I::Tuple{Any}) = I +Base.@pure flatten{N}(I::Tuple{CartesianIndex{N}}) = I[1].I +Base.@pure flatten(I) = _flatten(I...) Base.@pure _flatten() = () Base.@pure _flatten(i, I...) = (i, _flatten(I...)...) Base.@pure _flatten(i::CartesianIndex, I...) = (i.I..., _flatten(I...)...) From d43a2b1930f0602bf5c662888678606b5103ce4d Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 18 Jul 2016 06:38:23 -0700 Subject: [PATCH 0500/1117] Update mbedtls to 2.3.0 --- deps/Versions.make | 2 +- deps/checksums/mbedtls-2.2.1-apache.tgz/md5 | 1 - deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 | 1 - deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 | 1 - deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 | 1 - deps/checksums/mbedtls-2.3.0-apache.tgz/md5 | 1 + deps/checksums/mbedtls-2.3.0-apache.tgz/sha512 | 1 + deps/checksums/mbedtls-2.3.0-gpl.tgz/md5 | 1 + deps/checksums/mbedtls-2.3.0-gpl.tgz/sha512 | 1 + 9 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 deps/checksums/mbedtls-2.2.1-apache.tgz/md5 delete mode 100644 deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 delete mode 100644 deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 delete mode 100644 deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 create mode 100644 deps/checksums/mbedtls-2.3.0-apache.tgz/md5 create mode 100644 deps/checksums/mbedtls-2.3.0-apache.tgz/sha512 create mode 100644 deps/checksums/mbedtls-2.3.0-gpl.tgz/md5 create mode 100644 deps/checksums/mbedtls-2.3.0-gpl.tgz/sha512 diff --git a/deps/Versions.make b/deps/Versions.make index 5a662941892c8..4d6c5fc751b79 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -12,4 +12,4 @@ GMP_VER = 6.1.0 MPFR_VER = 3.1.4 PATCHELF_VER = 0.9 VIRTUALENV_VER = 15.0.0 -MBEDTLS_VER = 2.2.1 +MBEDTLS_VER = 2.3.0 diff --git a/deps/checksums/mbedtls-2.2.1-apache.tgz/md5 b/deps/checksums/mbedtls-2.2.1-apache.tgz/md5 deleted file mode 100644 index 6a06d0ebc2b38..0000000000000 --- a/deps/checksums/mbedtls-2.2.1-apache.tgz/md5 +++ /dev/null @@ -1 +0,0 @@ -77751c0e370ed2ab01934e4c5e1d380f diff --git a/deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 b/deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 deleted file mode 100644 index f8b91b02ac065..0000000000000 --- a/deps/checksums/mbedtls-2.2.1-apache.tgz/sha512 +++ /dev/null @@ -1 +0,0 @@ -6a74abc4ea225eb6bcf20894bb1a6faa82dbaff11129c41849151e2654570609efeee70d0644ce63c4d2c11e6142b2db262b88f3a22fdceff0a215a64a5d6eb0 diff --git a/deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 b/deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 deleted file mode 100644 index 5b8fbecd5b1d7..0000000000000 --- a/deps/checksums/mbedtls-2.2.1-gpl.tgz/md5 +++ /dev/null @@ -1 +0,0 @@ -b6909d94600fc5f644e9bca52e96fb27 diff --git a/deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 b/deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 deleted file mode 100644 index 79f2f47dbb2d8..0000000000000 --- a/deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512 +++ /dev/null @@ -1 +0,0 @@ -03c32b46369d35e61acfeac83f417d09307d2f1e4358f9ea6af275d9a9a2319ca7a244733b5950a48d59f6305e4adcad6a66e589b42ee7417d1d4362c2595a4f diff --git a/deps/checksums/mbedtls-2.3.0-apache.tgz/md5 b/deps/checksums/mbedtls-2.3.0-apache.tgz/md5 new file mode 100644 index 0000000000000..0b072f92db44b --- /dev/null +++ b/deps/checksums/mbedtls-2.3.0-apache.tgz/md5 @@ -0,0 +1 @@ +4dab982481441ed0a0235bc06d40eb17 diff --git a/deps/checksums/mbedtls-2.3.0-apache.tgz/sha512 b/deps/checksums/mbedtls-2.3.0-apache.tgz/sha512 new file mode 100644 index 0000000000000..902ec8bb91d07 --- /dev/null +++ b/deps/checksums/mbedtls-2.3.0-apache.tgz/sha512 @@ -0,0 +1 @@ +96b55c0b159e5f2a0895670b2db693e52d2ee70198c4ff1f50c62d07a9f66e683b9862564059f2c0c955ebefefb98e02ad19c9df5ee18aebeff120bd15f43e27 diff --git a/deps/checksums/mbedtls-2.3.0-gpl.tgz/md5 b/deps/checksums/mbedtls-2.3.0-gpl.tgz/md5 new file mode 100644 index 0000000000000..665e79bab0357 --- /dev/null +++ b/deps/checksums/mbedtls-2.3.0-gpl.tgz/md5 @@ -0,0 +1 @@ +3f396d21b9c86c0a11ac9ba0b6e6f999 diff --git a/deps/checksums/mbedtls-2.3.0-gpl.tgz/sha512 b/deps/checksums/mbedtls-2.3.0-gpl.tgz/sha512 new file mode 100644 index 0000000000000..3b04779fbc308 --- /dev/null +++ b/deps/checksums/mbedtls-2.3.0-gpl.tgz/sha512 @@ -0,0 +1 @@ +f86ec87735b919e23ad7eb960a290013e7dfc2a5953d662ecee4eca7b990384c095aea74a313eaa0e1febb74e0c015879f4d680d51bc5eae811ab9e5063ba38a From ead3985001f3f5f89c4210b24b71075744ccbe5e Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Fri, 15 Jul 2016 13:19:51 -0500 Subject: [PATCH 0501/1117] Make `size` throw an error for arrays with non-1 indexing --- base/abstractarray.jl | 30 ++-- base/abstractarraymath.jl | 13 +- base/arraymath.jl | 11 +- base/bitarray.jl | 3 +- base/broadcast.jl | 88 +++++----- base/deprecated.jl | 14 +- base/multidimensional.jl | 125 ++++++++------- base/operators.jl | 36 +++-- base/permuteddimsarray.jl | 5 +- base/range.jl | 4 +- base/reduce.jl | 37 +++-- base/reducedim.jl | 69 ++++---- base/reshapedarray.jl | 3 +- base/serialize.jl | 4 +- base/show.jl | 30 ++-- base/sort.jl | 6 +- base/statistics.jl | 23 ++- base/subarray.jl | 89 +++++------ doc/devdocs/julia.rst | 1 + doc/devdocs/offset-arrays.rst | 293 ++++++++++++++++++++++++++++++++++ doc/manual/interfaces.rst | 9 +- test/offsetarray.jl | 52 +++--- 22 files changed, 640 insertions(+), 305 deletions(-) create mode 100644 doc/devdocs/offset-arrays.rst diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 0497852f798ff..5c70411c7d950 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -6,8 +6,6 @@ typealias AbstractVector{T} AbstractArray{T,1} typealias AbstractMatrix{T} AbstractArray{T,2} typealias AbstractVecOrMat{T} Union{AbstractVector{T}, AbstractMatrix{T}} typealias RangeIndex Union{Int, Range{Int}, AbstractUnitRange{Int}, Colon} -typealias Indices{N} NTuple{N,AbstractUnitRange} -typealias IndicesOne{N} NTuple{N,OneTo} typealias DimOrInd Union{Integer, AbstractUnitRange} typealias DimsOrInds{N} NTuple{N,DimOrInd} @@ -29,6 +27,7 @@ end size{T,N}(t::AbstractArray{T,N}, d) = d <= N ? size(t)[d] : 1 size{N}(x, d1::Integer, d2::Integer, dx::Vararg{Integer, N}) = (size(x, d1), size(x, d2), ntuple(k->size(x, dx[k]), Val{N})...) + """ indices(A, d) @@ -66,7 +65,7 @@ is `indices(A, 1)`. Calling this function is the "safe" way to write algorithms that exploit linear indexing. """ -linearindices(A) = (@_inline_meta; 1:length(A)) +linearindices(A) = (@_inline_meta; OneTo(unsafe_length(A))) linearindices(A::AbstractVector) = (@_inline_meta; indices1(A)) eltype{T}(::Type{AbstractArray{T}}) = T eltype{T,N}(::Type{AbstractArray{T,N}}) = T @@ -74,7 +73,8 @@ elsize{T}(::AbstractArray{T}) = sizeof(T) ndims{T,N}(::AbstractArray{T,N}) = N ndims{T,N}(::Type{AbstractArray{T,N}}) = N ndims{T<:AbstractArray}(::Type{T}) = ndims(supertype(T)) -length(t::AbstractArray) = prod(size(t))::Int +length(t::AbstractArray) = prod(size(t)) +unsafe_length(A::AbstractArray) = prod(map(unsafe_length, indices(A))) # internal method endof(a::AbstractArray) = length(a) first(a::AbstractArray) = a[first(eachindex(a))] @@ -132,6 +132,10 @@ function trailingsize(A, n) end return s end +function trailingsize(inds::Indices) + @_inline_meta + prod(map(unsafe_length, inds)) +end ## Traits for array types ## @@ -230,7 +234,7 @@ function checkbounds_indices(::Type{Bool}, IA::Tuple{Any}, I::Tuple{Any}) end function checkbounds_indices(::Type{Bool}, IA::Tuple, I::Tuple{Any}) @_inline_meta - checkindex(Bool, 1:prod(map(dimlength, IA)), I[1]) # linear indexing + checkindex(Bool, OneTo(trailingsize(IA)), I[1]) # linear indexing end """ @@ -264,16 +268,6 @@ end throw_boundserror(A, I) = (@_noinline_meta; throw(BoundsError(A, I))) -@generated function trailingsize{T,N,n}(A::AbstractArray{T,N}, ::Type{Val{n}}) - (isa(n, Int) && isa(N, Int)) || error("Must have concrete type") - n > N && return 1 - ex = :(size(A, $n)) - for m = n+1:N - ex = :($ex * size(A, $m)) - end - Expr(:block, Expr(:meta, :inline), ex) -end - # check along a single dimension """ checkindex(Bool, inds::AbstractUnitRange, index) @@ -357,7 +351,7 @@ to_shape(dims::DimsOrInds) = map(to_shape, dims) to_shape(i::Int) = i to_shape(i::Integer) = Int(i) to_shape(r::OneTo) = Int(last(r)) -to_shape(r::UnitRange) = convert(UnitRange{Int}, r) +to_shape(r::AbstractUnitRange) = r """ similar(storagetype, indices) @@ -492,7 +486,7 @@ function copy!(::LinearIndexing, dest::AbstractArray, ::LinearSlow, src::Abstrac end function copy!(dest::AbstractArray, dstart::Integer, src::AbstractArray) - copy!(dest, dstart, src, first(linearindices(src)), length(src)) + copy!(dest, dstart, src, first(linearindices(src)), unsafe_length(src)) end function copy!(dest::AbstractArray, dstart::Integer, src::AbstractArray, sstart::Integer) @@ -618,7 +612,7 @@ function _maxlength(A, B, C...) max(length(A), _maxlength(B, C...)) end -isempty(a::AbstractArray) = (length(a) == 0) +isempty(a::AbstractArray) = (unsafe_length(a) == 0) ## Conversions ## diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index d4b2e48e885d8..d05abd4f4281f 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -11,7 +11,7 @@ transpose(a::AbstractArray) = error("transpose not implemented for $(typeof(a)). ## Constructors ## -vec(a::AbstractArray) = reshape(a,length(a)) +vec(a::AbstractArray) = reshape(a,unsafe_length(a)) vec(a::AbstractVector) = a _sub(::Tuple{}, ::Tuple{}) = () @@ -74,22 +74,23 @@ function flipdim(A::AbstractArray, d::Integer) if d > nd || isempty(A) return copy(A) end + inds = indices(A) B = similar(A) nnd = 0 for i = 1:nd - nnd += Int(size(A,i)==1 || i==d) + nnd += Int(length(inds[i])==1 || i==d) end - inds = indices(A, d) - sd = first(inds)+last(inds) + indsd = inds[d] + sd = first(indsd)+last(indsd) if nnd==nd # flip along the only non-singleton dimension - for i in inds + for i in indsd B[i] = A[sd-i] end return B end alli = [ indices(B,n) for n in 1:nd ] - for i in inds + for i in indsd B[[ n==d ? sd-i : alli[n] for n in 1:nd ]...] = slicedim(A, d, i) end return B diff --git a/base/arraymath.jl b/base/arraymath.jl index 785735d37bcb0..7ec3eacf6e5b2 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -252,19 +252,20 @@ end const transposebaselength=64 function transpose_f!(f,B::AbstractMatrix,A::AbstractMatrix) - indices(B,1) == indices(A,2) && indices(B,2) == indices(A,1) || throw(DimensionMismatch(string(f))) + inds = indices(A) + indices(B,1) == inds[2] && indices(B,2) == inds[1] || throw(DimensionMismatch(string(f))) - m, n = size(A) + m, n = length(inds[1]), length(inds[2]) if m*n<=4*transposebaselength @inbounds begin - for j = indices(A,2) - for i = indices(A,1) + for j = inds[2] + for i = inds[1] B[j,i] = f(A[i,j]) end end end else - transposeblock!(f,B,A,m,n,first(indices(A,1))-1,first(indices(A,2))-1) + transposeblock!(f,B,A,m,n,first(inds[1])-1,first(inds[2])-1) end return B end diff --git a/base/bitarray.jl b/base/bitarray.jl index e0754de709666..ecb6efd9ae16a 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -8,8 +8,7 @@ type BitArray{N} <: DenseArray{Bool, N} chunks::Vector{UInt64} len::Int dims::NTuple{N,Int} - function BitArray(dims::Int...) - length(dims) == N || throw(ArgumentError("number of dimensions must be $N, got $(length(dims))")) + function BitArray(dims::Vararg{Int,N}) n = 1 i = 1 for d in dims diff --git a/base/broadcast.jl b/base/broadcast.jl index d7e11afcd1a9d..3a66787f7f980 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -3,7 +3,7 @@ module Broadcast using Base.Cartesian -using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, to_shape, tail, dimlength, OneTo +using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, tail, OneTo, to_shape import Base: .+, .-, .*, ./, .\, .//, .==, .<, .!=, .<=, .÷, .%, .<<, .>>, .^ export broadcast, broadcast!, bitbroadcast export broadcast_getindex, broadcast_setindex! @@ -20,7 +20,7 @@ broadcast(f, x::Number...) = f(x...) broadcast_shape() = () broadcast_shape(A) = indices(A) @inline broadcast_shape(A, B...) = broadcast_shape((), indices(A), map(indices, B)...) -# shape inputs +# shape (i.e., tuple-of-indices) inputs broadcast_shape(shape::Tuple) = shape @inline broadcast_shape(shape::Tuple, shape1::Tuple, shapes::Tuple...) = broadcast_shape(_bcs((), shape, shape1), shapes...) # _bcs consolidates two shapes into a single output shape @@ -37,7 +37,7 @@ _bcs1(a::Integer, b) = a == 1 ? b : (first(b) == 1 && last(b) == a ? b : throw(D _bcs1(a, b::Integer) = _bcs1(b, a) _bcs1(a, b) = _bcsm(b, a) ? b : (_bcsm(a, b) ? a : throw(DimensionMismatch("arrays could not be broadcast to a common size"))) # _bcsm tests whether the second index is consistent with the first -_bcsm(a, b) = a == b || (dimlength(b) == 1 && first(b) == first(a)) +_bcsm(a, b) = a == b || (length(b) == 1 && first(b) == 1) _bcsm(a, b::Number) = b == 1 _bcsm(a::Number, b::Number) = a == b || b == 1 @@ -62,22 +62,30 @@ end end ## Indexing manipulations -# newindex(I, rule) replaces a CartesianIndex with something that is -# appropriate for a particular array/scalar. `rule` is a tuple that -# describes the manipulations that should be made. -@inline newindex(I::CartesianIndex, ::Tuple{}) = 1 # for scalars -@inline newindex(I::CartesianIndex, indexmap) = CartesianIndex(_newindex((), I.I, indexmap...)) -@inline _newindex(out, I) = out # can truncate if indexmap is shorter than I -@inline _newindex(out, I, keep::Bool, indexmap...) = _newindex((out..., ifelse(keep, I[1], 1)), tail(I), indexmap...) - -newindexer(sz, x::Number) = () -@inline newindexer(sz, A) = _newindexer(sz, size(A)) -@inline _newindexer(sz, szA::Tuple{}) = () -@inline _newindexer(sz, szA) = (sz[1] == szA[1], _newindexer(tail(sz), tail(szA))...) -# map(x->newindexer(sz, x), As), but see #15276 -map_newindexer(sz, ::Tuple{}) = () -@inline map_newindexer(sz, As) = (newindexer(sz, As[1]), map_newindexer(sz, tail(As))...) +# newindex(I, keep) replaces a CartesianIndex `I` with something that +# is appropriate for a particular broadcast array/scalar. `keep` is a +# NTuple{N,Bool}, where keep[d] == true means that one should preserve +# I[d]; if false, replace it with 1. In other words, this is +# equivalent to map((k,i)->k ? i : 1, keep, I.I) (but see #17126). +@inline newindex(I::CartesianIndex, ::Tuple{}) = 1 # for scalars +@inline newindex(I::CartesianIndex, indexmap) = CartesianIndex(_newindex(I.I, indexmap)) +@inline _newindex(I, indexmap) = + (ifelse(indexmap[1], I[1], 1), _newindex(tail(I), tail(indexmap))...) +@inline _newindex(I, indexmap::Tuple{}) = () # truncate if indexmap is shorter than I + +# newindexer(shape, A) generates `keep` (for use by `newindex` above) +# for a particular array `A`, given the broadcast_shape `shape` +# Equivalent to map(==, indices(A), shape) (but see #17126) +newindexer(shape, x::Number) = () +@inline newindexer(shape, A) = newindexer(shape, indices(A)) +@inline newindexer(shape, indsA::Tuple{}) = () +@inline newindexer(shape, indsA::Tuple) = + (shape[1] == indsA[1], newindexer(tail(shape), tail(indsA))...) + +# Equivalent to map(x->newindexer(shape, x), As) (but see #17126) +map_newindexer(shape, ::Tuple{}) = () +@inline map_newindexer(shape, As) = (newindexer(shape, As[1]), map_newindexer(shape, tail(As))...) # For output BitArrays const bitcache_chunks = 64 # this can be changed @@ -140,9 +148,9 @@ end end @inline function broadcast!{nargs}(f, B::AbstractArray, As::Vararg{Any,nargs}) - check_broadcast_shape(indices(B), As...) - sz = size(B) - mapindex = map(x->newindexer(sz, x), As) + shape = indices(B) + check_broadcast_shape(shape, As...) + mapindex = map_newindexer(shape, As) _broadcast!(f, B, mapindex, As, Val{nargs}) B end @@ -183,18 +191,17 @@ end end function broadcast_t(f, ::Type{Any}, As...) - shp = broadcast_shape(As...) - iter = CartesianRange(shp) + shape = broadcast_shape(As...) + iter = CartesianRange(shape) if isempty(iter) - return similar(Array{Union{}}, shp) + return similar(Array{Union{}}, shape) end nargs = length(As) - sz = size(iter) - indexmaps = map(x->newindexer(sz, x), As) + indexmaps = map_newindexer(shape, As) st = start(iter) I, st = next(iter, st) val = f([ As[i][newindex(I, indexmaps[i])] for i=1:nargs ]...) - B = similar(Array{typeof(val)}, shp) + B = similar(Array{typeof(val)}, shape) B[I] = val return _broadcast!(f, B, indexmaps, As, Val{nargs}, iter, st, 1) end @@ -213,10 +220,9 @@ end end function broadcast(f, As...) - shp = broadcast_shape(As...) - iter = CartesianRange(shp) - sz = size(iter) - indexmaps = map(x->newindexer(sz, x), As) + shape = broadcast_shape(As...) + iter = CartesianRange(shape) + indexmaps = map_newindexer(shape, As) naT = Val{nfields(As)} _broadcast(f, indexmaps, As, naT, iter) end @@ -224,7 +230,7 @@ end @inline bitbroadcast(f, As...) = broadcast!(f, similar(BitArray, broadcast_shape(As...)), As...) -broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex!(Array{eltype(src)}(to_shape(broadcast_shape(I...))), src, I...) +broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex!(similar(Array{eltype(src)}, broadcast_shape(I...)), src, I...) @generated function broadcast_getindex!(dest::AbstractArray, src::AbstractArray, I::AbstractArray...) N = length(I) Isplat = Expr[:(I[$d]) for d = 1:N] @@ -232,7 +238,8 @@ broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex @nexprs $N d->(I_d = I[d]) check_broadcast_shape(indices(dest), $(Isplat...)) # unnecessary if this function is never called directly checkbounds(src, $(Isplat...)) - @nloops $N i dest d->(@nexprs $N k->(j_d_k = size(I_k, d) == 1 ? 1 : i_d)) begin + @nexprs $N d->(@nexprs $N k->(Ibcast_d_k = indices(I_k, d) == OneTo(1))) + @nloops $N i dest d->(@nexprs $N k->(j_d_k = Ibcast_d_k ? 1 : i_d)) begin @nexprs $N k->(@inbounds J_k = @nref $N I_k d->j_d_k) @inbounds (@nref $N dest i) = (@nref $N src J) end @@ -248,18 +255,19 @@ end checkbounds(A, $(Isplat...)) shape = broadcast_shape($(Isplat...)) @nextract $N shape d->(length(shape) < d ? OneTo(1) : shape[d]) + @nexprs $N d->(@nexprs $N k->(Ibcast_d_k = indices(I_k, d) == 1:1)) if !isa(x, AbstractArray) xA = convert(eltype(A), x) - @nloops $N i d->shape_d d->(@nexprs $N k->(j_d_k = size(I_k, d) == 1 ? 1 : i_d)) begin + @nloops $N i d->shape_d d->(@nexprs $N k->(j_d_k = Ibcast_d_k ? 1 : i_d)) begin @nexprs $N k->(@inbounds J_k = @nref $N I_k d->j_d_k) @inbounds (@nref $N A J) = xA end else X = x - @nexprs $N d->(shapelen_d = dimlength(shape_d)) + @nexprs $N d->(shapelen_d = length(shape_d)) @ncall $N Base.setindex_shape_check X shapelen Xstate = start(X) - @inbounds @nloops $N i d->shape_d d->(@nexprs $N k->(j_d_k = size(I_k, d) == 1 ? 1 : i_d)) begin + @inbounds @nloops $N i d->shape_d d->(@nexprs $N k->(j_d_k = Ibcast_d_k ? 1 : i_d)) begin @nexprs $N k->(J_k = @nref $N I_k d->j_d_k) x_el, Xstate = next(X, Xstate) (@nref $N A J) = x_el @@ -287,11 +295,11 @@ end function broadcast_bitarrays(scalarf, bitf, A::AbstractArray{Bool}, B::AbstractArray{Bool}) local shape try - shape = promote_shape(size(A), size(B)) + shape = promote_shape(indices(A), indices(B)) catch return bitbroadcast(scalarf, A, B) end - F = BitArray(shape) + F = BitArray(to_shape(shape)) Fc = F.chunks Ac = BitArray(A).chunks Bc = BitArray(B).chunks @@ -378,11 +386,11 @@ end function (.^){T<:Integer}(A::BitArray, B::Array{T}) local shape try - shape = promote_shape(size(A), size(B)) + shape = promote_shape(indices(A), indices(B)) catch return bitbroadcast(^, A, B) end - F = BitArray(shape) + F = BitArray(to_shape(shape)) l = length(F) l == 0 && return F Ac = A.chunks diff --git a/base/deprecated.jl b/base/deprecated.jl index 9f0247a809e21..816bcea7f0b66 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -752,16 +752,26 @@ function checkbounds{N,T}(::Type{Bool}, sz::NTuple{N,Integer}, I1::T, I...) end function first(::Colon) - depwarn("first(:) is no longer unambiguous, call Base._first(:, A, dim)", :first) + depwarn("first(:) is deprecated, see http://docs.julialang.org/en/latest/devdocs/offset-arrays/", :first) 1 end +function _first(i, A, d) + depwarn("_first is deprecated, see http://docs.julialang.org/en/latest/devdocs/offset-arrays/", :_first) + __first(i, A, d) +end +__first(::Colon, P, ::Colon) = first(linearindices(P)) +__first(i, P, ::Colon) = first(i) +__first(::Colon, P, d) = first(indices(P, d)) +__first(i, P, d) = first(i) -# Not exported, but may be useful just in case +# Not exported, but deprecation may be useful just in case function Broadcast.check_broadcast_shape(sz::Dims, As::Union{AbstractArray,Number}...) depwarn("check_broadcast_shape(size(A), B...) should be replaced with check_broadcast_shape(indices(A), B...)", :check_broadcast_shape) Broadcast.check_broadcast_shape(map(OneTo, sz), As...) end +@deprecate trailingsize{n}(A::AbstractArray, ::Type{Val{n}}) trailingsize(A, n) + @deprecate slice view @deprecate sub view diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 9637892bd2877..7226e893faeba 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -192,7 +192,7 @@ index_lengths(A::AbstractArray, I::Colon) = (length(A),) @inline index_lengths(A::AbstractArray, I...) = index_lengths_dim(A, 1, I...) index_lengths_dim(A, dim) = () index_lengths_dim(A, dim, ::Colon) = (trailingsize(A, dim),) -@inline index_lengths_dim(A, dim, ::Colon, i, I...) = (size(A, dim), index_lengths_dim(A, dim+1, i, I...)...) +@inline index_lengths_dim(A, dim, ::Colon, i, I...) = (unsafe_length(indices(A, dim)), index_lengths_dim(A, dim+1, i, I...)...) @inline index_lengths_dim(A, dim, ::Real, I...) = (1, index_lengths_dim(A, dim+1, I...)...) @inline index_lengths_dim{N}(A, dim, ::CartesianIndex{N}, I...) = (1, index_lengths_dim(A, dim+N, I...)...) @inline index_lengths_dim(A, dim, i::AbstractArray, I...) = (length(i), index_lengths_dim(A, dim+1, I...)...) @@ -200,32 +200,41 @@ index_lengths_dim(A, dim, ::Colon) = (trailingsize(A, dim),) @inline index_lengths_dim{N}(A, dim, i::AbstractArray{CartesianIndex{N}}, I...) = (length(i), index_lengths_dim(A, dim+N, I...)...) # shape of array to create for getindex() with indexes I, dropping scalars -# Rather than use an Integer dim, we grow a tuple (true, true, ...) -# whose length is equal to the dimension we're to process next. This -# allows us to dispatch, which is important for the type-stability of -# the lines involving Colon as the final index. -index_shape(A::AbstractVector, I::Colon) = indices(A) -index_shape(A::AbstractArray, I::Colon) = (length(A),) -@inline index_shape(A::AbstractArray, I...) = index_shape_dim(A, (true,), I...) -@inline index_shape_dim(A, dim, ::Colon) = (trailingsize(A, length(dim)),) -@inline index_shape_dim{T,N}(A::AbstractArray{T,N}, dim::NTuple{N}, ::Colon) = (indices(A, N),) -@inline index_shape_dim(A, dim, I::Real...) = () -@inline index_shape_dim(A, dim, ::Colon, i, I...) = (indices(A, length(dim)), index_shape_dim(A, (dim...,true), i, I...)...) -@inline index_shape_dim(A, dim, ::Real, I...) = (index_shape_dim(A, (dim...,true), I...)...) -@inline index_shape_dim{N}(A, dim, ::CartesianIndex{N}, I...) = (index_shape_dim(A, (dim...,ntuple(d->true,Val{N})...), I...)...) -@inline index_shape_dim(A, dim, i::AbstractArray, I...) = (indices(i)..., index_shape_dim(A, (dim...,true), I...)...) -@inline index_shape_dim(A, dim, i::AbstractArray{Bool}, I...) = (sum(i), index_shape_dim(A, (dim...,true), I...)...) -@inline index_shape_dim{N}(A, dim, i::AbstractArray{CartesianIndex{N}}, I...) = (indices(i)..., index_shape_dim(A, (dim...,ntuple(d->true,Val{N})...), I...)...) - -@inline decolon(A::AbstractVector, ::Colon) = (indices(A,1),) -@inline decolon(A::AbstractArray, ::Colon) = (1:length(A),) -@inline decolon(A::AbstractArray, I...) = decolon_dim(A, (true,), I...) -@inline decolon_dim(A::AbstractArray, dim) = () -@inline decolon_dim{T,N}(A::AbstractArray{T,N}, dim::NTuple{N}, ::Colon) = (indices(A, N),) -@inline decolon_dim(A, dim, ::Colon) = (1:trailingsize(A, length(dim)),) -@inline decolon_dim(A::AbstractArray, dim, i1, I...) = (i1, decolon_dim(A, (dim...,true), I...)...) -@inline decolon_dim{N}(A::AbstractArray, dim, i1::AbstractArray{CartesianIndex{N}}, I...) = (i1, decolon_dim(A, (dim...,ntuple(d->true,Val{N})...), I...)...) -@inline decolon_dim(A::AbstractArray, dim, ::Colon, I...) = (indices(A, length(dim)), decolon_dim(A, (dim...,true), I...)...) +# returns a Tuple{Vararg{AbstractUnitRange}} of indices +index_shape(A::AbstractArray, I::Colon) = (linearindices(A),) +@inline index_shape(A::AbstractArray, I...) = index_shape_dim(indices(A), I...) +@inline index_shape_dim(inds::Tuple{Any}, ::Colon) = inds +@inline index_shape_dim(inds, ::Colon) = (OneTo(trailingsize(inds)),) +@inline index_shape_dim(inds, ::Colon, i, I...) = + (inds[1], index_shape_dim(tail(inds), i, I...)...) +@inline index_shape_dim(inds, ::Real...) = () +@inline index_shape_dim(inds, ::Real, I...) = index_shape_dim(tail(inds), I...) +@inline index_shape_dim(inds, i::AbstractArray, I...) = + (indices(i)..., index_shape_dim(tail(inds), I...)...) +@inline index_shape_dim(inds, i::AbstractArray{Bool}, I...) = + (OneTo(sum(i)), index_shape_dim(tail(inds), I...)...) +@inline function index_shape_dim{N}(inds, ::CartesianIndex{N}, I...) + indsN, indstail = IterationMD.split(inds, Val{N}) + index_shape_dim(indstail, I...) +end +@inline function index_shape_dim{N}(A, i::AbstractArray{CartesianIndex{N}}, I...) + indsN, indstail = IterationMD.split(inds, Val{N}) + (indices(i)..., index_shape_dim(indstail, I...)...) +end + +# Convert Colon indices into explicit indices +@inline decolon(A::AbstractArray, ::Colon) = (linearindices(A),) +@inline decolon(A::AbstractArray, I...) = decolon_dim(indices(A), I...) +@inline decolon_dim(inds) = () +@inline decolon_dim(inds::Tuple{Any}, ::Colon) = inds +@inline decolon_dim(inds, ::Colon) = (OneTo(trailingsize(inds)),) +@inline decolon_dim(inds, ::Colon, I...) = + (inds[1], decolon_dim(tail(inds), I...)...) +@inline decolon_dim(inds, i1, I...) = (i1, decolon_dim(tail(inds), I...)...) +@inline function decolon_dim{N}(inds, i1::AbstractArray{CartesianIndex{N}}, I...) + indsN, indstail = IterationMD.split(inds, Val{N}) + (i1, decolon_dim(indstail, I...)...) +end ### From abstractarray.jl: Internal multidimensional indexing definitions ### # These are not defined on directly on getindex to avoid @@ -258,7 +267,7 @@ end @nexprs $N d->(I_d = to_index(I[d])) shape = @ncall $N index_shape A I dest = similar(A, shape) - size(dest) == map(dimlength, shape) || throw_checksize_error(dest, shape) + map(unsafe_length, indices(dest)) == map(unsafe_length, shape) || throw_checksize_error(dest, shape) @ncall $N _unsafe_getindex! dest A I end end @@ -267,7 +276,7 @@ end function _unsafe_getindex(::LinearIndexing, src::AbstractArray, I::AbstractArray{Bool}) shape = index_shape(src, I) dest = similar(src, shape) - size(dest) == map(dimlength, shape) || throw_checksize_error(dest, shape) + map(unsafe_length, indices(dest)) == map(unsafe_length, shape) || throw_checksize_error(dest, shape) D = eachindex(dest) Ds = start(D) @@ -284,7 +293,7 @@ end function _unsafe_getindex(::LinearFast, src::AbstractArray, I::AbstractArray{Bool}) shape = index_shape(src, I) dest = similar(src, shape) - size(dest) == shape || throw_checksize_error(dest, shape) + map(unsafe_length, indices(dest)) == map(unsafe_length, shape) || throw_checksize_error(dest, shape) D = eachindex(dest) Ds = start(D) @@ -437,20 +446,24 @@ for (f, fmod, op) = ((:cummin, :_cummin!, :min), (:cummax, :_cummax!, :max)) @eval function ($f)(A::AbstractArray, axis::Integer) res = similar(A) - if size(A, axis) < 1 + axis > ndims(A) && return copy!(res, A) + inds = indices(A) + if isempty(inds[axis]) return res end - R1 = CartesianRange(size(A)[1:axis-1]) - R2 = CartesianRange(size(A)[axis+1:end]) + R1 = CartesianRange(inds[1:axis-1]) + R2 = CartesianRange(inds[axis+1:end]) ($fmod)(res, A, R1, R2, axis) end @eval @noinline function ($fmod)(res, A::AbstractArray, R1::CartesianRange, R2::CartesianRange, axis::Integer) + inds = indices(A, axis) + i1 = first(inds) for I2 in R2 for I1 in R1 - res[I1, 1, I2] = A[I1, 1, I2] + res[I1, i1, I2] = A[I1, i1, I2] end - for i = 2:size(A, axis) + for i = i1+1:last(inds) for I1 in R1 res[I1, i, I2] = ($op)(A[I1, i, I2], res[I1, i-1, I2]) end @@ -471,18 +484,15 @@ cumsum!(B, A, axis::Integer) = cumop!(+, B, A, axis) cumprod!(B, A, axis::Integer) = cumop!(*, B, A, axis) function cumop!(op, B, A, axis::Integer) - if size(B, axis) < 1 - return B - end - indices(B) == indices(A) || throw(DimensionMismatch("Shape of B must match A")) - if axis > ndims(A) - copy!(B, A) - return B - end + inds_t = indices(A) + indices(B) == inds_t || throw(DimensionMismatch("shape of B must match A")) + axis > ndims(A) && return copy!(B, A) + isempty(inds_t[axis]) && return B if axis == 1 - # We can accumulate to a temporary variable, which allows register usage and will be slightly faster - ind1 = indices(A,1) - @inbounds for I in CartesianRange(tail(indices(A))) + # We can accumulate to a temporary variable, which allows + # register usage and will be slightly faster + ind1 = inds_t[1] + @inbounds for I in CartesianRange(tail(inds_t)) tmp = convert(eltype(B), A[first(ind1), I]) B[first(ind1), I] = tmp for i_1 = first(ind1)+1:last(ind1) @@ -493,7 +503,7 @@ function cumop!(op, B, A, axis::Integer) else R1 = CartesianRange(indices(A)[1:axis-1]) # not type-stable R2 = CartesianRange(indices(A)[axis+1:end]) - _cumop!(op, B, A, R1, indices(A, axis), R2) # use function barrier + _cumop!(op, B, A, R1, inds_t[axis], R2) # use function barrier end return B end @@ -537,7 +547,7 @@ end # contiguous multidimensional indexing: if the first dimension is a range, # we can get some performance from using copy_chunks! @inline function _unsafe_getindex!(X::BitArray, B::BitArray, I0::Union{UnitRange{Int},Colon}) - copy_chunks!(X.chunks, 1, B.chunks, _first(I0, B, :), index_lengths(B, I0)[1]) + copy_chunks!(X.chunks, 1, B.chunks, indexoffset(I0)+1, index_lengths(B, I0)[1]) return X end @@ -548,7 +558,7 @@ end $(Expr(:meta, :inline)) @nexprs $N d->(I_d = I[d]) - f0 = _first(I0, B, 1) + f0 = indexoffset(I0)+1 l0 = size(X, 1) gap_lst_1 = 0 @@ -558,7 +568,7 @@ end @nexprs $N d->begin stride *= size(B, d) stride_lst_d = stride - ind += stride * (_first(I_d, B, d) - 1) + ind += stride * indexoffset(I_d) gap_lst_{d+1} *= stride end @@ -607,7 +617,7 @@ end l0 = index_lengths(B, I0)[1] setindex_shape_check(X, l0) l0 == 0 && return B - f0 = _first(I0, B, :) + f0 = indexoffset(I0)+1 copy_to_bitarray_chunks!(B.chunks, f0, X, 1, l0) return B end @@ -617,7 +627,7 @@ end y = Bool(x) l0 = index_lengths(B, I0)[1] l0 == 0 && return B - f0 = _first(I0, B, :) + f0 = indexoffset(I0)+1 fill_chunks!(B.chunks, y, f0, l0) return B end @@ -633,7 +643,7 @@ end idxlens = @ncall $N index_lengths B I0 d->I[d] @ncall $N setindex_shape_check X idxlens[1] d->idxlens[d+1] isempty(X) && return B - f0 = _first(I0, B, 1) + f0 = indexoffset(I0)+1 l0 = idxlens[1] gap_lst_1 = 0 @@ -643,7 +653,7 @@ end @nexprs $N d->begin stride *= size(B, d) stride_lst_d = stride - ind += stride * (_first(I[d], B, d) - 1) + ind += stride * indexoffset(I[d]) gap_lst_{d+1} *= stride end @@ -672,7 +682,7 @@ end y = Bool(x) idxlens = @ncall $N index_lengths B I0 d->I[d] - f0 = _first(I0, B, 1) + f0 = indexoffset(I0)+1 l0 = idxlens[1] l0 == 0 && return B @nexprs $N d->(isempty(I[d]) && return B) @@ -684,7 +694,7 @@ end @nexprs $N d->begin stride *= size(B, d) stride_lst_d = stride - ind += stride * (_first(I[d], B, d) - 1) + ind += stride * indexoffset(I[d]) gap_lst_{d+1} *= stride end @@ -877,3 +887,6 @@ If `dim` is specified, returns unique regions of the array `itr` along `dim`. @nref $N A d->d == dim ? sort!(uniquerows) : (indices(A, d)) end end + +indexoffset(i) = first(i)-1 +indexoffset(::Colon) = 0 diff --git a/base/operators.jl b/base/operators.jl index fd997396d3fd4..2c8fd21de95ee 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -2,6 +2,10 @@ ## types ## +typealias Dims{N} NTuple{N,Int} +typealias DimsInteger{N} NTuple{N,Integer} +typealias Indices{N} NTuple{N,AbstractUnitRange} + const (<:) = issubtype supertype(T::DataType) = T.super @@ -332,6 +336,8 @@ eltype(x) = eltype(typeof(x)) # array shape rules +promote_shape(::Tuple{}, ::Tuple{}) = () + function promote_shape(a::Tuple{Int,}, b::Tuple{Int,}) if a[1] != b[1] throw(DimensionMismatch("dimensions must match")) @@ -373,20 +379,24 @@ function promote_shape(a::Dims, b::Dims) end function promote_shape(a::AbstractArray, b::AbstractArray) - if ndims(a) < ndims(b) + promote_shape(indices(a), indices(b)) +end + +function promote_shape(a::Indices, b::Indices) + if length(a) < length(b) return promote_shape(b, a) end - for i=1:ndims(b) - if indices(a, i) != indices(b, i) + for i=1:length(b) + if a[i] != b[i] throw(DimensionMismatch("dimensions must match")) end end - for i=ndims(b)+1:ndims(a) - if indices(a, i) != 1:1 + for i=length(b)+1:length(a) + if a[i] != 1:1 throw(DimensionMismatch("dimensions must match")) end end - return indices(a) + return a end function throw_setindex_mismatch(X, I) @@ -407,12 +417,12 @@ function setindex_shape_check(X::AbstractArray, I::Integer...) lj = length(I) i = j = 1 while true - ii = size(X,i) + ii = length(indices(X,i)) jj = I[j] if i == li || j == lj while i < li i += 1 - ii *= size(X,i) + ii *= length(indices(X,i)) end while j < lj j += 1 @@ -437,22 +447,22 @@ function setindex_shape_check(X::AbstractArray, I::Integer...) end setindex_shape_check(X::AbstractArray) = - (length(X)==1 || throw_setindex_mismatch(X,())) + (unsafe_length(X)==1 || throw_setindex_mismatch(X,())) setindex_shape_check(X::AbstractArray, i::Integer) = - (length(X)==i || throw_setindex_mismatch(X, (i,))) + (unsafe_length(X)==i || throw_setindex_mismatch(X, (i,))) setindex_shape_check{T}(X::AbstractArray{T,1}, i::Integer) = - (length(X)==i || throw_setindex_mismatch(X, (i,))) + (unsafe_length(X)==i || throw_setindex_mismatch(X, (i,))) setindex_shape_check{T}(X::AbstractArray{T,1}, i::Integer, j::Integer) = - (length(X)==i*j || throw_setindex_mismatch(X, (i,j))) + (unsafe_length(X)==i*j || throw_setindex_mismatch(X, (i,j))) function setindex_shape_check{T}(X::AbstractArray{T,2}, i::Integer, j::Integer) if length(X) != i*j throw_setindex_mismatch(X, (i,j)) end - sx1 = size(X,1) + sx1 = length(indices(X,1)) if !(i == 1 || i == sx1 || sx1 == 1) throw_setindex_mismatch(X, (i,j)) end diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index 8ab7d60f26301..aa2944f7c4bb0 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -7,13 +7,12 @@ export permutedims # Some day we will want storage-order-aware iteration, so put perm in the parameters immutable PermutedDimsArray{T,N,perm,iperm,AA<:AbstractArray} <: AbstractArray{T,N} parent::AA - dims::NTuple{N,Int} function PermutedDimsArray(data::AA) (isa(perm, NTuple{N,Int}) && isa(iperm, NTuple{N,Int})) || error("perm and iperm must both be NTuple{$N,Int}") isperm(perm) || throw(ArgumentError(string(perm, " is not a valid permutation of dimensions 1:", N))) all(map(d->iperm[perm[d]]==d, 1:N)) || throw(ArgumentError(string(perm, " and ", iperm, " must be inverses"))) - new(data, genperm(size(data), perm)) + new(data) end end @@ -24,7 +23,7 @@ function PermutedDimsArray{T,N}(data::AbstractArray{T,N}, perm) end Base.parent(A::PermutedDimsArray) = A.parent -Base.size(A::PermutedDimsArray) = A.dims +Base.size{T,N,perm}(A::PermutedDimsArray{T,N,perm}) = genperm(size(parent(A)), perm) Base.indices{T,N,perm}(A::PermutedDimsArray{T,N,perm}) = genperm(indices(parent(A)), perm) @inline function Base.getindex{T,N,perm,iperm}(A::PermutedDimsArray{T,N,perm,iperm}, I::Vararg{Int,N}) diff --git a/base/range.jl b/base/range.jl index 48a591438ebd3..62997616404a4 100644 --- a/base/range.jl +++ b/base/range.jl @@ -2,9 +2,6 @@ ## 1-dimensional ranges ## -typealias Dims{N} NTuple{N,Int} -typealias DimsInteger{N} NTuple{N,Integer} - abstract Range{T} <: AbstractArray{T,1} ## ordinal ranges @@ -715,6 +712,7 @@ convert{T<:Real}(::Type{OneTo{T}}, r::OneTo) = OneTo{T}(r.stop) promote_rule{T1,UR<:AbstractUnitRange}(::Type{UnitRange{T1}}, ::Type{UR}) = UnitRange{promote_type(T1,eltype(UR))} convert{T<:Real}(::Type{UnitRange{T}}, r::AbstractUnitRange) = UnitRange{T}(first(r), last(r)) +convert(::Type{UnitRange}, r::AbstractUnitRange) = UnitRange(first(r), last(r)) promote_rule{T1a,T1b,T2a,T2b}(::Type{StepRange{T1a,T1b}},::Type{StepRange{T2a,T2b}}) = StepRange{promote_type(T1a,T2a),promote_type(T1b,T2b)} diff --git a/base/reduce.jl b/base/reduce.jl index 679fa4b9bb7d8..4d26a67eee985 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -93,7 +93,7 @@ foldr(op, itr) = mapfoldr(identity, op, itr) ## reduce & mapreduce -function mapreduce_impl(f, op, A::AbstractArray, ifirst::Int, ilast::Int, blksize::Int=pairwise_blocksize(f, op)) +function mapreduce_impl(f, op, A::AbstractArray, ifirst::Integer, ilast::Integer, blksize::Int=pairwise_blocksize(f, op)) if ifirst + blksize > ilast # sequential portion fx1 = r_promote(op, f(A[ifirst])) @@ -141,23 +141,26 @@ mr_empty(f, op::typeof(|), T) = false _mapreduce(f, op, A::AbstractArray) = _mapreduce(f, op, linearindexing(A), A) function _mapreduce{T}(f, op, ::LinearFast, A::AbstractArray{T}) - n = Int(length(A)) - if n == 0 - return mr_empty(f, op, T) - elseif n == 1 - return r_promote(op, f(A[1])) - elseif n < 16 - fx1 = r_promote(op, f(A[1])) - fx2 = r_promote(op, f(A[2])) - s = op(fx1, fx2) - i = 2 - while i < n - @inbounds Ai = A[i+=1] - s = op(s, f(Ai)) + inds = linearindices(A) + n = length(inds) + @inbounds begin + if n == 0 + return mr_empty(f, op, T) + elseif n == 1 + return r_promote(op, f(A[inds[1]])) + elseif n < 16 + fx1 = r_promote(op, f(A[inds[1]])) + fx2 = r_promote(op, f(A[inds[2]])) + s = op(fx1, fx2) + i = inds[2] + while i < last(inds) + Ai = A[i+=1] + s = op(s, f(Ai)) + end + return s + else + return mapreduce_impl(f, op, A, first(inds), last(inds)) end - return s - else - return mapreduce_impl(f, op, A, 1, n) end end diff --git a/base/reducedim.jl b/base/reducedim.jl index 7f82a176cac98..15ee8c67512ff 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -159,15 +159,16 @@ function check_reducedims(R, A) # it will be size(A, 1) or size(A, 1) * size(A, 2). # - Otherwise, e.g. sum(A, 2) or sum(A, (1, 3)), it returns 0. # - ndims(R) <= ndims(A) || throw(DimensionMismatch("Cannot reduce $(ndims(A))-dimensional array to $(ndims(R)) dimensions")) + ndims(R) <= ndims(A) || throw(DimensionMismatch("cannot reduce $(ndims(A))-dimensional array to $(ndims(R)) dimensions")) lsiz = 1 had_nonreduc = false for i = 1:ndims(A) - sRi = size(R, i) - sAi = size(A, i) + Ri, Ai = indices(R, i), indices(A, i) + sRi, sAi = length(Ri), length(Ai) if sRi == 1 - first(indices(R, i)) == first(indices(A, i)) || - throw(DimensionMismatch("Reduction along dimension $i must use lower indices")) + if Ri != Ai + Ri == 1:1 || throw(DimensionMismatch("reduction along dimension $i must use indices 1:1; got $(convert(UnitRange, Ri)) for reducing $(convert(UnitRange, Ai))")) + end if sAi > 1 if had_nonreduc lsiz = 0 # to reduce along i, but some previous dimensions were non-reducing @@ -176,8 +177,7 @@ function check_reducedims(R, A) end end else - indices(R, i) == indices(A, i) || - throw(DimensionMismatch("Reduction on array with indices $(indices(A)) with output with indices $(indices(R))")) + Ri == Ai || throw(DimensionMismatch("reduction on array with indices $(indices(A)) with output with indices $(indices(R))")) had_nonreduc = true end end @@ -187,11 +187,10 @@ end function _mapreducedim!{T,N}(f, op, R::AbstractArray, A::AbstractArray{T,N}) lsiz = check_reducedims(R,A) isempty(A) && return R - sizA1 = size(A, 1) if has_fast_linear_indexing(A) && lsiz > 16 # use mapreduce_impl, which is probably better tuned to achieve higher performance - nslices = div(length(A), lsiz) + nslices = div(unsafe_length(A), lsiz) ibase = first(linearindices(A))-1 for i = 1:nslices @inbounds R[i] = op(R[i], mapreduce_impl(f, op, A, ibase+1, ibase+lsiz)) @@ -199,21 +198,21 @@ function _mapreducedim!{T,N}(f, op, R::AbstractArray, A::AbstractArray{T,N}) end return R end - IRmax = dims_tail(map(last, indices(R)), A) - if size(R, 1) == 1 && sizA1 > 1 + indsAt, indsRt = safe_tail(indices(A)), safe_tail(indices(R)) # handle d=1 manually + imap = Broadcast.newindexer(indsAt, indsRt) + if reducedim1(R, A) # keep the accumulator as a local variable when reducing along the first dimension - i1 = first(indices(A, 1)) - @inbounds for IA in CartesianRange(tail(indices(A))) - IR = min(IA, IRmax) - r = R[i1,IR] + @inbounds for IA in CartesianRange(indsAt) + IR = Broadcast.newindex(IA, imap) + r = R[1,IR] @simd for i in indices(A, 1) r = op(r, f(A[i, IA])) end - R[i1,IR] = r + R[1,IR] = r end else - @inbounds for IA in CartesianRange(tail(indices(A))) - IR = min(IA, IRmax) + @inbounds for IA in CartesianRange(indsAt) + IR = Broadcast.newindex(IA, imap) @simd for i in indices(A, 1) R[i,IR] = op(R[i,IR], f(A[i,IA])) end @@ -273,20 +272,20 @@ end function findminmax!{T,N}(f, Rval, Rind, A::AbstractArray{T,N}) (isempty(Rval) || isempty(A)) && return Rval, Rind - check_reducedims(Rval, A) + lsiz = check_reducedims(Rval, A) for i = 1:N indices(Rval, i) == indices(Rind, i) || throw(DimensionMismatch("Find-reduction: outputs must have the same indices")) end # If we're reducing along dimension 1, for efficiency we can make use of a temporary. # Otherwise, keep the result in Rval/Rind so that we traverse A in storage order. - IRmax = dims_tail(map(last, indices(Rval)), A) + indsAt, indsRt = safe_tail(indices(A)), safe_tail(indices(Rval)) + imap = Broadcast.newindexer(indsAt, indsRt) k = 0 - if size(Rval, 1) < size(A, 1) - i1 = first(indices(A, 1)) - @inbounds for IA in CartesianRange(tail(indices(A))) - IR = min(IRmax, IA) - tmpRv = Rval[i1,IR] - tmpRi = Rind[i1,IR] + if reducedim1(Rval, A) + @inbounds for IA in CartesianRange(indsAt) + IR = Broadcast.newindex(IA, imap) + tmpRv = Rval[1,IR] + tmpRi = Rind[1,IR] for i in indices(A,1) k += 1 tmpAv = A[i,IA] @@ -295,12 +294,12 @@ function findminmax!{T,N}(f, Rval, Rind, A::AbstractArray{T,N}) tmpRi = k end end - Rval[i1,IR] = tmpRv - Rind[i1,IR] = tmpRi + Rval[1,IR] = tmpRv + Rind[1,IR] = tmpRi end else - @inbounds for IA in CartesianRange(tail(indices(A))) - IR = min(IRmax, IA) + @inbounds for IA in CartesianRange(indsAt) + IR = Broadcast.newindex(IA, imap) for i in indices(A, 1) k += 1 tmpAv = A[i,IA] @@ -359,6 +358,10 @@ function findmax{T}(A::AbstractArray{T}, region) zeros(Int, reduced_dims0(A, region)), A) end -dims_tail{T}(dims::Tuple{}, Aref::AbstractArray{T,0}) = CartesianIndex(()) -dims_tail{T,N}(dims::NTuple{N,Int}, Aref::AbstractArray{T,N}) = CartesianIndex(tail(dims)) -@inline dims_tail{T,M,N}(dims::NTuple{M,Int}, Aref::AbstractArray{T,N}) = dims_tail(tuple(dims..., 1), Aref) +safe_tail(t::Tuple) = tail(t) +safe_tail(t::Tuple{}) = () + +function reducedim1(R, A) + iR1 = indices(R, 1) + iR1 == 1:1 && iR1 != indices(A, 1) +end diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index 053a554eb821a..47365a6340059 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -52,7 +52,8 @@ end @pure rdims{N}(out::Tuple, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = rdims((out..., first(inds)), tail(inds), Val{N}) function _reshape(parent::AbstractArray, dims::Dims) - prod(dims) == length(parent) || throw(DimensionMismatch("parent has $(length(parent)) elements, which is incompatible with size $dims")) + n = unsafe_length(parent) + prod(dims) == n || throw(DimensionMismatch("parent has $n elements, which is incompatible with size $dims")) __reshape((parent, linearindexing(parent)), dims) end _reshape(R::ReshapedArray, dims::Dims) = _reshape(R.parent, dims) diff --git a/base/serialize.jl b/base/serialize.jl index 30c38312a1f2b..f0e6d938fba49 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -227,8 +227,8 @@ end trimmedsize(V) = index_lengths(V.parent, V.indexes...) function _trimmedsubarray{T,N,P,I,LD}(A, V::SubArray{T,N,P,I,LD}, newindexes) - LD && return SubArray{T,N,P,I,LD}(A, newindexes, size(V), Base.compute_offset1(A, 1, newindexes), 1) - SubArray{T,N,P,I,LD}(A, newindexes, size(V), 0, 0) + LD && return SubArray{T,N,P,I,LD}(A, newindexes, Base.compute_offset1(A, 1, newindexes), 1) + SubArray{T,N,P,I,LD}(A, newindexes, 0, 0) end _trimmedsubarray(A, V, newindexes, index::ViewIndex, indexes...) = _trimmedsubarray(A, V, (newindexes..., trimmedindex(V.parent, length(newindexes)+1, index)), indexes...) diff --git a/base/show.jl b/base/show.jl index fe9db2c308e44..51b04e057ae47 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1286,7 +1286,7 @@ function alignment(io::IO, X::AbstractVecOrMat, break end end - if 1 < length(a) < size(X,2) + if 1 < length(a) < length(indices(X,2)) while sum(map(sum,a)) + sep*length(a) >= cols_otherwise pop!(a) end @@ -1384,8 +1384,9 @@ function print_matrix(io::IO, X::AbstractVecOrMat, postsp = "" @assert strwidth(hdots) == strwidth(ddots) sepsize = length(sep) - m, n = size(X,1), size(X,2) - rowsA, colsA = collect(indices(X,1)), collect(indices(X,2)) + inds1, inds2 = indices(X,1), indices(X,2) + m, n = length(inds1), length(inds2) + rowsA, colsA = collect(inds1), collect(inds2) # To figure out alignments, only need to look at as many rows as could # fit down screen. If screen has at least as many rows as A, look at A. # If not, then we only need to look at the first and last chunks of A, @@ -1488,19 +1489,19 @@ function show_nd(io::IO, a::AbstractArray, print_matrix, label_slices) if isempty(a) return end - tail = indices(a)[3:end] + tailinds = tail(tail(indices(a))) nd = ndims(a)-2 - for I in CartesianRange(tail) + for I in CartesianRange(tailinds) idxs = I.I if limit for i = 1:nd ii = idxs[i] - ind = tail[i] + ind = tailinds[i] if length(ind) > 10 - if ii == ind[4] && all(d->idxs[d]==first(tail[d]),1:i-1) + if ii == ind[4] && all(d->idxs[d]==first(tailinds[d]),1:i-1) for j=i+1:nd szj = size(a,j+2) - indj = tail[j] + indj = tailinds[j] if szj>10 && first(indj)+2 < idxs[j] <= last(indj)-3 @goto skip end @@ -1522,7 +1523,7 @@ function show_nd(io::IO, a::AbstractArray, print_matrix, label_slices) end slice = view(a, indices(a,1), indices(a,2), idxs...) print_matrix(io, slice) - print(io, idxs == map(last,tail) ? "" : "\n\n") + print(io, idxs == map(last,tailinds) ? "" : "\n\n") @label skip end end @@ -1536,10 +1537,11 @@ function print_matrix_repr(io, X::AbstractArray) if compact && !haskey(io, :compact) io = IOContext(io, :compact => compact) end - nr, nc = size(X,1), size(X,2) + indr, indc = indices(X,1), indices(X,2) + nr, nc = length(indr), length(indc) rdots, cdots = false, false - rr1, rr2 = indices(X,1), 1:0 - cr1, cr2 = indices(X,2), 1:0 + rr1, rr2 = UnitRange{Int}(indr), 1:0 + cr1, cr2 = UnitRange{Int}(indc), 1:0 if limit if nr > 4 rr1, rr2 = rr1[1:2], rr1[nr-1:nr] @@ -1563,8 +1565,8 @@ function print_matrix_repr(io, X::AbstractArray) show(io, el) end end - if last(cr) == last(indices(X,2)) - i < last(indices(X,1)) && print(io, "; ") + if last(cr) == last(indc) + i < last(indr) && print(io, "; ") elseif cdots print(io, " \u2026 ") end diff --git a/base/sort.jl b/base/sort.jl index 99900be146ef4..d388e55f6647c 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -482,16 +482,16 @@ function sort(A::AbstractArray, dim::Integer; order::Ordering=Forward, initialized::Bool=false) order = ord(lt,by,rev,order) + n = length(indices(A, dim)) if dim != 1 pdims = (dim, setdiff(1:ndims(A), dim)...) # put the selected dimension first - Ap = permutedims(A, pdims) # note Ap is an Array, no matter what A is - n = size(Ap, 1) + Ap = permutedims(A, pdims) Av = vec(Ap) sort_chunks!(Av, n, alg, order) ipermutedims(Ap, pdims) else Av = A[:] - sort_chunks!(Av, size(A,1), alg, order) + sort_chunks!(Av, n, alg, order) reshape(Av, indices(A)) end end diff --git a/base/statistics.jl b/base/statistics.jl index c7516f750ee00..e72f2f1cd6758 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -96,10 +96,9 @@ function centralize_sumabs2!{S,T,N}(R::AbstractArray{S}, A::AbstractArray{T,N}, lsiz = check_reducedims(R,A) isempty(R) || fill!(R, zero(S)) isempty(A) && return R - sizA1 = size(A, 1) if has_fast_linear_indexing(A) && lsiz > 16 - nslices = div(length(A), lsiz) + nslices = div(unsafe_length(A), lsiz) ibase = first(linearindices(A))-1 for i = 1:nslices @inbounds R[i] = centralize_sumabs2(A, means[i], ibase+1, ibase+lsiz) @@ -107,21 +106,21 @@ function centralize_sumabs2!{S,T,N}(R::AbstractArray{S}, A::AbstractArray{T,N}, end return R end - IRmax = dims_tail(map(last, indices(R)), A) - if size(R, 1) == 1 && sizA1 > 1 - i1 = first(indices(A, 1)) - @inbounds for IA in CartesianRange(tail(indices(A))) - IR = min(IA, IRmax) - r = R[i1,IR] - m = means[i1,IR] + indsAt, indsRt = safe_tail(indices(A)), safe_tail(indices(R)) # handle d=1 manually + imap = Broadcast.newindexer(indsAt, indsRt) + if reducedim1(R, A) + @inbounds for IA in CartesianRange(indsAt) + IR = Broadcast.newindex(IA, imap) + r = R[1,IR] + m = means[1,IR] @simd for i in indices(A, 1) r += abs2(A[i,IA] - m) end - R[i1,IR] = r + R[1,IR] = r end else - @inbounds for IA in CartesianRange(tail(indices(A))) - IR = min(IA, IRmax) + @inbounds for IA in CartesianRange(indsAt) + IR = Broadcast.newindex(IA, imap) @simd for i in indices(A, 1) R[i,IR] += abs2(A[i,IA] - means[i,IR]) end diff --git a/base/subarray.jl b/base/subarray.jl index 48446fcc60cb4..390b6fc5aa07e 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -8,25 +8,24 @@ abstract AbstractCartesianIndex{N} # This is a hacky forward declaration for Car immutable SubArray{T,N,P,I,L} <: AbstractArray{T,N} parent::P indexes::I - dims::NTuple{N,Int} offset1::Int # for linear indexing and pointer, only valid when L==true stride1::Int # used only for linear indexing - function SubArray(parent, indexes, dims, offset1, stride1) + function SubArray(parent, indexes, offset1, stride1) check_parent_index_match(parent, indexes) - new(parent, indexes, dims, offset1, stride1) + new(parent, indexes, offset1, stride1) end end # Compute the linear indexability of the indices, and combine it with the linear indexing of the parent function SubArray(parent::AbstractArray, indexes::Tuple, dims::Tuple) - SubArray(linearindexing(viewindexing(indexes), linearindexing(parent)), parent, indexes, map(dimlength, dims)) + SubArray(linearindexing(viewindexing(indexes), linearindexing(parent)), parent, indexes, dims) end -function SubArray{P, I, N}(::LinearSlow, parent::P, indexes::I, dims::NTuple{N, Int}) - SubArray{eltype(P), N, P, I, false}(parent, indexes, dims, 0, 0) +function SubArray{P, I, N}(::LinearSlow, parent::P, indexes::I, dims::NTuple{N}) + SubArray{eltype(P), N, P, I, false}(parent, indexes, 0, 0) end -function SubArray{P, I, N}(::LinearFast, parent::P, indexes::I, dims::NTuple{N, Int}) +function SubArray{P, I, N}(::LinearFast, parent::P, indexes::I, dims::NTuple{N}) # Compute the stride and offset stride1 = compute_stride1(parent, indexes) - SubArray{eltype(P), N, P, I, true}(parent, indexes, dims, compute_offset1(parent, stride1, indexes), stride1) + SubArray{eltype(P), N, P, I, true}(parent, indexes, compute_offset1(parent, stride1, indexes), stride1) end check_parent_index_match{T,N}(parent::AbstractArray{T,N}, indexes::NTuple{N}) = nothing @@ -47,12 +46,8 @@ viewindexing(I::Tuple{Vararg{Any}}) = LinearSlow() # Of course, all other array types are slow viewindexing(I::Tuple{AbstractArray, Vararg{Any}}) = LinearSlow() -dimlength(r::Range) = length(r) -dimlength(i::Integer) = Int(i) - # Simple utilities -size(V::SubArray) = V.dims -length(V::SubArray) = prod(V.dims) +size(V::SubArray) = (@_inline_meta; map(n->Int(unsafe_length(n)), indices(V))) similar(V::SubArray, T::Type, dims::Dims) = similar(V.parent, T, dims) @@ -83,12 +78,12 @@ end function unsafe_view{T,N}(A::AbstractArray{T,N}, I::Vararg{ViewIndex,N}) @_inline_meta J = to_indexes(I...) - SubArray(A, J, map(dimlength, index_shape(A, J...))) + SubArray(A, J, map(unsafe_length, index_shape(A, J...))) end function unsafe_view{T,N}(V::SubArray{T,N}, I::Vararg{ViewIndex,N}) @_inline_meta idxs = reindex(V, V.indexes, to_indexes(I...)) - SubArray(V.parent, idxs, map(dimlength, (index_shape(V.parent, idxs...)))) + SubArray(V.parent, idxs, map(unsafe_length, (index_shape(V.parent, idxs...)))) end # Re-indexing is the heart of a view, transforming A[i, j][x, y] to A[i[x], j[y]] @@ -199,34 +194,25 @@ substrides(s, parent, dim, I::Tuple{Any, Vararg{Any}}) = throw(ArgumentError("st stride(V::SubArray, d::Integer) = d <= ndims(V) ? strides(V)[d] : strides(V)[end] * size(V)[end] -compute_stride1(parent, I::Tuple) = compute_stride1(1, parent, 1, I) -compute_stride1(s, parent, dim, I::Tuple{}) = s -compute_stride1(s, parent, dim, I::Tuple{DroppedScalar, Vararg{Any}}) = - (@_inline_meta; compute_stride1(s*size(parent, dim), parent, dim+1, tail(I))) -compute_stride1(s, parent, dim, I::Tuple{Range, Vararg{Any}}) = s*step(I[1]) -compute_stride1(s, parent, dim, I::Tuple{Colon, Vararg{Any}}) = s -compute_stride1(s, parent, dim, I::Tuple{Any, Vararg{Any}}) = throw(ArgumentError("invalid strided index type $(typeof(I[1]))")) +compute_stride1{N}(parent::AbstractArray, I::NTuple{N}) = + compute_stride1(1, fill_to_length(indices(parent), OneTo(1), Val{N}), I) +compute_stride1(s, inds, I::Tuple{}) = s +compute_stride1(s, inds, I::Tuple{DroppedScalar, Vararg{Any}}) = + (@_inline_meta; compute_stride1(s*unsafe_length(inds[1]), tail(inds), tail(I))) +compute_stride1(s, inds, I::Tuple{Range, Vararg{Any}}) = s*step(I[1]) +compute_stride1(s, inds, I::Tuple{Colon, Vararg{Any}}) = s +compute_stride1(s, inds, I::Tuple{Any, Vararg{Any}}) = throw(ArgumentError("invalid strided index type $(typeof(I[1]))")) iscontiguous(A::SubArray) = iscontiguous(typeof(A)) iscontiguous{S<:SubArray}(::Type{S}) = false iscontiguous{F<:FastContiguousSubArray}(::Type{F}) = true -# Fast linear SubArrays have their first index cached -first_index(V::FastSubArray) = V.offset1 + V.stride1 -first_index(V::SubArray) = first_index(V.parent, V.indexes) -function first_index(P::AbstractArray, indexes::Tuple) - f = first(linearindices(P)) - s = 1 - for i = 1:length(indexes) - f += (_first(indexes[i], P, i)-first(indices(P, i)))*s - s *= size(P, i) - end - f +first_index(V::FastSubArray) = V.offset1 + V.stride1 # cached for fast linear SubArrays +function first_index(V::SubArray) + P, I = parent(V), V.indexes + s1 = compute_stride1(P, I) + s1 + compute_offset1(P, s1, I) end -_first(::Colon, P, ::Colon) = first(linearindices(P)) -_first(i, P, ::Colon) = first(i) -_first(::Colon, P, d) = first(indices(P, d)) -_first(i, P, d) = first(i) # Computing the first index simply steps through the indices, accumulating the # sum of index each multiplied by the parent's stride. @@ -238,17 +224,26 @@ compute_offset1(parent, stride1::Integer, I::Tuple) = (@_inline_meta; compute_of compute_offset1(parent, stride1::Integer, dims::Tuple{Int}, inds::Tuple{Colon}, I::Tuple) = compute_linindex(parent, I) - stride1*first(indices(parent, dims[1])) # index-preserving case compute_offset1(parent, stride1::Integer, dims, inds, I::Tuple) = compute_linindex(parent, I) - stride1 # linear indexing starts with 1 -compute_linindex(parent, I) = compute_linindex(1, 1, parent, 1, I) -compute_linindex(f, s, parent, dim, I::Tuple{Real, Vararg{Any}}) = - (@_inline_meta; compute_linindex(f + (I[1]-first(indices(parent,dim)))*s, s*size(parent, dim), parent, dim+1, tail(I))) +function compute_linindex{N}(parent, I::NTuple{N}) + IP = fill_to_length(indices(parent), OneTo(1), Val{N}) + compute_linindex(1, 1, IP, I) +end +function compute_linindex(f, s, IP::Tuple, I::Tuple{Real, Vararg{Any}}) + @_inline_meta + Δi = I[1]-first(IP[1]) + compute_linindex(f + Δi*s, s*unsafe_length(IP[1]), tail(IP), tail(I)) +end # Just splat out the cartesian indices and continue -compute_linindex(f, s, parent, dim, I::Tuple{AbstractCartesianIndex, Vararg{Any}}) = - (@_inline_meta; compute_linindex(f, s, parent, dim, (I[1].I..., tail(I)...))) -compute_linindex(f, s, parent, dim, I::Tuple{Colon, Vararg{Any}}) = - (@_inline_meta; compute_linindex(f, s*size(parent, dim), parent, dim+1, tail(I))) -compute_linindex(f, s, parent, dim, I::Tuple{Any, Vararg{Any}}) = - (@_inline_meta; compute_linindex(f + (first(I[1])-first(indices(parent,dim)))*s, s*size(parent, dim), parent, dim+1, tail(I))) -compute_linindex(f, s, parent, dim, I::Tuple{}) = f +compute_linindex(f, s, IP::Tuple, I::Tuple{AbstractCartesianIndex, Vararg{Any}}) = + (@_inline_meta; compute_linindex(f, s, IP, (I[1].I..., tail(I)...))) +compute_linindex(f, s, IP::Tuple, I::Tuple{Colon, Vararg{Any}}) = + (@_inline_meta; compute_linindex(f, s*unsafe_length(IP[1]), tail(IP), tail(I))) +function compute_linindex(f, s, IP::Tuple, I::Tuple{Any, Vararg{Any}}) + @_inline_meta + Δi = first(I[1])-first(IP[1]) + compute_linindex(f + Δi*s, s*unsafe_length(IP[1]), tail(IP), tail(I)) +end +compute_linindex(f, s, IP::Tuple, I::Tuple{}) = f find_extended_dims(I) = (@_inline_meta; _find_extended_dims((), (), 1, I...)) _find_extended_dims(dims, inds, dim) = dims, inds diff --git a/doc/devdocs/julia.rst b/doc/devdocs/julia.rst index 32c1c212caebb..141c9f4933b81 100644 --- a/doc/devdocs/julia.rst +++ b/doc/devdocs/julia.rst @@ -25,3 +25,4 @@ promote-op boundscheck locks + offset-arrays diff --git a/doc/devdocs/offset-arrays.rst b/doc/devdocs/offset-arrays.rst new file mode 100644 index 0000000000000..5d6beb8563251 --- /dev/null +++ b/doc/devdocs/offset-arrays.rst @@ -0,0 +1,293 @@ +.. currentmodule:: Base + +.. _devdocs-offsetarrays: + +************************** +Arrays with custom indices +************************** + +Julia 0.5 adds experimental support for arrays with arbitrary indices. +Conventionally, Julia's arrays are indexed starting at 1, whereas some +other languages start numbering at 0, and yet others (e.g., Fortran) +allow you to specify arbitrary starting indices. While there is much +merit in picking a standard (i.e., 1 for Julia), there are some +algorithms which simplify considerably if you can index outside the +range ``1:size(A,d)`` (and not just ``0:size(A,d)-1``, either). Such +array types are expected to be supplied through packages. + +The purpose of this page is to address the question, "what do I +have to do to support such arrays in my own code?" First, let's +address the simplest case: if you know that your code will never need +to handle arrays with unconventional indexing, hopefully the answer is +"nothing." Old code, on conventional arrays, should function +essentially without alteration as long as it was using the exported +interfaces of Julia. + +Generalizing existing code +-------------------------- + +As an overview, the steps are: + +- replace many uses of ``size`` with ``indices`` +- replace ``1:length(A)`` with ``linearindices(A)``, and ``length(A)`` with ``length(linearindices(A))`` +- replace explicit allocations like ``Array{Int}(size(B))`` with ``similar(Array{Int}, indices(B))`` + +These are described in more detail below. + +Background +~~~~~~~~~~ + +Because unconventional indexing breaks deeply-held assumptions +throughout the Julia ecosystem, early adopters running code that has +not been updated are likely to experience errors. The most +frustrating bugs would be incorrect results or segfaults (total +crashes of Julia). For example, consider the following function:: + + function mycopy!(dest::AbstractVector, src::AbstractVector) + length(dest) == length(src) || throw(DimensionMismatch("vectors must match")) + # OK, now we're safe to use @inbounds, right? (not anymore!) + for i = 1:length(src) + @inbounds dest[i] = src[i] + end + dest + end + +This code implicitly assumes that vectors are indexed from 1. +Previously that was a safe assumption, so this code was fine, but +(depending on what types the user passes to this function) it may no +longer be safe. If this code continued to work when passed a vector +with non-1 indices, it would either produce an incorrect answer or it +would segfault. (If you do get segfaults, to help locate the +cause try running julia with the option ``--check-bounds=yes``.) + +To ensure that such errors are caught, in Julia 0.5 both ``length`` +and ``size`` **should** throw an error when passed an array with +non-1 indexing. This is designed to force users of such arrays to +check the code, and inspect it for whether it needs to be generalized. + +Using ``indices`` for bounds checks and loop iteration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``indices(A)`` (reminiscent of ``size(A)``) returns a tuple of +``AbstractUnitRange`` objects, specifying the range of valid indices +along each dimension of ``A``. When ``A`` has unconventional +indexing, the ranges may not start at 1. If you just want the range +for a particular dimension ``d``, there is ``indices(A, d)``. + +Base implements a custom range type, ``OneTo``, where ``OneTo(n)`` +means the same thing as ``1:n`` but in a form that guarantees (via the +type system) that the lower index is 1. For any new ``AbstractArray`` +type, this is the default returned by ``indices``, and it indicates +that this array type uses "conventional" 1-based indexing. Note that +if you don't want to be bothered supporting arrays with non-1 +indexing, you can add the following line:: + + @assert all(x->isa(x, Base.OneTo), indices(A)) + +at the top of any function. + +For bounds checking, note that there are dedicated functions +``checkbounds`` and ``checkindex`` which can sometimes simplify such +tests. + +Linear indexing (``linearindices``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some algorithms are most conveniently (or efficiently) written in +terms of a single linear index, ``A[i]`` even if ``A`` is +multi-dimensional. In "true" linear indexing, the indices always +range from ``1:length(A)``. However, this raises an ambiguity for +one-dimensional arrays (a.k.a., ``AbstractVector``): does ``v[i]`` +mean linear indexing, or Cartesian indexing with the array's native +indices? + +For this reason, if you want to use linear indexing in an algorithm, +your best option is to get the index range by calling +``linearindices(A)``. This will return ``indices(A, 1)`` if ``A`` is an +``AbstractVector``, and the equivalent of ``1:length(A)`` otherwise. + +In a sense, one can say that 1-dimensional arrays always use Cartesian +indexing. To help enforce this, it's worth noting that +``sub2ind(shape, i...)`` and ``ind2sub(shape, ind)`` will throw an +error if ``shape`` indicates a 1-dimensional array with unconventional +indexing (i.e., is a ``Tuple{UnitRange}`` rather than a tuple of +``OneTo``). For arrays with conventional indexing, these functions +continue to work the same as always. + +Using ``indices`` and ``linearindices``, here is one way you could +rewrite ``mycopy!``:: + + function mycopy!(dest::AbstractVector, src::AbstractVector) + indices(dest) == indices(src) || throw(DimensionMismatch("vectors must match")) + for i in linearindices(src) + @inbounds dest[i] = src[i] + end + dest + end + +Allocating storage using generalizations of ``similar`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Storage is often allocated with ``Array{Int}(dims)`` or ``similar(A, args...)``. +When the result needs to match the indices of some other array, this may not always suffice. +The generic replacement for such patterns is to use +``similar(storagetype, shape)``. ``storagetype`` indicates the kind +of underlying "conventional" behavior you'd like, e.g., ``Array{Int}`` +or ``BitArray`` or even ``dims->zeros(Float32, dims)`` (which would +allocate an all-zeros array). ``shape`` is a tuple of ``Integer`` or +``AbstractUnitRange`` values, specifying the indices +that you want the result to use. + +Let's walk through a couple of explicit examples. First, if ``A`` has +conventional indices, then ``similar(Array{Int}, indices(A))`` +would end up calling ``Array{Int}(size(A))``, and thus return an +array. If ``A`` is an ``AbstractArray`` type with unconventional +indexing, then ``similar(Array{Int}, indices(A))`` should return +something that "behaves like" an ``Array{Int}`` but with a shape +(including indices) that matches ``A``. (The most obvious +implementation is to allocate an ``Array{Int}(size(A))`` and then +"wrap" it in a type that shifts the indices.) + +Note also that ``similar(Array{Int}, (indices(A, 2),))`` would allocate +an ``AbstractVector{Int}`` (i.e., 1-dimensional array) that matches the +indices of the columns of ``A``. + + +Deprecations +~~~~~~~~~~~~ + +In generalizing Julia's code base, at least one deprecation was +unavoidable: earlier versions of Julia defined ``first(::Colon) = 1``, +meaning that the first index along a dimension indexed by ``:`` is 1. +This definition can no longer be justified, so it was deprecated. +There is no provided replacement, because the proper replacement +depends on what you are doing and might need to know more about the +array. However, it appears that many uses of ``first(::Colon)`` are +really about computing an index offset; when that is the case, a +candidate replacement is:: + + indexoffset(r::AbstractVector) = first(r) - 1 + indexoffset(::Colon) = 0 + +In other words, while ``first(:)`` does not itself make sense, in +general you can say that the offset associated with a colon-index is +zero. + +Writing custom array types with non-1 indexing +---------------------------------------------- + +Most of the methods you'll need to define are standard for any +``AbstractArray`` type, see :ref:`man-interfaces-abstractarray`. +This page focuses on the steps needed to define unconventional +indexing. + +Do **not** implement ``size`` or ``length`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Perhaps the majority of pre-existing code that uses ``size`` will not +work properly for arrays with non-1 indices. For that reason, it is +much better to avoid implementing these methods, and use the resulting +``MethodError`` to identify code that needs to be audited and perhaps +generalized. + +Do **not** annotate bounds checks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Julia 0.5 includes ``@boundscheck`` to annotate code that can be +removed for callers that exploit ``@inbounds``. Initially, it seems +far preferable to run with bounds checking always enabled (i.e., omit +the ``@boundscheck`` annotation so the check always runs). + +Custom ``AbstractUnitRange`` types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you're writing a non-1 indexed array type, you will want to +specialize ``indices`` so it returns a ``UnitRange``, or (perhaps +better) a custom ``AbstractUnitRange``. The advantage of a custom +type is that it "signals" the allocation type for functions like +``similar``. If we're writing an array type for which indexing will +start at 0, we likely want to begin by creating a new +``AbstractUnitRange``, ``ZeroRange``, where ``ZeroRange(n)`` is equivalent +to ``0:n-1``. + +In general, you should probably *not* export ``ZeroRange`` from your +package: there may be other packages that implement their own +``ZeroRange``, and having multiple distinct ``ZeroRange`` types is (perhaps +counterintuitively) an advantage: ``ModuleA.ZeroRange`` indicates that +``similar`` should create a ``ModuleA.ZeroArray``, whereas +``ModuleB.ZeroRange`` indicates a ``ModuleB.ZeroArray`` type. This +design allows peaceful coexistence among many different custom array +types. + +Note that the Julia package ``CustomUnitRanges.jl`` can sometimes be used to +avoid the need to write your own ``ZeroRange`` type. + +Specializing ``indices`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +Once you have your ``AbstractUnitRange`` type, then specialize ``indices``:: + + Base.indices(A::ZeroArray) = map(n->ZeroRange(n), A.size) + +where here we imagine that ``ZeroArray`` has a field called ``size`` +(there would be other ways to implement this). + +In some cases, the fallback definition for ``indices(A, d)``:: + + indices{T,N}(A::AbstractArray{T,N}, d) = d <= N ? indices(A)[d] : OneTo(1) + +may not be what you want: you may need to specialize it to return +something other than ``OneTo(1)`` when ``d > ndims(A)``. Likewise, in +``Base`` there is a dedicated function ``indices1`` which is +equivalent to ``indices(A, 1)`` but which avoids checking (at +runtime) whether ``ndims(A) > 0``. (This is purely a performance +optimization.) It is defined as:: + + indices1{T}(A::AbstractArray{T,0}) = OneTo(1) + indices1{T}(A::AbstractArray{T}) = indices(A)[1] + +If the first of these (the zero-dimensional case) is problematic for +your custom array type, be sure to specialize it appropriately. + +Specializing ``similar`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +Given your custom ``ZeroRange`` type, then you should also add the following two specializations for ``similar``:: + + function Base.similar(A::AbstractArray, T::Type, shape::Tuple{ZeroRange,Vararg{ZeroRange}}) + # body + end + + function Base.similar(f::Union{Function,DataType}, shape::Tuple{ZeroRange,Vararg{ZeroRange}}) + # body + end + +Both of these should allocate your custom array type. + +Specializing ``reshape`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +Optionally, define a method +:: + + Base.reshape(A::AbstractArray, shape::Tuple{ZeroRange,Vararg{ZeroRange}}) = ... + +and you can ``reshape`` an array so that the result has custom indices. + +Summary +------- + +Writing code that doesn't make assumptions about indexing requires a +few extra abstractions, but hopefully the necessary changes are +relatively straightforward. + +As a reminder, this support is still experimental. While much of +Julia's base code has been updated to support unconventional indexing, +without a doubt there are many omissions that will be discovered only +through usage. Moreover, at the time of this writing, most packages +do not support unconventional indexing. As a consequence, early +adopters should be prepared to identify and/or fix bugs. On the other +hand, only through practical usage will it become clear whether this +experimental feature should be retained in future versions of Julia; +consequently, interested parties are encouraged to accept some +ownership for putting it through its paces. diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index b0f0886da6c22..15465047b2402 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -166,8 +166,9 @@ Methods to implement :func:`similar(A, dims::NTuple{Int}) <similar>` ``similar(A, eltype(A), dims)`` Return a mutable array with the same element type and size `dims` :func:`similar(A, ::Type{S}, dims::NTuple{Int}) <similar>` ``Array{S}(dims)`` Return a mutable array with the specified element type and size **Non-traditional indices** **Default definition** **Brief description** -:func:`indices(A, d) <indices>` ``OneTo(size(A, d))`` Return the ``AbstractUnitRange`` of valid indices along dimension ``d`` -:func:`Base.similar(A, ::Type{S}, inds::NTuple{Ind}) <similar>` ``similar(A, S, map(Base.dimlength, inds))`` Return a mutable array with the specified indices ``inds`` (see below) +:func:`indices(A) <indices>` ``map(OneTo, size(A))`` Return the ``AbstractUnitRange`` of valid indices +:func:`Base.similar(A, ::Type{S}, inds::NTuple{Ind}) <similar>` ``similar(A, S, Base.to_shape(inds))`` Return a mutable array with the specified indices ``inds`` (see below) +:func:`Base.similar(T::Union{Type,Function}, inds) <similar>` ``T(Base.to_shape(inds))`` Return an array similar to ``T`` with the specified indices ``inds`` (see below) ===================================================================== ============================================ ======================================================================================= If a type is defined as a subtype of ``AbstractArray``, it inherits a very large set of rich behaviors including iteration and multidimensional indexing built on top of single-element access. See the :ref:`arrays manual page <man-arrays>` and :ref:`standard library section <stdlib-arrays>` for more supported methods. @@ -292,6 +293,4 @@ If you are defining an array type that allows non-traditional indexing ``indices``. You should also specialize ``similar`` so that the ``dims`` argument (ordinarily a ``Dims`` size-tuple) can accept ``AbstractUnitRange`` objects, perhaps range-types ``Ind`` of your own -design. For example, if indexing always starts with 0 for your -arrays, you likely want to define a ``ZeroTo`` range type. Otherwise, -you can use standard ``UnitRange``. +design. For more information, see :ref:`devdocs-offsetarrays`. diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 4a91bf6a29f26..4e83a3299226d 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -7,7 +7,7 @@ module OAs -using Base: DimOrInd, DimsOrInds, Indices, LinearSlow, LinearFast +using Base: Indices, LinearSlow, LinearFast, tail export OffsetArray @@ -20,7 +20,7 @@ typealias OffsetVector{T,AA<:AbstractArray} OffsetArray{T,1,AA} OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::NTuple{N,Int}) = OffsetArray{T,N,typeof(A)}(A, offsets) OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::Vararg{Int,N}) = OffsetArray(A, offsets) -(::Type{OffsetArray{T,N}}){T,N}(inds::Indices{N}) = OffsetArray{T,N,Array{T,N}}(Array{T,N}(map(Base.dimlength, inds)), map(indsoffset, inds)) +(::Type{OffsetArray{T,N}}){T,N}(inds::Indices{N}) = OffsetArray{T,N,Array{T,N}}(Array{T,N}(map(length, inds)), map(indsoffset, inds)) (::Type{OffsetArray{T}}){T,N}(inds::Indices{N}) = OffsetArray{T,N}(inds) Base.linearindexing{T<:OffsetArray}(::Type{T}) = Base.linearindexing(parenttype(T)) @@ -28,59 +28,62 @@ parenttype{T,N,AA}(::Type{OffsetArray{T,N,AA}}) = AA parenttype(A::OffsetArray) = parenttype(typeof(A)) Base.parent(A::OffsetArray) = A.parent -Base.size(A::OffsetArray) = size(parent(A)) + +errmsg(A) = error("size not supported for arrays with indices $(indices(A)); see http://docs.julialang.org/en/latest/devdocs/offset-arrays/") +Base.size(A::OffsetArray) = errmsg(A) +Base.size(A::OffsetArray, d) = errmsg(A) +Base.eachindex(::LinearSlow, A::OffsetArray) = CartesianRange(indices(A)) +Base.eachindex(::LinearFast, A::OffsetVector) = indices(A, 1) Base.summary(A::OffsetArray) = string(typeof(A))*" with indices "*string(indices(A)) # Implementations of indices and indices1. Since bounds-checking is # performance-critical and relies on indices, these are usually worth # optimizing thoroughly. -@inline Base.indices(A::OffsetArray, d) = 1 <= d <= length(A.offsets) ? (o = A.offsets[d]; (1+o:size(parent(A),d)+o)) : (1:1) -@inline Base.indices(A::OffsetArray) = _indices((), A) # would rather use ntuple, but see #15276 -@inline _indices{T,N}(out::NTuple{N}, A::OffsetArray{T,N}) = out -@inline _indices(out, A::OffsetArray) = (d = length(out)+1; o = A.offsets[d]; _indices((out..., (1+o:size(parent(A),d)+o)), A)) -# By optimizing indices1 we can avoid a branch on the dim-check -Base.indices1{T}(A::OffsetArray{T,0}) = 1:1 -@inline Base.indices1(A::OffsetArray) = (o = A.offsets[1]; 1+o:size(parent(A),1)+o) +@inline Base.indices(A::OffsetArray, d) = 1 <= d <= length(A.offsets) ? indices(parent(A))[d] + A.offsets[d] : (1:1) +@inline Base.indices(A::OffsetArray) = _indices(indices(parent(A)), A.offsets) # would rather use ntuple, but see #15276 +@inline _indices(inds, offsets) = (inds[1]+offsets[1], _indices(tail(inds), tail(offsets))...) +_indices(::Tuple{}, ::Tuple{}) = () +Base.indices1{T}(A::OffsetArray{T,0}) = 1:1 # we only need to specialize this one function Base.similar(A::OffsetArray, T::Type, dims::Dims) B = similar(parent(A), T, dims) end function Base.similar(A::AbstractArray, T::Type, inds::Tuple{UnitRange,Vararg{UnitRange}}) - B = similar(A, T, map(Base.dimlength, inds)) + B = similar(A, T, map(length, inds)) OffsetArray(B, map(indsoffset, inds)) end -Base.similar(f::Union{Function,Type}, shape::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(f(map(Base.dimlength, shape)), map(indsoffset, shape)) +Base.similar(f::Union{Function,DataType}, shape::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(f(map(length, shape)), map(indsoffset, shape)) -Base.reshape(A::AbstractArray, inds::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(reshape(A, map(Base.dimlength, inds)), map(indsoffset, inds)) +Base.reshape(A::AbstractArray, inds::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(reshape(A, map(length, inds)), map(indsoffset, inds)) @inline function Base.getindex{T,N}(A::OffsetArray{T,N}, I::Vararg{Int,N}) - @boundscheck checkbounds(A, I...) + checkbounds(A, I...) @inbounds ret = parent(A)[offset(A.offsets, I)...] ret end @inline function Base._getindex(::LinearFast, A::OffsetVector, i::Int) - @boundscheck checkbounds(A, i) + checkbounds(A, i) @inbounds ret = parent(A)[offset(A.offsets, (i,))[1]] ret end @inline function Base._getindex(::LinearFast, A::OffsetArray, i::Int) - @boundscheck checkbounds(A, i) + checkbounds(A, i) @inbounds ret = parent(A)[i] ret end @inline function Base.setindex!{T,N}(A::OffsetArray{T,N}, val, I::Vararg{Int,N}) - @boundscheck checkbounds(A, I...) + checkbounds(A, I...) @inbounds parent(A)[offset(A.offsets, I)...] = val val end @inline function Base._setindex!(::LinearFast, A::OffsetVector, val, i::Int) - @boundscheck checkbounds(A, i) + checkbounds(A, i) @inbounds parent(A)[offset(A.offsets, (i,))[1]] = val val end @inline function Base._setindex!(::LinearFast, A::OffsetArray, val, i::Int) - @boundscheck checkbounds(A, i) + checkbounds(A, i) @inbounds parent(A)[i] = val val end @@ -101,8 +104,12 @@ let # Basics A0 = [1 3; 2 4] A = OffsetArray(A0, (-1,2)) # LinearFast -S = OffsetArray(view(A0, 1:2, 1:2), (-1,2)) # LinearSlow +S = OffsetArray(view(A0, 1:2, 1:2), (-1,2)) # LinearSlow @test indices(A) == indices(S) == (0:1, 3:4) +@test_throws ErrorException size(A) +@test_throws ErrorException size(A, 1) + +# Scalar indexing @test A[0,3] == A[1] == S[0,3] == S[1] == 1 @test A[1,3] == A[2] == S[1,3] == S[2] == 2 @test A[0,4] == A[3] == S[0,4] == S[3] == 3 @@ -204,7 +211,6 @@ cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,10^3), (10,-9))) # neithe # Similar B = similar(A, Float32) @test isa(B, OffsetArray{Float32,2}) -@test size(B) == size(A) @test indices(B) === indices(A) B = similar(A, (3,4)) @test isa(B, Array{Int,2}) @@ -307,10 +313,10 @@ cumsum!(C, A, 1) @test parent(cumsum(A, 1)) == cumsum(parent(A), 1) cumsum!(C, A, 2) @test parent(C) == cumsum(parent(A), 2) -R = similar(A, (-2:-2, 6:9)) +R = similar(A, (1:1, 6:9)) maximum!(R, A) @test parent(R) == maximum(parent(A), 1) -R = similar(A, (-2:1, 6:6)) +R = similar(A, (-2:1, 1:1)) maximum!(R, A) @test parent(R) == maximum(parent(A), 2) amin, iamin = findmin(A) From 721cc6f12310fcb271c09395ff606881379538a9 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 18 Jul 2016 11:36:01 -0500 Subject: [PATCH 0502/1117] unsafe_length(A) -> _length(A) --- base/abstractarray.jl | 8 ++++---- base/abstractarraymath.jl | 2 +- base/multidimensional.jl | 2 +- base/operators.jl | 8 ++++---- base/reducedim.jl | 2 +- base/reshapedarray.jl | 2 +- base/statistics.jl | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 5c70411c7d950..15cdb1b816a99 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -65,7 +65,7 @@ is `indices(A, 1)`. Calling this function is the "safe" way to write algorithms that exploit linear indexing. """ -linearindices(A) = (@_inline_meta; OneTo(unsafe_length(A))) +linearindices(A) = (@_inline_meta; OneTo(_length(A))) linearindices(A::AbstractVector) = (@_inline_meta; indices1(A)) eltype{T}(::Type{AbstractArray{T}}) = T eltype{T,N}(::Type{AbstractArray{T,N}}) = T @@ -74,7 +74,7 @@ ndims{T,N}(::AbstractArray{T,N}) = N ndims{T,N}(::Type{AbstractArray{T,N}}) = N ndims{T<:AbstractArray}(::Type{T}) = ndims(supertype(T)) length(t::AbstractArray) = prod(size(t)) -unsafe_length(A::AbstractArray) = prod(map(unsafe_length, indices(A))) # internal method +_length(A::AbstractArray) = prod(map(unsafe_length, indices(A))) # circumvent missing size endof(a::AbstractArray) = length(a) first(a::AbstractArray) = a[first(eachindex(a))] @@ -486,7 +486,7 @@ function copy!(::LinearIndexing, dest::AbstractArray, ::LinearSlow, src::Abstrac end function copy!(dest::AbstractArray, dstart::Integer, src::AbstractArray) - copy!(dest, dstart, src, first(linearindices(src)), unsafe_length(src)) + copy!(dest, dstart, src, first(linearindices(src)), _length(src)) end function copy!(dest::AbstractArray, dstart::Integer, src::AbstractArray, sstart::Integer) @@ -612,7 +612,7 @@ function _maxlength(A, B, C...) max(length(A), _maxlength(B, C...)) end -isempty(a::AbstractArray) = (unsafe_length(a) == 0) +isempty(a::AbstractArray) = (_length(a) == 0) ## Conversions ## diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index d05abd4f4281f..43efa5063b48c 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -11,7 +11,7 @@ transpose(a::AbstractArray) = error("transpose not implemented for $(typeof(a)). ## Constructors ## -vec(a::AbstractArray) = reshape(a,unsafe_length(a)) +vec(a::AbstractArray) = reshape(a,_length(a)) vec(a::AbstractVector) = a _sub(::Tuple{}, ::Tuple{}) = () diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 7226e893faeba..ca6340ec9c86f 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -192,7 +192,7 @@ index_lengths(A::AbstractArray, I::Colon) = (length(A),) @inline index_lengths(A::AbstractArray, I...) = index_lengths_dim(A, 1, I...) index_lengths_dim(A, dim) = () index_lengths_dim(A, dim, ::Colon) = (trailingsize(A, dim),) -@inline index_lengths_dim(A, dim, ::Colon, i, I...) = (unsafe_length(indices(A, dim)), index_lengths_dim(A, dim+1, i, I...)...) +@inline index_lengths_dim(A, dim, ::Colon, i, I...) = (_length(indices(A, dim)), index_lengths_dim(A, dim+1, i, I...)...) @inline index_lengths_dim(A, dim, ::Real, I...) = (1, index_lengths_dim(A, dim+1, I...)...) @inline index_lengths_dim{N}(A, dim, ::CartesianIndex{N}, I...) = (1, index_lengths_dim(A, dim+N, I...)...) @inline index_lengths_dim(A, dim, i::AbstractArray, I...) = (length(i), index_lengths_dim(A, dim+1, I...)...) diff --git a/base/operators.jl b/base/operators.jl index 2c8fd21de95ee..3f3a8fc95977c 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -447,16 +447,16 @@ function setindex_shape_check(X::AbstractArray, I::Integer...) end setindex_shape_check(X::AbstractArray) = - (unsafe_length(X)==1 || throw_setindex_mismatch(X,())) + (_length(X)==1 || throw_setindex_mismatch(X,())) setindex_shape_check(X::AbstractArray, i::Integer) = - (unsafe_length(X)==i || throw_setindex_mismatch(X, (i,))) + (_length(X)==i || throw_setindex_mismatch(X, (i,))) setindex_shape_check{T}(X::AbstractArray{T,1}, i::Integer) = - (unsafe_length(X)==i || throw_setindex_mismatch(X, (i,))) + (_length(X)==i || throw_setindex_mismatch(X, (i,))) setindex_shape_check{T}(X::AbstractArray{T,1}, i::Integer, j::Integer) = - (unsafe_length(X)==i*j || throw_setindex_mismatch(X, (i,j))) + (_length(X)==i*j || throw_setindex_mismatch(X, (i,j))) function setindex_shape_check{T}(X::AbstractArray{T,2}, i::Integer, j::Integer) if length(X) != i*j diff --git a/base/reducedim.jl b/base/reducedim.jl index 15ee8c67512ff..71c3844c307d1 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -190,7 +190,7 @@ function _mapreducedim!{T,N}(f, op, R::AbstractArray, A::AbstractArray{T,N}) if has_fast_linear_indexing(A) && lsiz > 16 # use mapreduce_impl, which is probably better tuned to achieve higher performance - nslices = div(unsafe_length(A), lsiz) + nslices = div(_length(A), lsiz) ibase = first(linearindices(A))-1 for i = 1:nslices @inbounds R[i] = op(R[i], mapreduce_impl(f, op, A, ibase+1, ibase+lsiz)) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index 47365a6340059..d1a4b0961a5f6 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -52,7 +52,7 @@ end @pure rdims{N}(out::Tuple, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = rdims((out..., first(inds)), tail(inds), Val{N}) function _reshape(parent::AbstractArray, dims::Dims) - n = unsafe_length(parent) + n = _length(parent) prod(dims) == n || throw(DimensionMismatch("parent has $n elements, which is incompatible with size $dims")) __reshape((parent, linearindexing(parent)), dims) end diff --git a/base/statistics.jl b/base/statistics.jl index e72f2f1cd6758..5cdff1c8982d1 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -98,7 +98,7 @@ function centralize_sumabs2!{S,T,N}(R::AbstractArray{S}, A::AbstractArray{T,N}, isempty(A) && return R if has_fast_linear_indexing(A) && lsiz > 16 - nslices = div(unsafe_length(A), lsiz) + nslices = div(_length(A), lsiz) ibase = first(linearindices(A))-1 for i = 1:nslices @inbounds R[i] = centralize_sumabs2(A, means[i], ibase+1, ibase+lsiz) From f9662dafd2ae3bb12398218e68d087a1040c1e83 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 17 Jul 2016 09:46:27 -0500 Subject: [PATCH 0503/1117] Fixes and tests to index_shape, decolon, and indexing with CartesianIndex --- base/multidimensional.jl | 64 +++++++++++++++++++++++++--------------- base/reducedim.jl | 3 -- base/tuple.jl | 4 +++ test/arrayops.jl | 52 +++++++++++++++++++------------- 4 files changed, 76 insertions(+), 47 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index ca6340ec9c86f..04cbe195a3cbc 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -186,6 +186,18 @@ function checkindex{N}(::Type{Bool}, inds::Tuple, I::AbstractArray{CartesianInde b end +# combined dimensionality of all indices, including CartesianIndex and +# AbstractArray{CartesianIndex} +# rather than returning N, it returns an NTuple{N,Bool} so the result is inferrable +@inline index_ndims(i1, I...) = (true, index_ndims(I...)...) +@inline function index_ndims{N}(i1::CartesianIndex{N}, I...) + (map(x->true, i1.I)..., index_ndims(I...)...) +end +@inline function index_ndims{N}(i1::AbstractArray{CartesianIndex{N}}, I...) + (ntuple(x->true, Val{N})..., index_ndims(I...)...) +end +index_ndims() = () + # Recursively compute the lengths of a list of indices, without dropping scalars # These need to be inlined for more than 3 indexes index_lengths(A::AbstractArray, I::Colon) = (length(A),) @@ -205,36 +217,40 @@ index_shape(A::AbstractArray, I::Colon) = (linearindices(A),) @inline index_shape(A::AbstractArray, I...) = index_shape_dim(indices(A), I...) @inline index_shape_dim(inds::Tuple{Any}, ::Colon) = inds @inline index_shape_dim(inds, ::Colon) = (OneTo(trailingsize(inds)),) -@inline index_shape_dim(inds, ::Colon, i, I...) = - (inds[1], index_shape_dim(tail(inds), i, I...)...) +@inline function index_shape_dim(inds, ::Colon, i, I...) + inds1, indstail = IteratorsMD.split(inds, Val{1}) + (inds1..., index_shape_dim(indstail, i, I...)...) +end @inline index_shape_dim(inds, ::Real...) = () -@inline index_shape_dim(inds, ::Real, I...) = index_shape_dim(tail(inds), I...) +@inline index_shape_dim(inds, ::Real, I...) = index_shape_dim(safe_tail(inds), I...) @inline index_shape_dim(inds, i::AbstractArray, I...) = - (indices(i)..., index_shape_dim(tail(inds), I...)...) + (indices(i)..., index_shape_dim(safe_tail(inds), I...)...) @inline index_shape_dim(inds, i::AbstractArray{Bool}, I...) = - (OneTo(sum(i)), index_shape_dim(tail(inds), I...)...) -@inline function index_shape_dim{N}(inds, ::CartesianIndex{N}, I...) - indsN, indstail = IterationMD.split(inds, Val{N}) - index_shape_dim(indstail, I...) -end -@inline function index_shape_dim{N}(A, i::AbstractArray{CartesianIndex{N}}, I...) - indsN, indstail = IterationMD.split(inds, Val{N}) + (OneTo(sum(i)), index_shape_dim(safe_tail(inds), I...)...) +# single CartesianIndex version not needed because of call to flatten in _getindex... +# ...but array of CartesianIndex is not covered +@inline function index_shape_dim{N}(inds, i::AbstractArray{CartesianIndex{N}}, I...) + indsN, indstail = IteratorsMD.split(inds, Val{N}) (indices(i)..., index_shape_dim(indstail, I...)...) end # Convert Colon indices into explicit indices @inline decolon(A::AbstractArray, ::Colon) = (linearindices(A),) -@inline decolon(A::AbstractArray, I...) = decolon_dim(indices(A), I...) +@inline decolon(A::AbstractArray, I...) = decolon_dim(indices(A), I...) @inline decolon_dim(inds) = () @inline decolon_dim(inds::Tuple{Any}, ::Colon) = inds @inline decolon_dim(inds, ::Colon) = (OneTo(trailingsize(inds)),) -@inline decolon_dim(inds, ::Colon, I...) = - (inds[1], decolon_dim(tail(inds), I...)...) -@inline decolon_dim(inds, i1, I...) = (i1, decolon_dim(tail(inds), I...)...) +@inline function decolon_dim(inds, ::Colon, I...) + inds1, indstail = IteratorsMD.split(inds, Val{1}) + (maybe_oneto(inds1...), decolon_dim(indstail, I...)...) +end +@inline decolon_dim(inds, i1, I...) = (i1, decolon_dim(safe_tail(inds), I...)...) @inline function decolon_dim{N}(inds, i1::AbstractArray{CartesianIndex{N}}, I...) - indsN, indstail = IterationMD.split(inds, Val{N}) + indsN, indstail = IteratorsMD.split(inds, Val{N}) (i1, decolon_dim(indstail, I...)...) end +maybe_oneto(i) = i +maybe_oneto() = OneTo(1) ### From abstractarray.jl: Internal multidimensional indexing definitions ### # These are not defined on directly on getindex to avoid @@ -248,16 +264,18 @@ end # Explicitly allow linear indexing with one non-scalar index @inline function _getindex(l::LinearIndexing, A::AbstractArray, i::Union{Real, AbstractArray, Colon}) @boundscheck checkbounds(A, i) - _unsafe_getindex(l, _maybe_linearize(l, A), i) + _unsafe_getindex(l, _maybe_reshape(l, A, (i,)), i) end # But we can speed up LinearSlow arrays by reshaping them to vectors: -_maybe_linearize(::LinearFast, A::AbstractArray) = A -_maybe_linearize(::LinearSlow, A::AbstractVector) = A -_maybe_linearize(::LinearSlow, A::AbstractArray) = reshape(A, length(A)) +_maybe_reshape(::LinearFast, A::AbstractArray, i) = A +_maybe_reshape(::LinearSlow, A::AbstractVector, i) = A +@inline _maybe_reshape(::LinearSlow, A::AbstractArray, i) = _maybe_reshape(LinearSlow(), index_ndims(i...), A) +@inline _maybe_reshape{T,N}(::LinearIndexing, ::NTuple{N}, A::AbstractArray{T,N}) = A +@inline _maybe_reshape{N}(::LinearIndexing, ::NTuple{N}, A) = reshape(A, Val{N}) @inline function _getindex{N}(l::LinearIndexing, A::AbstractArray, I::Vararg{Union{Real, AbstractArray, Colon},N}) # TODO: DEPRECATE FOR #14770 @boundscheck checkbounds(A, I...) - _unsafe_getindex(l, reshape(A, Val{N}), I...) + _unsafe_getindex(l, _maybe_reshape(l, A, I), I...) end @generated function _unsafe_getindex(::LinearIndexing, A::AbstractArray, I::Union{Real, AbstractArray, Colon}...) @@ -340,12 +358,12 @@ _iterable(v) = repeated(v) end @inline function _setindex!(l::LinearIndexing, A::AbstractArray, x, j::Union{Real,AbstractArray,Colon}) @boundscheck checkbounds(A, j) - _unsafe_setindex!(l, _maybe_linearize(l, A), x, j) + _unsafe_setindex!(l, _maybe_reshape(l, A, (j,)), x, j) A end @inline function _setindex!{N}(l::LinearIndexing, A::AbstractArray, x, J::Vararg{Union{Real, AbstractArray, Colon},N}) # TODO: DEPRECATE FOR #14770 @boundscheck checkbounds(A, J...) - _unsafe_setindex!(l, reshape(A, Val{N}), x, J...) + _unsafe_setindex!(l, _maybe_reshape(l, A, J), x, J...) A end diff --git a/base/reducedim.jl b/base/reducedim.jl index 71c3844c307d1..3f35bd73d79cb 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -358,9 +358,6 @@ function findmax{T}(A::AbstractArray{T}, region) zeros(Int, reduced_dims0(A, region)), A) end -safe_tail(t::Tuple) = tail(t) -safe_tail(t::Tuple{}) = () - function reducedim1(R, A) iR1 = indices(R, 1) iR1 == 1:1 && iR1 != indices(A, 1) diff --git a/base/tuple.jl b/base/tuple.jl index 08c245f837977..eabbce23e4f18 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -43,6 +43,10 @@ first(t::Tuple) = t[1] eltype(::Type{Tuple{}}) = Bottom eltype{T}(::Type{Tuple{Vararg{T}}}) = T +# version of tail that doesn't throw on empty tuples (used in array indexing) +safe_tail(t::Tuple) = tail(t) +safe_tail(t::Tuple{}) = () + # front (the converse of tail: it skips the last entry) function front(t::Tuple) diff --git a/test/arrayops.jl b/test/arrayops.jl index 741fac3fb5816..00e4ca331153c 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1218,27 +1218,37 @@ a = ones(5,0) b = view(a, :, :) @test mdsum(b) == 0 -a = copy(reshape(1:60, 3, 4, 5)) -@test a[CartesianIndex{3}(2,3,4)] == 44 -a[CartesianIndex{3}(2,3,3)] = -1 -@test a[CartesianIndex{3}(2,3,3)] == -1 -@test a[2,CartesianIndex{2}(3,4)] == 44 -a[1,CartesianIndex{2}(3,4)] = -2 -@test a[1,CartesianIndex{2}(3,4)] == -2 -@test a[CartesianIndex{1}(2),3,CartesianIndex{1}(4)] == 44 -a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] = -3 -@test a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] == -3 -@test a[:, :, CartesianIndex((1,))] == a[:,:,1] -@test a[CartesianIndex((1,)), [1,2], :] == a[1,[1,2],:] -@test a[CartesianIndex((2,)), 3:4, :] == a[2,3:4,:] - -a = view(zeros(3, 4, 5), :, :, :) -a[CartesianIndex{3}(2,3,3)] = -1 -@test a[CartesianIndex{3}(2,3,3)] == -1 -a[1,CartesianIndex{2}(3,4)] = -2 -@test a[1,CartesianIndex{2}(3,4)] == -2 -a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] = -3 -@test a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] == -3 +for a in (copy(reshape(1:60, 3, 4, 5)), + view(copy(reshape(1:60, 3, 4, 5)), 1:3, :, :)) + @test a[CartesianIndex{3}(2,3,4)] == 44 + a[CartesianIndex{3}(2,3,3)] = -1 + @test a[CartesianIndex{3}(2,3,3)] == -1 + @test a[2,CartesianIndex{2}(3,4)] == 44 + a[1,CartesianIndex{2}(3,4)] = -2 + @test a[1,CartesianIndex{2}(3,4)] == -2 + @test a[CartesianIndex{1}(2),3,CartesianIndex{1}(4)] == 44 + a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] = -3 + @test a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] == -3 + @test a[:, :, CartesianIndex((1,))] == a[:,:,1] + @test a[CartesianIndex((1,)), [1,2], :] == a[1,[1,2],:] + @test a[CartesianIndex((2,)), 3:4, :] == a[2,3:4,:] + @test a[[CartesianIndex(1,3),CartesianIndex(2,4)],3:3] == reshape([a[1,3,3]; a[2,4,3]], 2, 1) + @test_throws BoundsError a[[CartesianIndex(1,5),CartesianIndex(2,4)],3:3] + @test_throws BoundsError a[1:4, [CartesianIndex(1,3),CartesianIndex(2,4)]] +end + +for a in (view(zeros(3, 4, 5), :, :, :), + view(zeros(3, 4, 5), 1:3, :, :)) + a[CartesianIndex{3}(2,3,3)] = -1 + @test a[CartesianIndex{3}(2,3,3)] == -1 + a[1,CartesianIndex{2}(3,4)] = -2 + @test a[1,CartesianIndex{2}(3,4)] == -2 + a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] = -3 + @test a[CartesianIndex{1}(2),3,CartesianIndex{1}(3)] == -3 + a[[CartesianIndex(1,3),CartesianIndex(2,4)],3:3] = -4 + @test a[1,3,3] == -4 + @test a[2,4,3] == -4 +end I1 = CartesianIndex((2,3,0)) I2 = CartesianIndex((-1,5,2)) From 891eb6c041f2aad18243e4b77ce96f256bb65d9a Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 18 Jul 2016 11:36:42 -0500 Subject: [PATCH 0504/1117] Better error for array operations with axis < 1 --- base/multidimensional.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 04cbe195a3cbc..80697f87299a9 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -463,6 +463,7 @@ for (f, fmod, op) = ((:cummin, :_cummin!, :min), (:cummax, :_cummax!, :max)) end @eval function ($f)(A::AbstractArray, axis::Integer) + axis > 0 || throw(ArgumentError("axis must be a positive integer")) res = similar(A) axis > ndims(A) && return copy!(res, A) inds = indices(A) @@ -502,6 +503,7 @@ cumsum!(B, A, axis::Integer) = cumop!(+, B, A, axis) cumprod!(B, A, axis::Integer) = cumop!(*, B, A, axis) function cumop!(op, B, A, axis::Integer) + axis > 0 || throw(ArgumentError("axis must be a positive integer")) inds_t = indices(A) indices(B) == inds_t || throw(DimensionMismatch("shape of B must match A")) axis > ndims(A) && return copy!(B, A) From ac7da78528fc9fb8945ac56b2cbd2fe568a6df03 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 18 Jul 2016 04:54:50 -0500 Subject: [PATCH 0505/1117] reductions & broadcast: allow any dimension of size 1 (no specific indices required) --- base/broadcast.jl | 94 ++++++++++++++++++++++------------------ base/multidimensional.jl | 2 + base/reducedim.jl | 34 +++++++-------- base/statistics.jl | 13 +++--- test/broadcast.jl | 15 ++++--- 5 files changed, 84 insertions(+), 74 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 3a66787f7f980..9f50f4d9c0b2a 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -37,7 +37,7 @@ _bcs1(a::Integer, b) = a == 1 ? b : (first(b) == 1 && last(b) == a ? b : throw(D _bcs1(a, b::Integer) = _bcs1(b, a) _bcs1(a, b) = _bcsm(b, a) ? b : (_bcsm(a, b) ? a : throw(DimensionMismatch("arrays could not be broadcast to a common size"))) # _bcsm tests whether the second index is consistent with the first -_bcsm(a, b) = a == b || (length(b) == 1 && first(b) == 1) +_bcsm(a, b) = a == b || length(b) == 1 _bcsm(a, b::Number) = b == 1 _bcsm(a::Number, b::Number) = a == b || b == 1 @@ -63,29 +63,36 @@ end ## Indexing manipulations -# newindex(I, keep) replaces a CartesianIndex `I` with something that +# newindex(I, keep, Idefault) replaces a CartesianIndex `I` with something that # is appropriate for a particular broadcast array/scalar. `keep` is a # NTuple{N,Bool}, where keep[d] == true means that one should preserve -# I[d]; if false, replace it with 1. In other words, this is -# equivalent to map((k,i)->k ? i : 1, keep, I.I) (but see #17126). -@inline newindex(I::CartesianIndex, ::Tuple{}) = 1 # for scalars -@inline newindex(I::CartesianIndex, indexmap) = CartesianIndex(_newindex(I.I, indexmap)) -@inline _newindex(I, indexmap) = - (ifelse(indexmap[1], I[1], 1), _newindex(tail(I), tail(indexmap))...) -@inline _newindex(I, indexmap::Tuple{}) = () # truncate if indexmap is shorter than I - -# newindexer(shape, A) generates `keep` (for use by `newindex` above) -# for a particular array `A`, given the broadcast_shape `shape` -# Equivalent to map(==, indices(A), shape) (but see #17126) -newindexer(shape, x::Number) = () +# I[d]; if false, replace it with Idefault[d]. +@inline newindex(I::CartesianIndex, keep, Idefault) = CartesianIndex(_newindex(I.I, keep, Idefault)) +@inline _newindex(I, keep, Idefault) = + (ifelse(keep[1], I[1], Idefault[1]), _newindex(tail(I), tail(keep), tail(Idefault))...) +@inline _newindex(I, keep::Tuple{}, Idefault) = () # truncate if keep is shorter than I + +# newindexer(shape, A) generates `keep` and `Idefault` (for use by +# `newindex` above) for a particular array `A`, given the +# broadcast_shape `shape` +# `keep` is equivalent to map(==, indices(A), shape) (but see #17126) +newindexer(shape, x::Number) = (), () @inline newindexer(shape, A) = newindexer(shape, indices(A)) -@inline newindexer(shape, indsA::Tuple{}) = () -@inline newindexer(shape, indsA::Tuple) = - (shape[1] == indsA[1], newindexer(tail(shape), tail(indsA))...) +@inline newindexer(shape, indsA::Tuple{}) = (), () +@inline function newindexer(shape, indsA::Tuple) + ind1 = indsA[1] + keep, Idefault = newindexer(tail(shape), tail(indsA)) + (shape[1] == ind1, keep...), (first(ind1), Idefault...) +end # Equivalent to map(x->newindexer(shape, x), As) (but see #17126) -map_newindexer(shape, ::Tuple{}) = () -@inline map_newindexer(shape, As) = (newindexer(shape, As[1]), map_newindexer(shape, tail(As))...) +map_newindexer(shape, ::Tuple{}) = (), () +@inline function map_newindexer(shape, As) + A1 = As[1] + keeps, Idefaults = map_newindexer(shape, tail(As)) + keep, Idefault = newindexer(shape, A1) + (keep, keeps...), (Idefault, Idefaults...) +end # For output BitArrays const bitcache_chunks = 64 # this can be changed @@ -96,16 +103,17 @@ dumpbitcache(Bc::Vector{UInt64}, bind::Int, C::Vector{Bool}) = ## Broadcasting core # nargs encodes the number of As arguments (which matches the number -# of indexmaps). The first two type parameters are to ensure specialization. -@generated function _broadcast!{M,AT,nargs}(f, B::AbstractArray, indexmaps::M, As::AT, ::Type{Val{nargs}}) +# of keeps). The first two type parameters are to ensure specialization. +@generated function _broadcast!{K,ID,AT,nargs}(f, B::AbstractArray, keeps::K, Idefaults::ID, As::AT, ::Type{Val{nargs}}) quote $(Expr(:meta, :noinline)) - # destructure the indexmaps and As tuples + # destructure the keeps and As tuples @nexprs $nargs i->(A_i = As[i]) - @nexprs $nargs i->(imap_i = indexmaps[i]) + @nexprs $nargs i->(keep_i = keeps[i]) + @nexprs $nargs i->(Idefault_i = Idefaults[i]) @simd for I in CartesianRange(indices(B)) # reverse-broadcast the indices - @nexprs $nargs i->(I_i = newindex(I, imap_i)) + @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i)) # extract array values @nexprs $nargs i->(@inbounds val_i = A_i[I_i]) # call the function and store the result @@ -116,19 +124,20 @@ end # For BitArray outputs, we cache the result in a "small" Vector{Bool}, # and then copy in chunks into the output -@generated function _broadcast!{M,AT,nargs}(f, B::BitArray, indexmaps::M, As::AT, ::Type{Val{nargs}}) +@generated function _broadcast!{K,ID,AT,nargs}(f, B::BitArray, keeps::K, Idefaults::ID, As::AT, ::Type{Val{nargs}}) quote $(Expr(:meta, :noinline)) - # destructure the indexmaps and As tuples + # destructure the keeps and As tuples @nexprs $nargs i->(A_i = As[i]) - @nexprs $nargs i->(imap_i = indexmaps[i]) + @nexprs $nargs i->(keep_i = keeps[i]) + @nexprs $nargs i->(Idefault_i = Idefaults[i]) C = Vector{Bool}(bitcache_size) Bc = B.chunks ind = 1 cind = 1 @simd for I in CartesianRange(indices(B)) # reverse-broadcast the indices - @nexprs $nargs i->(I_i = newindex(I, imap_i)) + @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i)) # extract array values @nexprs $nargs i->(@inbounds val_i = A_i[I_i]) # call the function and store the result @@ -150,23 +159,24 @@ end @inline function broadcast!{nargs}(f, B::AbstractArray, As::Vararg{Any,nargs}) shape = indices(B) check_broadcast_shape(shape, As...) - mapindex = map_newindexer(shape, As) - _broadcast!(f, B, mapindex, As, Val{nargs}) + keeps, Idefaults = map_newindexer(shape, As) + _broadcast!(f, B, keeps, Idefaults, As, Val{nargs}) B end # broadcast with computed element type -@generated function _broadcast!{M,AT,nargs}(f, B::AbstractArray, indexmaps::M, As::AT, ::Type{Val{nargs}}, iter, st, count) +@generated function _broadcast!{K,ID,AT,nargs}(f, B::AbstractArray, keeps::K, Idefaults::ID, As::AT, ::Type{Val{nargs}}, iter, st, count) quote $(Expr(:meta, :noinline)) - # destructure the indexmaps and As tuples + # destructure the keeps and As tuples @nexprs $nargs i->(A_i = As[i]) - @nexprs $nargs i->(imap_i = indexmaps[i]) + @nexprs $nargs i->(keep_i = keeps[i]) + @nexprs $nargs i->(Idefault_i = Idefaults[i]) while !done(iter, st) I, st = next(iter, st) # reverse-broadcast the indices - @nexprs $nargs i->(I_i = newindex(I, imap_i)) + @nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i)) # extract array values @nexprs $nargs i->(@inbounds val_i = A_i[I_i]) # call the function @@ -182,7 +192,7 @@ end new[II] = B[II] end new[I] = V - return _broadcast!(f, new, indexmaps, As, Val{nargs}, iter, st, count+1) + return _broadcast!(f, new, keeps, Idefaults, As, Val{nargs}, iter, st, count+1) end count += 1 end @@ -197,13 +207,13 @@ function broadcast_t(f, ::Type{Any}, As...) return similar(Array{Union{}}, shape) end nargs = length(As) - indexmaps = map_newindexer(shape, As) + keeps, Idefaults = map_newindexer(shape, As) st = start(iter) I, st = next(iter, st) - val = f([ As[i][newindex(I, indexmaps[i])] for i=1:nargs ]...) + val = f([ As[i][newindex(I, keeps[i], Idefaults[i])] for i=1:nargs ]...) B = similar(Array{typeof(val)}, shape) B[I] = val - return _broadcast!(f, B, indexmaps, As, Val{nargs}, iter, st, 1) + return _broadcast!(f, B, keeps, Idefaults, As, Val{nargs}, iter, st, 1) end @inline broadcast_t(f, T, As...) = broadcast!(f, similar(Array{T}, broadcast_shape(As...)), As...) @@ -213,18 +223,18 @@ end # alternate, more compact implementation; unfortunately slower. # also the `collect` machinery doesn't yet support arbitrary index bases. #= -@generated function _broadcast{nargs}(f, indexmaps, As, ::Type{Val{nargs}}, iter) +@generated function _broadcast{nargs}(f, keeps, As, ::Type{Val{nargs}}, iter) quote - collect((@ncall $nargs f i->As[i][newindex(I, indexmaps[i])]) for I in iter) + collect((@ncall $nargs f i->As[i][newindex(I, keeps[i])]) for I in iter) end end function broadcast(f, As...) shape = broadcast_shape(As...) iter = CartesianRange(shape) - indexmaps = map_newindexer(shape, As) + keeps, Idefaults = map_newindexer(shape, As) naT = Val{nfields(As)} - _broadcast(f, indexmaps, As, naT, iter) + _broadcast(f, keeps, Idefaults, As, naT, iter) end =# diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 80697f87299a9..8b0aa1aa54bbd 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -253,6 +253,8 @@ maybe_oneto(i) = i maybe_oneto() = OneTo(1) ### From abstractarray.jl: Internal multidimensional indexing definitions ### +getindex(x::Number, i::CartesianIndex{0}) = x + # These are not defined on directly on getindex to avoid # ambiguities for AbstractArray subtypes. See the note in abstractarray.jl diff --git a/base/reducedim.jl b/base/reducedim.jl index 3f35bd73d79cb..f6fa07169a511 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -166,9 +166,6 @@ function check_reducedims(R, A) Ri, Ai = indices(R, i), indices(A, i) sRi, sAi = length(Ri), length(Ai) if sRi == 1 - if Ri != Ai - Ri == 1:1 || throw(DimensionMismatch("reduction along dimension $i must use indices 1:1; got $(convert(UnitRange, Ri)) for reducing $(convert(UnitRange, Ai))")) - end if sAi > 1 if had_nonreduc lsiz = 0 # to reduce along i, but some previous dimensions were non-reducing @@ -199,20 +196,21 @@ function _mapreducedim!{T,N}(f, op, R::AbstractArray, A::AbstractArray{T,N}) return R end indsAt, indsRt = safe_tail(indices(A)), safe_tail(indices(R)) # handle d=1 manually - imap = Broadcast.newindexer(indsAt, indsRt) + keep, Idefault = Broadcast.newindexer(indsAt, indsRt) if reducedim1(R, A) # keep the accumulator as a local variable when reducing along the first dimension + i1 = first(indices1(R)) @inbounds for IA in CartesianRange(indsAt) - IR = Broadcast.newindex(IA, imap) - r = R[1,IR] + IR = Broadcast.newindex(IA, keep, Idefault) + r = R[i1,IR] @simd for i in indices(A, 1) r = op(r, f(A[i, IA])) end - R[1,IR] = r + R[i1,IR] = r end else @inbounds for IA in CartesianRange(indsAt) - IR = Broadcast.newindex(IA, imap) + IR = Broadcast.newindex(IA, keep, Idefault) @simd for i in indices(A, 1) R[i,IR] = op(R[i,IR], f(A[i,IA])) end @@ -279,13 +277,14 @@ function findminmax!{T,N}(f, Rval, Rind, A::AbstractArray{T,N}) # If we're reducing along dimension 1, for efficiency we can make use of a temporary. # Otherwise, keep the result in Rval/Rind so that we traverse A in storage order. indsAt, indsRt = safe_tail(indices(A)), safe_tail(indices(Rval)) - imap = Broadcast.newindexer(indsAt, indsRt) + keep, Idefault = Broadcast.newindexer(indsAt, indsRt) k = 0 if reducedim1(Rval, A) + i1 = first(indices1(Rval)) @inbounds for IA in CartesianRange(indsAt) - IR = Broadcast.newindex(IA, imap) - tmpRv = Rval[1,IR] - tmpRi = Rind[1,IR] + IR = Broadcast.newindex(IA, keep, Idefault) + tmpRv = Rval[i1,IR] + tmpRi = Rind[i1,IR] for i in indices(A,1) k += 1 tmpAv = A[i,IA] @@ -294,12 +293,12 @@ function findminmax!{T,N}(f, Rval, Rind, A::AbstractArray{T,N}) tmpRi = k end end - Rval[1,IR] = tmpRv - Rind[1,IR] = tmpRi + Rval[i1,IR] = tmpRv + Rind[i1,IR] = tmpRi end else @inbounds for IA in CartesianRange(indsAt) - IR = Broadcast.newindex(IA, imap) + IR = Broadcast.newindex(IA, keep, Idefault) for i in indices(A, 1) k += 1 tmpAv = A[i,IA] @@ -358,7 +357,4 @@ function findmax{T}(A::AbstractArray{T}, region) zeros(Int, reduced_dims0(A, region)), A) end -function reducedim1(R, A) - iR1 = indices(R, 1) - iR1 == 1:1 && iR1 != indices(A, 1) -end +reducedim1(R, A) = length(indices1(R)) == 1 diff --git a/base/statistics.jl b/base/statistics.jl index 5cdff1c8982d1..1545ce0d28e2f 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -107,20 +107,21 @@ function centralize_sumabs2!{S,T,N}(R::AbstractArray{S}, A::AbstractArray{T,N}, return R end indsAt, indsRt = safe_tail(indices(A)), safe_tail(indices(R)) # handle d=1 manually - imap = Broadcast.newindexer(indsAt, indsRt) + keep, Idefault = Broadcast.newindexer(indsAt, indsRt) if reducedim1(R, A) + i1 = first(indices1(R)) @inbounds for IA in CartesianRange(indsAt) - IR = Broadcast.newindex(IA, imap) - r = R[1,IR] - m = means[1,IR] + IR = Broadcast.newindex(IA, keep, Idefault) + r = R[i1,IR] + m = means[i1,IR] @simd for i in indices(A, 1) r += abs2(A[i,IA] - m) end - R[1,IR] = r + R[i1,IR] = r end else @inbounds for IA in CartesianRange(indsAt) - IR = Broadcast.newindex(IA, imap) + IR = Broadcast.newindex(IA, keep, Idefault) @simd for i in indices(A, 1) R[i,IR] += abs2(A[i,IA] - means[i,IR]) end diff --git a/test/broadcast.jl b/test/broadcast.jl index 2668e21e68e05..2d6adca165282 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -42,13 +42,14 @@ check_broadcast_shape((-1:1, 6:9), (1, 6:9)) check_broadcast_shape((-1:1, 6:9), 1) ci(x) = CartesianIndex(x) -@test @inferred(newindex(ci((2,2)), (true, true))) == ci((2,2)) -@test @inferred(newindex(ci((2,2)), (true, false))) == ci((2,1)) -@test @inferred(newindex(ci((2,2)), (false, true))) == ci((1,2)) -@test @inferred(newindex(ci((2,2)), (false, false))) == ci((1,1)) -@test @inferred(newindex(ci((2,2)), (true,))) == ci((2,)) -@test @inferred(newindex(ci((2,2)), (false,))) == ci((1,)) -@test @inferred(newindex(ci((2,2)), ())) == 1 +@test @inferred(newindex(ci((2,2)), (true, true), (-1,-1))) == ci((2,2)) +@test @inferred(newindex(ci((2,2)), (true, false), (-1,-1))) == ci((2,-1)) +@test @inferred(newindex(ci((2,2)), (false, true), (-1,-1))) == ci((-1,2)) +@test @inferred(newindex(ci((2,2)), (false, false), (-1,-1))) == ci((-1,-1)) +@test @inferred(newindex(ci((2,2)), (true,), (-1,-1))) == ci((2,)) +@test @inferred(newindex(ci((2,2)), (true,), (-1,))) == ci((2,)) +@test @inferred(newindex(ci((2,2)), (false,), (-1,))) == ci((-1,)) +@test @inferred(newindex(ci((2,2)), (), ())) == ci(()) end From 2447335b36352ed4f7362ca1b3b8a991dbb52a90 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 13 Jul 2016 18:08:14 -0400 Subject: [PATCH 0506/1117] fix #16793, assertion failure in `at-test abstract Foo` also type expressions in value position sometimes had an invalid result value --- src/julia-syntax.scm | 8 +++++--- test/core.jl | 8 ++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 4cfcf09e11fb6..61cf3853e0682 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -832,7 +832,7 @@ (receive (params bounds) (sparam-name-bounds params '() '()) `(block - (const ,name) + (global ,name) (const ,name) (scope-block (block ,@(map (lambda (v) `(local ,v)) params) @@ -843,7 +843,7 @@ (receive (params bounds) (sparam-name-bounds params '() '()) `(block - (const ,name) + (global ,name) (const ,name) (scope-block (block ,@(map (lambda (v) `(local ,v)) params) @@ -3235,7 +3235,9 @@ f(x) = yt(x) ;; top level expressions returning values ((abstract_type bits_type composite_type thunk toplevel module) - (if tail (emit-return e) (emit e))) + (emit e) + (if tail (emit-return '(null))) + '(null)) ;; other top level expressions and metadata ((import importall using export line meta inbounds boundscheck simdloop) diff --git a/test/core.jl b/test/core.jl index 1e153aba58154..1133a7aa779d6 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4458,6 +4458,14 @@ let g = f16340(1) @test isa(typeof(g).name.mt.defs.tvars, TypeVar) end +# issue #16793 +try + abstract T16793 +catch +end +@test isa(T16793, Type) +@test isa(abstract T16793_2, Void) + # issue #17147 f17147(::Tuple) = 1 f17147{N}(::Vararg{Tuple,N}) = 2 From 4a8b88c104f96270dd7acf775b0ebcc58f465677 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Mon, 18 Jul 2016 15:01:13 -0400 Subject: [PATCH 0507/1117] Update embeding flags (#17454) * Include threading flags. Fix #16868 * Add document about exporting symbols from main executable. Fix #17451 --- contrib/julia-config.jl | 7 +++++-- doc/manual/embedding.rst | 4 ++++ src/sys.c | 9 +++++++++ test/threads.jl | 9 +++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/contrib/julia-config.jl b/contrib/julia-config.jl index a3abff9d3a21f..6e98c5c0938fe 100755 --- a/contrib/julia-config.jl +++ b/contrib/julia-config.jl @@ -8,6 +8,8 @@ const options = "--ldlibs" ]; +threadingOn() = ccall(:jl_threading_enabled, Cint, ()) != 0 + function imagePath() opts = Base.JLOptions() unsafe_string(opts.image_file) @@ -57,10 +59,11 @@ end function cflags() arg1 = replace(initDir(),"\\","\\\\\\\\") arg2 = replace(includeDir(),"\\","\\\\") + threading_def = threadingOn() ? "-DJULIA_ENABLE_THREADING=1 " : "" if is_unix() - return """-fPIC -DJULIA_INIT_DIR=\\"$arg1\\" -I$arg2""" + return """$(threading_def)-fPIC -DJULIA_INIT_DIR=\\"$arg1\\" -I$arg2""" else - return """-DJULIA_INIT_DIR=\\"$arg1\\" -I$arg2""" + return """$(threading_def)-DJULIA_INIT_DIR=\\"$arg1\\" -I$arg2""" end end diff --git a/doc/manual/embedding.rst b/doc/manual/embedding.rst index dd943f19a2142..ab385176ffccc 100644 --- a/doc/manual/embedding.rst +++ b/doc/manual/embedding.rst @@ -56,6 +56,10 @@ Before the program terminates, it is strongly recommended to call ``jl_atexit_ho >>> julia.jl_init('.') 250593296 +.. note:: + + If the julia program needs to access symbols from the main executable, it may be necessary to add ``-Wl,--export-dynamic`` linker flag at compile time on Linux in addition to the ones generated by ``julia-config.jl`` described below. This is not necessary when compiling a shared library. + Using julia-config to automatically determine build parameters -------------------------------------------------------------- diff --git a/src/sys.c b/src/sys.c index afc61db38683e..3406a8a7e11dc 100644 --- a/src/sys.c +++ b/src/sys.c @@ -742,6 +742,15 @@ JL_DLLEXPORT size_t jl_maxrss(void) #endif } +JL_DLLEXPORT int jl_threading_enabled(void) +{ +#ifdef JULIA_ENABLE_THREADING + return 1; +#else + return 0; +#endif +} + #ifdef __cplusplus } #endif diff --git a/test/threads.jl b/test/threads.jl index d2130690c5a70..4fb50d6ba080a 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -343,3 +343,12 @@ let async = Base.AsyncCondition(), t @test_throws EOFError wait(t) @test_throws EOFError wait(async) end + +# Compare the two ways of checking if threading is enabled. +# `jl_tls_states` should only be defined on non-threading build. +if ccall(:jl_threading_enabled, Cint, ()) == 0 + @test nthreads() == 1 + cglobal(:jl_tls_states) != C_NULL +else + @test_throws ErrorException cglobal(:jl_tls_states) +end From d55a329dd861261643f84e611435c9a051c84d01 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 9 Jun 2016 13:47:31 -0400 Subject: [PATCH 0508/1117] ensure the array constructors are generally using leaftypes this makes for faster dynamic dispatch and apply-type construction ref #16128 --- base/abstractarray.jl | 30 +++++++++++++------------- base/array.jl | 50 +++++++++++++++++++++---------------------- base/boot.jl | 4 ++-- base/sysimg.jl | 2 +- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 15cdb1b816a99..e7294238e4220 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -15,14 +15,14 @@ end ## Basic functions ## -vect() = Array{Any}(0) +vect() = Array{Any,1}(0) vect{T}(X::T...) = T[ X[i] for i=1:length(X) ] function vect(X...) T = promote_typeof(X...) #T[ X[i] for i=1:length(X) ] # TODO: this is currently much faster. should figure out why. not clear. - copy!(Array{T}(length(X)), X) + copy!(Array{T,1}(length(X)), X) end size{T,N}(t::AbstractArray{T,N}, d) = d <= N ? size(t)[d] : 1 @@ -342,7 +342,7 @@ similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, to_shap similar( a::AbstractArray, T::Type, dims::DimOrInd...) = similar(a, T, to_shape(dims)) similar( a::AbstractArray, T::Type, dims) = similar(a, T, to_shape(dims)) # similar creates an Array by default -similar( a::AbstractArray, T::Type, dims::Dims) = Array{T}(dims) +similar( a::AbstractArray, T::Type, dims::Dims) = Array{T,nfields(dims)}(dims) to_shape(::Tuple{}) = () to_shape(dims::Dims) = dims @@ -864,12 +864,12 @@ promote_eltype() = Bottom promote_eltype(v1, vs...) = promote_type(eltype(v1), promote_eltype(vs...)) #TODO: ERROR CHECK -cat(catdim::Integer) = Array{Any}(0) +cat(catdim::Integer) = Array{Any,1}(0) -vcat() = Array{Any}(0) -hcat() = Array{Any}(0) -typed_vcat{T}(::Type{T}) = Array{T}(0) -typed_hcat{T}(::Type{T}) = Array{T}(0) +vcat() = Array{Any,1}(0) +hcat() = Array{Any,1}(0) +typed_vcat{T}(::Type{T}) = Array{T,1}(0) +typed_hcat{T}(::Type{T}) = Array{T,1}(0) ## cat: special cases vcat{T}(X::T...) = T[ X[i] for i=1:length(X) ] @@ -879,8 +879,8 @@ hcat{T<:Number}(X::T...) = T[ X[j] for i=1:1, j=1:length(X) ] vcat(X::Number...) = hvcat_fill(Array{promote_typeof(X...)}(length(X)), X) hcat(X::Number...) = hvcat_fill(Array{promote_typeof(X...)}(1,length(X)), X) -typed_vcat{T}(::Type{T}, X::Number...) = hvcat_fill(Array{T}(length(X)), X) -typed_hcat{T}(::Type{T}, X::Number...) = hvcat_fill(Array{T}(1,length(X)), X) +typed_vcat{T}(::Type{T}, X::Number...) = hvcat_fill(Array{T,1}(length(X)), X) +typed_hcat{T}(::Type{T}, X::Number...) = hvcat_fill(Array{T,2}(1,length(X)), X) vcat(V::AbstractVector...) = typed_vcat(promote_eltype(V...), V...) vcat{T}(V::AbstractVector{T}...) = typed_vcat(T, V...) @@ -1098,13 +1098,13 @@ function typed_hvcat{T}(::Type{T}, rows::Tuple{Vararg{Int}}, as::AbstractMatrix. end hvcat(rows::Tuple{Vararg{Int}}) = [] -typed_hvcat{T}(::Type{T}, rows::Tuple{Vararg{Int}}) = Array{T}(0) +typed_hvcat{T}(::Type{T}, rows::Tuple{Vararg{Int}}) = Array{T,1}(0) function hvcat{T<:Number}(rows::Tuple{Vararg{Int}}, xs::T...) nr = length(rows) nc = rows[1] - a = Array{T}(nr, nc) + a = Array{T,2}(nr, nc) if length(a) != length(xs) throw(ArgumentError("argument count does not match specified shape (expected $(length(a)), got $(length(xs)))")) end @@ -1147,13 +1147,13 @@ function typed_hvcat{T}(::Type{T}, rows::Tuple{Vararg{Int}}, xs::Number...) if nr*nc != len throw(ArgumentError("argument count $(len) does not match specified shape $((nr,nc))")) end - hvcat_fill(Array{T}(nr, nc), xs) + hvcat_fill(Array{T,2}(nr, nc), xs) end # fallback definition of hvcat in terms of hcat and vcat function hvcat(rows::Tuple{Vararg{Int}}, as...) nbr = length(rows) # number of block rows - rs = Array{Any}(nbr) + rs = Array{Any,1}(nbr) a = 1 for i = 1:nbr rs[i] = hcat(as[a:a-1+rows[i]]...) @@ -1164,7 +1164,7 @@ end function typed_hvcat{T}(::Type{T}, rows::Tuple{Vararg{Int}}, as...) nbr = length(rows) # number of block rows - rs = Array{Any}(nbr) + rs = Array{Any,1}(nbr) a = 1 for i = 1:nbr rs[i] = hcat(as[a:a-1+rows[i]]...) diff --git a/base/array.jl b/base/array.jl index 2a19d201cabda..7aa4f03ec4d56 100644 --- a/base/array.jl +++ b/base/array.jl @@ -114,38 +114,38 @@ end ## Constructors ## -similar(a::Array, T::Type, dims::Dims) = Array{T}(dims) -similar{T}(a::Array{T,1}) = Array{T}(size(a,1)) -similar{T}(a::Array{T,2}) = Array{T}(size(a,1), size(a,2)) -similar{T}(a::Array{T,1}, dims::Dims) = Array{T}(dims) -similar{T}(a::Array{T,1}, m::Int) = Array{T}(m) -similar{T}(a::Array{T,1}, S::Type) = Array{S}(size(a,1)) -similar{T}(a::Array{T,2}, dims::Dims) = Array{T}(dims) -similar{T}(a::Array{T,2}, m::Int) = Array{T}(m) -similar{T}(a::Array{T,2}, S::Type) = Array{S}(size(a,1), size(a,2)) +similar(a::Array, T::Type, dims::Dims) = Array{T,nfields(dims)}(dims) +similar{T}(a::Array{T,1}) = Array{T,1}(size(a,1)) +similar{T}(a::Array{T,2}) = Array{T,2}(size(a,1), size(a,2)) +similar{T}(a::Array{T,1}, dims::Dims) = Array{T,nfields(dims)}(dims) +similar{T}(a::Array{T,1}, m::Int) = Array{T,1}(m) +similar{T}(a::Array{T,1}, S::Type) = Array{S,1}(size(a,1)) +similar{T}(a::Array{T,2}, dims::Dims) = Array{T,nfields(dims)}(dims) +similar{T}(a::Array{T,2}, m::Int) = Array{T,1}(m) +similar{T}(a::Array{T,2}, S::Type) = Array{S,2}(size(a,1), size(a,2)) # T[x...] constructs Array{T,1} function getindex(T::Type, vals...) - a = Array{T}(length(vals)) + a = Array{T,1}(length(vals)) @inbounds for i = 1:length(vals) a[i] = vals[i] end return a end -getindex(T::Type) = Array{T}(0) -getindex(T::Type, x) = (a = Array{T}(1); @inbounds a[1] = x; a) -getindex(T::Type, x, y) = (a = Array{T}(2); @inbounds (a[1] = x; a[2] = y); a) -getindex(T::Type, x, y, z) = (a = Array{T}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a) +getindex(T::Type) = Array{T,1}(0) +getindex(T::Type, x) = (a = Array{T,1}(1); @inbounds a[1] = x; a) +getindex(T::Type, x, y) = (a = Array{T,1}(2); @inbounds (a[1] = x; a[2] = y); a) +getindex(T::Type, x, y, z) = (a = Array{T,1}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a) function getindex(::Type{Any}, vals::ANY...) - a = Array{Any}(length(vals)) + a = Array{Any,1}(length(vals)) @inbounds for i = 1:length(vals) a[i] = vals[i] end return a end -getindex(::Type{Any}) = Array{Any}(0) +getindex(::Type{Any}) = Array{Any,1}(0) function fill!(a::Union{Array{UInt8}, Array{Int8}}, x::Integer) ccall(:memset, Ptr{Void}, (Ptr{Void}, Cint, Csize_t), a, x, length(a)) @@ -195,7 +195,7 @@ convert{T,n}(::Type{Array{T}}, x::Array{T,n}) = x convert{T,n}(::Type{Array{T,n}}, x::Array{T,n}) = x convert{T,n,S}(::Type{Array{T}}, x::AbstractArray{S, n}) = convert(Array{T, n}, x) -convert{T,n,S}(::Type{Array{T,n}}, x::AbstractArray{S,n}) = copy!(Array{T}(size(x)), x) +convert{T,n,S}(::Type{Array{T,n}}, x::AbstractArray{S,n}) = copy!(Array{T,n}(size(x)), x) promote_rule{T,n,S}(::Type{Array{T,n}}, ::Type{Array{S,n}}) = Array{promote_type(T,S),n} @@ -647,7 +647,7 @@ function vcat{T}(arrays::Vector{T}...) for a in arrays n += length(a) end - arr = Array{T}(n) + arr = Array{T,1}(n) ptr = pointer(arr) if isbits(T) elsz = Core.sizeof(T) @@ -763,13 +763,13 @@ findlast(testf::Function, A) = findprev(testf, A, length(A)) function find(testf::Function, A) # use a dynamic-length array to store the indexes, then copy to a non-padded # array for the return - tmpI = Array{Int}(0) + tmpI = Array{Int,1}(0) for (i,a) = enumerate(A) if testf(a) push!(tmpI, i) end end - I = Array{Int}(length(tmpI)) + I = Array{Int,1}(length(tmpI)) copy!(I, tmpI) return I end @@ -787,8 +787,8 @@ function find(A) return I end -find(x::Number) = x == 0 ? Array{Int}(0) : [1] -find(testf::Function, x::Number) = !testf(x) ? Array{Int}(0) : [1] +find(x::Number) = x == 0 ? Array{Int,1}(0) : [1] +find(testf::Function, x::Number) = !testf(x) ? Array{Int,1}(0) : [1] findn(A::AbstractVector) = find(A) @@ -811,7 +811,7 @@ function findnz{T}(A::AbstractMatrix{T}) nnzA = countnz(A) I = zeros(Int, nnzA) J = zeros(Int, nnzA) - NZs = Array{T}(nnzA) + NZs = Array{T,1}(nnzA) count = 1 if nnzA > 0 for j=1:size(A,2), i=1:size(A,1) @@ -874,7 +874,7 @@ function indexin(a::AbstractArray, b::AbstractArray) end function findin(a, b) - ind = Array{Int}(0) + ind = Array{Int,1}(0) bset = Set(b) @inbounds for (i,ai) in enumerate(a) ai in bset && push!(ind, i) @@ -969,7 +969,7 @@ end function setdiff(a, b) args_type = promote_type(eltype(a), eltype(b)) bset = Set(b) - ret = Array{args_type}(0) + ret = Array{args_type,1}(0) seen = Set{eltype(a)}() for a_elem in a if !in(a_elem, seen) && !in(a_elem, bset) diff --git a/base/boot.jl b/base/boot.jl index 170815e85a7e0..88d7ed85972fe 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -325,8 +325,8 @@ typealias NTuple{N,T} Tuple{Vararg{T,N}} (::Type{Array{T,2}}){T}() = Array{T,2}(0, 0) # TODO: possibly turn these into deprecations -Array{T,N}(::Type{T}, d::NTuple{N,Int}) = Array{T}(d) -Array{T}(::Type{T}, d::Int...) = Array{T}(d) +Array{T,N}(::Type{T}, d::NTuple{N,Int}) = Array{T,N}(d) +Array{T}(::Type{T}, d::Int...) = Array(T, d) Array{T}(::Type{T}, m::Int) = Array{T,1}(m) Array{T}(::Type{T}, m::Int,n::Int) = Array{T,2}(m,n) Array{T}(::Type{T}, m::Int,n::Int,o::Int) = Array{T,3}(m,n,o) diff --git a/base/sysimg.jl b/base/sysimg.jl index a2ec4b59a0cd5..89217dbf71b8a 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -85,7 +85,7 @@ include("array.jl") (::Type{Matrix})(m::Integer, n::Integer) = Matrix{Any}(Int(m), Int(n)) # TODO: possibly turn these into deprecations -Array{T}(::Type{T}, d::Integer...) = Array{T}(convert(Tuple{Vararg{Int}}, d)) +Array{T}(::Type{T}, d::Integer...) = Array(T, convert(Tuple{Vararg{Int}}, d)) Array{T}(::Type{T}, m::Integer) = Array{T,1}(Int(m)) Array{T}(::Type{T}, m::Integer,n::Integer) = Array{T,2}(Int(m),Int(n)) Array{T}(::Type{T}, m::Integer,n::Integer,o::Integer) = Array{T,3}(Int(m),Int(n),Int(o)) From 88568d0a1171dad39aaa8f78743b6ca909f89f52 Mon Sep 17 00:00:00 2001 From: pabloferz <pabloferz@yahoo.com.mx> Date: Fri, 15 Jul 2016 18:46:20 +0200 Subject: [PATCH 0509/1117] Use type-parameter on Dims in similar helps ensure it is properly inferrable --- base/abstractarray.jl | 2 +- base/array.jl | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index e7294238e4220..cb6ecc9e68f8f 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -342,7 +342,7 @@ similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, to_shap similar( a::AbstractArray, T::Type, dims::DimOrInd...) = similar(a, T, to_shape(dims)) similar( a::AbstractArray, T::Type, dims) = similar(a, T, to_shape(dims)) # similar creates an Array by default -similar( a::AbstractArray, T::Type, dims::Dims) = Array{T,nfields(dims)}(dims) +similar{N}(a::AbstractArray, T::Type, dims::Dims{N}) = Array{T,N}(dims) to_shape(::Tuple{}) = () to_shape(dims::Dims) = dims diff --git a/base/array.jl b/base/array.jl index 7aa4f03ec4d56..0381f26b63829 100644 --- a/base/array.jl +++ b/base/array.jl @@ -114,15 +114,13 @@ end ## Constructors ## -similar(a::Array, T::Type, dims::Dims) = Array{T,nfields(dims)}(dims) -similar{T}(a::Array{T,1}) = Array{T,1}(size(a,1)) -similar{T}(a::Array{T,2}) = Array{T,2}(size(a,1), size(a,2)) -similar{T}(a::Array{T,1}, dims::Dims) = Array{T,nfields(dims)}(dims) -similar{T}(a::Array{T,1}, m::Int) = Array{T,1}(m) -similar{T}(a::Array{T,1}, S::Type) = Array{S,1}(size(a,1)) -similar{T}(a::Array{T,2}, dims::Dims) = Array{T,nfields(dims)}(dims) -similar{T}(a::Array{T,2}, m::Int) = Array{T,1}(m) -similar{T}(a::Array{T,2}, S::Type) = Array{S,2}(size(a,1), size(a,2)) +similar{T}(a::Array{T,1}) = Array{T,1}(size(a,1)) +similar{T}(a::Array{T,2}) = Array{T,2}(size(a,1), size(a,2)) +similar{T}(a::Array{T,1}, S::Type) = Array{S,1}(size(a,1)) +similar{T}(a::Array{T,2}, S::Type) = Array{S,2}(size(a,1), size(a,2)) +similar{T}(a::Array{T}, m::Int) = Array{T,1}(m) +similar{N}(a::Array, T::Type, dims::Dims{N}) = Array{T,N}(dims) +similar{T,N}(a::Array{T}, dims::Dims{N}) = Array{T,N}(dims) # T[x...] constructs Array{T,1} function getindex(T::Type, vals...) From ac5416d9f74a55a64b70b18fe94b0024f47d6dad Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman <benditlikeranjan@gmail.com> Date: Tue, 19 Jul 2016 02:58:42 +0530 Subject: [PATCH 0510/1117] Add test for Issue #17153 and PR #17154 (#17164) * Add test for Issue #17153 and PR #17154 * Fix whitespaces --- test/statistics.jl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/statistics.jl b/test/statistics.jl index d0c5032de39d1..412d6da5824e3 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -341,3 +341,24 @@ let v = varm([1.0+2.0im], 0; corrected = false) @test v ≈ 5 @test isa(v, Float64) end + +# Issue #17153 and PR #17154 +let a = rand(10,10) + b = deepcopy(a) + x = median(a, 1) + @test b == a + x = median(a, 2) + @test b == a + x = mean(a, 1) + @test b == a + x = mean(a, 2) + @test b == a + x = var(a, 1) + @test b == a + x = var(a, 2) + @test b == a + x = std(a, 1) + @test b == a + x = std(a, 2) + @test b == a +end From 0d925dc3dc393f51d8e5f1868b4f4381fc501b8b Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Mon, 18 Jul 2016 22:59:01 -0400 Subject: [PATCH 0511/1117] Error if CC or CXX was specified in Make.user without override This is a very common mistake to make and can lead to different dependencies being built with different compilers, leading to all sorts of subtle errors. Add a check to catch this instead. --- Make.inc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Make.inc b/Make.inc index c8d5beaa09532..3c23c3faa868a 100644 --- a/Make.inc +++ b/Make.inc @@ -483,6 +483,10 @@ ifeq (exists, $(shell [ -e $(BUILDROOT)/Make.user ] && echo exists )) include $(BUILDROOT)/Make.user endif +ifneq ($(CC_BASE)$(CXX_BASE),$(shell echo $(CC) | cut -d' ' -f1)$(shell echo $(CXX) | cut -d' ' -f1)) + $(error Forgot override directive on CC or CXX in Make.user? Cowardly refusing to build) +endif + ifeq ($(SANITIZE),1) ifeq ($(SANITIZE_MEMORY),1) SANITIZE_OPTS := -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer From 3e844366223ecc59a5769d545d7e6401d8c05883 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 18 Jul 2016 20:21:02 -0700 Subject: [PATCH 0512/1117] Revert "temporarily distclean libgit2 and deps on travis" (#17485) This reverts commit 854d905b1f7a939c540555a5d019fdd40dd7f028. [av skip] --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d5f64bf36e2bd..94edf04dbd460 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,7 +95,6 @@ before_install: script: - make -C moreutils mispipe - make $BUILDOPTS -C base version_git.jl.phony - - make $BUILDOPTS NO_GIT=1 -C deps distclean-libgit2 distclean-libssh2 distclean-mbedtls - moreutils/mispipe "make $BUILDOPTS NO_GIT=1 -C deps" bar > deps.log || cat deps.log - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia install | moreutils/ts -s "%.s" - make $BUILDOPTS NO_GIT=1 build-stats From 29672e213424122451f9faa50e0471e733fe4cc6 Mon Sep 17 00:00:00 2001 From: Simon Kornblith <simon@simonster.com> Date: Mon, 18 Jul 2016 23:21:55 -0400 Subject: [PATCH 0513/1117] Return to previous strategy for @ inferred without kwargs (#17482) While the new code worked, it slowed things down too much. --- base/test.jl | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/base/test.jl b/base/test.jl index e82bb3a7b8ec6..e2e8956d6a724 100644 --- a/base/test.jl +++ b/base/test.jl @@ -914,20 +914,24 @@ julia> @inferred max(1,2) """ macro inferred(ex) Meta.isexpr(ex, :call)|| error("@inferred requires a call expression") - args = gensym() - kwargs = gensym() - - # Need to only construct a call with a kwargs if there are actually kwargs, since - # the inferred type depends on the presence/absence of kwargs - if any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex.args) - typecall = :($(ex.args[1])($(args)...; $(kwargs)...)) - else - typecall = :($(ex.args[1])($(args)...)) - end Base.remove_linenums!(quote - $(esc(args)), $(esc(kwargs)), result = $(esc(Expr(:call, _args_and_call, ex.args[2:end]..., ex.args[1]))) - inftypes = $(Base.gen_call_with_extracted_types(Base.return_types, typecall)) + $(if any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex.args) + # Has keywords + args = gensym() + kwargs = gensym() + quote + $(esc(args)), $(esc(kwargs)), result = $(esc(Expr(:call, _args_and_call, ex.args[2:end]..., ex.args[1]))) + inftypes = $(Base.gen_call_with_extracted_types(Base.return_types, :($(ex.args[1])($(args)...; $(kwargs)...)))) + end + else + # No keywords + quote + args = ($([esc(ex.args[i]) for i = 2:length(ex.args)]...),) + result = $(esc(ex.args[1]))(args...) + inftypes = Base.return_types($(esc(ex.args[1])), Base.typesof(args...)) + end + end) @assert length(inftypes) == 1 rettype = isa(result, Type) ? Type{result} : typeof(result) rettype == inftypes[1] || error("return type $rettype does not match inferred return type $(inftypes[1])") From 179a34cb7a85b149f7d058d54d465d952509e5c1 Mon Sep 17 00:00:00 2001 From: Felipe Noronha <felipenoris@users.noreply.github.com> Date: Tue, 19 Jul 2016 03:48:38 -0300 Subject: [PATCH 0514/1117] docs: remove direct calls to require (#17277) --- doc/manual/parallel-computing.rst | 2 +- doc/manual/workflow-tips.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 10270911bcd5a..f36682002a570 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -282,7 +282,7 @@ The function ``count_heads`` simply adds together ``n`` random bits. Here is how we can perform some trials on two machines, and add together the results:: - require("count_heads") + @everywhere include("count_heads.jl") a = @spawn count_heads(100000000) b = @spawn count_heads(100000000) diff --git a/doc/manual/workflow-tips.rst b/doc/manual/workflow-tips.rst index 874d6a5daae79..6eef93d22bc93 100644 --- a/doc/manual/workflow-tips.rst +++ b/doc/manual/workflow-tips.rst @@ -75,7 +75,7 @@ issuing the command:: If you further add the following to your ``.juliarc.jl`` file :: - isinteractive() && isfile("_init.jl") && require("_init.jl") + isfile("_init.jl") && include(joinpath(pwd(), "_init.jl")) then calling ``julia`` from that directory will run the initialization code without the additional command line argument. From 17bd99ac07c73f0df8bb7ea989b3ec67e619e7fb Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer+github@college.harvard.edu> Date: Tue, 19 Jul 2016 12:45:10 +0000 Subject: [PATCH 0515/1117] Don't introduce spurious libLLVM dependency into julia binary (#17486) * Don't introduce spurious libLLVM dependency into julia binary This causes problems with the new RPATH handling since we now omit lib/julia from the executable's RPATH. Fix this by not forcing the executable to link against libLLVM, since we don't require it. Fixes #17484. * Provide link-time rpath Unlike ld.so, ld does not understand `$ORIGIN`, so provide the full path to it at link time to make sure it can find the appropriate dependencies. --- Make.inc | 2 +- ui/Makefile | 16 +--------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/Make.inc b/Make.inc index 3c23c3faa868a..58ae3d09657f4 100644 --- a/Make.inc +++ b/Make.inc @@ -850,7 +850,7 @@ else ifeq ($(OS), Darwin) RPATH_ORIGIN := -Wl,-rpath,'@loader_path/' RPATH_LIB := -Wl,-rpath,'@loader_path/' -Wl,-rpath,'@loader_path/julia/' else - RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-z,origin + RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin RPATH_LIB := -Wl,-rpath,'$$ORIGIN' -Wl,-rpath,'$$ORIGIN/julia' -Wl,-z,origin endif diff --git a/ui/Makefile b/ui/Makefile index f869b429824e0..49c4dbac40167 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -23,21 +23,7 @@ OBJS := $(SRCS:%=$(BUILDDIR)/%.o) DOBJS := $(SRCS:%=$(BUILDDIR)/%.dbg.obj) DEBUGFLAGS += $(FLAGS) SHIPFLAGS += $(FLAGS) - -ifeq ($(USE_LLVM_SHLIB),1) -ifeq ($(LLVM_USE_CMAKE),1) -LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) -lLLVM -else -ifeq ($(OS),WINNT) -LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) -lLLVM-$(LLVM_VER_SHORT) -else -LLVMLINK := $(shell $(LLVM_CONFIG_HOST) --ldflags) -lLLVM-$(shell $(LLVM_CONFIG_HOST) --version) -endif # OS == WINNT -endif # LLVM_USE_CMAKE == 1 -else -LLVMLINK := -endif # USE_LLVM_SHLIB == 1 -JLDFLAGS += $(LDFLAGS) $(NO_WHOLE_ARCHIVE) $(OSLIBS) $(LLVMLINK) $(RPATH) +JLDFLAGS += $(LDFLAGS) $(NO_WHOLE_ARCHIVE) $(OSLIBS) $(RPATH) ifeq ($(USE_SYSTEM_LIBM),0) ifneq ($(UNTRUSTED_SYSTEM_LIBM),0) From c73c778c0ccc28bce6731d6078fb7dbbf15125ed Mon Sep 17 00:00:00 2001 From: Valentin Churavy <v.churavy@gmail.com> Date: Tue, 19 Jul 2016 06:54:37 +0900 Subject: [PATCH 0516/1117] Keep AddressSpace of global variables For backends, where AddressSpace information is important (NVPTX) we need to keep that information when moving global variables. --- src/ccall.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 4455312eafe1d..6b84e8898f338 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -795,7 +795,10 @@ class FunctionMover : public ValueMaterializer GV->isConstant(), GlobalVariable::ExternalLinkage, NULL, - GV->getName()); + GV->getName(), + NULL, + GV->getThreadLocalMode(), + GV->getType()->getPointerAddressSpace()); newGV->copyAttributesFrom(GV); newGV->setComdat(nullptr); if (GV->isDeclaration()) From efcc98cb1ce5d8e90075ef11217a157624ca11f1 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Tue, 19 Jul 2016 00:33:17 -0400 Subject: [PATCH 0517/1117] Fix LLVM svn build (now 4.0) --- src/llvm-version.h | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/llvm-version.h b/src/llvm-version.h index c006c926c474c..b63a397d4fb3c 100644 --- a/src/llvm-version.h +++ b/src/llvm-version.h @@ -2,16 +2,20 @@ #include <llvm/Config/llvm-config.h> -#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 9 +#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 4 && LLVM_VERSION_MINOR >= 0 +#define LLVM40 1 +#endif + +#if defined(LLVM40) || (defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 9) #define LLVM39 1 #endif -#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 8 +#if defined(LLVM40) || (defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 8) #define LLVM38 1 #define USE_ORCJIT #endif -#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 7 +#if defined(LLVM40) || (defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 7) #define LLVM37 1 // We enable ORCJIT only if we have our custom patches @@ -21,21 +25,23 @@ #endif -#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 6 +#if defined(LLVM40) || (defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 6) #define LLVM36 1 #endif -#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 5 +#if defined(LLVM40) || (defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 5) #define LLVM35 1 #endif -#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 4 +#if defined(LLVM40) || (defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 4) #define LLVM34 1 #define USE_MCJIT #endif -#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 3 +#if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR >= 3 +#if defined(LLVM40) || LLVM_VERSION_MINOR >= 3 #define LLVM33 1 +#endif #else #error LLVM versions < 3.3 are not supported by Julia #endif From 631705ff786febd246e04a222d40c5b18a1db4bf Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Tue, 19 Jul 2016 14:28:25 -0400 Subject: [PATCH 0518/1117] parse more dot operators, parse .= with assignment precedence (#17393) * parse more dot operators for #16285, parse .= with assignment precedence * include .|= and .&= in syntactic-operators list * reject .! operator * NEWS for new operators * tests * move tests to parse.jl --- NEWS.md | 7 +++++++ src/julia-parser.scm | 38 +++++++++++++++++++++++++------------- test/parse.jl | 8 ++++++++ 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/NEWS.md b/NEWS.md index df92f944d367d..c7c03bfa164f2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -36,6 +36,11 @@ New language features packages, but now Julia provides an API for writing generic algorithms for arbitrary indexing schemes ([#16260]). + * Many more operators now support `.` prefixes (e.g. `.≤`) ([#17393]). However, + users are discouraged from overloading these, since they are mainly parsed + in order to implement backwards compatibility with planned automatic + broadcasting of dot operators in Julia 0.6 ([#16285]). + New architectures ----------------- @@ -328,6 +333,7 @@ Deprecated or removed [#16107]: https://github.com/JuliaLang/julia/issues/16107 [#16219]: https://github.com/JuliaLang/julia/issues/16219 [#16260]: https://github.com/JuliaLang/julia/issues/16260 +[#16285]: https://github.com/JuliaLang/julia/issues/16285 [#16362]: https://github.com/JuliaLang/julia/issues/16362 [#16403]: https://github.com/JuliaLang/julia/issues/16403 [#16404]: https://github.com/JuliaLang/julia/issues/16404 @@ -346,5 +352,6 @@ Deprecated or removed [#17300]: https://github.com/JuliaLang/julia/issues/17300 [#17323]: https://github.com/JuliaLang/julia/issues/17323 [#17374]: https://github.com/JuliaLang/julia/issues/17374 +[#17393]: https://github.com/JuliaLang/julia/issues/17393 [#17402]: https://github.com/JuliaLang/julia/issues/17402 [#17404]: https://github.com/JuliaLang/julia/issues/17404 diff --git a/src/julia-parser.scm b/src/julia-parser.scm index bc4c96c98e7c9..3bad336c80bdc 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1,23 +1,31 @@ ;; Operator precedence table, lowest at top +; for most operators X there is a .X "elementwise" equivalent +(define (add-dots ops) (append! ops (map (lambda (op) (symbol (string "." op))) ops))) + ;; note: there are some strange-looking things in here because ;; the way the lexer works, every prefix of an operator must also ;; be an operator. (define prec-assignment - '(= := += -= *= /= //= .//= .*= ./= |\\=| |.\\=| ^= .^= ÷= .÷= %= .%= |\|=| &= $= => <<= >>= >>>= ~ |.+=| |.-=|)) + (append! (add-dots '(= += -= *= /= //= |\\=| ^= ÷= %= <<= >>= >>>= |\|=| &=)) + '(:= => ~ $=))) (define prec-conditional '(?)) -(define prec-arrow '(-- --> ← → ↔ ↚ ↛ ↠ ↣ ↦ ↮ ⇎ ⇏ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⭄ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ￩ ￫)) +(define prec-arrow (append! + '(-- -->) + (add-dots '(← → ↔ ↚ ↛ ↠ ↣ ↦ ↮ ⇎ ⇏ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⭄ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ￩ ￫)))) (define prec-lazy-or '(|\|\||)) (define prec-lazy-and '(&&)) (define prec-comparison - '(> < >= ≥ <= ≤ == === ≡ != ≠ !== ≢ |.>| |.<| |.>=| |.≥| |.<=| |.≤| |.==| |.!=| |.≠| |.=| |.!| |<:| |>:| ∈ ∉ ∋ ∌ ⊆ ⊈ ⊂ ⊄ ⊊ ∝ ∊ ∍ ∥ ∦ ∷ ∺ ∻ ∽ ∾ ≁ ≃ ≄ ≅ ≆ ≇ ≈ ≉ ≊ ≋ ≌ ≍ ≎ ≐ ≑ ≒ ≓ ≔ ≕ ≖ ≗ ≘ ≙ ≚ ≛ ≜ ≝ ≞ ≟ ≣ ≦ ≧ ≨ ≩ ≪ ≫ ≬ ≭ ≮ ≯ ≰ ≱ ≲ ≳ ≴ ≵ ≶ ≷ ≸ ≹ ≺ ≻ ≼ ≽ ≾ ≿ ⊀ ⊁ ⊃ ⊅ ⊇ ⊉ ⊋ ⊏ ⊐ ⊑ ⊒ ⊜ ⊩ ⊬ ⊮ ⊰ ⊱ ⊲ ⊳ ⊴ ⊵ ⊶ ⊷ ⋍ ⋐ ⋑ ⋕ ⋖ ⋗ ⋘ ⋙ ⋚ ⋛ ⋜ ⋝ ⋞ ⋟ ⋠ ⋡ ⋢ ⋣ ⋤ ⋥ ⋦ ⋧ ⋨ ⋩ ⋪ ⋫ ⋬ ⋭ ⋲ ⋳ ⋴ ⋵ ⋶ ⋷ ⋸ ⋹ ⋺ ⋻ ⋼ ⋽ ⋾ ⋿ ⟈ ⟉ ⟒ ⦷ ⧀ ⧁ ⧡ ⧣ ⧤ ⧥ ⩦ ⩧ ⩪ ⩫ ⩬ ⩭ ⩮ ⩯ ⩰ ⩱ ⩲ ⩳ ⩴ ⩵ ⩶ ⩷ ⩸ ⩹ ⩺ ⩻ ⩼ ⩽ ⩾ ⩿ ⪀ ⪁ ⪂ ⪃ ⪄ ⪅ ⪆ ⪇ ⪈ ⪉ ⪊ ⪋ ⪌ ⪍ ⪎ ⪏ ⪐ ⪑ ⪒ ⪓ ⪔ ⪕ ⪖ ⪗ ⪘ ⪙ ⪚ ⪛ ⪜ ⪝ ⪞ ⪟ ⪠ ⪡ ⪢ ⪣ ⪤ ⪥ ⪦ ⪧ ⪨ ⪩ ⪪ ⪫ ⪬ ⪭ ⪮ ⪯ ⪰ ⪱ ⪲ ⪳ ⪴ ⪵ ⪶ ⪷ ⪸ ⪹ ⪺ ⪻ ⪼ ⪽ ⪾ ⪿ ⫀ ⫁ ⫂ ⫃ ⫄ ⫅ ⫆ ⫇ ⫈ ⫉ ⫊ ⫋ ⫌ ⫍ ⫎ ⫏ ⫐ ⫑ ⫒ ⫓ ⫔ ⫕ ⫖ ⫗ ⫘ ⫙ ⫷ ⫸ ⫹ ⫺ ⊢ ⊣)) ;; plus `in` + (append! '(|<:| |>:| |.!|) + (add-dots '(> < >= ≥ <= ≤ == === ≡ != ≠ !== ≢ ∈ ∉ ∋ ∌ ⊆ ⊈ ⊂ ⊄ ⊊ ∝ ∊ ∍ ∥ ∦ ∷ ∺ ∻ ∽ ∾ ≁ ≃ ≄ ≅ ≆ ≇ ≈ ≉ ≊ ≋ ≌ ≍ ≎ ≐ ≑ ≒ ≓ ≔ ≕ ≖ ≗ ≘ ≙ ≚ ≛ ≜ ≝ ≞ ≟ ≣ ≦ ≧ ≨ ≩ ≪ ≫ ≬ ≭ ≮ ≯ ≰ ≱ ≲ ≳ ≴ ≵ ≶ ≷ ≸ ≹ ≺ ≻ ≼ ≽ ≾ ≿ ⊀ ⊁ ⊃ ⊅ ⊇ ⊉ ⊋ ⊏ ⊐ ⊑ ⊒ ⊜ ⊩ ⊬ ⊮ ⊰ ⊱ ⊲ ⊳ ⊴ ⊵ ⊶ ⊷ ⋍ ⋐ ⋑ ⋕ ⋖ ⋗ ⋘ ⋙ ⋚ ⋛ ⋜ ⋝ ⋞ ⋟ ⋠ ⋡ ⋢ ⋣ ⋤ ⋥ ⋦ ⋧ ⋨ ⋩ ⋪ ⋫ ⋬ ⋭ ⋲ ⋳ ⋴ ⋵ ⋶ ⋷ ⋸ ⋹ ⋺ ⋻ ⋼ ⋽ ⋾ ⋿ ⟈ ⟉ ⟒ ⦷ ⧀ ⧁ ⧡ ⧣ ⧤ ⧥ ⩦ ⩧ ⩪ ⩫ ⩬ ⩭ ⩮ ⩯ ⩰ ⩱ ⩲ ⩳ ⩴ ⩵ ⩶ ⩷ ⩸ ⩹ ⩺ ⩻ ⩼ ⩽ ⩾ ⩿ ⪀ ⪁ ⪂ ⪃ ⪄ ⪅ ⪆ ⪇ ⪈ ⪉ ⪊ ⪋ ⪌ ⪍ ⪎ ⪏ ⪐ ⪑ ⪒ ⪓ ⪔ ⪕ ⪖ ⪗ ⪘ ⪙ ⪚ ⪛ ⪜ ⪝ ⪞ ⪟ ⪠ ⪡ ⪢ ⪣ ⪤ ⪥ ⪦ ⪧ ⪨ ⪩ ⪪ ⪫ ⪬ ⪭ ⪮ ⪯ ⪰ ⪱ ⪲ ⪳ ⪴ ⪵ ⪶ ⪷ ⪸ ⪹ ⪺ ⪻ ⪼ ⪽ ⪾ ⪿ ⫀ ⫁ ⫂ ⫃ ⫄ ⫅ ⫆ ⫇ ⫈ ⫉ ⫊ ⫋ ⫌ ⫍ ⫎ ⫏ ⫐ ⫑ ⫒ ⫓ ⫔ ⫕ ⫖ ⫗ ⫘ ⫙ ⫷ ⫸ ⫹ ⫺ ⊢ ⊣)))) ;; (plus `in`) (define prec-pipe '(|\|>| |<\||)) (define prec-colon '(: |..|)) -(define prec-plus '(+ - ⊕ ⊖ ⊞ ⊟ |.+| |.-| |++| |\|| ∪ ∨ $ ⊔ ± ∓ ∔ ∸ ≂ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣)) -(define prec-bitshift '(<< >> >>> |.<<| |.>>| |.>>>|)) -(define prec-times '(* / |./| ÷ |.÷| % ⋅ ∘ × |.%| |.*| |\\| |.\\| & ∩ ∧ ⊗ ⊘ ⊙ ⊚ ⊛ ⊠ ⊡ ⊓ ∗ ∙ ∤ ⅋ ≀ ⊼ ⋄ ⋆ ⋇ ⋉ ⋊ ⋋ ⋌ ⋏ ⋒ ⟑ ⦸ ⦼ ⦾ ⦿ ⧶ ⧷ ⨇ ⨰ ⨱ ⨲ ⨳ ⨴ ⨵ ⨶ ⨷ ⨸ ⨻ ⨼ ⨽ ⩀ ⩃ ⩄ ⩋ ⩍ ⩎ ⩑ ⩓ ⩕ ⩘ ⩚ ⩜ ⩞ ⩟ ⩠ ⫛ ⊍ ▷ ⨝ ⟕ ⟖ ⟗)) -(define prec-rational '(// .//)) -(define prec-power '(^ |.^| ↑ ↓ ⇵ ⟰ ⟱ ⤈ ⤉ ⤊ ⤋ ⤒ ⤓ ⥉ ⥌ ⥍ ⥏ ⥑ ⥔ ⥕ ⥘ ⥙ ⥜ ⥝ ⥠ ⥡ ⥣ ⥥ ⥮ ⥯ ￪ ￬)) +(define prec-plus (append! '($) + (add-dots '(+ - |\|| ⊕ ⊖ ⊞ ⊟ |++| ∪ ∨ ⊔ ± ∓ ∔ ∸ ≂ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣)))) +(define prec-bitshift (add-dots '(<< >> >>>))) +(define prec-times (add-dots '(* / ÷ % & ⋅ ∘ × |\\| ∩ ∧ ⊗ ⊘ ⊙ ⊚ ⊛ ⊠ ⊡ ⊓ ∗ ∙ ∤ ⅋ ≀ ⊼ ⋄ ⋆ ⋇ ⋉ ⋊ ⋋ ⋌ ⋏ ⋒ ⟑ ⦸ ⦼ ⦾ ⦿ ⧶ ⧷ ⨇ ⨰ ⨱ ⨲ ⨳ ⨴ ⨵ ⨶ ⨷ ⨸ ⨻ ⨼ ⨽ ⩀ ⩃ ⩄ ⩋ ⩍ ⩎ ⩑ ⩓ ⩕ ⩘ ⩚ ⩜ ⩞ ⩟ ⩠ ⫛ ⊍ ▷ ⨝ ⟕ ⟖ ⟗))) +(define prec-rational (add-dots '(//))) +(define prec-power (add-dots '(^ ↑ ↓ ⇵ ⟰ ⟱ ⤈ ⤉ ⤊ ⤋ ⤒ ⤓ ⥉ ⥌ ⥍ ⥏ ⥑ ⥔ ⥕ ⥘ ⥙ ⥜ ⥝ ⥠ ⥡ ⥣ ⥥ ⥮ ⥯ ￪ ￬))) (define prec-decl '(|::|)) (define prec-dot '(|.|)) @@ -69,8 +77,8 @@ ; operators that are special forms, not function names (define syntactic-operators - '(= := += -= *= /= //= .//= .*= ./= |\\=| |.\\=| ^= .^= ÷= .÷= %= .%= |\|=| &= $= => - <<= >>= >>>= -> --> |\|\|| && |.| ... |.+=| |.-=|)) + (append! (add-dots '(= += -= *= /= //= |\\=| ^= ÷= %= <<= >>= >>>= |\|=| &=)) + '(:= --> $= => && |\|\|| |.| ... ->))) (define syntactic-unary-operators '($ & |::|)) (define syntactic-op? (Set syntactic-operators)) @@ -97,7 +105,11 @@ ;; characters that can be in an operator (define opchar? (Set op-chars)) ;; characters that can follow . in an operator -(define (dot-opchar? c) (and (char? c) (string.find ".*^/\\+-'<>!=%≥≤≠÷" c))) +(define dot-opchar? (Set + (delete-duplicates + (map (lambda (op) (string.char (string op) 1)) + (filter (lambda (op) (and (dotop? op) (not (eq? op '|.|)))) + operators))))) (define operator? (Set operators)) (define initial-reserved-words '(begin while if for try return break continue @@ -186,8 +198,8 @@ (loop newop (peek-char port))) str)) str)))) - (if (equal? str "--") - (error "invalid operator \"--\"")) + (if (or (equal? str "--") (equal? str ".!")) + (error (string "invalid operator \"" str "\""))) (string->symbol str)))) (define (accum-digits c pred port lz) diff --git a/test/parse.jl b/test/parse.jl index b9eda6f987956..bf1defd425645 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -555,3 +555,11 @@ end # error throwing branch from #10560 @test_throws ArgumentError Base.tryparse_internal(Bool, "foo", 1, 2, 10, true) + +# PR #17393 +for op in (:.==, :.&, :.|, :.≤) + @test parse("a $op b") == Expr(:call, op, :a, :b) +end +for op in (:.=, :.+=) + @test parse("a $op b") == Expr(op, :a, :b) +end From 0f9cf41907e8ac42bb9652e651b71963cccf06b5 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Tue, 19 Jul 2016 14:28:50 -0400 Subject: [PATCH 0519/1117] add back isvalid(::String), fixes #17467 (#17499) --- base/strings/basic.jl | 1 + test/strings/basic.jl | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 167bef7bcbd44..399d383f8be1e 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -289,6 +289,7 @@ byte_string_classify(s::String) = byte_string_classify(s.data) # 2: valid UTF-8 isvalid(::Type{String}, s::Union{Vector{UInt8},String}) = byte_string_classify(s) != 0 +isvalid(s::String) = isvalid(String, s) ## uppercase and lowercase transformations ## uppercase(s::AbstractString) = map(uppercase, s) diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 95419735d9c81..45f5f91605880 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -287,11 +287,11 @@ for (val, pass) in ( (b"\udc00\u0100", false), (b"\udc00\ud800", false) ) - @test isvalid(String, val) == pass + @test isvalid(String, val) == pass == isvalid(String(val)) end # Issue #11203 -@test isvalid(String, UInt8[]) == true +@test isvalid(String, UInt8[]) == true == isvalid("") # Check UTF-8 characters # Check ASCII range (true), From 73cdfea04a213e7672d88bb1eff36b9f0f764736 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 18 Jul 2016 16:57:25 -0400 Subject: [PATCH 0520/1117] avoid #unused# leaking into ArgumentError message fix #16536 --- src/toplevel.c | 26 ++++++++++++++++++-------- test/parse.jl | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/toplevel.c b/src/toplevel.c index 5a0a7a682a333..a402f06d27f9c 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -766,15 +766,25 @@ JL_DLLEXPORT void jl_method_def(jl_svec_t *argdata, jl_lambda_info_t *f, jl_valu f = m->lambda_template; // because jl_new_method makes a copy jl_check_static_parameter_conflicts(m, tvars); - // TODO - size_t na = jl_nparams(argtypes); - for(size_t i=0; i < na; i++) { - jl_value_t *elt = jl_tparam(argtypes,i); + size_t i, na = jl_nparams(argtypes); + for (i = 0; i < na; i++) { + jl_value_t *elt = jl_tparam(argtypes, i); if (!jl_is_type(elt) && !jl_is_typevar(elt)) { - jl_exceptionf(jl_argumenterror_type, "invalid type for argument %s in method definition for %s at %s:%d", - jl_symbol_name((jl_sym_t*)jl_array_ptr_ref(f->slotnames,i)), - jl_symbol_name(name), jl_symbol_name(m->file), - m->line); + jl_sym_t *argname = (jl_sym_t*)jl_array_ptr_ref(f->slotnames, i); + if (argname == unused_sym) + jl_exceptionf(jl_argumenterror_type, + "invalid type for argument number %d in method definition for %s at %s:%d", + i, + jl_symbol_name(name), + jl_symbol_name(m->file), + m->line); + else + jl_exceptionf(jl_argumenterror_type, + "invalid type for argument %s in method definition for %s at %s:%d", + jl_symbol_name(argname), + jl_symbol_name(name), + jl_symbol_name(m->file), + m->line); } } diff --git a/test/parse.jl b/test/parse.jl index bf1defd425645..6770d5bc25e2d 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +# tests for parser and syntax lowering + function parseall(str) pos = start(str) exs = [] @@ -563,3 +565,18 @@ end for op in (:.=, :.+=) @test parse("a $op b") == Expr(op, :a, :b) end + +# issue #17489 +let m_error, error_out, filename = Base.source_path() + m_error = try @eval method_c6(a::(:A)) = 1; catch e; e; end + error_out = sprint(showerror, m_error) + @test startswith(error_out, "ArgumentError: invalid type for argument a in method definition for method_c6 at $filename:") + + m_error = try @eval method_c6(::(:A)) = 2; catch e; e; end + error_out = sprint(showerror, m_error) + @test startswith(error_out, "ArgumentError: invalid type for argument number 1 in method definition for method_c6 at $filename:") + + m_error = try @eval method_c6(A; B) = 3; catch e; e; end + error_out = sprint(showerror, m_error) + @test error_out == "syntax: keyword argument \"B\" needs a default value" +end From 7ffb8700bbeb57219f68ad89676aaacf3a88ad85 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 19 Jul 2016 17:21:16 -0400 Subject: [PATCH 0521/1117] move a couple tests from core to parse --- test/core.jl | 63 --------------------------------------------------- test/parse.jl | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 63 deletions(-) diff --git a/test/core.jl b/test/core.jl index 1133a7aa779d6..7d8b3687a4270 100644 --- a/test/core.jl +++ b/test/core.jl @@ -528,12 +528,6 @@ let end @test glob_x3 == 12 -# issue #7272 -@test expand(parse("let - global x = 2 - local x = 1 - end")) == Expr(:error, "variable \"x\" declared both local and global") - # let - new variables, including undefinedness function let_undef() first = true @@ -1757,26 +1751,6 @@ let @test Test()() === nothing end -# make sure front end can correctly print values to error messages -let ex = expand(parse("\"a\"=1")) - @test ex == Expr(:error, "invalid assignment location \"\"a\"\"") -end - -# make sure that incomplete tags are detected correctly -# (i.e. error messages in src/julia-parser.scm must be matched correctly -# by the code in base/client.jl) -for (str, tag) in Dict("" => :none, "\"" => :string, "#=" => :comment, "'" => :char, - "`" => :cmd, "begin;" => :block, "quote;" => :block, - "let;" => :block, "for i=1;" => :block, "function f();" => :block, - "f() do x;" => :block, "module X;" => :block, "type X;" => :block, - "immutable X;" => :block, "(" => :other, "[" => :other, - "begin" => :other, "quote" => :other, - "let" => :other, "for" => :other, "function" => :other, - "f() do" => :other, "module" => :other, "type" => :other, - "immutable" => :other) - @test Base.incomplete_tag(parse(str, raise=false)) == tag -end - # issue #6031 macro m6031(x); x; end @test @m6031([2,4,6])[3] == 6 @@ -3347,9 +3321,6 @@ typealias PossiblyInvalidUnion{T} Union{T,Int} @test Symbol("x") === Symbol("x") @test split(string(gensym("abc")),'#')[3] == "abc" -# meta nodes for optional positional arguments -@test expand(:(@inline f(p::Int=2) = 3)).args[2].args[3].inlineable - # issue #13007 call13007{T,N}(::Type{Array{T,N}}) = 0 call13007(::Type{Array}) = 1 @@ -4161,20 +4132,6 @@ function f16023() end @test_throws UndefVarError f16023() -# issue #16096 -module M16096 -macro iter() - quote - @inline function foo(sub) - it = 1 - end - end -end -end -let ex = expand(:(@M16096.iter)) - @test !(isa(ex,Expr) && ex.head === :error) -end - # issue #16158 function f16158(x) bar(x) = length(x)==1 ? x : string(x, bar(x[1:end-1])) @@ -4393,26 +4350,6 @@ function trigger14878() end @test_throws UndefVarError trigger14878() -# issue #15838 -module A15838 - macro f() end - const x = :a -end -module B15838 - import A15838.@f - macro f(x); return :x; end - const x = :b -end -@test A15838.@f() === nothing -@test A15838.@f(1) === :b -let nometh = expand(:(A15838.@f(1, 2))) - @test (nometh::Expr).head === :error - @test length(nometh.args) == 1 - e = nometh.args[1]::MethodError - @test e.f === getfield(A15838, Symbol("@f")) - @test e.args === (1,2) -end - # issue #1090 function f1090(x)::Int if x == 1 diff --git a/test/parse.jl b/test/parse.jl index 6770d5bc25e2d..6714351c4016d 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -580,3 +580,66 @@ let m_error, error_out, filename = Base.source_path() error_out = sprint(showerror, m_error) @test error_out == "syntax: keyword argument \"B\" needs a default value" end + +# issue #7272 +@test expand(parse("let + global x = 2 + local x = 1 + end")) == Expr(:error, "variable \"x\" declared both local and global") + +# make sure front end can correctly print values to error messages +let ex = expand(parse("\"a\"=1")) + @test ex == Expr(:error, "invalid assignment location \"\"a\"\"") +end + +# make sure that incomplete tags are detected correctly +# (i.e. error messages in src/julia-parser.scm must be matched correctly +# by the code in base/client.jl) +for (str, tag) in Dict("" => :none, "\"" => :string, "#=" => :comment, "'" => :char, + "`" => :cmd, "begin;" => :block, "quote;" => :block, + "let;" => :block, "for i=1;" => :block, "function f();" => :block, + "f() do x;" => :block, "module X;" => :block, "type X;" => :block, + "immutable X;" => :block, "(" => :other, "[" => :other, + "begin" => :other, "quote" => :other, + "let" => :other, "for" => :other, "function" => :other, + "f() do" => :other, "module" => :other, "type" => :other, + "immutable" => :other) + @test Base.incomplete_tag(parse(str, raise=false)) == tag +end + +# meta nodes for optional positional arguments +@test expand(:(@inline f(p::Int=2) = 3)).args[2].args[3].inlineable + +# issue #16096 +module M16096 +macro iter() + quote + @inline function foo(sub) + it = 1 + end + end +end +end +let ex = expand(:(@M16096.iter)) + @test !(isa(ex,Expr) && ex.head === :error) +end + +# issue #15838 +module A15838 + macro f() end + const x = :a +end +module B15838 + import A15838.@f + macro f(x); return :x; end + const x = :b +end +@test A15838.@f() === nothing +@test A15838.@f(1) === :b +let nometh = expand(:(A15838.@f(1, 2))) + @test (nometh::Expr).head === :error + @test length(nometh.args) == 1 + e = nometh.args[1]::MethodError + @test e.f === getfield(A15838, Symbol("@f")) + @test e.args === (1,2) +end From 6b15abd11962f43c1ba5457137ca5d5371c197ce Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 18 Jul 2016 16:22:25 -0400 Subject: [PATCH 0522/1117] ensure reflection returns all names in a module fix #17140 --- base/REPLCompletions.jl | 2 +- base/docs/utils.jl | 2 +- base/interactiveutil.jl | 2 +- base/reflection.jl | 18 +++++++----------- base/test.jl | 3 ++- src/module.c | 4 ++-- test/reflection.jl | 40 +++++++++++++++++++++------------------- 7 files changed, 35 insertions(+), 36 deletions(-) diff --git a/base/REPLCompletions.jl b/base/REPLCompletions.jl index ff87762b6c1fb..8a1ea5e6b1b0a 100644 --- a/base/REPLCompletions.jl +++ b/base/REPLCompletions.jl @@ -57,7 +57,7 @@ function complete_symbol(sym, ffunc) # We will exclude the results that the user does not want, as well # as excluding Main.Main.Main, etc., because that's most likely not what # the user wants - p = s->(ffunc(mod, s) && s != module_name(mod)) + p = s->(!Base.isdeprecated(mod, s) && s != module_name(mod) && ffunc(mod, s)) # Looking for a binding in a module if mod == context_module # Also look in modules we got through `using` diff --git a/base/docs/utils.jl b/base/docs/utils.jl index a6caf2da11709..599b1c1ce1f9f 100644 --- a/base/docs/utils.jl +++ b/base/docs/utils.jl @@ -310,7 +310,7 @@ moduleusings(mod) = ccall(:jl_module_usings, Any, (Any,), mod) filtervalid(names) = filter(x->!ismatch(r"#", x), map(string, names)) accessible(mod::Module) = - [names(mod, true, true); + [filter!(s->Base.isdeprecated(mod, s), names(mod, true, true)); map(names, moduleusings(mod))...; builtins] |> unique |> filtervalid diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 1760cda164318..9cfb4d6616cbb 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -670,7 +670,7 @@ function summarysize(obj::Module, seen, excl) haskey(seen, obj) ? (return 0) : (seen[obj] = true) size::Int = Core.sizeof(obj) for binding in names(obj, true) - if isdefined(obj, binding) + if isdefined(obj, binding) && !isdeprecated(obj, binding) value = getfield(obj, binding) if !isa(value, Module) || module_parent(value) === obj size += summarysize(value, seen, excl)::Int diff --git a/base/reflection.jl b/base/reflection.jl index c81471b0801ef..1b2bcd37a7651 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -27,15 +27,11 @@ function fullname(m::Module) return tuple(fullname(mp)..., mn) end -names(m::Module, all::Bool, imported::Bool) = sort!(ccall(:jl_module_names, Array{Symbol,1}, (Any,Int32,Int32), m, all, imported)) -names(m::Module, all::Bool) = names(m, all, false) -names(m::Module) = names(m, false, false) +names(m::Module, all::Bool=false, imported::Bool=false) = sort!(ccall(:jl_module_names, Array{Symbol,1}, (Any,Int32,Int32), m, all, imported)) -isexported(m::Module, s::Symbol) = ccall(:jl_module_exports_p, Cint, (Any, Any), m, s)!=0 - -function isbindingresolved(m::Module, var::Symbol) - ccall(:jl_binding_resolved_p, Cint, (Any, Any), m, var) != 0 -end +isexported(m::Module, s::Symbol) = ccall(:jl_module_exports_p, Cint, (Any, Any), m, s) != 0 +isdeprecated(m::Module, s::Symbol) = ccall(:jl_is_binding_deprecated, Cint, (Any, Any), m, s) != 0 +isbindingresolved(m::Module, var::Symbol) = ccall(:jl_binding_resolved_p, Cint, (Any, Any), m, var) != 0 binding_module(s::Symbol) = binding_module(current_module(), s) function binding_module(m::Module, s::Symbol) @@ -158,8 +154,8 @@ function instances end # subtypes function _subtypes(m::Module, x::DataType, sts=Set(), visited=Set()) push!(visited, m) - for s in names(m,true) - if isdefined(m,s) + for s in names(m, true) + if isdefined(m, s) && !isdeprecated(m, s) t = getfield(m, s) if isa(t, DataType) && t.name.name == s && supertype(t).name == x.name ti = typeintersect(t, x) @@ -169,7 +165,7 @@ function _subtypes(m::Module, x::DataType, sts=Set(), visited=Set()) end end end - sts + return sts end subtypes(m::Module, x::DataType) = sort(collect(_subtypes(m, x)), by=string) subtypes(x::DataType) = subtypes(Main, x) diff --git a/base/test.jl b/base/test.jl index e2e8956d6a724..aa77ac6eb6f0f 100644 --- a/base/test.jl +++ b/base/test.jl @@ -986,6 +986,7 @@ function detect_ambiguities(mods...; imported::Bool=false) ambs = Set{Tuple{Method,Method}}() for mod in mods for n in names(mod, true, imported) + Base.isdeprecated(mod, n) && continue if !isdefined(mod, n) println("Skipping ", mod, '.', n) # typically stale exports continue @@ -1005,7 +1006,7 @@ function detect_ambiguities(mods...; imported::Bool=false) end end end - collect(ambs) + return collect(ambs) end """ diff --git a/src/module.c b/src/module.c index 93405223f8d29..28ca7f88bd1dc 100644 --- a/src/module.c +++ b/src/module.c @@ -563,12 +563,12 @@ JL_DLLEXPORT jl_value_t *jl_module_names(jl_module_t *m, int all, int imported) JL_GC_PUSH1(&a); size_t i; void **table = m->bindings.table; - for(i=1; i < m->bindings.size; i+=2) { + for (i = 1; i < m->bindings.size; i+=2) { if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; int hidden = jl_symbol_name(b->name)[0]=='#'; if ((b->exportp || ((imported || b->owner == m) && (all || m == jl_main_module))) && - !b->deprecated && !hidden) { + (all || (!b->deprecated && !hidden))) { jl_array_grow_end(a, 1); //XXX: change to jl_arrayset if array storage allocation for Array{Symbols,1} changes: jl_array_ptr_set(a, jl_array_dim0(a)-1, (jl_value_t*)b->name); diff --git a/test/reflection.jl b/test/reflection.jl index 6b4864c772ccc..00dc08b366a21 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -177,43 +177,44 @@ foo7648(x) = x a9475 = 5 b9475 = 7 let - @test Base.binding_module(:a9475)==current_module() - @test Base.binding_module(:c7648)==TestMod7648 - @test Base.module_name(current_module())==:TestModSub9475 - @test Base.fullname(current_module())==(:TestMod7648, :TestModSub9475) - @test Base.module_parent(current_module())==TestMod7648 + @test Base.binding_module(:a9475) == current_module() + @test Base.binding_module(:c7648) == TestMod7648 + @test Base.module_name(current_module()) == :TestModSub9475 + @test Base.fullname(current_module()) == (:TestMod7648, :TestModSub9475) + @test Base.module_parent(current_module()) == TestMod7648 end end # module TestModSub9475 using .TestModSub9475 let - @test Base.binding_module(:d7648)==current_module() - @test Base.binding_module(:a9475)==TestModSub9475 - @test Base.module_name(current_module())==:TestMod7648 - @test Base.module_parent(current_module())==Main + @test Base.binding_module(:d7648) == current_module() + @test Base.binding_module(:a9475) == TestModSub9475 + @test Base.module_name(current_module()) == :TestMod7648 + @test Base.module_parent(current_module()) == Main end end # module TestMod7648 let - @test Base.binding_module(TestMod7648, :d7648)==TestMod7648 - @test Base.binding_module(TestMod7648, :a9475)==TestMod7648.TestModSub9475 - @test Base.binding_module(TestMod7648.TestModSub9475, :b9475)==TestMod7648.TestModSub9475 - @test Set(names(TestMod7648))==Set([:TestMod7648, :a9475, :c7648, :foo7648]) + @test Base.binding_module(TestMod7648, :d7648) == TestMod7648 + @test Base.binding_module(TestMod7648, :a9475) == TestMod7648.TestModSub9475 + @test Base.binding_module(TestMod7648.TestModSub9475, :b9475) == TestMod7648.TestModSub9475 + @test Set(names(TestMod7648)) == Set([:TestMod7648, :a9475, :c7648, :foo7648]) + @test Set(names(TestMod7648, true)) == Set([:TestMod7648, :TestModSub9475, :a9475, :c7648, :d7648, :f7648, :foo7648, Symbol("#foo7648"), :eval, Symbol("#eval")]) @test isconst(TestMod7648, :c7648) @test !isconst(TestMod7648, :d7648) end let using TestMod7648 - @test Base.binding_module(:a9475)==TestMod7648.TestModSub9475 - @test Base.binding_module(:c7648)==TestMod7648 - @test Base.function_name(foo7648)==:foo7648 - @test Base.function_module(foo7648, (Any,))==TestMod7648 + @test Base.binding_module(:a9475) == TestMod7648.TestModSub9475 + @test Base.binding_module(:c7648) == TestMod7648 + @test Base.function_name(foo7648) == :foo7648 + @test Base.function_module(foo7648, (Any,)) == TestMod7648 @test basename(functionloc(foo7648, (Any,))[1]) == "reflection.jl" @test first(methods(TestMod7648.TestModSub9475.foo7648)) == @which foo7648(5) - @test TestMod7648==@which foo7648 - @test TestMod7648.TestModSub9475==@which a9475 + @test TestMod7648 == @which foo7648 + @test TestMod7648.TestModSub9475 == @which a9475 end @test_throws ArgumentError which(is, Tuple{Int, Int}) @@ -471,6 +472,7 @@ end function has_backslashes(mod::Module) for n in names(mod, true, true) isdefined(mod, n) || continue + Base.isdeprecated(mod, n) && continue f = getfield(mod, n) if isa(f, Module) && module_depth(Main, f) <= module_depth(Main, mod) continue From 33ddb5dc1d2f0e0df089d864855a94a837e635b6 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 19 Jul 2016 14:34:37 -0400 Subject: [PATCH 0523/1117] fix ambiguity test now that names(all=true) returns all names, can fix the ambiguity test to check all MethodTables --- base/test.jl | 4 ++-- test/ambiguous.jl | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/base/test.jl b/base/test.jl index aa77ac6eb6f0f..3752ea63f5539 100644 --- a/base/test.jl +++ b/base/test.jl @@ -992,8 +992,8 @@ function detect_ambiguities(mods...; imported::Bool=false) continue end f = getfield(mod, n) - if isa(f, Function) - mt = methods(f) + if isa(f, DataType) && isdefined(f.name, :mt) + mt = Base.MethodList(f.name.mt) for m in mt if m.ambig !== nothing for m2 in m.ambig diff --git a/test/ambiguous.jl b/test/ambiguous.jl index abf0cd6e6d8a3..4a34f57957781 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -178,4 +178,11 @@ end @test isempty(detect_ambiguities(Ambig6)) +module Ambig7 +immutable T end +(::T)(x::Int8, y) = 1 +(::T)(x, y::Int8) = 2 +end +@test length(detect_ambiguities(Ambig7)) == 1 + nothing From ebba40dbecdebbae8b739cab19e0e86a1b9206ab Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 19 Jul 2016 14:36:21 -0400 Subject: [PATCH 0524/1117] reuse some datatype layouts eventually we may want to hash-cons all of these. in the meantime, can avoid copying the layout struct for common known types --- src/alloc.c | 13 +++++++-- src/dump.c | 69 ++++++++++++++++++++++++++++++++--------------- src/interpreter.c | 5 +++- 3 files changed, 63 insertions(+), 24 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index e3d7225da1237..46c840146a4fc 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -969,6 +969,10 @@ void jl_compute_field_offsets(jl_datatype_t *st) uint32_t nfields = jl_svec_len(st->types); jl_fielddesc32_t* desc = (jl_fielddesc32_t*) alloca(nfields * sizeof(jl_fielddesc32_t)); int haspadding = 0; + assert(st->name == jl_tuple_typename || + st == jl_sym_type || + st == jl_simplevector_type || + nfields != 0); for (size_t i = 0; i < nfields; i++) { jl_value_t *ty = jl_field_type(st, i); @@ -1092,8 +1096,13 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super } else { t->uid = jl_assign_type_uid(); - if (t->types != NULL && t->isleaftype) - jl_compute_field_offsets(t); + if (t->types != NULL && t->isleaftype) { + static const jl_datatype_layout_t singleton_layout = {0, 1, 0, 1, 0}; + if (fnames == jl_emptysvec) + t->layout = &singleton_layout; + else + jl_compute_field_offsets(t); + } } JL_GC_POP(); return t; diff --git a/src/dump.c b/src/dump.c index c6dd9176ea905..96a7ec73d8218 100644 --- a/src/dump.c +++ b/src/dump.c @@ -567,15 +567,29 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) write_int32(s, dt->uid); } } - if (has_layout) { - size_t nf = dt->layout->nfields; - write_uint16(s, nf); - write_int8(s, dt->layout->fielddesc_type); - write_int32(s, dt->layout->alignment); - write_int8(s, dt->layout->haspadding); - write_int8(s, dt->layout->pointerfree); - size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type); - ios_write(s, (char*)(&dt->layout[1]), nf * fieldsize); + + if (has_layout) { + uint8_t layout = 0; + if (dt->layout == jl_array_type->layout) { + layout = 1; + } + else if (dt->layout == jl_void_type->layout) { + layout = 2; + } + else if (dt->layout == jl_pointer_type->layout) { + layout = 3; + } + write_uint8(s, layout); + if (layout == 0) { + size_t nf = dt->layout->nfields; + write_uint16(s, nf); + write_int8(s, dt->layout->fielddesc_type); + write_int32(s, dt->layout->alignment); + write_int8(s, dt->layout->haspadding); + write_int8(s, dt->layout->pointerfree); + size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type); + ios_write(s, (char*)(&dt->layout[1]), nf * fieldsize); + } } if (has_instance) @@ -1201,18 +1215,31 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) } if (has_layout) { - uint16_t nf = read_uint16(s); - uint8_t fielddesc_type = read_int8(s); - size_t fielddesc_size = nf > 0 ? jl_fielddesc_size(fielddesc_type) : 0; - jl_datatype_layout_t *layout = (jl_datatype_layout_t*)jl_gc_perm_alloc( - sizeof(jl_datatype_layout_t) + nf * fielddesc_size); - layout->nfields = nf; - layout->fielddesc_type = fielddesc_type; - layout->alignment = read_int32(s); - layout->haspadding = read_int8(s); - layout->pointerfree = read_int8(s); - ios_read(s, (char*)&layout[1], nf * fielddesc_size); - dt->layout = layout; + uint8_t layout = read_uint8(s); + if (layout == 1) { + dt->layout = jl_array_type->layout; + } + else if (layout == 2) { + dt->layout = jl_void_type->layout; + } + else if (layout == 3) { + dt->layout = jl_pointer_type->layout; + } + else { + assert(layout == 0); + uint16_t nf = read_uint16(s); + uint8_t fielddesc_type = read_int8(s); + size_t fielddesc_size = nf > 0 ? jl_fielddesc_size(fielddesc_type) : 0; + jl_datatype_layout_t *layout = (jl_datatype_layout_t*)jl_gc_perm_alloc( + sizeof(jl_datatype_layout_t) + nf * fielddesc_size); + layout->nfields = nf; + layout->fielddesc_type = fielddesc_type; + layout->alignment = read_int32(s); + layout->haspadding = read_int8(s); + layout->pointerfree = read_int8(s); + ios_read(s, (char*)&layout[1], nf * fielddesc_size); + dt->layout = layout; + } } if (tag == 5) { diff --git a/src/interpreter.c b/src/interpreter.c index c348349962b9d..67da253f7e6e0 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -395,7 +395,10 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) b->value = temp; jl_rethrow(); } - jl_compute_field_offsets(dt); + if (dt->name->names == jl_emptysvec) + dt->layout = jl_void_type->layout; // reuse the same layout for all singletons + else if (jl_is_leaf_type((jl_value_t*)dt)) + jl_compute_field_offsets(dt); if (para == (jl_value_t*)jl_emptysvec && jl_is_datatype_make_singleton(dt)) { dt->instance = jl_gc_alloc(ptls, 0, dt); jl_gc_wb(dt, dt->instance); From 9d22de5fe7f5ceb0c9b329e4e35803a5767bee6d Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 19 Jul 2016 14:38:01 -0400 Subject: [PATCH 0525/1117] fix and improve typetree.jl example make it recursive over all modules, update deprecations, format output a bit better (removing duplicates better) --- examples/typetree.jl | 113 ++++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 55 deletions(-) diff --git a/examples/typetree.jl b/examples/typetree.jl index ebbaf52395c20..a86bc7c3e4aaa 100644 --- a/examples/typetree.jl +++ b/examples/typetree.jl @@ -5,84 +5,75 @@ module TypeTrees # Generate a text graphic of Julia modules type tree ## -# The node type holds the type of the cuurent node and a dict of subtypes -type TTNode - strname::AbstractString +immutable Binding + mod::Module + sym::Symbol +end +Binding(tn::TypeName) = Binding(tn.module, tn.name) +Base.isless(a::Binding, b::Binding) = isless(a.sym, b.sym) + +# The node type holds the type of the current node and a dict of subtypes +immutable TTNode typ::Type - subtypes::Dict{AbstractString, TTNode} + subtypes::Dict{Binding, TTNode} - TTNode(sname::AbstractString, t::Type) = new(sname, t, Dict{AbstractString, TTNode}()) + TTNode(t::ANY) = new(t, Dict{Binding, TTNode}()) end # Add a node to a dict if not added -function add_ttnode(subtypes::Dict{AbstractString, TTNode}, sname::AbstractString, tnode::TTNode) - ret = get(subtypes, sname, nothing) - (nothing == ret) && (ret = subtypes[sname] = tnode) - ret +function add_ttnode(subtypes::Dict{Binding, TTNode}, sname::Binding, tnode::TTNode) + return get!(subtypes, sname, tnode) end -function add_ttnode(subtypes::Dict{AbstractString, TTNode}, sname::AbstractString, t::Type) - ret = get(subtypes, sname, nothing) - (nothing == ret) && (subtypes[sname] = ret = TTNode(sname, t)) - ret +function add_ttnode(subtypes::Dict{Binding, TTNode}, sname::Binding, t::Type) + return get!(subtypes, sname, TTNode(t)) end -# Get a string name for the type -typ_name(t::Union) = string(t) -typ_name(t::TypeConstructor) = string(t) -typ_name(t) = string(t.name) - # Store a type and its type hierarchy chain # Recurse till we reach the top level type -function store_type(sname::AbstractString, t::Union) +function store_type(sname::Binding, t::Union) suptype = Union - tnode = TTNode(sname, t) + tnode = TTNode(t) # store unions under Union type - subtypes = store_type(typ_name(suptype), suptype) + subtypes = store_type(Binding(suptype.name), suptype) add_ttnode(subtypes, sname, tnode) # unions are also in a sense related to the types of their components for suptype = t.types - subtypes = store_type(typ_name(suptype), suptype) - add_ttnode(subtypes, sname, tnode) + if isa(suptype, DataType) # ignore TypeConstructors + subtypes = store_type(Binding(suptype.name), suptype) + add_ttnode(subtypes, sname, tnode) + end end return tnode.subtypes end -function store_type(sname::AbstractString, t::TypeConstructor) +function store_type(sname::Binding, t::TypeConstructor) suptype = t.body - subtypes = store_type(typ_name(suptype), suptype) + subtypes = store_type(isa(suptype, DataType) ? Binding(suptype.name) : Binding(Main, string(suptype::Union)), suptype) tnode = add_ttnode(subtypes, sname, t) return tnode.subtypes end -function store_type(sname::AbstractString, t::DataType) +function store_type(sname::Binding, t::DataType) suptype = supertype(t) - subtypes = (suptype != t) ? store_type(typ_name(suptype), suptype) : types_tree - tnode = add_ttnode(subtypes, sname, t) - return tnode.subtypes -end - -function store_type(sname::AbstractString, t::Tuple) - tnode = add_ttnode(types_tree, sname, t) - return tnode.subtypes -end - -function store_type(sname::AbstractString, t) - suptype = supertype(t) - subtypes = (suptype != t) ? store_type(typ_name(suptype), suptype) : types_tree + subtypes = (suptype != t) ? store_type(Binding(suptype.name), suptype) : types_tree tnode = add_ttnode(subtypes, sname, t) return tnode.subtypes end # examine all symbols in module and store those that are types function store_all_from(m::Module) - for expr = names(m,true) - if isdefined(m,expr) - t = getfield(m, expr) - isa(t, Type) && store_type(string(expr), t) + for s in names(m, true) + if isdefined(m, s) && !Base.isdeprecated(m, s) + t = getfield(m, s) + if isa(t, Type) + store_type(Binding(m, s), t) + elseif isa(t, Module) && module_name(t) === s && module_parent(t) === m && t !== m + store_all_from(t) + end end end end @@ -91,31 +82,43 @@ type_props(typ) = "" type_props(typ::DataType) = string("<<", typ.abstract ? " abstract" : " concrete", typ.mutable ? " mutable" : " immutable", - typ.pointerfree ? " pointerfree" : "", - " size:", typ.size, + typ.layout != C_NULL ? string( + Base.datatype_pointerfree(typ) ? " pointerfree" : "", + Base.datatype_haspadding(typ) ? " haspadding" : "", + " nfields:", Core.nfields(typ), + " size:", typ.size, + ", align:", Base.datatype_alignment(typ)) : "", " >>") -function print_tree(subtypes::Dict{AbstractString, TTNode}, pfx::AbstractString="") - for n in sort!([keys(subtypes)...]) - v = subtypes[n] - if(n == string(v.typ)) +function print_tree(subtypes::Dict{Binding, TTNode}, pfx::String="") + for b in sort!(collect(keys(subtypes))) + v = subtypes[b] + if b.mod === Main + n = string(b.sym) + elseif !isa(v.typ, DataType) || v.typ.name.module != b.mod || v.typ.name.name != b.sym + n = string(b.mod, '.', b.sym, (isa(v.typ, TypeConstructor) ? ("{", join(v.typ.parameters, ","), "}") : ())...) + else + n = string(v.typ) + end + ishidden = unsafe_load(Base.unsafe_convert(Ptr{UInt8}, b.sym)) == UInt8('#') + if ishidden && supertype(v.typ) === Function + continue + elseif n == string(v.typ) println(pfx, "+- ", n, " ", type_props(v.typ)) else - println(pfx, "+- ", n, " = ", v.typ, " ", type_props(v.typ)) + println(pfx, "+- ", n, " = ", v.typ, " ", type_props(v.typ)) end + v.typ === Function && println(pfx, ". ## hiding implicit Function subtypes ##") print_tree(v.subtypes, pfx * ". ") end end # TODO: optionally take module names in command line -# TODO: sort output # TODO: option to list subtrees of type tree, or other symbol types -const types_tree = Dict{AbstractString, TTNode}() +const types_tree = Dict{Binding, TTNode}() -for m in (Base, Core, Main) - store_all_from(m) -end +store_all_from(Main) # print_tree(types_tree) From d2c3cbc3388cf2216b84f6d5f62c45571e6876a2 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 19 Jul 2016 15:51:03 -0400 Subject: [PATCH 0526/1117] fix dumptypes this method was broken and extremely slow make it more correct and add tests --- base/show.jl | 72 ++++++++++++++++++++++++++-------------------------- test/show.jl | 26 ++++++++++++++++--- 2 files changed, 58 insertions(+), 40 deletions(-) diff --git a/base/show.jl b/base/show.jl index 51b04e057ae47..c584a50f5a6a0 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1158,12 +1158,13 @@ function dump(io::IO, x::DataType, n::Int, indent) print(io, " <: ", supertype(x)) end if !(x <: Tuple) + tvar_io = IOContext(io, :tvar_env => Any[x.parameters...]) fields = fieldnames(x) if n > 0 for idx in 1:length(fields) println(io) print(io, indent, " ", fields[idx], "::") - print(io, fieldtype(x,idx)) + print(tvar_io, fieldtype(x, idx)) end end end @@ -1171,50 +1172,49 @@ function dump(io::IO, x::DataType, n::Int, indent) nothing end -# dumptype is for displaying abstract type hierarchies like Jameson -# Nash's wiki page: https://github.com/JuliaLang/julia/wiki/Types-Hierarchy - +# dumptype is for displaying abstract type hierarchies, +# based on Jameson Nash's examples/typetree.jl function dumptype(io::IO, x::ANY, n::Int, indent) - # based on Jameson Nash's examples/typetree.jl println(io, x) - if n == 0 # too deeply nested - return - end - typargs(t) = split(string(t), "{")[1] - # todo: include current module? - for m in (Core, Base) - for s in fieldnames(m) - if isdefined(m,s) - t = eval(m,s) - if isa(t, TypeConstructor) - if string(x.name) == typargs(t) || - ("Union" == split(string(t), "(")[1] && - any(map(tt -> string(x.name) == typargs(tt), t.body.types))) - targs = join(t.parameters, ",") - println(io, indent, " ", s, - !isempty(t.parameters) ? "{$targs}" : "", - " = ", t) - end - elseif isa(t, Union) - if any(tt -> string(x.name) == typargs(tt), t.types) - println(io, indent, " ", s, " = ", t) - end - elseif isa(t, DataType) && supertype(t).name == x.name - # type aliases - if string(s) != string(t.name) - println(io, indent, " ", s, " = ", t.name) - elseif t != Any - print(io, indent, " ") - dump(io, t, n - 1, string(indent, " ")) - end + n == 0 && return # too deeply nested + isa(x, DataType) && x.abstract && dumpsubtypes(io, x, Main, n, indent) + nothing +end + +directsubtype(a::DataType, b::DataType) = supertype(a).name === b.name +directsubtype(a::TypeConstructor, b::DataType) = directsubtype(a.body, b) +directsubtype(a::Union, b::DataType) = any(t->directsubtype(t, b), a.types) +function dumpsubtypes(io::IO, x::DataType, m::Module, n::Int, indent) + for s in names(m, true) + if isdefined(m, s) && !isdeprecated(m, s) + t = getfield(m, s) + if t === x || t === m + continue + elseif isa(t, Module) && module_name(t) === s && module_parent(t) === m + # recurse into primary module bindings + dumpsubtypes(io, x, t, n, indent) + elseif isa(t, TypeConstructor) && directsubtype(t, x) + print(io, indent, " ", m, ".", s) + isempty(t.parameters) || print(io, "{", join(t.parameters, ","), "}") + println(io, " = ", t) + elseif isa(t, Union) && directsubtype(t, x) + println(io, indent, " ", m, ".", s, " = ", t) + elseif isa(t, DataType) && directsubtype(t, x) + if t.name.module !== m || t.name.name != s + # aliases to types + println(io, indent, " ", m, ".", s, " = ", t) + else + # primary type binding + print(io, indent, " ") + dumptype(io, t, n - 1, string(indent, " ")) end end end end - isempty(indent) && println(io) nothing end + # For abstract types, use _dumptype only if it's a form that will be called # interactively. dflt_io() = IOContext(STDOUT::IO, :limit => true) diff --git a/test/show.jl b/test/show.jl index 3eadf5e1e434c..5c34d8d898583 100644 --- a/test/show.jl +++ b/test/show.jl @@ -501,10 +501,28 @@ let s = IOBuffer(Array{UInt8}(0), true, true) @test String(resize!(s.data, s.size)) == " 1\n 2\n 3" end -# The `dump` function should always have a trailing newline -let io = IOBuffer() - dump(io, :(x = 1)) - @test takebuf_string(io)[end] == '\n' +let repr = sprint(dump, :(x = 1)) + @test repr == "Expr\n head: Symbol =\n args: Array{Any}((2,))\n 1: Symbol x\n 2: $Int 1\n typ: Any\n" +end +let repr = sprint(dump, Pair) + @test repr == "Pair{A,B} <: Any\n first::A\n second::B\n" +end +let repr = sprint(dump, Tuple) + @test repr == "Tuple <: Any\n" +end +let repr = sprint(dump, Int64) + @test repr == "Int64 <: Signed\n" +end +let repr = sprint(dump, Any) + @test length(repr) > 100000 + @test startswith(repr, "Any\n Base.") + @test endswith(repr, '\n') + @test contains(repr, " Base.Vector{T} = Array{T,1}\n") + @test !contains(repr, "Core.Vector{T}") +end +let repr = sprint(dump, Integer) + @test contains(repr, "UInt128") + @test !contains(repr, "Any") end # issue #17338 From b6d66607fcbe7ff00be5ed9b19ee8f7a74ec649f Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 19 Jul 2016 15:52:33 -0400 Subject: [PATCH 0527/1117] fix a numbers tests to be non-empty subtypes(Complex) is the empty set since 7f9325cf improved it --- test/numbers.jl | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/test/numbers.jl b/test/numbers.jl index cc14ff0156848..f318a7c2f8e38 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2476,14 +2476,34 @@ for x in [1.23, 7, e, 4//5] #[FP, Int, Irrational, Rat] @test isreal(x) == true end -#eltype{T<:Number}(::Type{T}) = T -for T in [subtypes(Complex); subtypes(Real)] - @test eltype(T) == T +function allsubtypes!(m::Module, x::DataType, sts::Set) + for s in names(m, true) + if isdefined(m, s) && !Base.isdeprecated(m, s) + t = getfield(m, s) + if isa(t, Type) && t <: x + push!(sts, t) + elseif isa(t, Module) && t !== m && module_name(t) === s && module_parent(t) === m + allsubtypes!(t, x, sts) + end + end + end end -#ndims{T<:Number}(::Type{T}) = 0 -for x in [subtypes(Complex); subtypes(Real)] - @test ndims(x) == 0 +let number_types = Set() + allsubtypes!(Base, Number, number_types) + allsubtypes!(Core, Number, number_types) + + @test !isempty(number_types) + + #eltype{T<:Number}(::Type{T}) = T + for T in number_types + @test eltype(T) == T + end + + #ndims{T<:Number}(::Type{T}) = 0 + for x in number_types + @test ndims(x) == 0 + end end #getindex(x::Number) = x From d4dd36a3bc667ae152c48906ec5f455292dd82ac Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 19 Jul 2016 16:33:47 -0400 Subject: [PATCH 0528/1117] improve testing coverage and implementation of dump --- base/show.jl | 37 +++++++++++++------------------------ test/show.jl | 34 +++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/base/show.jl b/base/show.jl index c584a50f5a6a0..8a872ff0a2bca 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1072,7 +1072,6 @@ function dump(io::IO, x::SimpleVector, n::Int, indent) end end end - isempty(indent) && println(io) nothing end @@ -1098,11 +1097,13 @@ function dump(io::IO, x::ANY, n::Int, indent) else !isa(x,Function) && print(io, " ", x) end - isempty(indent) && println(io) nothing end dump(io::IO, x::Module, n::Int, indent) = print(io, "Module ", x) +dump(io::IO, x::String, n::Int, indent) = (print(io, "String "); show(io, x)) +dump(io::IO, x::Symbol, n::Int, indent) = print(io, typeof(x), " ", x) +dump(io::IO, x::Union, n::Int, indent) = print(io, x) function dump_elts(io::IO, x::Array, n::Int, indent, i0, i1) for i in i0:i1 @@ -1136,22 +1137,10 @@ function dump(io::IO, x::Array, n::Int, indent) end end end - isempty(indent) && println(io) - nothing -end -function dump(io::IO, x::Symbol, n::Int, indent) - print(io, typeof(x), " ", x) - isempty(indent) && println(io) nothing end # Types -function dump(io::IO, x::Union, n::Int, indent) - print(io, x) - isempty(indent) && println(io) - nothing -end - function dump(io::IO, x::DataType, n::Int, indent) print(io, x) if x !== Any @@ -1168,14 +1157,13 @@ function dump(io::IO, x::DataType, n::Int, indent) end end end - isempty(indent) && println(io) nothing end # dumptype is for displaying abstract type hierarchies, # based on Jameson Nash's examples/typetree.jl function dumptype(io::IO, x::ANY, n::Int, indent) - println(io, x) + print(io, x) n == 0 && return # too deeply nested isa(x, DataType) && x.abstract && dumpsubtypes(io, x, Main, n, indent) nothing @@ -1194,15 +1182,18 @@ function dumpsubtypes(io::IO, x::DataType, m::Module, n::Int, indent) # recurse into primary module bindings dumpsubtypes(io, x, t, n, indent) elseif isa(t, TypeConstructor) && directsubtype(t, x) + println(io) print(io, indent, " ", m, ".", s) isempty(t.parameters) || print(io, "{", join(t.parameters, ","), "}") - println(io, " = ", t) + print(io, " = ", t) elseif isa(t, Union) && directsubtype(t, x) - println(io, indent, " ", m, ".", s, " = ", t) + println(io) + print(io, indent, " ", m, ".", s, " = ", t) elseif isa(t, DataType) && directsubtype(t, x) + println(io) if t.name.module !== m || t.name.name != s # aliases to types - println(io, indent, " ", m, ".", s, " = ", t) + print(io, indent, " ", m, ".", s, " = ", t) else # primary type binding print(io, indent, " ") @@ -1217,12 +1208,10 @@ end # For abstract types, use _dumptype only if it's a form that will be called # interactively. -dflt_io() = IOContext(STDOUT::IO, :limit => true) -dump(io::IO, x::DataType; maxdepth=8) = (x.abstract ? dumptype : dump)(io, x, maxdepth, "") -dump(x::DataType; maxdepth=8) = (x.abstract ? dumptype : dump)(dflt_io(), x, maxdepth, "") +dump(io::IO, x::DataType; maxdepth=8) = ((x.abstract ? dumptype : dump)(io, x, maxdepth, ""); println(io)) -dump(io::IO, arg; maxdepth=8) = dump(io, arg, maxdepth, "") -dump(arg; maxdepth=8) = dump(dflt_io(), arg, maxdepth, "") +dump(io::IO, arg; maxdepth=8) = (dump(io, arg, maxdepth, ""); println(io)) +dump(arg; maxdepth=8) = dump(IOContext(STDOUT::IO, :limit => true), arg; maxdepth=maxdepth) """ diff --git a/test/show.jl b/test/show.jl index 5c34d8d898583..bdd5eba4e9685 100644 --- a/test/show.jl +++ b/test/show.jl @@ -278,9 +278,6 @@ end let oldout = STDOUT, olderr = STDERR local rdout, wrout, rderr, wrerr, out, err, rd, wr try - rd, wr = redirect_stdout() - @test dump(STDERR) == nothing - # pr 16917 rdout, wrout = redirect_stdout() @test wrout === STDOUT @@ -288,10 +285,12 @@ let oldout = STDOUT, olderr = STDERR rderr, wrerr = redirect_stderr() @test wrerr === STDERR err = @async readstring(rderr) + @test dump(Int64) == nothing if !is_windows() close(wrout) close(wrerr) end + for io in (Core.STDOUT, Core.STDERR) Core.println(io, "TESTA") println(io, "TESTB") @@ -308,7 +307,7 @@ let oldout = STDOUT, olderr = STDERR redirect_stderr(olderr) close(wrout) close(wrerr) - @test wait(out) == "TESTA\nTESTB\nΑ1Β2\"A\"\nA\n123\"C\"\n" + @test wait(out) == "Int64 <: Signed\nTESTA\nTESTB\nΑ1Β2\"A\"\nA\n123\"C\"\n" @test wait(err) == "TESTA\nTESTB\nΑ1Β2\"A\"\n" finally redirect_stdout(oldout) @@ -496,7 +495,7 @@ end # PR 17117 # test show array -let s = IOBuffer(Array{UInt8}(0), true, true) +let s = IOBuffer(Array{UInt8}(0), true, true) Base.showarray(s, [1,2,3], false, header = false) @test String(resize!(s.data, s.size)) == " 1\n 2\n 3" end @@ -524,6 +523,31 @@ let repr = sprint(dump, Integer) @test contains(repr, "UInt128") @test !contains(repr, "Any") end +let repr = sprint(dump, Union{Integer, Float32}) + @test repr == "Union{Integer,Float32}\n" || repr == "Union{Float32,Integer}\n" +end +let repr = sprint(dump, Core.svec()) + @test repr == "empty SimpleVector\n" +end +let sv = Core.svec(:a, :b, :c) + # unsafe replacement of :c with #undef to test handling of incomplete SimpleVectors + unsafe_store!(convert(Ptr{Ptr{Void}}, Base.data_pointer_from_objref(sv)) + 3 * sizeof(Ptr), C_NULL) + repr = sprint(dump, sv) + @test repr == "SimpleVector\n 1: Symbol a\n 2: Symbol b\n 3: #undef\n" +end +let repr = sprint(dump, sin) + @test repr == "sin (function of type Base.#sin)\n" +end +let repr = sprint(dump, Base.Test) + @test repr == "Module Base.Test\n" +end +let a = Array{Any}(10000) + a[2] = "elemA" + a[4] = "elemB" + a[11] = "elemC" + repr = sprint(0, dump, a; env= (:limit => true)) + @test repr == "Array{Any}((10000,))\n 1: #undef\n 2: String \"elemA\"\n 3: #undef\n 4: String \"elemB\"\n 5: #undef\n ...\n 9996: #undef\n 9997: #undef\n 9998: #undef\n 9999: #undef\n 10000: #undef\n" +end # issue #17338 @test repr(Core.svec(1,2)) == "svec(1,2)" From 74f2bfbe1906071fced701e6483c35ae7c2c1d8f Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Tue, 19 Jul 2016 18:50:30 -0400 Subject: [PATCH 0529/1117] Remove EmptyCredentials and fix ssh-agent during Pkg.clone Also make setindex! fail if requested on a credential type that does not have the given field. --- base/libgit2/callbacks.jl | 11 ++++------- base/libgit2/types.jl | 24 ++++++++++-------------- test/libgit2.jl | 7 ------- 3 files changed, 14 insertions(+), 28 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index b24612d4dcd82..ce2f34529bb8b 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -63,7 +63,7 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, schema = schema === nothing ? "" : schema*"://" # get credentials object from payload pointer - creds = EmptyCredentials() + creds = nothing if payload_ptr != C_NULL tmpobj = unsafe_pointer_to_objref(payload_ptr) if isa(tmpobj, AbstractCredentials) @@ -74,15 +74,11 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, # use ssh key or ssh-agent if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) + creds == nothing && (creds = SSHCredentials()) credid = "ssh://$host" - # set ssh-agent trigger for first use - if creds[:usesshagent, credid] === nothing - creds[:usesshagent, credid] = "Y" - end - # first try ssh-agent if credentials support its usage - if creds[:usesshagent, credid] == "Y" + if creds[:usesshagent, credid] === nothing || creds[:usesshagent, credid] == "Y" err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring), cred, username_ptr) creds[:usesshagent, credid] = "U" # used ssh-agent only one time @@ -163,6 +159,7 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, end if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) + creds == nothing && (creds = UserPasswordCredentials()) credid = "$schema$host" username = creds[:user, credid] if username === nothing || isusedcreds diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 919c2866bc4f3..5a5de6d843fa3 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -56,23 +56,21 @@ abstract AbstractPayload "Abstract credentials payload" abstract AbstractCredentials <: AbstractPayload "Returns a credentials parameter" -function Base.getindex(p::AbstractCredentials, keys...) - for k in keys - ks = Symbol(k) - isdefined(p, ks) && return getfield(p, ks) - end +function Base.getindex(p::AbstractCredentials, key, _=nothing) + ks = Symbol(key) + isdefined(p, ks) && return getfield(p, ks) return nothing end "Sets credentials with `key` parameter to a value" -function Base.setindex!(p::AbstractCredentials, val, keys...) - for k in keys - ks = Symbol(k) - isdefined(p, ks) && setfield!(p, ks, val) - end +function Base.setindex!(p::AbstractCredentials, val, key, _=nothing) + ks = Symbol(key) + @assert isdefined(p, ks) + setfield!(p, ks, val) return p end "Checks if credentials were used" checkused!(p::AbstractCredentials) = true +checkused!(p::Void) = false "Resets credentials for another use" reset!(p::AbstractCredentials, cnt::Int=3) = nothing @@ -707,9 +705,6 @@ function getobjecttype(obj_type::Cint) end end -"Empty credentials" -type EmptyCredentials <: AbstractCredentials end - "Credentials that support only `user` and `password` parameters" type UserPasswordCredentials <: AbstractCredentials user::AbstractString @@ -768,7 +763,8 @@ function Base.setindex!(p::CachedCredentials, val, keys...) p.cred[host] = SSHCredentials() end creds = p.cred[host] - isdefined(creds,key) && setfield!(creds, key, val) + @assert isdefined(creds,key) + setfield!(creds, key, val) return p end "Checks if credentials were used or failed authentication, see `LibGit2.credentials_callback`" diff --git a/test/libgit2.jl b/test/libgit2.jl index 35032040ea099..f5752130ecda0 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -496,13 +496,6 @@ mktempdir() do dir #end #@testset "Credentials" begin - creds = LibGit2.EmptyCredentials() - @test LibGit2.checkused!(creds) - @test LibGit2.reset!(creds) === nothing - @test creds[:user] === nothing - @test creds[:pass] === nothing - @test creds[:pubkey, "localhost"] === nothing - creds_user = "USER" creds_pass = "PASS" creds = LibGit2.UserPasswordCredentials(creds_user, creds_pass) From 4e466ac1105ca1cf7fc7611412622fa0ec2047fb Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Tue, 19 Jul 2016 20:49:19 -0400 Subject: [PATCH 0530/1117] Use `musttail` only as an optimization in the PLT. Fix #17373 --- src/ccall.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 6b84e8898f338..54806368df0b4 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -194,13 +194,15 @@ static DenseMap<AttributeSet, std::map<std::tuple<GlobalVariable*,FunctionType*, CallingConv::ID>,GlobalVariable*>> allPltMap; -#ifdef LLVM37 // needed for musttail // Emit a "PLT" entry that will be lazily initialized // when being called the first time. static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, CallingConv::ID cc, const char *f_lib, const char *f_name) { assert(imaging_mode); + // Don't do this for vararg functions so that the `musttail` is only + // an optimization and is not required to function correctly. + assert(!functype->isVarArg()); GlobalVariable *libptrgv; GlobalVariable *llvmgv; void *symaddr; @@ -261,7 +263,12 @@ static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, builder.CreateUnreachable(); } else { + // musttail support is very bad on ARM, PPC, PPC64 (as of LLVM 3.9) + // Known failures includes vararg (not needed here) and sret. +#if defined(LLVM37) && (defined(_CPU_X86_) || defined(_CPU_X86_64_) || \ + defined(_CPU_AARCH64_)) ret->setTailCallKind(CallInst::TCK_MustTail); +#endif if (functype->getReturnType() == T_void) { builder.CreateRetVoid(); } @@ -287,7 +294,6 @@ static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, } return builder.CreateBitCast(builder.CreateLoad(got), funcptype); } -#endif // --- ABI Implementations --- // Partially based on the LDC ABI implementations licensed under the BSD 3-clause license @@ -1703,16 +1709,12 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) PointerType *funcptype = PointerType::get(functype,0); if (imaging_mode) { -#ifdef LLVM37 - // ARM, PPC, PPC64 (as of LLVM 3.9) don't support `musttail` for vararg functions. - // And musttail can't precede unreachable, but is required for vararg (https://llvm.org/bugs/show_bug.cgi?id=23766) + // vararg requires musttail, + // but musttail is incompatible with noreturn. if (functype->isVarArg()) llvmf = runtime_sym_lookup(funcptype, f_lib, f_name, ctx->f); else llvmf = emit_plt(functype, attrs, cc, f_lib, f_name); -#else - llvmf = runtime_sym_lookup(funcptype, f_lib, f_name, ctx->f); -#endif } else { void *symaddr = jl_dlsym_e(jl_get_library(f_lib), f_name); From 2c6bc1dd72065602137d0adad9bf44c3e021f784 Mon Sep 17 00:00:00 2001 From: Erik Schnetter <schnetter@gmail.com> Date: Tue, 19 Jul 2016 21:31:00 -0400 Subject: [PATCH 0531/1117] Add missing type conversion Closes #17501. --- src/gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gc.c b/src/gc.c index ffe93a799099c..9245c27b299d0 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1004,7 +1004,7 @@ static void gc_sweep_other(jl_ptls_t ptls, int sweep_full) static void gc_pool_sync_nfree(jl_gc_pagemeta_t *pg, jl_taggedvalue_t *last) { - assert(pg->fl_begin_offset != -1); + assert(pg->fl_begin_offset != (uint16_t)-1); char *cur_pg = gc_page_data(last); // Fast path for page that has no allocation jl_taggedvalue_t *fl_beg = (jl_taggedvalue_t*)(cur_pg + pg->fl_begin_offset); From 32d8fb5ba541cc6bdfaf921119d5f1768d9fc954 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Wed, 20 Jul 2016 06:17:28 -0500 Subject: [PATCH 0532/1117] De-purify IteratorsMD.flatten --- base/multidimensional.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 8b0aa1aa54bbd..8cf6e59fdb8f8 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -25,13 +25,13 @@ CartesianIndex{N}(index::NTuple{N,Integer}) = CartesianIndex{N}(index) (::Type{CartesianIndex{N}}){N}() = CartesianIndex{N}(()) # Un-nest passed CartesianIndexes CartesianIndex(index::Union{Integer, CartesianIndex}...) = CartesianIndex(flatten(index)) -Base.@pure flatten(I::Tuple{}) = I -Base.@pure flatten(I::Tuple{Any}) = I -Base.@pure flatten{N}(I::Tuple{CartesianIndex{N}}) = I[1].I -Base.@pure flatten(I) = _flatten(I...) -Base.@pure _flatten() = () -Base.@pure _flatten(i, I...) = (i, _flatten(I...)...) -Base.@pure _flatten(i::CartesianIndex, I...) = (i.I..., _flatten(I...)...) +flatten(I::Tuple{}) = I +flatten(I::Tuple{Any}) = I +flatten{N}(I::Tuple{CartesianIndex{N}}) = I[1].I +@inline flatten(I) = _flatten(I...) +@inline _flatten() = () +@inline _flatten(i, I...) = (i, _flatten(I...)...) +@inline _flatten(i::CartesianIndex, I...) = (i.I..., _flatten(I...)...) CartesianIndex(index::Tuple{Vararg{Union{Integer, CartesianIndex}}}) = CartesianIndex(index...) # length From aaa25cd8560a212322caf97e24b164e9f4ecb4ab Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 20 Jul 2016 08:38:13 -0500 Subject: [PATCH 0533/1117] Update exception line numbers for doctests --- base/docs/helpdb/Base.jl | 2 +- doc/manual/conversion-and-promotion.rst | 2 +- doc/manual/mathematical-operations.rst | 4 ++-- doc/manual/stacktraces.rst | 2 +- doc/manual/strings.rst | 8 ++++---- doc/stdlib/collections.rst | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 698602b5f1854..fce5280854b3b 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5449,7 +5449,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:511 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:536 ... ``` """ diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 6c64950d0c3ee..1ee291d4dddf7 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -142,7 +142,7 @@ to one and zero: julia> convert(Bool, 1im) ERROR: InexactError() - in convert(::Type{Bool}, ::Complex{Int64}) at ./complex.jl:18 + in convert(::Type{Bool}, ::Complex{Int64}) at ./complex.jl:23 ... julia> convert(Bool, 0im) diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index c08c8b115003d..c7925dea544e6 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -427,8 +427,8 @@ The following examples show the different forms. julia> round(Int8,127.6) ERROR: InexactError() - in trunc(::Type{Int8}, ::Float64) at ./float.jl:425 - in round(::Type{Int8}, ::Float64) at ./float.jl:180 + in trunc(::Type{Int8}, ::Float64) at ./float.jl:456 + in round(::Type{Int8}, ::Float64) at ./float.jl:211 ... See :ref:`man-conversion-and-promotion` for how to define your own diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index 3886cb0f8b0da..b86f0362ff1c4 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -36,7 +36,7 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example in eval_user_input(::Any, ::Bool) at client.jl:117 in eval(::Module, ::Any) at boot.jl:234 in eval_user_input(::Any, ::Bool) at client.jl:117 - in _start() at client.jl:359 + in _start() at client.jl:355 julia> @noinline child() = stacktrace() child (generic function with 1 method) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 5628c1d415e1d..4da392ec1edec 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -288,15 +288,15 @@ such an invalid byte index, an error is thrown: julia> s[2] ERROR: UnicodeError: invalid character index - in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:69 - in next at ./strings/string.jl:94 [inlined] + in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:67 + in next at ./strings/string.jl:92 [inlined] in getindex(::String, ::Int64) at ./strings/basic.jl:70 ... julia> s[3] ERROR: UnicodeError: invalid character index - in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:69 - in next at ./strings/string.jl:94 [inlined] + in slow_utf8_next(::Array{UInt8,1}, ::UInt8, ::Int64) at ./strings/string.jl:67 + in next at ./strings/string.jl:92 [inlined] in getindex(::String, ::Int64) at ./strings/basic.jl:70 ... diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 54482d0e4136c..2c896cdbac013 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1134,7 +1134,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:511 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:536 ... .. function:: splice!(collection, index, [replacement]) -> item From d323a3471574a5edba04e7377155fbeb7cece1e3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 20 Jul 2016 08:43:48 -0500 Subject: [PATCH 0534/1117] Update vectorize_1arg doctest --- doc/manual/arrays.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index f4244b1fad44f..ed6ea4bf864f2 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -513,13 +513,11 @@ the name of the function to vectorize. Here is a simple example: square (generic function with 1 method) julia> @vectorize_1arg Number square - square (generic function with 4 methods) + square (generic function with 2 methods) julia> methods(square) - # 4 methods for generic function "square": - square{T<:Number}(x::AbstractArray{T,1}) at operators.jl:... - square{T<:Number}(x::AbstractArray{T,2}) at operators.jl:... - square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:... + # 2 methods for generic function "square": + square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:540 square(x) at none:1 julia> square([1 2 4; 5 6 7]) From d69b2191510168f214c3bc0c66271e97b1ac471d Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 20 Jul 2016 08:44:07 -0500 Subject: [PATCH 0535/1117] Update structinfo doctest --- base/reflection.jl | 2 +- doc/stdlib/base.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index 1b2bcd37a7651..b1b944c8b71e9 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -122,7 +122,7 @@ use it in the following manner to summarize information about a struct type: julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:nfields(T)]; julia> structinfo(Base.Filesystem.StatStruct) -12-element Array{Tuple{UInt64,Union{Int64,Symbol},Type{_}},1}: +12-element Array{Tuple{UInt64,Symbol,DataType},1}: (0x0000000000000000,:device,UInt64) (0x0000000000000008,:inode,UInt64) (0x0000000000000010,:mode,UInt64) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index f961e60fa2270..56e6ebdb8fb4b 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -535,7 +535,7 @@ Types julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:nfields(T)]; julia> structinfo(Base.Filesystem.StatStruct) - 12-element Array{Tuple{UInt64,Union{Int64,Symbol},Type{_}},1}: + 12-element Array{Tuple{UInt64,Symbol,DataType},1}: (0x0000000000000000,:device,UInt64) (0x0000000000000008,:inode,UInt64) (0x0000000000000010,:mode,UInt64) From 9483e2fe82f6319d26cd6f7695c50589b7b8ca34 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 20 Jul 2016 14:31:37 -0500 Subject: [PATCH 0536/1117] Fix problematic doctest --- base/docs/helpdb/Base.jl | 2 +- doc/stdlib/collections.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index fce5280854b3b..3f73fbe8d0004 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5449,7 +5449,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:536 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:534 ... ``` """ diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 2c896cdbac013..4af354530908a 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1134,7 +1134,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:536 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:534 ... .. function:: splice!(collection, index, [replacement]) -> item From 5befc7859e8a56e693a76196e211855381f64efb Mon Sep 17 00:00:00 2001 From: Valentin Churavy <v.churavy@gmail.com> Date: Thu, 21 Jul 2016 04:45:18 +0900 Subject: [PATCH 0537/1117] Don't assume that /usr/bin/python2 is correct This assumption can be wrong on cluster enviroments. --- deps/llvm.mk | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 4f050e7f7e744..ac200ad6a7216 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -262,13 +262,14 @@ $(LLVM_SRC_DIR)/configure: $(LLVM_SRC_DIR)/tools/lldb endif # LLDB still relies on plenty of python 2.x infrastructure, without checking +llvm_python_location=$(shell /usr/bin/env python2 -c 'import sys; print(sys.executable)') llvm_python_workaround=$(SRCDIR)/srccache/python2_path $(llvm_python_workaround): mkdir -p $@ -python -c 'import sys; sys.exit(not sys.version_info > (3, 0))' && \ - /usr/bin/python2 -c 'import sys; sys.exit(not sys.version_info < (3, 0))' && \ - ln -sf /usr/bin/python2 "$@/python" && \ - ln -sf /usr/bin/python2-config "$@/python-config" + /usr/bin/env python2 -c 'import sys; sys.exit(not sys.version_info < (3, 0))' && \ + ln -sf $(llvm_python_location) "$@/python" && \ + ln -sf $(llvm_python_location)-config "$@/python-config" LLVM_FLAGS += --with-python="$(shell $(SRCDIR)/tools/find_python2)" ifeq ($(BUILD_CUSTOM_LIBCXX),1) From d61ec114c63a47114f88b6f3df47f42cab2f53e9 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Wed, 20 Jul 2016 12:46:23 -0700 Subject: [PATCH 0538/1117] update llvm to 3.8.1 on power. --- Make.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index 58ae3d09657f4..7d7d4a0b10a3c 100644 --- a/Make.inc +++ b/Make.inc @@ -646,7 +646,7 @@ endif ifneq (,$(filter $(ARCH), powerpc64 ppc64le)) JCFLAGS += -fsigned-char -override LLVM_VER:=3.8.0 +override LLVM_VER:=3.8.1 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=POWER8 From 9fe72479caac4745313559405ca2808a7766c503 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Wed, 20 Jul 2016 14:46:56 -0400 Subject: [PATCH 0539/1117] Make the FDE processing a little more robust * Do not load offset and stop processing when a terminator is found. * Add assertion to check the length is valid. --- src/debuginfo.cpp | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 4901eabb98bf5..454c53e738747 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -1356,29 +1356,26 @@ extern "C" jl_lambda_info_t *jl_gdblookuplinfo(void *p) extern "C" void __register_frame(void*); extern "C" void __deregister_frame(void*); -template <typename callback> -static const char *processFDE(const char *Entry, callback f) -{ - const char *P = Entry; - uint32_t Length = *((const uint32_t *)P); - P += 4; - uint32_t Offset = *((const uint32_t *)P); - // Offset == 0: CIE - // Length == 0: Terminator - if (Offset != 0 && Length != 0) { - f(Entry); - } - return P + Length; -} - template <typename callback> static void processFDEs(const char *EHFrameAddr, size_t EHFrameSize, callback f) { - const char *P = (const char*)EHFrameAddr; + const char *P = EHFrameAddr; const char *End = P + EHFrameSize; - do { - P = processFDE(P, f); - } while(P != End); + do { + const char *Entry = P; + P += 4; + assert(P <= End); + uint32_t Length = *(const uint32_t*)Entry; + // Length == 0: Terminator + if (Length == 0) + break; + assert(P + Length <= End); + uint32_t Offset = *(const uint32_t*)P; + // Offset == 0: CIE + if (Offset != 0) + f(Entry); + P += Length; + } while (P != End); } #endif From 460f96c9ae3d9a7e5eec9a67c587fefd553e0b2f Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 20 Jul 2016 16:01:02 -0400 Subject: [PATCH 0540/1117] document `iteratorsize` and `iteratoreltype` fixes #15977 --- base/generator.jl | 31 +++++++++++++++++++++++++++++++ doc/manual/interfaces.rst | 22 ++++++++++++---------- doc/stdlib/collections.rst | 28 ++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/base/generator.jl b/base/generator.jl index 75625c2c40525..e598f9318c29f 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -35,6 +35,23 @@ immutable HasLength <: IteratorSize end immutable HasShape <: IteratorSize end immutable IsInfinite <: IteratorSize end +""" + iteratorsize(itertype::Type) -> IteratorSize + +Given the type of an iterator, returns one of the following values: + +* `SizeUnknown()` if the length (number of elements) cannot be determined in advance. +* `HasLength()` if there is a fixed, finite length. +* `HasShape()` if there is a known length plus a notion of multidimensional shape (as for an array). + In this case the `size` function is valid for the iterator. +* `IsInfinite()` if the iterator yields values forever. + +The default value (for iterators that do not define this function) is `HasLength()`. +This means that most iterators are assumed to implement `length`. + +This trait is generally used to select between algorithms that pre-allocate space for their +result, and algorithms that resize their result incrementally. +""" iteratorsize(x) = iteratorsize(typeof(x)) iteratorsize(::Type) = HasLength() # HasLength is the default @@ -47,6 +64,20 @@ abstract IteratorEltype immutable EltypeUnknown <: IteratorEltype end immutable HasEltype <: IteratorEltype end +""" + iteratoreltype(itertype::Type) -> IteratorEltype + +Given the type of an iterator, returns one of the following values: + +* `EltypeUnknown()` if the type of elements yielded by the iterator is not known in advance. +* `HasEltype()` if the element type is known, and `eltype` would return a meaningful value. + +`HasEltype()` is the default, since iterators are assumed to implement `eltype`. + +This trait is generally used to select between algorithms that pre-allocate a specific +type of result, and algorithms that pick a result type based on the types of yielded +values. +""" iteratoreltype(x) = iteratoreltype(typeof(x)) iteratoreltype(::Type) = HasEltype() # HasEltype is the default diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 15465047b2402..92af7faf5f5ea 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -11,16 +11,18 @@ A lot of the power and extensibility in Julia comes from a collection of informa Iteration --------- -================================= ======================== =========================================== -Required methods Brief description -================================= ======================== =========================================== -:func:`start(iter) <start>` Returns the initial iteration state -:func:`next(iter, state) <next>` Returns the current item and the next state -:func:`done(iter, state) <done>` Tests if there are any items remaining -**Important optional methods** **Default definition** **Brief description** -:func:`eltype(IterType) <eltype>` ``Any`` The type the items returned by :func:`next` -:func:`length(iter) <length>` (*undefined*) The number of items, if known -================================= ======================== =========================================== +================================================== ======================== =========================================== +Required methods Brief description +================================================== ======================== =========================================== +:func:`start(iter) <start>` Returns the initial iteration state +:func:`next(iter, state) <next>` Returns the current item and the next state +:func:`done(iter, state) <done>` Tests if there are any items remaining +**Important optional methods** **Default definition** **Brief description** +:func:`eltype(IterType) <eltype>` ``Any`` The type the items returned by :func:`next` +:func:`length(iter) <length>` (*undefined*) The number of items, if known +:func:`iteratorsize(IterType) <iteratorsize>` ``HasLength()`` +:func:`iteratoreltype(IterType) <iteratoreltype>` ``HasEltype()`` +================================================== ======================== =========================================== Sequential iteration is implemented by the methods :func:`start`, :func:`done`, and :func:`next`. Instead of mutating objects as they are iterated over, Julia provides these three methods to keep track of the iteration state externally from the object. The :func:`start(iter) <start>` method returns the initial state for the iterable object ``iter``. That state gets passed along to :func:`done(iter, state) <done>`, which tests if there are any elements remaining, and :func:`next(iter, state) <next>`, which returns a tuple containing the current element and an updated ``state``. The ``state`` object can be anything, and is generally considered to be an implementation detail private to the iterable object. diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 4af354530908a..23c9aafca8b65 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -108,6 +108,34 @@ type. An iterator that generates the value ``x`` forever. If ``n`` is specified, generates ``x`` that many times (equivalent to ``take(repeated(x), n)``\ ). +.. function:: iteratorsize(itertype::Type) -> IteratorSize + + .. Docstring generated from Julia source + + Given the type of an iterator, returns one of the following values: + + * ``SizeUnknown()`` if the length (number of elements) cannot be determined in advance. + * ``HasLength()`` if there is a fixed, finite length. + * ``HasShape()`` if there is a known length plus a notion of multidimensional shape (as for an array). In this case the ``size`` function is valid for the iterator. + * ``IsInfinite()`` if the iterator yields values forever. + + The default value (for iterators that do not define this function) is ``HasLength()``\ . This means that most iterators are assumed to implement ``length``\ . + + This trait is generally used to select between algorithms that pre-allocate space for their result, and algorithms that resize their result incrementally. + +.. function:: iteratoreltype(itertype::Type) -> IteratorEltype + + .. Docstring generated from Julia source + + Given the type of an iterator, returns one of the following values: + + * ``EltypeUnknown()`` if the type of elements yielded by the iterator is not known in advance. + * ``HasEltype()`` if the element type is known, and ``eltype`` would return a meaningful value. + + ``HasEltype()`` is the default, since iterators are assumed to implement ``eltype``\ . + + This trait is generally used to select between algorithms that pre-allocate a specific type of result, and algorithms that pick a result type based on the types of yielded values. + Fully implemented by: - :obj:`Range` From b9bf6e185c938d51d5299077a6ec224cff68d06e Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Wed, 20 Jul 2016 14:14:06 -0700 Subject: [PATCH 0541/1117] Augment convert's documentation with a brief section noting that convert's result may alias part or all of convert's second argument when that argument is a collection or composite. --- base/docs/helpdb/Base.jl | 32 ++++++++++++++++++++++++++++++++ doc/stdlib/base.rst | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 3f73fbe8d0004..7f9125fb3e03d 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -8129,6 +8129,38 @@ julia> convert(Rational{Int32}, x) julia> convert(Rational{Int64}, x) 6004799503160661//18014398509481984 ``` + +If `T` is a collection type and `x` a collection, the result of `convert(T, x)` may alias +`x`. +```jldoctest +julia> x = Int[1,2,3]; + +julia> y = convert(Vector{Int}, x); + +julia> y === x +true +``` +Similarly, if `T` is a composite type and `x` a related instance, the result of +`convert(T, x)` may alias part or all of `x`. +```jldoctest +julia> x = speye(5); + +julia> typeof(x) +SparseMatrixCSC{Float64,Int64} + +julia> y = convert(SparseMatrixCSC{Float64,Int64}, x); + +julia> z = convert(SparseMatrixCSC{Float32,Int64}, y); + +julia> y === x +true + +julia> z === x +false + +julia> z.colptr === x.colptr +true +``` """ convert diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 56e6ebdb8fb4b..bb8b0580aa7a0 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -387,6 +387,39 @@ All Objects julia> convert(Rational{Int64}, x) 6004799503160661//18014398509481984 + If ``T`` is a collection type and ``x`` a collection, the result of ``convert(T, x)`` may alias ``x``\ . + + .. doctest:: + + julia> x = Int[1,2,3]; + + julia> y = convert(Vector{Int}, x); + + julia> y === x + true + + Similarly, if ``T`` is a composite type and ``x`` a related instance, the result of ``convert(T, x)`` may alias part or all of ``x``\ . + + .. doctest:: + + julia> x = speye(5); + + julia> typeof(x) + SparseMatrixCSC{Float64,Int64} + + julia> y = convert(SparseMatrixCSC{Float64,Int64}, x); + + julia> z = convert(SparseMatrixCSC{Float32,Int64}, y); + + julia> y === x + true + + julia> z === x + false + + julia> z.colptr === x.colptr + true + .. function:: promote(xs...) .. Docstring generated from Julia source From 3eef68b2e7253234a9717dc6d13e88bf36fc43d0 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Wed, 20 Jul 2016 22:13:43 +0000 Subject: [PATCH 0542/1117] Update LLVM to 3.8.1 for arm --- Make.inc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Make.inc b/Make.inc index 7d7d4a0b10a3c..2264fea33433c 100644 --- a/Make.inc +++ b/Make.inc @@ -645,7 +645,6 @@ endif # If we are running on powerpc64 or ppc64, set certain options automatically ifneq (,$(filter $(ARCH), powerpc64 ppc64le)) JCFLAGS += -fsigned-char - override LLVM_VER:=3.8.1 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=POWER8 @@ -656,9 +655,7 @@ endif ifneq (,$(findstring arm,$(ARCH))) JCFLAGS += -fsigned-char -LLVM_ASSERTIONS=1 -#LLVM_FLAGS+="--with-float=hard --with-abi=aapcs-vfp" - +override LLVM_VER:=3.8.1 override USE_BLAS64:=0 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=ARMV7 From 7a9bc35d84d21aedaeb7d69835a79c25fed789a9 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 19:20:40 -0400 Subject: [PATCH 0543/1117] make new dumptypes test more robust --- test/show.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/show.jl b/test/show.jl index bdd5eba4e9685..36494817c9fe2 100644 --- a/test/show.jl +++ b/test/show.jl @@ -514,7 +514,7 @@ let repr = sprint(dump, Int64) end let repr = sprint(dump, Any) @test length(repr) > 100000 - @test startswith(repr, "Any\n Base.") + @test startswith(repr, "Any\n Main.") || startswith(repr, "Any\n Base.") @test endswith(repr, '\n') @test contains(repr, " Base.Vector{T} = Array{T,1}\n") @test !contains(repr, "Core.Vector{T}") From 7e2a99e069914459637ae99629acf42351b67926 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Wed, 20 Jul 2016 20:02:46 -0400 Subject: [PATCH 0544/1117] Make failure to connect to ssh-agent non-fatal --- base/libgit2/callbacks.jl | 8 ++++---- base/libgit2/libgit2.jl | 6 +++--- deps/libgit2.mk | 6 +++++- deps/patches/libgit2-agent-nonfatal.patch | 21 +++++++++++++++++++++ 4 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 deps/patches/libgit2-agent-nonfatal.patch diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index ce2f34529bb8b..aa102546d067f 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -114,8 +114,8 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, if keydefpath !== nothing && !isusedcreds keydefpath # use cached value else - keydefpath = if keydefpath === nothing - joinpath(homedir(),".ssh","id_rsa") + if isempty(keydefpath) || keydefpath === nothing + keydefpath = joinpath(homedir(),".ssh","id_rsa") end prompt("Private key location for '$schema$username@$host'", default=keydefpath) end @@ -130,8 +130,8 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, if keydefpath !== nothing && !isusedcreds keydefpath # use cached value else - keydefpath = if keydefpath === nothing - privatekey*".pub" + if isempty(keydefpath) || keydefpath === nothing + keydefpath = privatekey*".pub" end if isfile(keydefpath) keydefpath diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index 5294cb8fb9b74..48e76a89e43d7 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -148,7 +148,7 @@ function fetch{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; remote::AbstractString="origin", remoteurl::AbstractString="", refspecs::Vector{T}=AbstractString[], - payload::Nullable{P}=Nullable{AbstractPayload}()) + payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else @@ -168,7 +168,7 @@ function push{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; remoteurl::AbstractString="", refspecs::Vector{T}=AbstractString[], force::Bool=false, - payload::Nullable{P}=Nullable{AbstractPayload}()) + payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else @@ -295,7 +295,7 @@ function clone{P<:AbstractPayload}(repo_url::AbstractString, repo_path::Abstract branch::AbstractString="", isbare::Bool = false, remote_cb::Ptr{Void} = C_NULL, - payload::Nullable{P}=Nullable{AbstractPayload}()) + payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) # setup clone options lbranch = Base.cconvert(Cstring, branch) fetch_opts=FetchOptions(callbacks = RemoteCallbacks(credentials_cb(), payload)) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 75d27f195a376..17e4517f36a89 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -33,10 +33,14 @@ $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied: $(S cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch echo 1 > $@ +$(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied | $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-agent-nonfatal.patch + echo 1 > $@ + ifeq ($(OS),Linux) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied endif -$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(LIBSSH2_OBJ_TARGET) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied $(LIBSSH2_OBJ_TARGET) mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) diff --git a/deps/patches/libgit2-agent-nonfatal.patch b/deps/patches/libgit2-agent-nonfatal.patch new file mode 100644 index 0000000000000..3fab1d4c0cee9 --- /dev/null +++ b/deps/patches/libgit2-agent-nonfatal.patch @@ -0,0 +1,21 @@ +commit 70020247d1903c7a1262d967cf205a44dc6f6ebe +Author: Keno Fischer <kfischer@college.harvard.edu> +Date: Wed Jul 20 19:59:00 2016 -0400 + + Make failure to connect to ssh-agent non-fatal + +diff --git a/src/transports/ssh.c b/src/transports/ssh.c +index cfd5736..82d2c63 100644 +--- a/src/transports/ssh.c ++++ b/src/transports/ssh.c +@@ -296,8 +296,10 @@ static int ssh_agent_auth(LIBSSH2_SESSION *session, git_cred_ssh_key *c) { + + rc = libssh2_agent_connect(agent); + +- if (rc != LIBSSH2_ERROR_NONE) ++ if (rc != LIBSSH2_ERROR_NONE) { ++ rc = LIBSSH2_ERROR_AUTHENTICATION_FAILED; + goto shutdown; ++ } + + rc = libssh2_agent_list_identities(agent); From 2c379c85e6613ac20a47a2873d37e6b35c88cd13 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 13:47:16 -0400 Subject: [PATCH 0545/1117] ensure datatype alignments are powers-of-two fix #16336 --- src/alloc.c | 4 +++- src/runtime_intrinsics.c | 13 ------------- src/support/dtypes.h | 13 +++++++++++++ src/support/hashing.c | 13 ------------- src/support/htable.c | 4 ++-- src/typemap.c | 13 ------------- 6 files changed, 18 insertions(+), 42 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 46c840146a4fc..4ce6635352b1f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -1114,7 +1114,9 @@ JL_DLLEXPORT jl_datatype_t *jl_new_bitstype(jl_value_t *name, jl_datatype_t *sup jl_datatype_t *bt = jl_new_datatype((jl_sym_t*)name, super, parameters, jl_emptysvec, jl_emptysvec, 0, 0, 0); uint32_t nbytes = (nbits + 7) / 8; - uint32_t alignm = nbytes > MAX_ALIGN ? MAX_ALIGN : nbytes; + uint32_t alignm = next_power_of_two(nbytes); + if (alignm > MAX_ALIGN) + alignm = MAX_ALIGN; bt->size = nbytes; bt->layout = jl_get_layout(0, alignm, 0, NULL); return bt; diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index cebd18b9c1230..6ea3a9d7dfcc2 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -76,19 +76,6 @@ JL_DLLEXPORT jl_value_t *jl_pointerset(jl_value_t *p, jl_value_t *x, jl_value_t return p; } -static inline unsigned int next_power_of_two(unsigned int val) -{ - /* this function taken from libuv src/unix/core.c */ - val -= 1; - val |= val >> 1; - val |= val >> 2; - val |= val >> 4; - val |= val >> 8; - val |= val >> 16; - val += 1; - return val; -} - static inline char signbitbyte(void *a, unsigned bytes) { // sign bit of an signed number of n bytes, as a byte diff --git a/src/support/dtypes.h b/src/support/dtypes.h index 67584a204f681..56b5ddfb93612 100644 --- a/src/support/dtypes.h +++ b/src/support/dtypes.h @@ -147,6 +147,19 @@ typedef uint32_t uint_t; typedef int32_t int_t; #endif +STATIC_INLINE unsigned int next_power_of_two(unsigned int val) +{ + /* this function taken from libuv src/unix/core.c */ + val -= 1; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + val += 1; + return val; +} + #define LLT_ALIGN(x, sz) (((x) + (sz)-1) & -(sz)) // branch prediction annotations diff --git a/src/support/hashing.c b/src/support/hashing.c index 7a48b710c3ad8..da76f3a793867 100644 --- a/src/support/hashing.c +++ b/src/support/hashing.c @@ -14,19 +14,6 @@ extern "C" { #endif -uint_t nextipow2(uint_t i) -{ - if (i==0) return 1; - if ((i&(i-1))==0) return i; - if (i&TOP_BIT) return TOP_BIT; - - // repeatedly clear bottom bit - while (i&(i-1)) - i = i&(i-1); - - return i<<1; -} - uint32_t int32hash(uint32_t a) { a = (a+0x7ed55d16) + (a<<12); diff --git a/src/support/htable.c b/src/support/htable.c index 0d084c5604561..3e1c9022ca6c7 100644 --- a/src/support/htable.c +++ b/src/support/htable.c @@ -25,7 +25,7 @@ htable_t *htable_new(htable_t *h, size_t size) h->table = &h->_space[0]; } else { - size = nextipow2(size); + size = next_power_of_two(size); size *= 2; // 2 pointers per key/value pair size *= 2; // aim for 50% occupancy h->size = size; @@ -47,7 +47,7 @@ void htable_free(htable_t *h) // empty and reduce size void htable_reset(htable_t *h, size_t sz) { - sz = nextipow2(sz); + sz = next_power_of_two(sz); if (h->size > sz*4 && h->size > HT_N_INLINE) { size_t newsz = sz*4; void **newtab = (void**)LLT_REALLOC(h->table, newsz*sizeof(void*)); diff --git a/src/typemap.c b/src/typemap.c index b47d8a00b1347..06a4b2627caf3 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -264,19 +264,6 @@ union jl_typemap_t mtcache_hash_lookup(const struct jl_ordereddict_t *a, jl_valu return ml; } -static inline unsigned int next_power_of_two(unsigned int val) -{ - /* this function taken from libuv src/unix/core.c */ - val -= 1; - val |= val >> 1; - val |= val >> 2; - val |= val >> 4; - val |= val >> 8; - val |= val >> 16; - val += 1; - return val; -} - static void mtcache_rehash(struct jl_ordereddict_t *pa, size_t newlen, jl_value_t *parent, int8_t tparam, int8_t offs) { size_t i, nval = jl_array_len(pa->values); From e088fa54c176d70264e1ad3a4018a07180786f04 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 13:58:16 -0400 Subject: [PATCH 0546/1117] handle errors in calling code_reflections methods on non-generic functions fix #15606 --- base/reflection.jl | 9 +++++++++ src/codegen.cpp | 7 +++++++ test/reflection.jl | 4 ++++ 3 files changed, 20 insertions(+) diff --git a/base/reflection.jl b/base/reflection.jl index b1b944c8b71e9..f311a3b109a01 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -325,6 +325,9 @@ uncompressed_ast(l::LambdaInfo) = # Printing code representations in IR and assembly function _dump_function(f, t::ANY, native, wrapper, strip_ir_metadata, dump_module) ccall(:jl_is_in_pure_context, Bool, ()) && error("native reflection cannot be used from generated functions") + if isa(f, Core.Builtin) + throw(ArgumentError("argument is not a generic function")) + end t = tt_cons(Core.Typeof(f), to_tuple_type(t)) llvmf = ccall(:jl_get_llvmf, Ptr{Void}, (Any, Bool, Bool), t, wrapper, native) @@ -363,6 +366,9 @@ end function code_typed(f::ANY, types::ANY=Tuple; optimize=true) ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions") + if isa(f, Core.Builtin) + throw(ArgumentError("argument is not a generic function")) + end types = to_tuple_type(types) asts = [] for x in _methods(f,types,-1) @@ -380,6 +386,9 @@ end function return_types(f::ANY, types::ANY=Tuple) ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions") + if isa(f, Core.Builtin) + throw(ArgumentError("argument is not a generic function")) + end types = to_tuple_type(types) rt = [] for x in _methods(f,types,-1) diff --git a/src/codegen.cpp b/src/codegen.cpp index a794ab72b7a7a..b08ba94e9fd99 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1058,9 +1058,16 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) } } if (linfo == NULL) { + // no function found for argument tuple type JL_GC_POP(); return NULL; } + if (linfo->def->lambda_template->code == jl_nothing) { + // not a generic function + JL_GC_POP(); + return NULL; + } + // make sure to compile this normally first, // since `emit_function` doesn't handle recursive compilation correctly linfo = jl_compile_for_dispatch(linfo); diff --git a/test/reflection.jl b/test/reflection.jl index 00dc08b366a21..13f2e004435b3 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -218,6 +218,10 @@ let end @test_throws ArgumentError which(is, Tuple{Int, Int}) +@test_throws ArgumentError code_typed(is, Tuple{Int, Int}) +@test_throws ArgumentError code_llvm(is, Tuple{Int, Int}) +@test_throws ArgumentError code_native(is, Tuple{Int, Int}) +@test_throws ArgumentError Base.return_types(is, Tuple{Int, Int}) module TestingExported using Base.Test From 3b5f43e8404f1bc679b5336c1cb286a629f12b12 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 14:12:16 -0400 Subject: [PATCH 0547/1117] fix repr of binary operators called with ... fix #15387 --- base/show.jl | 2 +- test/show.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/base/show.jl b/base/show.jl index 8a872ff0a2bca..a1f532d1f1eb7 100644 --- a/base/show.jl +++ b/base/show.jl @@ -742,7 +742,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) # binary operator (i.e. "x + y") elseif func_prec > 0 # is a binary operator na = length(func_args) - if na == 2 || (na > 2 && func in (:+, :++, :*)) + if (na == 2 || (na > 2 && func in (:+, :++, :*))) && all(!isa(a, Expr) || a.head !== :... for a in func_args) sep = " $func " if func_prec <= prec show_enclosed_list(io, '(', func_args, sep, ')', indent, func_prec, true) diff --git a/test/show.jl b/test/show.jl index 36494817c9fe2..8966b75683c8e 100644 --- a/test/show.jl +++ b/test/show.jl @@ -80,6 +80,8 @@ end @test_repr "a & (b && c)" @test_repr "(a => b) in c" @test_repr "a => b in c" +@test_repr "*(a..., b)" +@test_repr "+(a, b, c...)" # precedence tie resolution @test_repr "(a * b) * (c * d)" From 94c3e686d7fb8b83a314a98a783edad3ea035022 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 15:30:48 -0400 Subject: [PATCH 0548/1117] make printing random LambdaInfo objects less segfault-y --- base/methodshow.jl | 11 ++++++----- base/replutil.jl | 17 +++++++++++++++++ base/show.jl | 45 +++++++++++++++------------------------------ base/stacktraces.jl | 21 +++++++++------------ src/alloc.c | 9 +++++---- src/gf.c | 13 ++++++++++--- src/jltypes.c | 2 +- test/reflection.jl | 16 +++++++++++++--- test/stacktraces.jl | 11 +++++++++++ 9 files changed, 87 insertions(+), 58 deletions(-) diff --git a/base/methodshow.jl b/base/methodshow.jl index fa03de7d232d1..79737b4f596f4 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -28,8 +28,6 @@ function argtype_decl(env, n, sig, i, nargs, isva) # -> (argname, argtype) end end return s, string_with_env(env, "Vararg{", tt, ",", tn, "}") - elseif t == String - return s, "String" end return s, string_with_env(env, t) end @@ -43,7 +41,7 @@ function arg_decl_parts(m::Method) end li = m.lambda_template file, line = "", 0 - if li !== nothing + if li !== nothing && isdefined(li, :slotnames) argnames = li.slotnames[1:li.nargs] decls = Any[argtype_decl(:tvar_env => tv, argnames[i], m.sig, i, li.nargs, li.isva) for i = 1:li.nargs] @@ -51,7 +49,7 @@ function arg_decl_parts(m::Method) file, line = li.def.file, li.def.line end else - decls = Any["" for i = 1:length(m.sig.parameters)] + decls = Any[("", "") for i = 1:length(m.sig.parameters)] end return tv, decls, file, line end @@ -76,7 +74,10 @@ function show(io::IO, m::Method; kwtype::Nullable{DataType}=Nullable{DataType}() tv, decls, file, line = arg_decl_parts(m) ft = m.sig.parameters[1] d1 = decls[1] - if ft <: Function && + if m.sig === Tuple + print(io, m.name) + decls = Any[(), ("...", "")] + elseif ft <: Function && isdefined(ft.name.module, ft.name.mt.name) && ft == typeof(getfield(ft.name.module, ft.name.mt.name)) print(io, ft.name.mt.name) diff --git a/base/replutil.jl b/base/replutil.jl index 478b8cfe4c3e8..14282e8128343 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -113,6 +113,23 @@ function show(io::IO, ::MIME"text/plain", f::Function) end end +function show(io::IO, ::MIME"text/plain", l::LambdaInfo) + show(io, l) + # Fix slot names and types in function body + ast = uncompressed_ast(l) + if ast !== nothing + println(io) + lambda_io = IOContext(io, :LAMBDAINFO => l) + if isdefined(l, :slotnames) + lambda_io = IOContext(lambda_io, :LAMBDA_SLOTNAMES => lambdainfo_slotnames(l)) + end + body = Expr(:body) + body.args = ast + body.typ = l.rettype + show(lambda_io, body) + end +end + function show(io::IO, ::MIME"text/plain", r::LinSpace) # show for linspace, e.g. # linspace(1,3,7) diff --git a/base/show.jl b/base/show.jl index a1f532d1f1eb7..d85312643efbc 100644 --- a/base/show.jl +++ b/base/show.jl @@ -232,6 +232,7 @@ function show(io::IO, p::Pair) isa(p.second,Pair) && print(io, "(") show(io, p.second) isa(p.second,Pair) && print(io, ")") + nothing end function show(io::IO, m::Module) @@ -259,30 +260,21 @@ function lambdainfo_slotnames(l::LambdaInfo) end printnames[i] = printname end - printnames + return printnames end function show(io::IO, l::LambdaInfo) if isdefined(l, :def) - if (l === l.def.lambda_template) + if l === l.def.lambda_template print(io, "LambdaInfo template for ") show(io, l.def) - println(io) else print(io, "LambdaInfo for ") - show_lambda_types(io, l.specTypes.parameters) - println(io) + show_lambda_types(io, l) end else - println(io, "Toplevel LambdaInfo thunk") - end - # Fix slot names and types in function body - lambda_io = IOContext(IOContext(io, :LAMBDAINFO => l), - :LAMBDA_SLOTNAMES => lambdainfo_slotnames(l)) - body = Expr(:body) - body.args = uncompressed_ast(l) - body.typ = l.rettype - show(lambda_io, body) + print(io, "Toplevel LambdaInfo thunk") + end end function show_delim_array(io::IO, itr::Union{AbstractArray,SimpleVector}, op, delim, cl, @@ -563,15 +555,6 @@ show_unquoted(io::IO, ex::LabelNode, ::Int, ::Int) = print(io, ex.label, ": show_unquoted(io::IO, ex::GotoNode, ::Int, ::Int) = print(io, "goto ", ex.label) show_unquoted(io::IO, ex::GlobalRef, ::Int, ::Int) = print(io, ex.mod, '.', ex.name) -function show_unquoted(io::IO, ex::LambdaInfo, ::Int, ::Int) - if isdefined(ex, :specTypes) - print(io, "LambdaInfo for ") - show_lambda_types(io, ex.specTypes.parameters) - else - show(io, ex) - end -end - function show_unquoted(io::IO, ex::Slot, ::Int, ::Int) typ = isa(ex,TypedSlot) ? ex.typ : Any slotid = ex.id @@ -981,11 +964,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) show(io, ex.head) for arg in args print(io, ", ") - if isa(arg, LambdaInfo) && isdefined(arg, :specTypes) - show_lambda_types(io, arg.specTypes.parameters) - else - show(io, arg) - end + show(io, arg) end print(io, "))") end @@ -993,8 +972,14 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) nothing end -function show_lambda_types(io::IO, sig::SimpleVector) - # print a method signature tuple +function show_lambda_types(io::IO, li::LambdaInfo) + # print a method signature tuple for a lambda definition + if li.specTypes === Tuple + print(io, li.def.name, "(...)") + return + end + + sig = li.specTypes.parameters ft = sig[1] if ft <: Function && isempty(ft.parameters) && isdefined(ft.name.module, ft.name.mt.name) && diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 9fddd537351c3..b4be771e44628 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -17,7 +17,7 @@ Stack information representing execution context, with the following fields: The name of the function containing the execution context. -- `outer_linfo::Nullable{LambdaInfo}` +- `linfo::Nullable{LambdaInfo}` The LambdaInfo containing the execution context (if it could be found). @@ -29,18 +29,14 @@ Stack information representing execution context, with the following fields: The line number in the file containing the execution context. -- `inlined_file::Symbol` - - The path to the file containing the context for inlined code. - -- `inlined_line::Int` - - The line number in the file containing the context for inlined code. - - `from_c::Bool` True if the code is from C. +- `inlined::Bool` + + True if the code is from an inlined frame. + - `pointer::Int64` Representation of the pointer to the execution context as returned by `backtrace`. @@ -57,6 +53,7 @@ immutable StackFrame # this type should be kept platform-agnostic so that profil linfo::Nullable{LambdaInfo} "true if the code is from C" from_c::Bool + "true if the code is from an inlined frame" inlined::Bool "representation of the pointer to the execution context as returned by `backtrace`" pointer::Int64 # Large enough to be read losslessly on 32- and 64-bit machines. @@ -195,10 +192,10 @@ function show_spec_linfo(io::IO, frame::StackFrame) end else linfo = get(frame.linfo) - if isdefined(linfo, :specTypes) - Base.show_lambda_types(io, linfo.specTypes.parameters) + if isdefined(linfo, :def) + Base.show_lambda_types(io, linfo) else - print(io, linfo.name) + Base.show(io, linfo) end end end diff --git a/src/alloc.c b/src/alloc.c index 4ce6635352b1f..fc2da73d86992 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -394,8 +394,10 @@ JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info_uninit(void) (jl_lambda_info_t*)jl_gc_alloc(ptls, sizeof(jl_lambda_info_t), jl_lambda_info_type); li->code = NULL; - li->slotnames = li->slotflags = NULL; - li->slottypes = li->ssavaluetypes = NULL; + li->slotnames = NULL; + li->slotflags = NULL; + li->slottypes = NULL; + li->ssavaluetypes = NULL; li->rettype = (jl_value_t*)jl_any_type; li->sparam_syms = jl_emptysvec; li->sparam_vals = jl_emptysvec; @@ -586,7 +588,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(void) m->specializations.unknown = jl_nothing; m->sig = NULL; m->tvars = NULL; - m->ambig = NULL; + m->ambig = jl_nothing; m->roots = NULL; m->module = ptls->current_module; m->lambda_template = NULL; @@ -612,7 +614,6 @@ jl_method_t *jl_new_method(jl_lambda_info_t *definition, jl_sym_t *name, jl_tupl if (jl_svec_len(tvars) == 1) tvars = (jl_svec_t*)jl_svecref(tvars, 0); m->tvars = tvars; - m->ambig = jl_nothing; JL_GC_PUSH1(&m); // the front end may add this lambda to multiple methods; make a copy if so jl_method_t *oldm = definition->def; diff --git a/src/gf.c b/src/gf.c index 11d9a0ed913eb..99d26eb5d2588 100644 --- a/src/gf.c +++ b/src/gf.c @@ -163,12 +163,19 @@ jl_value_t *jl_mk_builtin_func(const char *name, jl_fptr_t fptr) jl_value_t *f = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0); jl_lambda_info_t *li = jl_new_lambda_info_uninit(); li->fptr = fptr; - // TODO jb/functions: what should li->code be? - li->code = jl_nothing; jl_gc_wb(li, li->code); + li->code = jl_nothing; + li->slottypes = jl_nothing; + li->specTypes = jl_anytuple_type; + li->ssavaluetypes = jl_box_long(0); + jl_gc_wb(li, li->ssavaluetypes); + li->def = jl_new_method_uninit(); li->def->name = sname; + // li->def->module will be set to jl_core_module by init.c li->def->lambda_template = li; - li->def->ambig = jl_nothing; + li->def->sig = jl_anytuple_type; + li->def->tvars = jl_emptysvec; + jl_methtable_t *mt = jl_gf_mtable(f); jl_typemap_insert(&mt->cache, (jl_value_t*)mt, jl_anytuple_type, jl_emptysvec, NULL, jl_emptysvec, (jl_value_t*)li, 0, &lambda_cache, NULL); return f; diff --git a/src/jltypes.c b/src/jltypes.c index ccf8fc116a778..a2f54d88b929d 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3833,7 +3833,7 @@ void jl_init_types(void) jl_bool_type, jl_bool_type, jl_bool_type), - 0, 1, 8); + 0, 1, 9); jl_lambda_info_type = jl_new_datatype(jl_symbol("LambdaInfo"), diff --git a/test/reflection.jl b/test/reflection.jl index 13f2e004435b3..773e734ab49cd 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -375,8 +375,8 @@ function test_typed_ast_printing(f::ANY, types::ANY, must_used_vars) for name in must_used_vars @test name in slotnames end - for str in (sprint(io->code_warntype(io, f, types)), - sprint(io->show(io, li))) + for str in (sprint(code_warntype, f, types), + stringmime("text/plain", li)) # Test to make sure the clearing of file path below works @test string(li.def.file) == @__FILE__ for var in must_used_vars @@ -401,7 +401,7 @@ function test_typed_ast_printing(f::ANY, types::ANY, must_used_vars) end end # Make sure printing an AST outside LambdaInfo still works. - str = sprint(io->show(io, Base.uncompressed_ast(li))) + str = sprint(show, Base.uncompressed_ast(li)) # Check that we are printing the slot numbers when we don't have the context # Use the variable names that we know should be present in the optimized AST for i in 2:length(li.slotnames) @@ -418,6 +418,16 @@ test_typed_ast_printing(g15714, Tuple{Vector{Float32}}, @test used_dup_var_tested15714 @test used_unique_var_tested15714 +let li = typeof(getfield).name.mt.cache.func::LambdaInfo, + lrepr = string(li), + mrepr = string(li.def), + lmime = stringmime("text/plain", li) + + @test lrepr == "LambdaInfo template for getfield(...)" + @test mrepr == "getfield(...)" +end + + # Linfo Tracing test tracefoo(x, y) = x+y didtrace = false diff --git a/test/stacktraces.jl b/test/stacktraces.jl index ec351a4373663..2f36384a11112 100644 --- a/test/stacktraces.jl +++ b/test/stacktraces.jl @@ -98,3 +98,14 @@ for (frame, func, inlined) in zip(trace, [g,h,f], (can_inline, can_inline, false @test frame.inlined === inlined end end + +let li = expand(quote let x = 1 end end).args[1]::LambdaInfo, + sf = StackFrame(:a, :b, 3, li, false, false, 0), + repr = string(sf) + @test repr == " in Toplevel LambdaInfo thunk at b:3" +end +let li = typeof(getfield).name.mt.cache.func::LambdaInfo, + sf = StackFrame(:a, :b, 3, li, false, false, 0), + repr = string(sf) + @test repr == " in getfield(...) at b:3" +end From d3f39ed0cede65b7cbee3a05385c43a3b65b3423 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 15:33:15 -0400 Subject: [PATCH 0549/1117] fix show_method_candidates for calling a Tuple this was bitrotted code from inline_incompletematch (disabled) fix #17502 --- base/replutil.jl | 6 +----- test/replutil.jl | 4 ++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/replutil.jl b/base/replutil.jl index 14282e8128343..b3be4ce49dc0d 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -406,11 +406,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs::Vector=Any[]) arg_types_param = Any[arg_types.parameters...] # Displays the closest candidates of the given function by looping over the # functions methods and counting the number of matching arguments. - if isa(ex.f, Tuple) - f = ex.f[1] - else - f = ex.f - end + f = ex.f ft = typeof(f) lines = [] # These functions are special cased to only show if first argument is matched. diff --git a/test/replutil.jl b/test/replutil.jl index e44ef80ddee1b..08f7ccd27d4c3 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -34,6 +34,10 @@ test_have_color(buf, Base.show_method_candidates(buf, Base.MethodError(method_c1,(1, 1, 1))) test_have_color(buf, "", "") +# matches the implicit constructor -> convert method +Base.show_method_candidates(buf, Base.MethodError(Tuple{}, (1, 1, 1))) +@test contains(takebuf_string(buf), "\nClosest candidates are:\n Tuple{}{T}(") + method_c2(x::Int32, args...) = true method_c2(x::Int32, y::Float64, args...) = true method_c2(x::Int32, y::Float64) = true From 39a35545fa4a4ca4d47f8c8b9ca7384b9e2977df Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 16:16:22 -0400 Subject: [PATCH 0550/1117] fix repl-completion behavior with ans fix #13955 --- base/REPL.jl | 26 +++++++++++++------------- base/client.jl | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/base/REPL.jl b/base/REPL.jl index deebc606df6a5..e7e29bb82a2c5 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -37,31 +37,32 @@ abstract AbstractREPL answer_color(::AbstractREPL) = "" type REPLBackend + "channel for AST" repl_channel::Channel + "channel for results: (value, nothing) or (error, backtrace)" response_channel::Channel + "flag indicating the state of this backend" in_eval::Bool - ans + "current backend task" backend_task::Task - REPLBackend(repl_channel, response_channel, in_eval, ans) = - new(repl_channel, response_channel, in_eval, ans) + + REPLBackend(repl_channel, response_channel, in_eval) = + new(repl_channel, response_channel, in_eval) end function eval_user_input(ast::ANY, backend::REPLBackend) - iserr, lasterr, bt = false, (), nothing + iserr, lasterr = false, ((), nothing) while true try if iserr - put!(backend.response_channel, (lasterr, bt)) + put!(backend.response_channel, lasterr) iserr, lasterr = false, () else - ans = backend.ans - # note: value wrapped in a non-syntax value to avoid evaluating - # possibly-invalid syntax (issue #6763). - eval(Main, :(ans = $(getindex)($(Any[ans]), 1))) backend.in_eval = true value = eval(Main, ast) backend.in_eval = false - backend.ans = value + # note: value wrapped in a closure to ensure it doesn't get passed through expand + eval(Main, Expr(:(=), :ans, Expr(:call, ()->value))) put!(backend.response_channel, (value, nothing)) end break @@ -70,14 +71,13 @@ function eval_user_input(ast::ANY, backend::REPLBackend) println("SYSTEM ERROR: Failed to report error to REPL frontend") println(err) end - iserr, lasterr = true, err - bt = catch_backtrace() + iserr, lasterr = true, (err, catch_backtrace()) end end end function start_repl_backend(repl_channel::Channel, response_channel::Channel) - backend = REPLBackend(repl_channel, response_channel, false, nothing) + backend = REPLBackend(repl_channel, response_channel, false) backend.backend_task = @schedule begin # include looks at this to determine the relative include path # nothing means cwd diff --git a/base/client.jl b/base/client.jl index 84adfff03ec13..19cc480e4d624 100644 --- a/base/client.jl +++ b/base/client.jl @@ -114,8 +114,8 @@ function eval_user_input(ast::ANY, show_value) errcount, lasterr = 0, () else ast = expand(ast) - value = eval(Main,ast) - eval(Main, :(ans = $(Expr(:quote, value)))) + value = eval(Main, ast) + eval(Main, Expr(:(=), :ans, Expr(:call, ()->value))) if !is(value,nothing) && show_value if have_color print(answer_color()) From 67c8eeb698ac0247a57073defa8fd1e7cba0a0d6 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Wed, 20 Jul 2016 23:22:13 -0400 Subject: [PATCH 0551/1117] Fix dump(Any) test. --- test/show.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/show.jl b/test/show.jl index 8966b75683c8e..91cbc7824e2b5 100644 --- a/test/show.jl +++ b/test/show.jl @@ -516,7 +516,7 @@ let repr = sprint(dump, Int64) end let repr = sprint(dump, Any) @test length(repr) > 100000 - @test startswith(repr, "Any\n Main.") || startswith(repr, "Any\n Base.") + @test ismatch(r"^Any\n [^ \t\n]", repr) @test endswith(repr, '\n') @test contains(repr, " Base.Vector{T} = Array{T,1}\n") @test !contains(repr, "Core.Vector{T}") From 1bfb70e4792fabb2594a8d445a9b38732eec527e Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sun, 17 Jul 2016 14:12:49 -0500 Subject: [PATCH 0552/1117] fix lowering bug in closure sparams ordering --- src/julia-syntax.scm | 2 +- test/core.jl | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 61cf3853e0682..4d41fb98054df 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2838,7 +2838,7 @@ f(x) = yt(x) (if lam2 (lam:sp lam2) '()) (map (lambda (methdef) (lam:sp (cadddr methdef))) alldefs)))) - (capt-sp (intersect cvs sps)) + (capt-sp (simple-sort (intersect cvs sps))) (capt-vars (diff cvs capt-sp)) (method-sp (map (lambda (s) (make-ssavalue)) capt-sp)) (typedef ;; expression to define the type diff --git a/test/core.jl b/test/core.jl index 7d8b3687a4270..65018c25aa85e 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4424,3 +4424,11 @@ function h17449(v) return r[] end @test h17449(true) === :k + +# make sure lowering agrees on sp order +function captsp{T, S}(x::T, y::S) + subf(x2::Int) = T + subf(x2::UInt) = S + return subf(Int(1)), subf(UInt(1)) +end +@test captsp(1, 2.0) == (Int, Float64) From e311761d839c1a3fa7e788c1cb9545c92d97b20c Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sun, 17 Jul 2016 14:14:38 -0500 Subject: [PATCH 0553/1117] allow inner functions signatures to depend on static parameters there is logic present to also handle all locals, but this causes some broken toplevel lowering to fail (notably, some gensym locals are getting pulled into global scope and some toplevel scopes are incorrectly identified as local scope) fix #15068 --- src/julia-syntax.scm | 122 ++++++++++++++++++++++++++++--------------- test/core.jl | 19 +++++++ 2 files changed, 100 insertions(+), 41 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 4d41fb98054df..99772f4219375 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2478,24 +2478,21 @@ end f(x) = yt(x) |# -;; template for generating a closure type with parameters -(define (type-for-closure-parameterized name P fields types super) - (let ((n (length P))) - `(thunk +(define (type-for-closure-parameterized name P names fields types super) + (let ((n (length P))) + `((thunk (lambda () - ((,@(map (lambda (p) `(,p Any 18)) P)) - () 0 ()) + (() () 0 ()) (body (global ,name) (const ,name) - ,@(map (lambda (p) `(= ,p (call (core TypeVar) ',p (core Any) false))) P) + ,@(map (lambda (p n) `(= ,p (call (core TypeVar) ',n (core Any) false))) P names) (composite_type ,name (call (core svec) ,@P) (call (core svec) ,@(map (lambda (v) `',v) fields)) ,super (call (core svec) ,@types) false ,(length fields)) - (return (null))))))) + (return (null)))))))) -;; ... and without parameters (define (type-for-closure name fields super) - `(thunk (lambda () + `((thunk (lambda () (() () 0 ()) (body (global ,name) (const ,name) (composite_type ,name (call (core svec)) @@ -2503,7 +2500,33 @@ f(x) = yt(x) ,super (call (core svec) ,@(map (lambda (v) 'Any) fields)) false ,(length fields)) - (return (null)))))) + (return (null))))))) + + +;; better versions of above, but they gets handled wrong in many places +;; need to fix that in order to handle #265 fully (and use the definitions) + +;; template for generating a closure type with parameters +;(define (type-for-closure-parameterized name P names fields types super) +; (let ((n (length P))) +; `((global ,name) +; (const ,name) +; ,@(map (lambda (p n) `(= ,p (call (core TypeVar) ',n (core Any) false))) P names) +; (composite_type ,name (call (core svec) ,@P) +; (call (core svec) ,@(map (lambda (v) `',v) fields)) +; ,super +; (call (core svec) ,@types) false ,(length fields))))) + +;; ... and without parameters +;(define (type-for-closure name fields super) +; `((global ,name) +; (const ,name) +; (composite_type ,name (call (core svec)) +; (call (core svec) ,@(map (lambda (v) `',v) fields)) +; ,super +; (call (core svec) ,@(map (lambda (v) 'Any) fields)) +; false ,(length fields)))) + (define (vinfo:not-capt vi) (list (car vi) (cadr vi) (logand (caddr vi) (lognot 5)))) @@ -2773,9 +2796,10 @@ f(x) = yt(x) (lam2 (if short #f (cadddr e))) (vis (if short '(() () ()) (lam:vinfo lam2))) (cvs (map car (cadr vis))) - (local? (and lam (symbol? name) - (or (assq name (car (lam:vinfo lam))) - (assq name (cadr (lam:vinfo lam)))))) + (local? (lambda (s) (and (symbol? s) + (or (assq s (car (lam:vinfo lam))) + (assq s (cadr (lam:vinfo lam))))))) + (local (and lam (local? name))) (sig (and (not short) (caddr e))) (sp-inits (if (or short (not (eq? (car sig) 'block))) '() @@ -2784,14 +2808,14 @@ f(x) = yt(x) (sig (and sig (if (eq? (car sig) 'block) (last sig) sig)))) - (if local? + (if local (begin (if (memq name (lam:args lam)) (error (string "cannot add method to function argument " name))) (if (eqv? (string.char (string name) 0) #\@) (error "macro definition not allowed inside a local scope")))) (if lam2 (lambda-optimize-vars! lam2)) - (if (not local?) ;; not a local function; will not be closure converted to a new type + (if (not local) ;; not a local function; will not be closure converted to a new type (cond (short e) ((null? cvs) `(block @@ -2840,18 +2864,51 @@ f(x) = yt(x) alldefs)))) (capt-sp (simple-sort (intersect cvs sps))) (capt-vars (diff cvs capt-sp)) + (find-locals (lambda (methdef) + (expr-find-all + (lambda (s) (and (not (eq? name s)) + (not (memq s capt-sp)) + (or ;(local? s) ; TODO: make this work for local variables too + (memq s (lam:sp lam))))) + (caddr methdef) + identity))) + (sig-locals (simple-sort + (delete-duplicates ;; locals used in sig from all definitions + (apply append ;; will convert these into sparams for dispatch + (if lam2 (find-locals e) '()) + (map find-locals alldefs))))) + (capt-sp (append capt-sp sig-locals)) ; sparams for the closure method declaration (method-sp (map (lambda (s) (make-ssavalue)) capt-sp)) (typedef ;; expression to define the type (let* ((fieldtypes (map (lambda (v) (if (is-var-boxed? v lam) '(core Box) - (gensy))) - capt-vars)) - (para (append capt-sp - (filter (lambda (v) (symbol? v)) fieldtypes)))) + (make-ssavalue))) + capt-vars)) + (para (append method-sp + (filter (lambda (v) (ssavalue? v)) fieldtypes))) + (fieldnames (append capt-sp (filter (lambda (v) (not (is-var-boxed? v lam))) capt-vars)))) (if (null? para) (type-for-closure tname capt-vars '(core Function)) - (type-for-closure-parameterized tname para capt-vars fieldtypes '(core Function))))) + (type-for-closure-parameterized tname para fieldnames capt-vars fieldtypes '(core Function))))) + (mk-method ;; expression to make the method + (if short '() + (let* ((iskw ;; TODO jb/functions need more robust version of this + (contains (lambda (x) (eq? x 'kwftype)) sig)) + (renamemap (map cons capt-sp method-sp)) + (arg-defs (replace-vars + (fix-function-arg-type sig tname iskw namemap method-sp) + renamemap))) + (append (map (lambda (gs tvar) + (make-assignment gs `(call (core TypeVar) ',tvar (core Any) true))) + method-sp capt-sp) + `((method #f ,(cl-convert arg-defs fname lam namemap toplevel interp) + ,(convert-lambda lam2 + (if iskw + (caddr (lam:args lam2)) + (car (lam:args lam2))) + #f capt-sp) + ,(last e))))))) (mk-closure ;; expression to make the closure (let* ((var-exprs (map (lambda (v) (let ((cv (assq v (cadr (lam:vinfo lam))))) @@ -2871,31 +2928,14 @@ f(x) = yt(x) `(new ,(if (null? P) tname `(call (core apply_type) ,tname ,@P)) - ,@var-exprs))) - (iskw ;; TODO jb/functions need more robust version of this - (contains (lambda (x) (eq? x 'kwftype)) sig))) + ,@var-exprs)))) `(toplevel-butlast ,@(if exists '() - (list (begin (and name (put! namemap name tname)) - typedef))) + typedef)) ,@sp-inits - ,@(if short '() - (map (lambda (gs tvar) - (make-assignment gs `(call (core TypeVar) ',tvar (core Any) true))) - method-sp capt-sp)) - ,@(if short '() - `((method #f - ,(cl-convert - (fix-function-arg-type sig tname iskw namemap method-sp) - fname lam namemap toplevel interp) - ,(convert-lambda lam2 - (if iskw - (caddr (lam:args lam2)) - (car (lam:args lam2))) - #f capt-sp) - ,(last e)))) + ,@mk-method ,(if exists '(null) (convert-assignment name mk-closure fname lam interp))))))) diff --git a/test/core.jl b/test/core.jl index 65018c25aa85e..5ce801a800b70 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4432,3 +4432,22 @@ function captsp{T, S}(x::T, y::S) return subf(Int(1)), subf(UInt(1)) end @test captsp(1, 2.0) == (Int, Float64) + +# issue #15068 +function sp_innersig{T}(x::T) + subf(x2::T) = (x, x2, :a) + subf(x2) = (x, x2, :b) + return (subf(one(T)), subf(unsigned(one(T)))) +end +@test sp_innersig(2) == ((2, 1, :a), (2, UInt(1), :b)) + +# TODO: broken - also with local variables +#function local_innersig{T}(x::T) +# V = typeof(x) +# U = unsigned(T) +# subf(x2::T, x3::Complex{V}) = (x, x2, x3) +# subf(x2::U) = (x, x2) +# return (subf(one(T), x * im), subf(unsigned(one(T)))) +#end +#@test local_innersig(Int32(2)) == ((Int32(2), Int32(1), Int32(2)im), (Int32(2), UInt32(1))) +#@test local_innersig(Int64(3)) == ((Int64(3), Int64(1), Int64(3)im), (Int64(3), UInt64(1))) From e558c25c04599ea44e6386fedd33a3a72e545d39 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 21 Jul 2016 00:06:03 -0400 Subject: [PATCH 0554/1117] lowering: use more readable variable names, and a few comments --- src/julia-syntax.scm | 80 ++++++++++++++++++++++---------------------- test/core.jl | 2 +- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 99772f4219375..06300e28c3837 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2840,9 +2840,9 @@ f(x) = yt(x) ,(last e)))))) ;; local case - lift to a new type at top level (let* ((exists (get namemap name #f)) - (tname (or exists - (and name - (symbol (string "#" name "#" (current-julia-module-counter)))))) + (type-name (or exists + (and name + (symbol (string "#" name "#" (current-julia-module-counter)))))) (alldefs (expr-find-all (lambda (ex) (and (eq? (car ex) 'method) (not (eq? ex e)) @@ -2851,63 +2851,63 @@ f(x) = yt(x) (lam:body lam) identity (lambda (x) (and (pair? x) (not (eq? (car x) 'lambda)))))) - (cvs (delete-duplicates - (apply append ;; merge captured vars from all definitions - cvs - (map (lambda (methdef) - (map car (cadr (lam:vinfo (cadddr methdef))))) - alldefs)))) - (sps (delete-duplicates ;; static params from all definitions - (apply append - (if lam2 (lam:sp lam2) '()) - (map (lambda (methdef) (lam:sp (cadddr methdef))) - alldefs)))) - (capt-sp (simple-sort (intersect cvs sps))) - (capt-vars (diff cvs capt-sp)) - (find-locals (lambda (methdef) - (expr-find-all - (lambda (s) (and (not (eq? name s)) - (not (memq s capt-sp)) - (or ;(local? s) ; TODO: make this work for local variables too - (memq s (lam:sp lam))))) - (caddr methdef) - identity))) + (all-capt-vars (delete-duplicates + (apply append ;; merge captured vars from all definitions + cvs + (map (lambda (methdef) + (map car (cadr (lam:vinfo (cadddr methdef))))) + alldefs)))) + (all-sparams (delete-duplicates ;; static params from all definitions + (apply append + (if lam2 (lam:sp lam2) '()) + (map (lambda (methdef) (lam:sp (cadddr methdef))) + alldefs)))) + (capt-sp (simple-sort (intersect all-capt-vars all-sparams))) ; the intersection is the list of sparams that need to be captured + (capt-vars (diff all-capt-vars capt-sp)) ; remove capt-sp from capt-vars + (find-locals-in-method-sig (lambda (methdef) + (expr-find-all + (lambda (s) (and (not (eq? name s)) + (not (memq s capt-sp)) + (or ;(local? s) ; TODO: make this work for local variables too? + (memq s (lam:sp lam))))) + (caddr methdef) + identity))) (sig-locals (simple-sort (delete-duplicates ;; locals used in sig from all definitions (apply append ;; will convert these into sparams for dispatch - (if lam2 (find-locals e) '()) - (map find-locals alldefs))))) - (capt-sp (append capt-sp sig-locals)) ; sparams for the closure method declaration - (method-sp (map (lambda (s) (make-ssavalue)) capt-sp)) + (if lam2 (find-locals-in-method-sig e) '()) + (map find-locals-in-method-sig alldefs))))) + (closure-param-names (append capt-sp sig-locals)) ; sparams for the closure method declaration + (closure-param-syms (map (lambda (s) (make-ssavalue)) closure-param-names)) (typedef ;; expression to define the type (let* ((fieldtypes (map (lambda (v) (if (is-var-boxed? v lam) '(core Box) (make-ssavalue))) capt-vars)) - (para (append method-sp + (para (append closure-param-syms (filter (lambda (v) (ssavalue? v)) fieldtypes))) - (fieldnames (append capt-sp (filter (lambda (v) (not (is-var-boxed? v lam))) capt-vars)))) + (fieldnames (append closure-param-names (filter (lambda (v) (not (is-var-boxed? v lam))) capt-vars)))) (if (null? para) - (type-for-closure tname capt-vars '(core Function)) - (type-for-closure-parameterized tname para fieldnames capt-vars fieldtypes '(core Function))))) + (type-for-closure type-name capt-vars '(core Function)) + (type-for-closure-parameterized type-name para fieldnames capt-vars fieldtypes '(core Function))))) (mk-method ;; expression to make the method (if short '() (let* ((iskw ;; TODO jb/functions need more robust version of this (contains (lambda (x) (eq? x 'kwftype)) sig)) - (renamemap (map cons capt-sp method-sp)) + (renamemap (map cons closure-param-names closure-param-syms)) (arg-defs (replace-vars - (fix-function-arg-type sig tname iskw namemap method-sp) + (fix-function-arg-type sig type-name iskw namemap closure-param-syms) renamemap))) (append (map (lambda (gs tvar) (make-assignment gs `(call (core TypeVar) ',tvar (core Any) true))) - method-sp capt-sp) + closure-param-syms closure-param-names) `((method #f ,(cl-convert arg-defs fname lam namemap toplevel interp) ,(convert-lambda lam2 (if iskw (caddr (lam:args lam2)) (car (lam:args lam2))) - #f capt-sp) + #f closure-param-names) ,(last e))))))) (mk-closure ;; expression to make the closure (let* ((var-exprs (map (lambda (v) @@ -2919,20 +2919,20 @@ f(x) = yt(x) v))) capt-vars)) (P (append - capt-sp + closure-param-names (filter identity (map (lambda (v ve) (if (is-var-boxed? v lam) #f `(call (core typeof) ,ve))) capt-vars var-exprs))))) `(new ,(if (null? P) - tname - `(call (core apply_type) ,tname ,@P)) + type-name + `(call (core apply_type) ,type-name ,@P)) ,@var-exprs)))) `(toplevel-butlast ,@(if exists '() - (begin (and name (put! namemap name tname)) + (begin (and name (put! namemap name type-name)) typedef)) ,@sp-inits ,@mk-method diff --git a/test/core.jl b/test/core.jl index 5ce801a800b70..7d4d46bd8444b 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4441,7 +4441,7 @@ function sp_innersig{T}(x::T) end @test sp_innersig(2) == ((2, 1, :a), (2, UInt(1), :b)) -# TODO: broken - also with local variables +# TODO: also allow local variables? #function local_innersig{T}(x::T) # V = typeof(x) # U = unsigned(T) From fcebf8e15a906637922db40c8e94a64fa3388879 Mon Sep 17 00:00:00 2001 From: Darwin Darakananda <darwindarak@gmail.com> Date: Wed, 20 Jul 2016 21:22:57 -0700 Subject: [PATCH 0555/1117] Make `watch_file` return values consistent with docs --- base/poll.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/poll.jl b/base/poll.jl index 1425b6ee59be4..cce9a9af06f9e 100644 --- a/base/poll.jl +++ b/base/poll.jl @@ -392,13 +392,13 @@ function watch_file(s::AbstractString, timeout_s::Real=-1) if timeout_s >= 0 result = fetimeout() - @schedule (try result = wait(fm); catch e; notify_error(wt, e); end; notify(wt)) + @schedule (try _, result = wait(fm); catch e; notify_error(wt, e); end; notify(wt)) @schedule (sleep(timeout_s); notify(wt)) wait(wt) return result else - return wait(fm) + return wait(fm)[2] end finally close(fm) From 2d49a5e0da21fb1007474b6a83dc1b2d0209027f Mon Sep 17 00:00:00 2001 From: Darwin Darakananda <darwindarak@gmail.com> Date: Wed, 20 Jul 2016 23:50:45 -0700 Subject: [PATCH 0556/1117] Add tests for `watch_file` --- test/file.jl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/file.jl b/test/file.jl index ef60ff00fd69c..956b54d2952b4 100644 --- a/test/file.jl +++ b/test/file.jl @@ -212,6 +212,19 @@ function test_touch(slval) @test ispath(tr[1]) && ispath(tr[2]) end +function test_watch_file_timeout(tval) + watch = @async watch_file(file, tval) + @test wait(watch) == Base.Filesystem.FileEvent(false, false, true) +end + +function test_watch_file_change(tval) + watch = @async watch_file(file, tval) + sleep(tval/3) + open(file, "a") do f + write(f, "small change\n") + end + @test wait(watch) == Base.Filesystem.FileEvent(false, true, false) +end function test_monitor_wait(tval) fm = FileMonitor(file) @@ -247,6 +260,8 @@ test_monitor_wait(0.1) test_monitor_wait(0.1) test_monitor_wait_poll() test_monitor_wait_poll() +test_watch_file_timeout(0.1) +test_watch_file_change(6) @test_throws Base.UVError watch_file("nonexistantfile", 10) @test_throws Base.UVError poll_file("nonexistantfile", 2, 10) From 12ce6b66b8343a4fed965e2174b58b55f8997c02 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Thu, 21 Jul 2016 08:00:05 -0400 Subject: [PATCH 0557/1117] =?UTF-8?q?note=20that=20#17393=20now=20requires?= =?UTF-8?q?=20one=20to=20use=20Base.:=E2=89=A4=20instead=20of=20Base.?= =?UTF-8?q?=E2=89=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NEWS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index c7c03bfa164f2..0a09d07dd81ea 100644 --- a/NEWS.md +++ b/NEWS.md @@ -39,7 +39,9 @@ New language features * Many more operators now support `.` prefixes (e.g. `.≤`) ([#17393]). However, users are discouraged from overloading these, since they are mainly parsed in order to implement backwards compatibility with planned automatic - broadcasting of dot operators in Julia 0.6 ([#16285]). + broadcasting of dot operators in Julia 0.6 ([#16285]). Explicitly qualified + operator names like `Base.≤` should now use `Base.:≤` (prefixed by `@compat` + if you need 0.4 compatibility via the `Compat` package). New architectures ----------------- From 22ede9c8b886307a8ccd10e5ba79328371c2cc84 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sun, 10 Jul 2016 10:41:06 -0400 Subject: [PATCH 0558/1117] Make the gc uses mark robust against unreachable and noreturn functions. --- src/ccall.cpp | 49 ++++++++++++++++++++++++++++--------------------- src/codegen.cpp | 23 ++++++++++++++++++++++- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 54806368df0b4..6bcb44d61d5de 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -184,7 +184,7 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, runtime_lib); } -// Map from distinct callee's to it's GOT entry. +// Map from distinct callee's to its GOT entry. // In principle the attribute, function type and calling convention // don't need to be part of the key but it seems impossible to forward // all the arguments without writing assembly directly. @@ -1019,15 +1019,20 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c // the actual call builder.CreateCall(prepare_call(gcroot_flush_func)); - CallInst *inst = builder.CreateCall(f, ArrayRef<Value*>(&argvals[0], nargt)); - if (isString) - ctx->to_inline.push_back(inst); - - // after the llvmcall mark fake uses of all of the arguments to ensure the were live + SmallVector<Value*, 16> gc_uses; for (size_t i = 0; i < nargt; ++i) { const jl_cgval_t &arg = argv[i]; - mark_gc_use(arg); + push_gc_use(gc_uses, arg); } + // Mark GC use before **and** after the llvmcall to make sure the arguments + // are alive during the llvmcall even if the llvmcall has `unreachable`. + // If the llvmcall generates GC safepoint, it might need to emit its own + // gckill. + mark_gc_uses(gc_uses); + CallInst *inst = builder.CreateCall(f, ArrayRef<Value*>(&argvals[0], nargt)); + if (isString) + ctx->to_inline.push_back(inst); + mark_gc_uses(gc_uses); JL_GC_POP(); @@ -1757,6 +1762,21 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) //for (int i = 0; i < (nargs - 3) / 2 + sret; ++i) // argvals[i]->dump(); + // Mark GC use before **and** after the ccall to make sure the arguments + // are alive during the ccall even if the function called is `noreturn`. + SmallVector<Value*, 16> gc_uses; + for(i = 4; i < nargs + 1; i += 2) { + // Current C function parameter + size_t ai = (i - 4) / 2; + push_gc_use(gc_uses, argv[ai]); + + // Julia (expression) value of current parameter gcroot + jl_value_t *argi = args[i + 1]; + if (jl_is_long(argi)) continue; + jl_cgval_t arg = emit_expr(argi, ctx); + push_gc_use(gc_uses, arg); + } + mark_gc_uses(gc_uses); // the actual call Value *ret = builder.CreateCall(prepare_call(llvmf), ArrayRef<Value*>(&argvals[0], (nargs - 3) / 2 + sret)); @@ -1774,20 +1794,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) ctx->f->addFnAttr(Attribute::StackProtectReq); } - // after the ccall itself, mark fake uses of all of the arguments to ensure the were live, - // and run over the gcroot list and make give them a `mark_gc_use` - for(i = 4; i < nargs + 1; i += 2) { - // Current C function parameter - size_t ai = (i - 4) / 2; - mark_gc_use(argv[ai]); - - // Julia (expression) value of current parameter gcroot - jl_value_t *argi = args[i + 1]; - if (jl_is_long(argi)) continue; - jl_cgval_t arg = emit_expr(argi, ctx); - mark_gc_use(arg); - } - + mark_gc_uses(gc_uses); JL_GC_POP(); if (rt == jl_bottom_type) { // Do this after we marked all the GC uses. diff --git a/src/codegen.cpp b/src/codegen.cpp index b08ba94e9fd99..8216d43314222 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -555,7 +555,22 @@ static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, static Value *emit_condition(jl_value_t *cond, const std::string &msg, jl_codectx_t *ctx); static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx); static GlobalVariable *prepare_global(GlobalVariable *G, Module *M = jl_builderModule); +static llvm::Value *prepare_call(llvm::Value *Callee); +template<typename T> static void push_gc_use(T &&vec, const jl_cgval_t &v) +{ + if (v.gcroot) { + vec.push_back(v.gcroot); + } +} + +template<typename T> static void mark_gc_uses(T &&vec) +{ + auto f = prepare_call(gckill_func); + for (auto &v: vec) { + builder.CreateCall(f, v); + } +} // --- convenience functions for tagging llvm values with julia types --- @@ -1807,6 +1822,9 @@ static Value *emit_local_root(jl_codectx_t *ctx, jl_varinfo_t *vi) // Marks a use (and thus a potential kill) of a gcroot +// Note that if the operation that needs the root has terminating control flow +// (e.g. `unreachable`, `noreturn` functions) the use needs to be marked before +// the operation as well as after it. static void mark_gc_use(const jl_cgval_t &v) { if (v.gcroot) @@ -2615,6 +2633,7 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval argvals[idx] = result; idx++; } + SmallVector<Value*, 16> gc_uses; for(size_t i=0; i < nargs+1; i++) { jl_value_t *jt = jl_nth_slot_type(li->specTypes,i); bool isboxed; @@ -2637,7 +2656,7 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval jl_cgval_t arg = i==0 ? theF : emit_expr(args[i], ctx); assert(arg.ispointer()); argvals[idx] = data_pointer(arg, ctx, at); - mark_gc_use(arg); // TODO: must be after the jlcall + push_gc_use(gc_uses, arg); } else { assert(at == et); @@ -2649,8 +2668,10 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval idx++; } assert(idx == nfargs); + mark_gc_uses(gc_uses); CallInst *call = builder.CreateCall(prepare_call(cf), ArrayRef<Value*>(&argvals[0], nfargs)); call->setAttributes(cf->getAttributes()); + mark_gc_uses(gc_uses); return sret ? mark_julia_slot(result, jlretty, tbaa_stack) : mark_julia_type(call, retboxed, jlretty, ctx); } return mark_julia_type(emit_jlcall(theFptr, boxed(theF,ctx), &args[1], nargs, ctx), true, From 82df6a41052ebdd179282e3d7c4f6eb9be03beba Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sun, 10 Jul 2016 16:06:55 -0400 Subject: [PATCH 0559/1117] Towards being able to run GC frame lowering after LLVM optimizations * Make the pass more robust against dead code elimination * Remove module level setup code (so that the the GC frame lowering can be a `FunctionPass`) * Create PTLS lowering (module) pass since it needs to be run after the GC frame lowering * Set DLLImport attribute for `jl_tls_states` on windows, since it is not handled by module merger anymore --- src/Makefile | 4 +- src/codegen.cpp | 95 +--------------- src/jitlayers.cpp | 4 + src/jitlayers.h | 4 + src/llvm-gcroot.cpp | 266 ++++++++++++++++++++++++++------------------ src/llvm-ptls.cpp | 193 ++++++++++++++++++++++++++++++++ 6 files changed, 364 insertions(+), 202 deletions(-) create mode 100644 src/llvm-ptls.cpp diff --git a/src/Makefile b/src/Makefile index e1c18c8eeda04..ebf8219150674 100644 --- a/src/Makefile +++ b/src/Makefile @@ -40,7 +40,7 @@ SRCS := \ alloc dlload sys init task array dump toplevel jl_uv \ simplevector APInt-C runtime_intrinsics runtime_ccall \ threadgroup threading stackwalk gc gc-debug gc-pages \ - jlapi signal-handling safepoint jloptions timing + jlapi signal-handling safepoint jloptions timing ifeq ($(USEMSVC), 1) SRCS += getopt @@ -49,7 +49,7 @@ endif LLVMLINK := ifeq ($(JULIACODEGEN),LLVM) -SRCS += codegen jitlayers disasm debuginfo llvm-simdloop llvm-gcroot cgmemmgr +SRCS += codegen jitlayers disasm debuginfo llvm-simdloop llvm-ptls llvm-gcroot cgmemmgr FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir) LLVM_LIBS := all ifeq ($(USE_POLLY),1) diff --git a/src/codegen.cpp b/src/codegen.cpp index 8216d43314222..ce7cc3d0e5c54 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -85,8 +85,7 @@ #include <llvm/Support/PrettyStackTrace.h> #include <llvm/Support/CommandLine.h> -#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) || \ - (defined(LLVM37) && defined(JULIA_ENABLE_THREADING)) +#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) # include <llvm/IR/InlineAsm.h> #endif #if defined(USE_POLLY) @@ -257,7 +256,7 @@ static MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t static MDNode *tbaa_arraysize; // A size in a jl_array_t static MDNode *tbaa_arraylen; // The len in a jl_array_t static MDNode *tbaa_arrayflags; // The flags in a jl_array_t -static MDNode *tbaa_const; // Memory that is immutable by the time LLVM can see it +MDNode *tbaa_const; // Memory that is immutable by the time LLVM can see it // Basic DITypes #ifdef LLVM37 @@ -388,7 +387,6 @@ static Function *jlarray_data_owner_func; // placeholder functions static Function *gcroot_func; -static Function *gcstore_func; static Function *gckill_func; static Function *jlcall_frame_func; static Function *gcroot_flush_func; @@ -3406,87 +3404,15 @@ static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx) PointerType::get(T_psize, 0)); } -void jl_codegen_finalize_temp_arg(CallInst *ptlsStates, Type *T_pjlvalue, - MDNode *tbaa_gcframe); -static void finalize_gc_frame(Function *F) -{ - Module *M = F->getParent(); - M->getOrInsertFunction(gcroot_func->getName(), gcroot_func->getFunctionType()); - M->getOrInsertFunction(gckill_func->getName(), gckill_func->getFunctionType()); - M->getOrInsertFunction(gcstore_func->getName(), gcstore_func->getFunctionType()); - M->getOrInsertFunction(jlcall_frame_func->getName(), jlcall_frame_func->getFunctionType()); - M->getOrInsertFunction(gcroot_flush_func->getName(), gcroot_flush_func->getFunctionType()); - Function *jl_get_ptls_states = M->getFunction("jl_get_ptls_states"); - - CallInst *ptlsStates = NULL; - for (BasicBlock::iterator i = F->getEntryBlock().begin(), e = F->getEntryBlock().end(); i != e; ++i) { - if (CallInst *callInst = dyn_cast<CallInst>(&*i)) { - if (callInst->getCalledFunction() == jl_get_ptls_states) { - ptlsStates = callInst; - break; - } - } - } - if (!ptlsStates) - return; - - jl_codegen_finalize_temp_arg(ptlsStates, T_pjlvalue, tbaa_gcframe); - -#ifdef JULIA_ENABLE_THREADING - if (imaging_mode) { - GlobalVariable *GV = prepare_global(jltls_states_func_ptr, M); - Value *getter = tbaa_decorate(tbaa_const, - new LoadInst(GV, "", ptlsStates)); - ptlsStates->setCalledFunction(getter); - ptlsStates->setAttributes(jltls_states_func->getAttributes()); - } - else if (jl_tls_offset != -1) { -#ifdef LLVM37 - // Replace the function call with inline assembly if we know - // how to generate it. - const char *asm_str = nullptr; -# if defined(_CPU_X86_64_) - asm_str = "movq %fs:0, $0"; -# elif defined(_CPU_X86_) - asm_str = "movl %gs:0, $0"; -# elif defined(_CPU_AARCH64_) - asm_str = "mrs $0, tpidr_el0"; -# endif - assert(asm_str && "Cannot emit thread pointer for this architecture."); - static auto offset = ConstantInt::getSigned(T_size, jl_tls_offset); - static auto tp = InlineAsm::get(FunctionType::get(T_pint8, false), - asm_str, "=r", false); - Value *tls = CallInst::Create(tp, "thread_ptr", ptlsStates); - tls = GetElementPtrInst::Create(T_int8, tls, {offset}, - "ptls_i8", ptlsStates); - tls = new BitCastInst(tls, PointerType::get(T_ppjlvalue, 0), - "ptls", ptlsStates); - ptlsStates->replaceAllUsesWith(tls); - ptlsStates->eraseFromParent(); -#endif - } -#else - ptlsStates->replaceAllUsesWith(prepare_global(jltls_states_var, M)); - ptlsStates->eraseFromParent(); -#endif -} - +void jl_codegen_finalize_temp_arg(Function *F, MDNode *tbaa_gcframe); void finalize_gc_frame(Module *m) { for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { Function *F = &*I; if (F->isDeclaration()) continue; - finalize_gc_frame(F); + jl_codegen_finalize_temp_arg(F, tbaa_gcframe); } -#ifndef JULIA_ENABLE_THREADING - m->getFunction("jl_get_ptls_states")->eraseFromParent(); -#endif - m->getFunction("julia.gc_root_decl")->eraseFromParent(); - m->getFunction("julia.gc_root_kill")->eraseFromParent(); - m->getFunction("julia.gc_store")->eraseFromParent(); - m->getFunction("julia.jlcall_frame_decl")->eraseFromParent(); - m->getFunction("julia.gcroot_flush")->eraseFromParent(); } static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_tupletype_t *argt, @@ -5193,12 +5119,6 @@ static void init_julia_llvm_env(Module *m) jltls_states_func = Function::Create(FunctionType::get(PointerType::get(T_ppjlvalue, 0), false), Function::ExternalLinkage, "jl_get_ptls_states", m); - jltls_states_func->setAttributes( - jltls_states_func->getAttributes() - .addAttribute(jltls_states_func->getContext(), - AttributeSet::FunctionIndex, Attribute::ReadNone) - .addAttribute(jltls_states_func->getContext(), - AttributeSet::FunctionIndex, Attribute::NoUnwind)); add_named_global(jltls_states_func, jl_get_ptls_states_getter()); if (imaging_mode) { PointerType *pfunctype = jltls_states_func->getFunctionType()->getPointerTo(); @@ -5615,13 +5535,6 @@ static void init_julia_llvm_env(Module *m) "julia.gc_root_kill", m); add_named_global(gckill_func, (void*)NULL, /*dllimport*/false); - Type* gc_store_args[2] = { T_ppjlvalue, T_pjlvalue }; // [1] <= [2] - gcstore_func = - Function::Create(FunctionType::get(T_void, makeArrayRef(gc_store_args), false), - Function::ExternalLinkage, - "julia.gc_store", m); - add_named_global(gcstore_func, (void*)NULL, /*dllimport*/false); - jlcall_frame_func = Function::Create(FunctionType::get(T_ppjlvalue, ArrayRef<Type*>(T_int32), false), Function::ExternalLinkage, diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index bcab549ac9073..9047188084568 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -114,6 +114,7 @@ void addOptimizationPasses(PassManager *PM) # endif #endif if (jl_options.opt_level == 0) { + PM->add(createLowerPTLSPass(imaging_mode, tbaa_const)); return; } #ifdef LLVM37 @@ -140,6 +141,9 @@ void addOptimizationPasses(PassManager *PM) #ifndef INSTCOMBINE_BUG PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl. #endif + // Let the InstCombine pass remove the unnecessary load of + // safepoint address first + PM->add(createLowerPTLSPass(imaging_mode, tbaa_const)); PM->add(createSROAPass()); // Break up aggregate allocas #ifndef INSTCOMBINE_BUG PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl. diff --git a/src/jitlayers.h b/src/jitlayers.h index c4e2fe0fa5a31..786aaefe2e522 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -218,3 +218,7 @@ JL_DLLEXPORT extern LLVMContext jl_LLVMContext; #else JL_DLLEXPORT extern LLVMContext &jl_LLVMContext; #endif + +extern MDNode *tbaa_const; + +Pass *createLowerPTLSPass(bool imaging_mode, MDNode *tbaa_const); diff --git a/src/llvm-gcroot.cpp b/src/llvm-gcroot.cpp index b8b11f475906b..ba994bf729abc 100644 --- a/src/llvm-gcroot.cpp +++ b/src/llvm-gcroot.cpp @@ -37,6 +37,90 @@ static struct { } jl_gc_frame_stats = {0}; #endif +typedef std::pair<CallInst*, unsigned> frame_register; +struct liveness { + typedef unsigned id; + enum { + // an assignment to a gcroot exists in the basic-block + // (potentially no live-in from the predecessor basic-blocks) + assign = 1<<0, + // a use of a gcroot exists in the basic-block + // (potentially a "kill" and no live-out to the successor basic-blocks) + kill = 1<<1, + // the gcroot is live over the entire basic-block + // (the assign/kill are not dominating of entry/exit) + live = 1<<2 + + // live | kill | assign: + // a usage and assignment exist, but it is also live on exit, + // the entry liveness depends on whether a store or use is + // encountered first + // live | kill: + // a usage exists, + // but the value must be live for the entire basic-block + // since it is not the terminal usage in the domination tree + // kill | assign: + // a usage and definition exist in domination order, + // so the actual lifetime is only a subset of the basic-block + // live | assign: + // impossible (this would be strange) + }; +}; + +#ifndef NDEBUG // llvm assertions build +// gdb debugging code for inspecting the bb_uses map +void jl_dump_bb_uses(std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses) +{ + for (std::map<BasicBlock*, std::map<frame_register, liveness::id> >::iterator + live_reg = bb_uses.begin(), e = bb_uses.end(); live_reg != e; ++live_reg) { + BasicBlock *bb = live_reg->first; + errs() << '\n' << bb << '\n'; + for (std::map<frame_register, liveness::id>::iterator + regs = live_reg->second.begin(), regse = live_reg->second.end(); regs != regse; ++regs) { + errs() << regs->second << " #" << regs->first.second << ' ' << regs->first.first << '\n'; + } + } +} +#endif + +static void tbaa_decorate_gcframe(Instruction *inst, + std::set<Instruction*> &visited, + MDNode *tbaa_gcframe) +{ + if (visited.find(inst) != visited.end()) + return; + visited.insert(inst); +#ifdef LLVM35 + Value::user_iterator I = inst->user_begin(), E = inst->user_end(); +#else + Value::use_iterator I = inst->use_begin(), E = inst->use_end(); +#endif + for (;I != E;++I) { + Instruction *user = dyn_cast<Instruction>(*I); + if (!user) { + continue; + } else if (isa<GetElementPtrInst>(user)) { + if (__likely(user->getOperand(0) == inst)) { + tbaa_decorate_gcframe(user, visited, tbaa_gcframe); + } + } else if (isa<StoreInst>(user)) { + if (user->getOperand(1) == inst) { + user->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_gcframe); + } + } else if (isa<LoadInst>(user)) { + user->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_gcframe); + } else if (isa<BitCastInst>(user)) { + tbaa_decorate_gcframe(user, visited, tbaa_gcframe); + } + } +} + +static void tbaa_decorate_gcframe(Instruction *inst, MDNode *tbaa_gcframe) +{ + std::set<Instruction*> visited; + tbaa_decorate_gcframe(inst, visited, tbaa_gcframe); +} + class JuliaGCAllocator { public: JuliaGCAllocator(CallInst *ptlsStates, Type *T_pjlvalue, MDNode *tbaa) : @@ -51,7 +135,6 @@ class JuliaGCAllocator { gcframe(new AllocaInst(T_pjlvalue, ConstantInt::get(T_int32, 0))), gcroot_func(M.getFunction("julia.gc_root_decl")), gckill_func(M.getFunction("julia.gc_root_kill")), - gc_store_func(M.getFunction("julia.gc_store")), jlcall_frame_func(M.getFunction("julia.jlcall_frame_decl")), gcroot_flush_func(M.getFunction("julia.gcroot_flush")), tbaa_gcframe(tbaa) @@ -66,94 +149,40 @@ class JuliaGCAllocator { gcframe->setName("gcrootframe"); #endif gcframe->insertAfter(ptlsStates); - assert(gcroot_func && gckill_func && jlcall_frame_func && gc_store_func); } private: -Function &F; -Module &M; -Type *const T_int1; -Type *const T_int8; -Type *const T_int32; -Type *const T_int64; -Value *const V_null; -CallInst *const ptlsStates; -AllocaInst *const gcframe; -Function *const gcroot_func; -Function *const gckill_func; -Function *const gc_store_func; -Function *const jlcall_frame_func; -Function *const gcroot_flush_func; -MDNode *const tbaa_gcframe; - -typedef std::pair<CallInst*, unsigned> frame_register; -class liveness { + Function &F; + Module &M; + Type *const T_int1; + Type *const T_int8; + Type *const T_int32; + Type *const T_int64; + Value *const V_null; + CallInst *const ptlsStates; + AllocaInst *const gcframe; + Function *const gcroot_func; + Function *const gckill_func; + Function *const jlcall_frame_func; + Function *const gcroot_flush_func; + MDNode *const tbaa_gcframe; + + Instruction *get_pgcstack(Instruction *ptlsStates); + frame_register get_gcroot(Value *ptr); + void collapseRedundantRoots(); + bool record_usage(CallInst *callInst, + std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses, + std::map<BasicBlock*, SmallBitVector> ®s_used, + unsigned &offset, bool commit=true); + unsigned find_space_for(CallInst *callInst, + std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses, + std::map<BasicBlock*, SmallBitVector> ®s_used); + void rearrangeRoots(); public: - typedef unsigned id; - enum { - assign = 1<<0, // an assignment to a gcroot exists in the basic-block (potentially no live-in from the predecessor basic-blocks) - kill = 1<<1, // a use of a gcroot exists in the basic-block (potentially a "kill" and no live-out to the successor basic-blocks) - live = 1<<2 // the gcroot is live over the entire basic-block (the assign/kill are not dominating of entry/exit) - // live | kill | assign == a usage and assignment exist, but it is also live on exit, the entry liveness depends on whether a store or use is encountered first - // live | kill == a usage exists, but the value must be live for the entire basic-block since it is not the terminal usage in the domination tree - // kill | assign == a usage and definition exist in domination order, so the actual lifetime is only a subset of the basic-block - // live | assign == impossible (this would be strange) - }; + void allocate_frame(); }; - void tbaa_decorate_gcframe(Instruction *inst, std::set<Instruction*> &visited) - { - if (visited.find(inst) != visited.end()) - return; - visited.insert(inst); -#ifdef LLVM35 - Value::user_iterator I = inst->user_begin(), E = inst->user_end(); -#else - Value::use_iterator I = inst->use_begin(), E = inst->use_end(); -#endif - for (;I != E;++I) { - Instruction *user = dyn_cast<Instruction>(*I); - if (!user) { - continue; - } else if (isa<GetElementPtrInst>(user)) { - if (__likely(user->getOperand(0) == inst)) { - tbaa_decorate_gcframe(user, visited); - } - } else if (isa<StoreInst>(user)) { - if (user->getOperand(1) == inst) { - user->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_gcframe); - } - } else if (isa<LoadInst>(user)) { - user->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_gcframe); - } else if (isa<BitCastInst>(user)) { - tbaa_decorate_gcframe(user, visited); - } - } - } - - void tbaa_decorate_gcframe(Instruction *inst) - { - std::set<Instruction*> visited; - tbaa_decorate_gcframe(inst, visited); - } - -#ifndef NDEBUG // llvm assertions build -// gdb debugging code for inspecting the bb_uses map -void jl_dump_bb_uses(std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses) -{ - for (std::map<BasicBlock*, std::map<frame_register, liveness::id> >::iterator - live_reg = bb_uses.begin(), e = bb_uses.end(); live_reg != e; ++live_reg) { - BasicBlock *bb = live_reg->first; - errs() << '\n' << bb << '\n'; - for (std::map<frame_register, liveness::id>::iterator - regs = live_reg->second.begin(), regse = live_reg->second.end(); regs != regse; ++regs) { - errs() << regs->second << " #" << regs->first.second << ' ' << regs->first.first << '\n'; - } - } -} -#endif - -Instruction *get_pgcstack(Instruction *ptlsStates) +Instruction *JuliaGCAllocator::get_pgcstack(Instruction *ptlsStates) { Constant *offset = ConstantInt::getSigned(T_int32, offsetof(jl_tls_states_t, pgcstack) / sizeof(void*)); return GetElementPtrInst::Create( @@ -163,7 +192,7 @@ Instruction *get_pgcstack(Instruction *ptlsStates) "jl_pgcstack"); } -frame_register get_gcroot(Value *ptr) +frame_register JuliaGCAllocator::get_gcroot(Value *ptr) { frame_register frame; frame.first = dyn_cast<CallInst>(ptr); @@ -173,7 +202,7 @@ frame_register get_gcroot(Value *ptr) if (GetElementPtrInst *gepInst = dyn_cast<GetElementPtrInst>(ptr)) { if (gepInst->getNumIndices() == 1) { frame.first = dyn_cast<CallInst>(gepInst->getPointerOperand()); - if (frame.first && frame.first->getCalledFunction() == jlcall_frame_func) + if (frame.first && frame.first->getCalledValue() == jlcall_frame_func) frame.second = cast<ConstantInt>(gepInst->idx_begin()->get())->getZExtValue(); else frame.first = NULL; @@ -183,12 +212,12 @@ frame_register get_gcroot(Value *ptr) return frame; } -void collapseRedundantRoots() +void JuliaGCAllocator::collapseRedundantRoots() { for (BasicBlock::iterator I = gcframe->getParent()->begin(), E(gcframe); I != E; ) { CallInst* callInst = dyn_cast<CallInst>(&*I); ++I; - if (callInst && callInst->getCalledFunction() == gcroot_func) { + if (callInst && callInst->getCalledValue() == gcroot_func) { // see if a root is only used briefly for `store -> load -> store other` pattern or `store, store other` // such that the first store can be trivially replaced with just "other" and delete the chain // or if is used for store, but the value is never needed @@ -290,7 +319,7 @@ void collapseRedundantRoots() frame_register gcroot_other_gep = get_gcroot(theOther->getPointerOperand()); CallInst *gcroot_other = gcroot_other_gep.first; // it could be a gcroot... - if (gcroot_other && gcroot_other->getCalledFunction() == gcroot_func && theStore != NULL) { + if (gcroot_other && gcroot_other->getCalledValue() == gcroot_func && theStore != NULL) { // need to make sure there aren't any other uses of gcroot_other (including gckill) // between the initial store and the replacement store // TODO: do this better once we have liveness information for locals? @@ -339,7 +368,7 @@ void collapseRedundantRoots() } } // ...or it could be a jlcall frame - else if (gcroot_other && gcroot_other->getCalledFunction() == jlcall_frame_func) { + else if (gcroot_other && gcroot_other->getCalledValue() == jlcall_frame_func) { // jlcall_frame_func slots are effectively SSA, // so it's always safe to merge an earlier store into it // but do need to update liveness information for this slot @@ -386,10 +415,10 @@ void collapseRedundantRoots() } } -bool record_usage(CallInst *callInst, +bool JuliaGCAllocator::record_usage(CallInst *callInst, std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses, std::map<BasicBlock*, SmallBitVector> ®s_used, - unsigned &offset, bool commit=true) + unsigned &offset, bool commit) { /* record-usage(inst, bb-uses, regs-used, offset, commit=true) * for (arg-offset, operand) in enumerate(arguments(inst)) @@ -447,7 +476,7 @@ bool record_usage(CallInst *callInst, return true; } -unsigned find_space_for(CallInst *callInst, +unsigned JuliaGCAllocator::find_space_for(CallInst *callInst, std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses, std::map<BasicBlock*, SmallBitVector> ®s_used) { @@ -463,7 +492,7 @@ unsigned find_space_for(CallInst *callInst, return n; } -void rearrangeRoots() +void JuliaGCAllocator::rearrangeRoots() { for (auto BB = F.begin(), E(F.end()); BB != E; BB++) { auto terminst = BB->getTerminator(); @@ -481,14 +510,14 @@ void rearrangeRoots() if (LoadInst *loadInst = dyn_cast<LoadInst>(inst)) { CallInst *loadAddr = dyn_cast<CallInst>(loadInst->getPointerOperand()); - if (loadAddr && loadAddr->getCalledFunction() == gcroot_func) + if (loadAddr && loadAddr->getCalledValue() == gcroot_func) break; continue; } if (StoreInst *storeInst = dyn_cast<StoreInst>(inst)) { CallInst *storeAddr = dyn_cast<CallInst>(storeInst->getPointerOperand()); - if (storeAddr && storeAddr->getCalledFunction() == gcroot_func) + if (storeAddr && storeAddr->getCalledValue() == gcroot_func) toRemove.push_back(storeInst); continue; } @@ -505,8 +534,7 @@ void rearrangeRoots() } } -public: -void allocate_frame() +void JuliaGCAllocator::allocate_frame() { Instruction *last_gcframe_inst = gcframe; collapseRedundantRoots(); @@ -524,7 +552,7 @@ void allocate_frame() for (BasicBlock::iterator I = gcframe->getParent()->begin(), E(gcframe); I != E; ) { CallInst* callInst = dyn_cast<CallInst>(&*I); ++I; - if (callInst && callInst->getCalledFunction() == jlcall_frame_func) { + if (callInst && callInst->getCalledValue() == jlcall_frame_func) { BasicBlock *bb = NULL; unsigned arg_n = cast<ConstantInt>(callInst->getArgOperand(0))->getZExtValue(); frames.push(std::make_pair(arg_n, callInst)); @@ -579,7 +607,7 @@ void allocate_frame() if (StoreInst *storeInst = dyn_cast<StoreInst>(i)) { frame_register def = get_gcroot(storeInst->getPointerOperand()); if (CallInst *callInst = def.first) { - if (callInst->getCalledFunction() == jlcall_frame_func) { + if (callInst->getCalledValue() == jlcall_frame_func) { std::map<frame_register, liveness::id>::iterator inuse_reg = inuse_list.find(def); if (inuse_reg != inuse_list.end() && inuse_reg->second == liveness::kill) { inuse_reg->second |= liveness::assign; @@ -653,7 +681,7 @@ void allocate_frame() if (StoreInst *storeInst = dyn_cast<StoreInst>(&*i)) { frame_register def = get_gcroot(storeInst->getPointerOperand()); if (CallInst *callInst = def.first) { - if (callInst->getCalledFunction() == jlcall_frame_func) { + if (callInst->getCalledValue() == jlcall_frame_func) { std::map<frame_register, liveness::id>::iterator inuse_reg = inuse_list.find(def); if (inuse_reg != inuse_list.end() && (inuse_reg->second & liveness::live)) { inuse_reg->second |= liveness::assign; @@ -710,7 +738,7 @@ void allocate_frame() ++i; // delete the now unused gckill information if (CallInst* callInst = dyn_cast<CallInst>(inst)) { - Value *callee = callInst->getCalledFunction(); + Value *callee = callInst->getCalledValue(); if (callee == gckill_func || callee == gcroot_flush_func) { callInst->eraseFromParent(); } @@ -719,7 +747,7 @@ void allocate_frame() else if (StoreInst *storeInst = dyn_cast<StoreInst>(inst)) { frame_register def = get_gcroot(storeInst->getPointerOperand()); if (CallInst *gcroot = def.first) { - if (gcroot->getCalledFunction() == jlcall_frame_func) { + if (gcroot->getCalledValue() == jlcall_frame_func) { std::map<frame_register, liveness::id> &inuse_list = bb_uses[storeInst->getParent()]; std::map<frame_register, liveness::id>::iterator inuse_reg = inuse_list.find(def); if (inuse_reg == inuse_list.end()) @@ -746,7 +774,7 @@ void allocate_frame() // finalize all of the jlcall frames by replacing all of the frames with the appropriate gep(tempslot) for (std::map<CallInst*, unsigned>::iterator frame = frame_offsets.begin(), framee = frame_offsets.end(); frame != framee; ++frame) { CallInst *gcroot = frame->first; - tbaa_decorate_gcframe(gcroot); + tbaa_decorate_gcframe(gcroot, tbaa_gcframe); Value* offset[1] = {ConstantInt::get(T_int32, frame->second)}; GetElementPtrInst *gep = GetElementPtrInst::Create(LLVM37_param(NULL) tempSlot, makeArrayRef(offset)); gep->insertAfter(last_gcframe_inst); @@ -773,7 +801,7 @@ void allocate_frame() Instruction* inst = &*I; ++I; if (CallInst* callInst = dyn_cast<CallInst>(inst)) { - if (callInst->getCalledFunction() == gcroot_func) { + if (callInst->getCalledValue() == gcroot_func) { unsigned offset = 2 + argSpaceSize++; Instruction *argTempi = GetElementPtrInst::Create(LLVM37_param(NULL) gcframe, ArrayRef<Value*>(ConstantInt::get(T_int32, offset))); argTempi->insertAfter(last_gcframe_inst); @@ -806,7 +834,7 @@ void allocate_frame() } } #endif - tbaa_decorate_gcframe(callInst); + tbaa_decorate_gcframe(callInst, tbaa_gcframe); callInst->replaceAllUsesWith(argTempi); argTempi->takeName(callInst); callInst->eraseFromParent(); @@ -896,11 +924,31 @@ void allocate_frame() #endif } -}; - -void jl_codegen_finalize_temp_arg(CallInst *ptlsStates, Type *T_pjlvalue, - MDNode *tbaa) +void jl_codegen_finalize_temp_arg(Function *F, MDNode *tbaa) { + Module *M = F->getParent(); + + Function *ptls_getter = M->getFunction("jl_get_ptls_states"); + if (!ptls_getter) + return; + + CallInst *ptlsStates = NULL; + for (auto I = F->getEntryBlock().begin(), E = F->getEntryBlock().end(); + I != E; ++I) { + if (CallInst *callInst = dyn_cast<CallInst>(&*I)) { + if (callInst->getCalledValue() == ptls_getter) { + ptlsStates = callInst; + break; + } + } + } + if (!ptlsStates) + return; + + FunctionType *functype = ptls_getter->getFunctionType(); + auto T_ppjlvalue = + cast<PointerType>(functype->getReturnType())->getElementType(); + auto T_pjlvalue = cast<PointerType>(T_ppjlvalue)->getElementType(); JuliaGCAllocator allocator(ptlsStates, T_pjlvalue, tbaa); allocator.allocate_frame(); } diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp new file mode 100644 index 0000000000000..5ebb3ce5d3ed5 --- /dev/null +++ b/src/llvm-ptls.cpp @@ -0,0 +1,193 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + +#define DEBUG_TYPE "lower_ptls" +#undef DEBUG + +// LLVM pass to optimize TLS access and remove references to julia intrinsics + +#include "llvm-version.h" +#include "support/dtypes.h" + +#include <llvm/Pass.h> +#include <llvm/IR/Module.h> +#include <llvm/IR/Function.h> +#include <llvm/IR/Instructions.h> +#include <llvm/IR/Constants.h> +#include <llvm/IR/LLVMContext.h> + +#include "julia.h" +#include "julia_internal.h" + +#if defined(LLVM37) && defined(JULIA_ENABLE_THREADING) +# include <llvm/IR/InlineAsm.h> +#endif + +using namespace llvm; + +namespace { + +struct LowerPTLS: public ModulePass { + static char ID; + LowerPTLS(bool _imaging_mode=false, MDNode *_tbaa_const=nullptr) + : ModulePass(ID), + imaging_mode(_imaging_mode), + tbaa_const(_tbaa_const) + {} + +private: + bool imaging_mode; + MDNode *tbaa_const; // One `LLVMContext` only + bool runOnModule(Module &M) override; + void runOnFunction(LLVMContext &ctx, Module &M, Function *F, + Function *ptls_getter, Type *T_ppjlvalue); +}; + +static void ensure_global(const char *name, Type *t, Module &M, + bool dllimport=false) +{ + if (M.getNamedValue(name)) + return; + GlobalVariable *proto = new GlobalVariable(M, t, false, + GlobalVariable::ExternalLinkage, + NULL, name); +#ifdef _OS_WINDOWS_ + // setting JL_DLLEXPORT correctly only matters when building a binary + // (global_proto will strip this from the JIT) + if (dllimport) { +#ifdef LLVM35 + // add the __declspec(dllimport) attribute + proto->setDLLStorageClass(GlobalValue::DLLImportStorageClass); +#else + proto->setLinkage(GlobalValue::DLLImportLinkage); +#endif + } +#else // _OS_WINDOWS_ + (void)proto; +#endif // _OS_WINDOWS_ +} + +void LowerPTLS::runOnFunction(LLVMContext &ctx, Module &M, Function *F, + Function *ptls_getter, Type *T_ppjlvalue) +{ + CallInst *ptlsStates = NULL; + for (auto I = F->getEntryBlock().begin(), E = F->getEntryBlock().end(); + I != E; ++I) { + if (CallInst *callInst = dyn_cast<CallInst>(&*I)) { + if (callInst->getCalledValue() == ptls_getter) { + ptlsStates = callInst; + break; + } + } + } + if (!ptlsStates) + return; + + if (ptlsStates->use_empty()) { + ptlsStates->eraseFromParent(); + return; + } + +#ifdef JULIA_ENABLE_THREADING + if (imaging_mode) { + GlobalVariable *GV = cast<GlobalVariable>( + M.getNamedValue("jl_get_ptls_states.ptr")); + LoadInst *getter = new LoadInst(GV, "", ptlsStates); + getter->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_const); + ptlsStates->setCalledFunction(getter); + ptlsStates->addAttribute(AttributeSet::FunctionIndex, + Attribute::ReadNone); + ptlsStates->addAttribute(AttributeSet::FunctionIndex, + Attribute::NoUnwind); + } + else if (jl_tls_offset != -1) { +#ifdef LLVM37 + auto T_int8 = Type::getInt8Ty(ctx); + auto T_pint8 = PointerType::get(T_int8, 0); + auto T_size = (sizeof(size_t) == 8 ? Type::getInt64Ty(ctx) : + Type::getInt32Ty(ctx)); + // Replace the function call with inline assembly if we know + // how to generate it. + const char *asm_str = nullptr; +# if defined(_CPU_X86_64_) + asm_str = "movq %fs:0, $0"; +# elif defined(_CPU_X86_) + asm_str = "movl %gs:0, $0"; +# elif defined(_CPU_AARCH64_) + asm_str = "mrs $0, tpidr_el0"; +# endif + assert(asm_str && "Cannot emit thread pointer for this architecture."); + auto offset = ConstantInt::getSigned(T_size, jl_tls_offset); + auto tp = InlineAsm::get(FunctionType::get(T_pint8, false), + asm_str, "=r", false); + Value *tls = CallInst::Create(tp, "thread_ptr", ptlsStates); + tls = GetElementPtrInst::Create(T_int8, tls, {offset}, + "ptls_i8", ptlsStates); + tls = new BitCastInst(tls, PointerType::get(T_ppjlvalue, 0), + "ptls", ptlsStates); + ptlsStates->replaceAllUsesWith(tls); + ptlsStates->eraseFromParent(); +#endif + } + else { + ptlsStates->addAttribute(AttributeSet::FunctionIndex, + Attribute::ReadNone); + ptlsStates->addAttribute(AttributeSet::FunctionIndex, + Attribute::NoUnwind); + } +#else + ptlsStates->replaceAllUsesWith(M.getNamedValue("jl_tls_states")); + ptlsStates->eraseFromParent(); +#endif +} + +static void eraseFunction(Module &M, const char *name) +{ + if (Function *f = M.getFunction(name)) { + f->eraseFromParent(); + } +} + +bool LowerPTLS::runOnModule(Module &M) +{ + // Cleanup for GC frame lowering. + eraseFunction(M, "julia.gc_root_decl"); + eraseFunction(M, "julia.gc_root_kill"); + eraseFunction(M, "julia.jlcall_frame_decl"); + eraseFunction(M, "julia.gcroot_flush"); + + Function *ptls_getter = M.getFunction("jl_get_ptls_states"); + if (!ptls_getter) + return true; + LLVMContext &ctx = M.getContext(); + FunctionType *functype = ptls_getter->getFunctionType(); + auto T_ppjlvalue = + cast<PointerType>(functype->getReturnType())->getElementType(); +#ifdef JULIA_ENABLE_THREADING + if (imaging_mode) + ensure_global("jl_get_ptls_states.ptr", functype->getPointerTo(), M); +#else + ensure_global("jl_tls_states", T_ppjlvalue, M, imaging_mode); +#endif + for (auto F = M.begin(), E = M.end(); F != E; ++F) { + if (F->isDeclaration()) + continue; + runOnFunction(ctx, M, &*F, ptls_getter, T_ppjlvalue); + } +#ifndef JULIA_ENABLE_THREADING + ptls_getter->eraseFromParent(); +#endif + return true; +} + +char LowerPTLS::ID = 0; + +static RegisterPass<LowerPTLS> X("LowerPTLS", "LowerPTLS Pass", + false /* Only looks at CFG */, + false /* Analysis Pass */); + +} // anonymous namespace + +Pass *createLowerPTLSPass(bool imaging_mode, MDNode *tbaa_const) +{ + return new LowerPTLS(imaging_mode, tbaa_const); +} From 3b7f1112ba6deebf97bf3504e84422b97f985321 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sun, 10 Jul 2016 16:34:32 -0400 Subject: [PATCH 0560/1117] Make GC frame lowering a LLVM pass --- src/ccall.cpp | 2 +- src/codegen.cpp | 14 +-------- src/jitlayers.cpp | 6 ++-- src/jitlayers.h | 8 ++++- src/llvm-gcroot.cpp | 74 +++++++++++++++++++++++++++++++-------------- 5 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 6bcb44d61d5de..f078cd36d2ca3 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -761,7 +761,7 @@ class FunctionMover : public ValueMaterializer { Function *F = dyn_cast<Function>(V); if (F) { - if (F->isIntrinsic()) { + if (isIntrinsicFunction(F)) { return destModule->getOrInsertFunction(F->getName(),F->getFunctionType()); } if (F->isDeclaration() || F->getParent() != destModule) { diff --git a/src/codegen.cpp b/src/codegen.cpp index ce7cc3d0e5c54..e9650399eb65c 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -240,7 +240,7 @@ static Type *T_pppint8; static Type *T_void; // type-based alias analysis nodes. Indentation of comments indicates hierarchy. -static MDNode *tbaa_gcframe; // GC frame +MDNode *tbaa_gcframe; // GC frame // LLVM should have enough info for alias analysis of non-gcframe stack slot // this is mainly a place holder for `jl_cgval_t::tbaa` static MDNode *tbaa_stack; // stack slot @@ -1102,7 +1102,6 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) Function *f, *specf; jl_llvm_functions_t declarations; std::unique_ptr<Module> m = emit_function(temp ? temp : linfo, &declarations); - finalize_gc_frame(m.get()); jl_globalPM->run(*m.get()); f = (llvm::Function*)declarations.functionObject; specf = (llvm::Function*)declarations.specFunctionObject; @@ -3404,17 +3403,6 @@ static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx) PointerType::get(T_psize, 0)); } -void jl_codegen_finalize_temp_arg(Function *F, MDNode *tbaa_gcframe); -void finalize_gc_frame(Module *m) -{ - for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { - Function *F = &*I; - if (F->isDeclaration()) - continue; - jl_codegen_finalize_temp_arg(F, tbaa_gcframe); - } -} - static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_tupletype_t *argt, jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt) { diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 9047188084568..474f4e5feb59e 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -97,6 +97,7 @@ void addOptimizationPasses(legacy::PassManager *PM) void addOptimizationPasses(PassManager *PM) #endif { + PM->add(createLowerGCFramePass(tbaa_gcframe)); #ifdef JL_DEBUG_BUILD PM->add(createVerifierPass()); #endif @@ -481,7 +482,7 @@ void JuliaOJIT::addModule(std::unique_ptr<Module> M) if (F->isDeclaration()) { if (F->use_empty()) F->eraseFromParent(); - else if (!(F->isIntrinsic() || + else if (!(isIntrinsicFunction(F) || findUnmangledSymbol(F->getName()) || SectionMemoryManager::getSymbolAddressInProcess( F->getName()))) { @@ -742,7 +743,7 @@ static void jl_finalize_function(const std::string &F, Module *collector) if (!F->isDeclaration()) { module_for_fname.erase(F->getName()); } - else if (!F->isIntrinsic()) { + else if (!isIntrinsicFunction(F)) { to_finalize.push_back(F->getName().str()); } } @@ -784,7 +785,6 @@ void jl_finalize_function(Function *F, Module *collector) // and will add it to the execution engine when required (by jl_finalize_function) void jl_finalize_module(Module *m, bool shadow) { - finalize_gc_frame(m); #if !defined(USE_ORCJIT) jl_globalPM->run(*m); #endif diff --git a/src/jitlayers.h b/src/jitlayers.h index 786aaefe2e522..cc27c6a84cdba 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -64,7 +64,6 @@ GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name, void* jl_get_global(GlobalVariable *gv); GlobalVariable *jl_get_global_for(const char *cname, void *addr, Module *M); void jl_add_to_shadow(Module *m); -void finalize_gc_frame(Module *m); void jl_finalize_function(Function *F, Module *collector = NULL); void jl_finalize_module(Module *m, bool shadow); @@ -220,5 +219,12 @@ JL_DLLEXPORT extern LLVMContext &jl_LLVMContext; #endif extern MDNode *tbaa_const; +extern MDNode *tbaa_gcframe; Pass *createLowerPTLSPass(bool imaging_mode, MDNode *tbaa_const); +Pass *createLowerGCFramePass(MDNode *tbaa_gcframe); +// Whether the Function is an llvm or julia intrinsic. +static inline bool isIntrinsicFunction(Function *F) +{ + return F->isIntrinsic() || F->getName().startswith("julia."); +} diff --git a/src/llvm-gcroot.cpp b/src/llvm-gcroot.cpp index ba994bf729abc..74facedd86866 100644 --- a/src/llvm-gcroot.cpp +++ b/src/llvm-gcroot.cpp @@ -1,5 +1,8 @@ // This file is a part of Julia. License is MIT: http://julialang.org/license +#define DEBUG_TYPE "lower_gcroot" +#undef DEBUG + #include "llvm-version.h" #include <llvm/ADT/SmallBitVector.h> #include <llvm/IR/Value.h> @@ -29,6 +32,8 @@ using namespace llvm; +namespace { + #ifndef NDEBUG static struct { unsigned count; @@ -67,22 +72,6 @@ struct liveness { }; }; -#ifndef NDEBUG // llvm assertions build -// gdb debugging code for inspecting the bb_uses map -void jl_dump_bb_uses(std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses) -{ - for (std::map<BasicBlock*, std::map<frame_register, liveness::id> >::iterator - live_reg = bb_uses.begin(), e = bb_uses.end(); live_reg != e; ++live_reg) { - BasicBlock *bb = live_reg->first; - errs() << '\n' << bb << '\n'; - for (std::map<frame_register, liveness::id>::iterator - regs = live_reg->second.begin(), regse = live_reg->second.end(); regs != regse; ++regs) { - errs() << regs->second << " #" << regs->first.second << ' ' << regs->first.first << '\n'; - } - } -} -#endif - static void tbaa_decorate_gcframe(Instruction *inst, std::set<Instruction*> &visited, MDNode *tbaa_gcframe) @@ -924,16 +913,28 @@ void JuliaGCAllocator::allocate_frame() #endif } -void jl_codegen_finalize_temp_arg(Function *F, MDNode *tbaa) +struct LowerGCFrame: public FunctionPass { + static char ID; + LowerGCFrame(MDNode *_tbaa_gcframe=nullptr) + : FunctionPass(ID), + tbaa_gcframe(_tbaa_gcframe) + {} + +private: + MDNode *tbaa_gcframe; // One `LLVMContext` only + bool runOnFunction(Function &F) override; +}; + +bool LowerGCFrame::runOnFunction(Function &F) { - Module *M = F->getParent(); + Module *M = F.getParent(); Function *ptls_getter = M->getFunction("jl_get_ptls_states"); if (!ptls_getter) - return; + return true; CallInst *ptlsStates = NULL; - for (auto I = F->getEntryBlock().begin(), E = F->getEntryBlock().end(); + for (auto I = F.getEntryBlock().begin(), E = F.getEntryBlock().end(); I != E; ++I) { if (CallInst *callInst = dyn_cast<CallInst>(&*I)) { if (callInst->getCalledValue() == ptls_getter) { @@ -943,12 +944,41 @@ void jl_codegen_finalize_temp_arg(Function *F, MDNode *tbaa) } } if (!ptlsStates) - return; + return true; FunctionType *functype = ptls_getter->getFunctionType(); auto T_ppjlvalue = cast<PointerType>(functype->getReturnType())->getElementType(); auto T_pjlvalue = cast<PointerType>(T_ppjlvalue)->getElementType(); - JuliaGCAllocator allocator(ptlsStates, T_pjlvalue, tbaa); + JuliaGCAllocator allocator(ptlsStates, T_pjlvalue, tbaa_gcframe); allocator.allocate_frame(); + return true; +} + +char LowerGCFrame::ID = 0; + +static RegisterPass<LowerGCFrame> X("LowerGCFrame", "Lower GCFrame Pass", + false /* Only looks at CFG */, + false /* Analysis Pass */); +} + +#ifndef NDEBUG // llvm assertions build +// gdb debugging code for inspecting the bb_uses map +void jl_dump_bb_uses(std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses) +{ + for (std::map<BasicBlock*, std::map<frame_register, liveness::id> >::iterator + live_reg = bb_uses.begin(), e = bb_uses.end(); live_reg != e; ++live_reg) { + BasicBlock *bb = live_reg->first; + errs() << '\n' << bb << '\n'; + for (std::map<frame_register, liveness::id>::iterator + regs = live_reg->second.begin(), regse = live_reg->second.end(); regs != regse; ++regs) { + errs() << regs->second << " #" << regs->first.second << ' ' << regs->first.first << '\n'; + } + } +} +#endif + +Pass *createLowerGCFramePass(MDNode *tbaa_gcframe) +{ + return new LowerGCFrame(tbaa_gcframe); } From 269950fc17ef1d7d240055af41f536aada416c58 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 21 Jul 2016 08:30:38 -0400 Subject: [PATCH 0561/1117] Add fallback method to `directsubtype` to handle `TypeVar` Fix #17529 --- base/show.jl | 8 +++++--- test/show.jl | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/base/show.jl b/base/show.jl index d85312643efbc..6d37e300ebb93 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1157,6 +1157,8 @@ end directsubtype(a::DataType, b::DataType) = supertype(a).name === b.name directsubtype(a::TypeConstructor, b::DataType) = directsubtype(a.body, b) directsubtype(a::Union, b::DataType) = any(t->directsubtype(t, b), a.types) +# Fallback to handle TypeVar's +directsubtype(a, b::DataType) = false function dumpsubtypes(io::IO, x::DataType, m::Module, n::Int, indent) for s in names(m, true) if isdefined(m, s) && !isdeprecated(m, s) @@ -1166,15 +1168,15 @@ function dumpsubtypes(io::IO, x::DataType, m::Module, n::Int, indent) elseif isa(t, Module) && module_name(t) === s && module_parent(t) === m # recurse into primary module bindings dumpsubtypes(io, x, t, n, indent) - elseif isa(t, TypeConstructor) && directsubtype(t, x) + elseif isa(t, TypeConstructor) && directsubtype(t::TypeConstructor, x) println(io) print(io, indent, " ", m, ".", s) isempty(t.parameters) || print(io, "{", join(t.parameters, ","), "}") print(io, " = ", t) - elseif isa(t, Union) && directsubtype(t, x) + elseif isa(t, Union) && directsubtype(t::Union, x) println(io) print(io, indent, " ", m, ".", s, " = ", t) - elseif isa(t, DataType) && directsubtype(t, x) + elseif isa(t, DataType) && directsubtype(t::DataType, x) println(io) if t.name.module !== m || t.name.name != s # aliases to types diff --git a/test/show.jl b/test/show.jl index 91cbc7824e2b5..41b7ade6e5a2e 100644 --- a/test/show.jl +++ b/test/show.jl @@ -514,6 +514,8 @@ end let repr = sprint(dump, Int64) @test repr == "Int64 <: Signed\n" end +# Make sure a `TypeVar` in a `Union` doesn't break subtype dump. +typealias BreakDump17529{T} Union{T,Void} let repr = sprint(dump, Any) @test length(repr) > 100000 @test ismatch(r"^Any\n [^ \t\n]", repr) From 0e1f599da76091df32641de91ef3b2ab20fef3e4 Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Thu, 21 Jul 2016 00:23:31 +0200 Subject: [PATCH 0562/1117] Use literal_pointer_val for loading from global variables where possible. This avoids having to load from the variable when not in imaging mode, directly using its value instead. For example, for a throw(inexacterror) generated by codegen: Before: movabsq $140179715170792, %rax # imm = 0x7F7E2220A1E8 movq (%rax), %rdi movabsq $jl_throw, %rax callq *%rax After: movabsq $jl_throw, %rax movabsq $140289690340176, %rdi # imm = 0x7F97BD288B50 callq *%rax --- src/cgutils.cpp | 24 +++++++++++++++++------- src/codegen.cpp | 2 +- src/intrinsics.cpp | 24 ++++++++++++------------ 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index e5ab5f75eaed5..d24e2e475444e 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -214,6 +214,8 @@ static Value *literal_pointer_val(jl_value_t *p) // also, try to give it a nice name for gdb, for easy identification if (p == NULL) return ConstantPointerNull::get((PointerType*)T_pjlvalue); + if (!imaging_mode) + return literal_static_pointer_val(p, T_pjlvalue); // some common constant values if (p == jl_false) return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlfalse_var))); @@ -221,8 +223,17 @@ static Value *literal_pointer_val(jl_value_t *p) return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jltrue_var))); if (p == (jl_value_t*)jl_emptysvec) return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlemptysvec_var))); - if (!imaging_mode) - return literal_static_pointer_val(p, T_pjlvalue); + // exceptions + if (p == (jl_value_t*)jl_diverror_exception) + return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jldiverr_var))); + if (p == (jl_value_t*)jl_undefref_exception) + return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlundeferr_var))); + if (p == (jl_value_t*)jl_domain_exception) + return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jldomerr_var))); + if (p == (jl_value_t*)jl_overflow_exception) + return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlovferr_var))); + if (p == (jl_value_t*)jl_inexact_exception) + return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlinexacterr_var))); if (jl_is_datatype(p)) { jl_datatype_t *addr = (jl_datatype_t*)p; // DataTypes are prefixed with a + @@ -684,7 +695,7 @@ static void raise_exception_if(Value *cond, GlobalVariable *exc, jl_codectx_t *c static void null_pointer_check(Value *v, jl_codectx_t *ctx) { raise_exception_unless(builder.CreateICmpNE(v,Constant::getNullValue(v->getType())), - prepare_global(jlundeferr_var), ctx); + literal_pointer_val(jl_undefref_exception), ctx); } static void emit_type_error(const jl_cgval_t &x, jl_value_t *type, const std::string &msg, @@ -909,9 +920,8 @@ static void typed_store(Value *ptr, Value *idx_0based, const jl_cgval_t &rhs, static Value *julia_bool(Value *cond) { - return builder.CreateSelect(cond, - tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jltrue_var))), - tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlfalse_var)))); + return builder.CreateSelect(cond, literal_pointer_val(jl_true), + literal_pointer_val(jl_false)); } // --- get the inferred type of an AST node --- @@ -1069,7 +1079,7 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, Type *elty = julia_type_to_llvm(jfty); assert(elty != NULL); if (jfty == jl_bottom_type) { - raise_exception(prepare_global(jlundeferr_var), ctx); + raise_exception(literal_pointer_val(jl_undefref_exception), ctx); return jl_cgval_t(); // unreachable } if (type_is_ghost(elty)) diff --git a/src/codegen.cpp b/src/codegen.cpp index e9650399eb65c..1c2ab5200bbbf 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3034,7 +3034,7 @@ static Value *emit_condition(const jl_cgval_t &condV, const std::string &msg, } emit_typecheck(condV, (jl_value_t*)jl_bool_type, msg, ctx); if (condV.isboxed) { - return builder.CreateICmpEQ(boxed(condV, ctx), tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlfalse_var)))); + return builder.CreateICmpEQ(boxed(condV, ctx), literal_pointer_val(jl_false)); } // not a boolean return ConstantInt::get(T_int1,0); // TODO: replace with Undef diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 211d14f5fdb24..444a76b853f7d 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -627,7 +627,7 @@ static jl_cgval_t generic_trunc(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c Value *back = signd ? builder.CreateSExt(ans, ix->getType()) : builder.CreateZExt(ans, ix->getType()); raise_exception_unless(builder.CreateICmpEQ(back, ix), - prepare_global(jlinexacterr_var), ctx); + literal_pointer_val(jl_inexact_exception), ctx); } return mark_julia_type(ans, false, jlto, ctx); } @@ -700,10 +700,10 @@ static jl_cgval_t emit_checked_fptosi(jl_value_t *targ, jl_value_t *x, jl_codect raise_exception_unless (builder.CreateFCmpOEQ(builder.CreateFPExt(fx, T_float64), builder.CreateSIToFP(ans, T_float64)), - prepare_global(jlinexacterr_var), ctx); + literal_pointer_val(jl_inexact_exception), ctx); } else { - raise_exception_unless(emit_eqfsi(fx, ans), prepare_global(jlinexacterr_var), ctx); + raise_exception_unless(emit_eqfsi(fx, ans), literal_pointer_val(jl_inexact_exception), ctx); } return mark_julia_type(ans, false, jlto, ctx); } @@ -720,10 +720,10 @@ static jl_cgval_t emit_checked_fptoui(jl_value_t *targ, jl_value_t *x, jl_codect raise_exception_unless (builder.CreateFCmpOEQ(builder.CreateFPExt(fx, T_float64), builder.CreateUIToFP(ans, T_float64)), - prepare_global(jlinexacterr_var), ctx); + literal_pointer_val(jl_inexact_exception), ctx); } else { - raise_exception_unless(emit_eqfui(fx, ans), prepare_global(jlinexacterr_var), ctx); + raise_exception_unless(emit_eqfui(fx, ans), literal_pointer_val(jl_inexact_exception), ctx); } return mark_julia_type(ans, false, jlto, ctx); } @@ -866,7 +866,7 @@ static Value *emit_checked_srem_int(Value *x, Value *den, jl_codectx_t *ctx) { Type *t = den->getType(); raise_exception_unless(builder.CreateICmpNE(den, ConstantInt::get(t,0)), - prepare_global(jldiverr_var), ctx); + literal_pointer_val(jl_diverror_exception), ctx); BasicBlock *m1BB = BasicBlock::Create(jl_LLVMContext,"minus1",ctx->f); BasicBlock *okBB = BasicBlock::Create(jl_LLVMContext,"oksrem",ctx->f); BasicBlock *cont = BasicBlock::Create(jl_LLVMContext,"after_srem",ctx->f); @@ -1263,7 +1263,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Value *res = builder.CreateCall2(intr, ix, iy); #endif Value *obit = builder.CreateExtractValue(res, ArrayRef<unsigned>(1)); - raise_exception_if(obit, prepare_global(jlovferr_var), ctx); + raise_exception_if(obit, literal_pointer_val(jl_overflow_exception), ctx); return builder.CreateExtractValue(res, ArrayRef<unsigned>(0)); } @@ -1282,14 +1282,14 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, CreateICmpNE(den, ConstantInt::get(t,-1,true)), builder.CreateICmpNE(x, typemin))), - prepare_global(jldiverr_var), ctx); + literal_pointer_val(jl_diverror_exception), ctx); return builder.CreateSDiv(x, den); case checked_udiv_int: den = JL_INT(y); t = den->getType(); raise_exception_unless(builder.CreateICmpNE(den, ConstantInt::get(t,0)), - prepare_global(jldiverr_var), ctx); + literal_pointer_val(jl_diverror_exception), ctx); return builder.CreateUDiv(JL_INT(x), den); case checked_srem_int: @@ -1299,7 +1299,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, den = JL_INT(y); t = den->getType(); raise_exception_unless(builder.CreateICmpNE(den, ConstantInt::get(t,0)), - prepare_global(jldiverr_var), ctx); + literal_pointer_val(jl_diverror_exception), ctx); return builder.CreateURem(JL_INT(x), den); case check_top_bit: @@ -1309,7 +1309,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, CreateTrunc(builder. CreateLShr(x, ConstantInt::get(t, t->getPrimitiveSizeInBits()-1)), T_int1), - prepare_global(jlinexacterr_var), ctx); + literal_pointer_val(jl_inexact_exception), ctx); return x; case eq_int: *newtyp = jl_bool_type; return builder.CreateICmpEQ(JL_INT(x), JL_INT(y)); @@ -1510,7 +1510,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, case sqrt_llvm: { x = FP(x); raise_exception_unless(builder.CreateFCmpUGE(x, ConstantFP::get(x->getType(),0.0)), - prepare_global(jldomerr_var), ctx); + literal_pointer_val(jl_domain_exception), ctx); return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::sqrt, ArrayRef<Type*>(x->getType())), x); From 34fe9b4f9c86364c55c141bb3cbe48d19b8ca3ca Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Thu, 21 Jul 2016 15:21:55 +0200 Subject: [PATCH 0563/1117] Remove now unused raise methods directly using GVs. --- src/cgutils.cpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index d24e2e475444e..c1d7e52b05c89 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -656,12 +656,6 @@ static void raise_exception(Value *exc, jl_codectx_t *ctx, builder.SetInsertPoint(contBB); } -static void raise_exception(GlobalVariable *exc, jl_codectx_t *ctx) -{ - raise_exception((Value*)tbaa_decorate(tbaa_const, - builder.CreateLoad(exc)), ctx); -} - // DO NOT PASS IN A CONST CONDITION! static void raise_exception_unless(Value *cond, Value *exc, jl_codectx_t *ctx) { @@ -672,13 +666,6 @@ static void raise_exception_unless(Value *cond, Value *exc, jl_codectx_t *ctx) raise_exception(exc, ctx, passBB); } -// DO NOT PASS IN A CONST CONDITION! -static void raise_exception_unless(Value *cond, GlobalVariable *exc, - jl_codectx_t *ctx) -{ - raise_exception_unless(cond, (Value*)tbaa_decorate(tbaa_const,builder.CreateLoad(exc, false)), ctx); -} - // DO NOT PASS IN A CONST CONDITION! static void raise_exception_if(Value *cond, Value *exc, jl_codectx_t *ctx) { @@ -686,12 +673,6 @@ static void raise_exception_if(Value *cond, Value *exc, jl_codectx_t *ctx) exc, ctx); } -// DO NOT PASS IN A CONST CONDITION! -static void raise_exception_if(Value *cond, GlobalVariable *exc, jl_codectx_t *ctx) -{ - raise_exception_if(cond, (Value*)tbaa_decorate(tbaa_const, builder.CreateLoad(exc, false)), ctx); -} - static void null_pointer_check(Value *v, jl_codectx_t *ctx) { raise_exception_unless(builder.CreateICmpNE(v,Constant::getNullValue(v->getType())), From 5d543687a543ce3add48d8e5d4f512283540fa48 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Thu, 21 Jul 2016 10:22:24 -0400 Subject: [PATCH 0564/1117] treat .= as syntactic sugar for broadcast! (#17510) * treat .= as syntactic sugar for broadcast! * tests * optimized .= assignment of scalars and vector copies * .= documentation * fix show of .= ops * .-= tests * NEWS for .= --- NEWS.md | 6 +- base/broadcast.jl | 9 ++ base/show.jl | 6 +- doc/manual/arrays.rst | 2 +- doc/manual/functions.rst | 13 ++- doc/manual/performance-tips.rst | 5 +- src/julia-syntax.scm | 177 +++++++++++++++++--------------- test/broadcast.jl | 19 ++++ test/show.jl | 4 + 9 files changed, 153 insertions(+), 88 deletions(-) diff --git a/NEWS.md b/NEWS.md index 0a09d07dd81ea..95e5ce6a5ff93 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,8 +10,11 @@ New language features * Generators and comprehensions support filtering using `if` ([#550]) and nested iteration using multiple `for` keywords ([#4867]). - * Broadcasting syntax: ``f.(args...)`` is equivalent to ``broadcast(f, args...)`` ([#15032]), + * Fused broadcasting syntax: ``f.(args...)`` is equivalent to ``broadcast(f, args...)`` ([#15032]), and nested `f.(g.(args...))` calls are fused into a single `broadcast` loop ([#17300]). + Similarly, the syntax `x .= ...` is equivalent to a `broadcast!(identity, x, ...)` + call and fuses with nested "dot" calls; also, `x .+= y` and similar is now + equivalent to `x .= x .+ y`, rather than `=` ([#17510]). * Macro expander functions are now generic, so macros can have multiple definitions (e.g. for different numbers of arguments, or optional arguments) ([#8846], [#9627]). @@ -357,3 +360,4 @@ Deprecated or removed [#17393]: https://github.com/JuliaLang/julia/issues/17393 [#17402]: https://github.com/JuliaLang/julia/issues/17402 [#17404]: https://github.com/JuliaLang/julia/issues/17404 +[#17510]: https://github.com/JuliaLang/julia/issues/17510 diff --git a/base/broadcast.jl b/base/broadcast.jl index 9f50f4d9c0b2a..d577857b62937 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -15,6 +15,15 @@ export broadcast_getindex, broadcast_setindex! broadcast(f) = f() broadcast(f, x::Number...) = f(x...) +# special cases for "X .= ..." (broadcast!) assignments +broadcast!(::typeof(identity), X::AbstractArray, x::Number) = fill!(X, x) +broadcast!(f, X::AbstractArray) = fill!(X, f()) +broadcast!(f, X::AbstractArray, x::Number...) = fill!(X, f(x...)) +function broadcast!{T,S,N}(::typeof(identity), x::AbstractArray{T,N}, y::AbstractArray{S,N}) + check_broadcast_shape(size(x), size(y)) + copy!(x, y) +end + ## Calculate the broadcast shape of the arguments, or error if incompatible # array inputs broadcast_shape() = () diff --git a/base/show.jl b/base/show.jl index 6d37e300ebb93..5d0bf1cf6e036 100644 --- a/base/show.jl +++ b/base/show.jl @@ -400,8 +400,10 @@ show_unquoted(io::IO, ex, ::Int,::Int) = show(io, ex) const indent_width = 4 const quoted_syms = Set{Symbol}([:(:),:(::),:(:=),:(=),:(==),:(!=),:(===),:(!==),:(=>),:(>=),:(<=)]) const uni_ops = Set{Symbol}([:(+), :(-), :(!), :(¬), :(~), :(<:), :(>:), :(√), :(∛), :(∜)]) -const expr_infix_wide = Set{Symbol}([:(=), :(+=), :(-=), :(*=), :(/=), :(\=), :(&=), - :(|=), :($=), :(>>>=), :(>>=), :(<<=), :(&&), :(||), :(<:), :(=>), :(÷=)]) +const expr_infix_wide = Set{Symbol}([ + :(=), :(+=), :(-=), :(*=), :(/=), :(\=), :(^=), :(&=), :(|=), :(÷=), :(%=), :(>>>=), :(>>=), :(<<=), + :(.=), :(.+=), :(.-=), :(.*=), :(./=), :(.\=), :(.^=), :(.&=), :(.|=), :(.÷=), :(.%=), :(.>>>=), :(.>>=), :(.<<=), + :(&&), :(||), :(<:), :(=>), :($=)]) const expr_infix = Set{Symbol}([:(:), :(->), Symbol("::")]) const expr_infix_any = union(expr_infix, expr_infix_wide) const all_ops = union(quoted_syms, uni_ops, expr_infix_any) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index ed6ea4bf864f2..6a718926416ea 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -566,7 +566,7 @@ function elementwise: 1.71056 0.847604 1.73659 0.873631 -Elementwise operators such as ``.+`` and ``.*`` perform broadcasting if necessary. There is also a :func:`broadcast!` function to specify an explicit destination, and :func:`broadcast_getindex` and :func:`broadcast_setindex!` that broadcast the indices before indexing. Moreover, ``f.(args...)`` is equivalent to ``broadcast(f, args...)``, providing a convenient syntax to broadcast any function (:ref:`man-dot-vectorizing`:.). +Elementwise operators such as ``.+`` and ``.*`` perform broadcasting if necessary. There is also a :func:`broadcast!` function to specify an explicit destination, and :func:`broadcast_getindex` and :func:`broadcast_setindex!` that broadcast the indices before indexing. Moreover, ``f.(args...)`` is equivalent to ``broadcast(f, args...)``, providing a convenient syntax to broadcast any function (:ref:`man-dot-vectorizing`:). Implementation -------------- diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 0b1b77aef8102..e898f663201ef 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -652,9 +652,20 @@ the fusion stops as soon as a "non-dot" function is encountered; for example, in ``sin.(sort(cos.(X)))`` the ``sin`` and ``cos`` loops cannot be merged because of the intervening ``sort`` function. +Finally, the maximum efficiency is typically achieved when the output +array of a vectorized operation is *pre-allocated*, so that repeated +calls do not allocate new arrays over and over again for the results +(:ref:`man-preallocation`:). A convenient syntax for this is +``X .= ...``, which is equivalent to ``broadcast!(identity, X, ...)`` +except that, as above, the ``broadcast!`` loop is fused with any nested +"dot" calls. For example, ``X .= sin.(Y)`` is equivalent to +``broadcast!(sin, X, Y)``, overwriting ``X`` with ``sin.(Y)`` in-place. + (In future versions of Julia, operators like ``.*`` will also be handled with the same mechanism: they will be equivalent to ``broadcast`` calls and -will be fused with other nested "dot" calls.) +will be fused with other nested "dot" calls. ``x .+= y`` is equivalent +to ``x .= x .+ y`` and will eventually result in a fused in-place assignment. +Similarly for ``.*=`` etcetera.) Further Reading --------------- diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index eb092420787bc..f98db0d5b8130 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -944,7 +944,10 @@ above, we could have passed a :class:`SubArray` rather than an :class:`Array`, had we so desired. Taken to its extreme, pre-allocation can make your code uglier, so -performance measurements and some judgment may be required. +performance measurements and some judgment may be required. However, +for "vectorized" (element-wise) functions, the convenient syntax +``x .= f.(y)`` can be used for in-place operations with fused loops +and no temporary arrays (:ref:`dot-vectorizing`). Avoid string interpolation for I/O diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 06300e28c3837..76cf8956737af 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1418,12 +1418,12 @@ `(call ,(cadr e) ,(expand-forms a) ,(expand-forms b)))))) ;; convert `a+=b` to `a=a+b` -(define (expand-update-operator- op lhs rhs declT) +(define (expand-update-operator- op op= lhs rhs declT) (let ((e (remove-argument-side-effects lhs))) `(block ,@(cdr e) ,(if (null? declT) - `(= ,(car e) (call ,op ,(car e) ,rhs)) - `(= ,(car e) (call ,op (:: ,(car e) ,(car declT)) ,rhs)))))) + `(,op= ,(car e) (call ,op ,(car e) ,rhs)) + `(,op= ,(car e) (call ,op (:: ,(car e) ,(car declT)) ,rhs)))))) (define (partially-expand-ref e) (let ((a (cadr e)) @@ -1443,7 +1443,7 @@ ,@(append stmts stuff) (call getindex ,arr ,@new-idxs)))))) -(define (expand-update-operator op lhs rhs . declT) +(define (expand-update-operator op op= lhs rhs . declT) (cond ((and (pair? lhs) (eq? (car lhs) 'ref)) ;; expand indexing inside op= first, to remove "end" and ":" (let* ((ex (partially-expand-ref lhs)) @@ -1451,23 +1451,24 @@ (refex (last (cdr ex))) (nuref `(ref ,(caddr refex) ,@(cdddr refex)))) `(block ,@stmts - ,(expand-update-operator- op nuref rhs declT)))) + ,(expand-update-operator- op op= nuref rhs declT)))) ((and (pair? lhs) (eq? (car lhs) '|::|)) ;; (+= (:: x T) rhs) (let ((e (remove-argument-side-effects (cadr lhs))) (T (caddr lhs))) `(block ,@(cdr e) - ,(expand-update-operator op (car e) rhs T)))) + ,(expand-update-operator op op= (car e) rhs T)))) (else - (expand-update-operator- op lhs rhs declT)))) + (expand-update-operator- op op= lhs rhs declT)))) (define (lower-update-op e) (expand-forms - (expand-update-operator - (let ((str (string (car e)))) - (symbol (string.sub str 0 (- (length str) 1)))) - (cadr e) - (caddr e)))) + (let ((str (string (car e)))) + (expand-update-operator + (symbol (string.sub str 0 (- (length str) 1))) + (if (= (string.char str 0) #\.) '.= '=) + (cadr e) + (caddr e))))) (define (expand-and e) (let ((e (cdr (flatten-ex '&& e)))) @@ -1546,11 +1547,9 @@ (cadr expr) ;; eta reduce `x->f(x)` => `f` `(-> ,argname (block ,@splat ,expr))))) -(define (getfield-field? x) ; whether x from (|.| f x) is a getfield call - (or (eq? (car x) 'quote) (eq? (car x) 'inert) (eq? (car x) '$))) - -;; fuse nested calls to f.(args...) into a single broadcast call -(define (expand-fuse-broadcast f args) +; fuse nested calls to expr == f.(args...) into a single broadcast call, +; or a broadcast! call if lhs is non-null. +(define (expand-fuse-broadcast lhs rhs) (define (fuse? e) (and (pair? e) (eq? (car e) 'fuse))) (define (anyfuse? exprs) (if (null? exprs) #f (if (fuse? (car exprs)) #t (anyfuse? (cdr exprs))))) @@ -1594,28 +1593,31 @@ oldarg)) fargs args))) (let ,fbody ,@(reverse (fuse-lets fargs args '())))))) - (define (make-fuse f args) ; check for nested (fuse f args) exprs and combine - (define (split-kwargs args) ; return (cons keyword-args positional-args) extracted from args - (define (sk args kwargs pargs) - (if (null? args) - (cons kwargs pargs) - (if (kwarg? (car args)) - (sk (cdr args) (cons (car args) kwargs) pargs) - (sk (cdr args) kwargs (cons (car args) pargs))))) - (if (has-parameters? args) - (sk (reverse (cdr args)) (cdar args) '()) - (sk (reverse args) '() '()))) - (define (dot-to-fuse e) ; convert e == (. f (tuple args)) to (fuse f args) - (if (and (pair? e) (eq? (car e) '|.|) (not (getfield-field? (caddr e)))) - (make-fuse (cadr e) (cdaddr e)) - e)) - (let* ((kws.args (split-kwargs args)) - (kws (car kws.args)) - (args (cdr kws.args)) ; fusing occurs on positional args only - (args_ (map dot-to-fuse args))) - (if (anyfuse? args_) - `(fuse ,(fuse-funcs (to-lambda f args kws) args_) ,(fuse-args args_)) - `(fuse ,(to-lambda f args kws) ,args_)))) + (define (dot-to-fuse e) ; convert e == (. f (tuple args)) to (fuse f args) + (define (make-fuse f args) ; check for nested (fuse f args) exprs and combine + (define (split-kwargs args) ; return (cons keyword-args positional-args) extracted from args + (define (sk args kwargs pargs) + (if (null? args) + (cons kwargs pargs) + (if (kwarg? (car args)) + (sk (cdr args) (cons (car args) kwargs) pargs) + (sk (cdr args) kwargs (cons (car args) pargs))))) + (if (has-parameters? args) + (sk (reverse (cdr args)) (cdar args) '()) + (sk (reverse args) '() '()))) + (let* ((kws.args (split-kwargs args)) + (kws (car kws.args)) + (args (cdr kws.args)) ; fusing occurs on positional args only + (args_ (map dot-to-fuse args))) + (if (anyfuse? args_) + `(fuse ,(fuse-funcs (to-lambda f args kws) args_) ,(fuse-args args_)) + `(fuse ,(to-lambda f args kws) ,args_)))) + (if (and (pair? e) (eq? (car e) '|.|)) + (let ((f (cadr e)) (x (caddr e))) + (if (or (eq? (car x) 'quote) (eq? (car x) 'inert) (eq? (car x) '$)) + `(call (core getfield) ,f ,x) + (make-fuse f (cdr x)))) + e)) ; given e == (fuse lambda args), compress the argument list by removing (pure) ; duplicates in args, inlining literals, and moving any varargs to the end: (define (compress-fuse e) @@ -1623,43 +1625,51 @@ (if (eq? arg (car args)) (car fargs) (findfarg arg (cdr args) (cdr fargs)))) - (let ((f (cadr e)) - (args (caddr e))) - (define (cf old-fargs old-args new-fargs new-args renames varfarg vararg) - (if (null? old-args) - (let ((nfargs (if (null? varfarg) new-fargs (cons varfarg new-fargs))) - (nargs (if (null? vararg) new-args (cons vararg new-args)))) - `(fuse (-> (tuple ,@(reverse nfargs)) ,(replace-vars (caddr f) renames)) - ,(reverse nargs))) - (let ((farg (car old-fargs)) (arg (car old-args))) - (cond - ((and (vararg? farg) (vararg? arg)) ; arg... must be the last argument - (if (null? varfarg) - (cf (cdr old-fargs) (cdr old-args) - new-fargs new-args renames farg arg) - (if (eq? (cadr vararg) (cadr arg)) + (if (fuse? e) + (let ((f (cadr e)) + (args (caddr e))) + (define (cf old-fargs old-args new-fargs new-args renames varfarg vararg) + (if (null? old-args) + (let ((nfargs (if (null? varfarg) new-fargs (cons varfarg new-fargs))) + (nargs (if (null? vararg) new-args (cons vararg new-args)))) + `(fuse (-> (tuple ,@(reverse nfargs)) ,(replace-vars (caddr f) renames)) + ,(reverse nargs))) + (let ((farg (car old-fargs)) (arg (car old-args))) + (cond + ((and (vararg? farg) (vararg? arg)) ; arg... must be the last argument + (if (null? varfarg) (cf (cdr old-fargs) (cdr old-args) - new-fargs new-args (cons (cons (cadr farg) (cadr varfarg)) renames) - varfarg vararg) - (error "multiple splatted args cannot be fused into a single broadcast")))) - ((number? arg) ; inline numeric literals - (cf (cdr old-fargs) (cdr old-args) - new-fargs new-args - (cons (cons farg arg) renames) - varfarg vararg)) - ((and (symbol? arg) (memq arg new-args)) ; combine duplicate args - ; (note: calling memq for every arg is O(length(args)^2) ... - ; ... would be better to replace with a hash table if args is long) - (cf (cdr old-fargs) (cdr old-args) - new-fargs new-args - (cons (cons farg (findfarg arg new-args new-fargs)) renames) - varfarg vararg)) - (else - (cf (cdr old-fargs) (cdr old-args) - (cons farg new-fargs) (cons arg new-args) renames varfarg vararg)))))) - (cf (cdadr f) args '() '() '() '() '()))) - (let ((e (compress-fuse (make-fuse f args)))) ; an expression '(fuse func args) - (expand-forms `(call broadcast ,(from-lambda (cadr e)) ,@(caddr e))))) + new-fargs new-args renames farg arg) + (if (eq? (cadr vararg) (cadr arg)) + (cf (cdr old-fargs) (cdr old-args) + new-fargs new-args (cons (cons (cadr farg) (cadr varfarg)) renames) + varfarg vararg) + (error "multiple splatted args cannot be fused into a single broadcast")))) + ((number? arg) ; inline numeric literals + (cf (cdr old-fargs) (cdr old-args) + new-fargs new-args + (cons (cons farg arg) renames) + varfarg vararg)) + ((and (symbol? arg) (memq arg new-args)) ; combine duplicate args + ; (note: calling memq for every arg is O(length(args)^2) ... + ; ... would be better to replace with a hash table if args is long) + (cf (cdr old-fargs) (cdr old-args) + new-fargs new-args + (cons (cons farg (findfarg arg new-args new-fargs)) renames) + varfarg vararg)) + (else + (cf (cdr old-fargs) (cdr old-args) + (cons farg new-fargs) (cons arg new-args) renames varfarg vararg)))))) + (cf (cdadr f) args '() '() '() '() '())) + e)) ; (not (fuse? e)) + (let ((e (compress-fuse (dot-to-fuse rhs)))) ; an expression '(fuse func args) if expr is a dot call + (if (fuse? e) + (if (null? lhs) + (expand-forms `(call broadcast ,(from-lambda (cadr e)) ,@(caddr e))) + (expand-forms `(call broadcast! ,(from-lambda (cadr e)) ,lhs ,@(caddr e)))) + (if (null? lhs) + (expand-forms e) + (expand-forms `(call broadcast! identity ,lhs ,e)))))) ;; table mapping expression head to a function expanding that form (define expand-table @@ -1697,13 +1707,11 @@ '|.| (lambda (e) ; e = (|.| f x) - (let ((f (cadr e)) - (x (caddr e))) - (if (getfield-field? x) - `(call (core getfield) ,(expand-forms f) ,(expand-forms x)) - ; otherwise, came from f.(args...) --> broadcast(f, args...), - ; where we want to fuse with any nested broadcast calls. - (expand-fuse-broadcast f (cdr x))))) + (expand-fuse-broadcast '() e)) + + '.= + (lambda (e) + (expand-fuse-broadcast (cadr e) (caddr e))) '|<:| syntactic-op-to-call '|>:| syntactic-op-to-call @@ -2008,11 +2016,16 @@ '%= lower-update-op '.%= lower-update-op '|\|=| lower-update-op + '|.\|=| lower-update-op '&= lower-update-op + '.&= lower-update-op '$= lower-update-op '<<= lower-update-op + '.<<= lower-update-op '>>= lower-update-op + '.>>= lower-update-op '>>>= lower-update-op + '.>>>= lower-update-op ': (lambda (e) diff --git a/test/broadcast.jl b/test/broadcast.jl index 2d6adca165282..31a2166a4ee14 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -248,6 +248,25 @@ let x = [1:4;] @test sin.(f17300kw.(x, y=1)) == sin.(f17300kw.(x; y=1)) == sin.(x .+ 1) end +# PR #17510: Fused in-place assignment +let x = [1:4;], y = x + y .= 2:5 + @test y === x == [2:5;] + y .= factorial.(x) + @test y === x == [2,6,24,120] + y .= 7 + @test y === x == [7,7,7,7] + y .= factorial.(3) + @test y === x == [6,6,6,6] + f17510() = 9 + y .= f17510.() + @test y === x == [9,9,9,9] + y .-= 1 + @test y === x == [8,8,8,8] + y .-= 1:4 + @test y === x == [7,6,5,4] +end + # PR 16988 @test Base.promote_op(+, Bool) === Int @test isa(broadcast(+, [true]), Array{Int,1}) diff --git a/test/show.jl b/test/show.jl index 41b7ade6e5a2e..898b0ef625f36 100644 --- a/test/show.jl +++ b/test/show.jl @@ -561,3 +561,7 @@ end @test repr(:(x for x in y if aa for z in w if bb)) == ":(x for x = y if aa for z = w if bb)" @test repr(:([x for x = y])) == ":([x for x = y])" @test repr(:([x for x = y if z])) == ":([x for x = y if z])" + +for op in (:(.=), :(.+=), :(.&=)) + @test repr(parse("x $op y")) == ":(x $op y)" +end From 5a9359ff5269d307a89916f16b7b3615ddf5bcea Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 21 Jul 2016 11:49:21 -0400 Subject: [PATCH 0565/1117] typeassert syntax deprecations (#17445) deprecate `x::T` as type declaration syntax, ref #16071 deprecate `x::T = 0` where x is global and `global x::T` meaning typeassert, ref #964 also fix a bug where a typeassert was considered to be effect-free, causing it to be quasi-converted into a local variable declaration fix a bug in env.jl where a typeassert was intended --- base/env.jl | 10 +++++----- base/float16.jl | 10 +++++----- base/libuv.jl | 2 +- base/sparse/sparsevector.jl | 3 +-- src/julia-syntax.scm | 29 +++++++++++++++++------------ test/core.jl | 5 +++-- 6 files changed, 32 insertions(+), 27 deletions(-) diff --git a/base/env.jl b/base/env.jl index 0f41a5bbf170e..fc3fe87e78c70 100644 --- a/base/env.jl +++ b/base/env.jl @@ -86,10 +86,10 @@ if is_windows() start(hash::EnvHash) = (pos = ccall(:GetEnvironmentStringsW,stdcall,Ptr{UInt16},()); (pos,pos)) function done(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) if unsafe_load(block[1]) == 0 - ccall(:FreeEnvironmentStringsW,stdcall,Int32,(Ptr{UInt16},),block[2]) + ccall(:FreeEnvironmentStringsW, stdcall, Int32, (Ptr{UInt16},), block[2]) return true end - false + return false end function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) pos = block[1] @@ -102,7 +102,7 @@ function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) if m === nothing error("malformed environment entry: $env") end - (Pair{String,String}(m.captures[1], m.captures[2]), (pos+len*2, blk)) + return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+len*2, blk)) end else # !windows @@ -114,12 +114,12 @@ function next(::EnvHash, i) if env === nothing throw(BoundsError()) end - env::String + env = env::String m = match(r"^(.*?)=(.*)$"s, env) if m === nothing error("malformed environment entry: $env") end - (Pair{String,String}(m.captures[1], m.captures[2]), i+1) + return (Pair{String,String}(m.captures[1], m.captures[2]), i+1) end end # os-test diff --git a/base/float16.jl b/base/float16.jl index 46dcb8bfaa0d7..69fbbc20ba421 100644 --- a/base/float16.jl +++ b/base/float16.jl @@ -1,11 +1,11 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license function convert(::Type{Float32}, val::Float16) - ival::UInt32 = reinterpret(UInt16, val) - sign::UInt32 = (ival & 0x8000) >> 15 - exp::UInt32 = (ival & 0x7c00) >> 10 - sig::UInt32 = (ival & 0x3ff) >> 0 - ret::UInt32 + local ival::UInt32 = reinterpret(UInt16, val), + sign::UInt32 = (ival & 0x8000) >> 15, + exp::UInt32 = (ival & 0x7c00) >> 10, + sig::UInt32 = (ival & 0x3ff) >> 0, + ret::UInt32 if exp == 0 if sig == 0 diff --git a/base/libuv.jl b/base/libuv.jl index 9de035f4dd0d1..4cab2c213f4f6 100644 --- a/base/libuv.jl +++ b/base/libuv.jl @@ -70,7 +70,7 @@ show(io::IO, e::UVError) = print(io, e.prefix*": "*struverror(e)*" ("*uverrornam ## event loop ## -eventloop() = global uv_eventloop::Ptr{Void} +eventloop() = uv_eventloop::Ptr{Void} #mkNewEventLoop() = ccall(:jl_new_event_loop,Ptr{Void},()) # this would probably be fine, but is nowhere supported function run_event_loop() diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index b07360a707c93..db2a603232e33 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -1387,8 +1387,7 @@ end ### BLAS-2 / sparse A * sparse x -> dense y function densemv(A::SparseMatrixCSC, x::AbstractSparseVector; trans::Char='N') - xlen::Int - ylen::Int + local xlen::Int, ylen::Int m, n = size(A) if trans == 'N' || trans == 'n' xlen = n; ylen = m diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 76cf8956737af..c08d48843e963 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1321,16 +1321,11 @@ (define (remove-argument-side-effects e) (let ((a '())) (cond - ((and (decl? e) (symbol? (cadr e))) - (cons (cadr e) (list e))) ((not (pair? e)) (cons e '())) (else (cons (map (lambda (x) (cond - ((and (decl? x) (symbol? (cadr x))) - (set! a (cons x a)) - (cadr x)) ((not (effect-free? x)) (let ((g (make-ssavalue))) (if (or (eq? (car x) '...) (eq? (car x) '&)) @@ -1699,8 +1694,8 @@ (else `(block ,.(map (lambda (x) - (if (decl? x) - `(decl ,@(map expand-forms (cdr x))) + (if (and (decl? x) (length= (cdr x) 2) (symbol? (cadr x))) + `(impl-decl ,@(map expand-forms (cdr x))) (expand-forms x))) (butlast (cdr e))) ,(expand-forms (last (cdr e))))))) @@ -1951,9 +1946,9 @@ '|::| (lambda (e) - (if (length= e 2) + (if (not (length= e 3)) (error "invalid \"::\" syntax")) - (if (and (length= e 3) (not (symbol-like? (cadr e)))) + (if (not (symbol-like? (cadr e))) `(call (core typeassert) ,(expand-forms (cadr e)) ,(expand-forms (caddr e))) (map expand-forms e))) @@ -2439,7 +2434,7 @@ (vinfo:set-called! vi #t)) (for-each (lambda (x) (analyze-vars x env captvars sp)) (cdr e)))) - ((decl) + ((decl impl-decl) ;; handle var::T declaration by storing the type in the var-info ;; record. for non-symbols or globals, emit a type assertion. (let ((vi (var-info-for (cadr e) env))) @@ -2967,11 +2962,21 @@ f(x) = yt(x) (cl-convert `(call (core typeassert) ,@(cdr e)) fname lam namemap toplevel interp)) ;; remaining `decl` expressions are only type assertions if the ;; argument is global or a non-symbol. - ((decl) + ((impl-decl) (if (or (assq (cadr e) (car (lam:vinfo lam))) (assq (cadr e) (cadr (lam:vinfo lam)))) - '(null) + (let ((str-e (deparse `(|::| ,@(cdr e))))) + (syntax-deprecation #f str-e (string "local " str-e)) + '(null)) (cl-convert `(call (core typeassert) ,@(cdr e)) fname lam namemap toplevel interp))) + ((decl) + (cond ((not (symbol? (cadr e))) + (cl-convert `(call (core typeassert) ,@(cdr e)) fname lam namemap toplevel interp)) + ((or (assq (cadr e) (car (lam:vinfo lam))) + (assq (cadr e) (cadr (lam:vinfo lam)))) + '(null)) + (else (syntax-deprecation #f (string "global " (deparse `(|::| ,@(cdr e)))) "typeassert") + (cl-convert `(call (core typeassert) ,@(cdr e)) fname lam namemap toplevel interp)))) ;; `with-static-parameters` expressions can be removed now; used only by analyze-vars ((with-static-parameters) (cl-convert (cadr e) fname lam namemap toplevel interp)) diff --git a/test/core.jl b/test/core.jl index 7d4d46bd8444b..c2d85714956f4 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3134,8 +3134,9 @@ immutable Foo11874 end function bar11874(x) - y::Foo11874 - y=x + local y::Foo11874 + y = x + nothing end Base.convert(::Type{Foo11874},x::Int) = float(x) From 534e64d73c2d33087b97af2c6429d4323911cdca Mon Sep 17 00:00:00 2001 From: Art <wildart@users.noreply.github.com> Date: Thu, 21 Jul 2016 12:03:49 -0400 Subject: [PATCH 0566/1117] Pkg.free: correctly set target nullable object (#17483) * correctly set target nullable object * added: stage & stage tests added: file status, index entry search * throw if reset object wasn't found * removed trailing spaces --- base/libgit2/index.jl | 10 +++++++ base/libgit2/libgit2.jl | 12 ++++---- base/libgit2/repository.jl | 2 ++ base/libgit2/status.jl | 9 ++++++ test/libgit2.jl | 61 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 5 deletions(-) diff --git a/base/libgit2/index.jl b/base/libgit2/index.jl index c5af2191e7704..9778ebc820b8f 100644 --- a/base/libgit2/index.jl +++ b/base/libgit2/index.jl @@ -118,3 +118,13 @@ function Base.getindex(idx::GitIndex, i::Csize_t) return unsafe_load(convert(Ptr{IndexEntry}, ie_ptr), 1) end Base.getindex(idx::GitIndex, i::Int) = getindex(idx, Csize_t(i)) + +function Base.find(path::String, idx::GitIndex) + pos_ref = Ref{Csize_t}(0) + ret = ccall((:git_index_find, :libgit2), Cint, + (Ref{Csize_t}, Ptr{Void}, Cstring), pos_ref, idx.ptr, path) + ret == Error.ENOTFOUND && return Nullable{Csize_t}() + return Nullable(pos_ref[]+1) +end + +stage(ie::IndexEntry) = ccall((:git_index_entry_stage, :libgit2), Cint, (Ptr{IndexEntry},), Ref(ie)) diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index 48e76a89e43d7..80d4daf7d9036 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -310,19 +310,21 @@ end """ git reset [<committish>] [--] <pathspecs>... """ function reset!(repo::GitRepo, committish::AbstractString, pathspecs::AbstractString...) - target_obj = isempty(committish) ? Nullable{GitAnyObject}() : - Nullable(revparse(repo, committish)) + obj = revparse(repo, !isempty(committish) ? committish : Consts.HEAD_FILE) + # do not remove entries in the index matching the provided pathspecs with empty target commit tree + obj === nothing && throw(GitError(Error.Object, Error.ERROR, "`$committish` not found")) try - reset!(repo, target_obj, pathspecs...) + reset!(repo, Nullable(obj), pathspecs...) finally - !isnull(target_obj) && finalize(Base.get(target_obj)) + finalize(obj) end end """ git reset [--soft | --mixed | --hard] <commit> """ function reset!(repo::GitRepo, commit::Oid, mode::Cint = Consts.RESET_MIXED) obj = get(GitAnyObject, repo, commit) - obj === nothing && return + # object must exist for reset + obj === nothing && throw(GitError(Error.Object, Error.ERROR, "Commit `$(string(commit))` object not found")) try reset!(repo, obj, mode) finally diff --git a/base/libgit2/repository.jl b/base/libgit2/repository.jl index 0ada7cc7c1d43..ea81c04fe499c 100644 --- a/base/libgit2/repository.jl +++ b/base/libgit2/repository.jl @@ -165,6 +165,7 @@ function checkout_head(repo::GitRepo; options::CheckoutOptions = CheckoutOptions repo.ptr, Ref(options)) end +"""Updates some entries, determined by the `pathspecs`, in the index from the target commit tree.""" function reset!{T<:AbstractString, S<:GitObject}(repo::GitRepo, obj::Nullable{S}, pathspecs::T...) with(StrArrayStruct(pathspecs...)) do sa @check ccall((:git_reset_default, :libgit2), Cint, @@ -175,6 +176,7 @@ function reset!{T<:AbstractString, S<:GitObject}(repo::GitRepo, obj::Nullable{S} end end +"""Sets the current head to the specified commit oid and optionally resets the index and working tree to match.""" function reset!(repo::GitRepo, obj::GitObject, mode::Cint; checkout_opts::CheckoutOptions = CheckoutOptions()) @check ccall((:git_reset, :libgit2), Cint, diff --git a/base/libgit2/status.jl b/base/libgit2/status.jl index f3384705c6612..59d95bd7bf3f1 100644 --- a/base/libgit2/status.jl +++ b/base/libgit2/status.jl @@ -23,3 +23,12 @@ function Base.getindex(status::GitStatus, i::Csize_t) return unsafe_load(convert(Ptr{StatusEntry}, entry_ptr), 1) end Base.getindex(status::GitStatus, i::Int) = getindex(status, Csize_t(i)) + +function status(repo::GitRepo, path::String) + status_ptr = Ref{Cuint}(0) + ret = ccall((:git_status_file, :libgit2), Cint, + (Ref{Cuint}, Ptr{Void}, Cstring), + status_ptr, repo.ptr, path) + (ret == Cint(Error.ENOTFOUND) || ret == Cint(Error.EAMBIGUOUS)) && return Nullable{Cuint}() + return Nullable(status_ptr[]) +end diff --git a/test/libgit2.jl b/test/libgit2.jl index f5752130ecda0..8202fc4e9d39d 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -474,6 +474,67 @@ mktempdir() do dir #end #end + #@testset "Modify and reset repository" begin + repo = LibGit2.GitRepo(test_repo) + try + # check index for file + LibGit2.with(LibGit2.GitIndex(repo)) do idx + i = find(test_file, idx) + @test !isnull(i) + @test idx[get(i)] !== nothing + end + + # check non-existent file status + st = LibGit2.status(repo, "XYZ") + @test isnull(st) + + # check file status + st = LibGit2.status(repo, test_file) + @test !isnull(st) + @test LibGit2.isset(get(st), LibGit2.Consts.STATUS_CURRENT) + + # modify file + open(joinpath(test_repo, test_file), "a") do io + write(io, 0x41) + end + + # file modified but not staged + st_mod = LibGit2.status(repo, test_file) + @test !LibGit2.isset(get(st_mod), LibGit2.Consts.STATUS_INDEX_MODIFIED) + @test LibGit2.isset(get(st_mod), LibGit2.Consts.STATUS_WT_MODIFIED) + + # stage file + LibGit2.add!(repo, test_file) + + # modified file staged + st_stg = LibGit2.status(repo, test_file) + @test LibGit2.isset(get(st_stg), LibGit2.Consts.STATUS_INDEX_MODIFIED) + @test !LibGit2.isset(get(st_stg), LibGit2.Consts.STATUS_WT_MODIFIED) + + # try to unstage to unknown commit + @test_throws LibGit2.Error.GitError LibGit2.reset!(repo, "XYZ", test_file) + + # status should not change + st_new = LibGit2.status(repo, test_file) + @test get(st_new) == get(st_stg) + + # try to unstage to HEAD + LibGit2.reset!(repo, LibGit2.Consts.HEAD_FILE, test_file) + st_uns = LibGit2.status(repo, test_file) + @test get(st_uns) == get(st_mod) + + # reset repo + @test_throws LibGit2.Error.GitError LibGit2.reset!(repo, LibGit2.Oid(), LibGit2.Consts.RESET_HARD) + + LibGit2.reset!(repo, LibGit2.head_oid(repo), LibGit2.Consts.RESET_HARD) + open(joinpath(test_repo, test_file), "r") do io + @test read(io)[end] != 0x41 + end + finally + finalize(repo) + end + #end + #@testset "Transact test repository" begin repo = LibGit2.GitRepo(test_repo) try From a277e953db85b5a56686526853f9ab3ba2f20c72 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 21 Jul 2016 09:06:06 -0700 Subject: [PATCH 0567/1117] feature freeze master (#17503) no more breaking changes until we branch, only bugfixes, docs and tests ref https://github.com/JuliaLang/julia/issues/17418 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4259c642a39d0..4d64de0d547d2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0-dev +0.5.0-pre From f9eb8e6eb69ec3f9fc0e931cb34a3f520c65f37f Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 21 Jul 2016 09:08:17 -0700 Subject: [PATCH 0568/1117] Add a patch to fix LLVM 3.8.1 when built with configure (#17523) otherwise the version number says it's still 3.8.0 --- deps/llvm.mk | 3 ++ deps/patches/llvm-3.8.1-version.patch | 77 +++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 deps/patches/llvm-3.8.1-version.patch diff --git a/deps/llvm.mk b/deps/llvm.mk index ac200ad6a7216..2798acf817191 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -436,6 +436,9 @@ else ifeq ($(LLVM_VER_SHORT),3.8) ifeq ($(LLVM_VER),3.8.0) $(eval $(call LLVM_PATCH,llvm-D17326_unpack_load)) endif +ifeq ($(LLVM_VER),3.8.1) +$(eval $(call LLVM_PATCH,llvm-3.8.1-version)) +endif $(eval $(call LLVM_PATCH,llvm-3.7.1_3)) # Remove for 3.9 $(eval $(call LLVM_PATCH,llvm-D14260)) $(eval $(call LLVM_PATCH,llvm-3.8.0_bindir)) # Remove for 3.9 diff --git a/deps/patches/llvm-3.8.1-version.patch b/deps/patches/llvm-3.8.1-version.patch new file mode 100644 index 0000000000000..3bc365a596d9b --- /dev/null +++ b/deps/patches/llvm-3.8.1-version.patch @@ -0,0 +1,77 @@ +diff --git a/configure b/configure +index c94fb13..f146f2f 100755 +--- a/configure ++++ b/configure +@@ -1,6 +1,6 @@ + #! /bin/sh + # Guess values for system-dependent variables and create Makefiles. +-# Generated by GNU Autoconf 2.60 for LLVM 3.8.0. ++# Generated by GNU Autoconf 2.60 for LLVM 3.8.1. + # + # Report bugs to <http://llvm.org/bugs/>. + # +@@ -561,8 +561,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} + # Identity of this package. + PACKAGE_NAME='LLVM' + PACKAGE_TARNAME='llvm' +-PACKAGE_VERSION='3.8.0' +-PACKAGE_STRING='LLVM 3.8.0' ++PACKAGE_VERSION='3.8.1' ++PACKAGE_STRING='LLVM 3.8.1' + PACKAGE_BUGREPORT='http://llvm.org/bugs/' + + ac_unique_file="lib/IR/Module.cpp" +@@ -1334,7 +1334,7 @@ if test "$ac_init_help" = "long"; then + # Omit some internal or obsolete options to make the list less imposing. + # This message is too long to be a string in the A/UX 3.1 sh. + cat <<_ACEOF +-\`configure' configures LLVM 3.8.0 to adapt to many kinds of systems. ++\`configure' configures LLVM 3.8.1 to adapt to many kinds of systems. + + Usage: $0 [OPTION]... [VAR=VALUE]... + +@@ -1400,7 +1400,7 @@ fi + + if test -n "$ac_init_help"; then + case $ac_init_help in +- short | recursive ) echo "Configuration of LLVM 3.8.0:";; ++ short | recursive ) echo "Configuration of LLVM 3.8.1:";; + esac + cat <<\_ACEOF + +@@ -1584,7 +1584,7 @@ fi + test -n "$ac_init_help" && exit $ac_status + if $ac_init_version; then + cat <<\_ACEOF +-LLVM configure 3.8.0 ++LLVM configure 3.8.1 + generated by GNU Autoconf 2.60 + + Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, +@@ -1600,7 +1600,7 @@ cat >config.log <<_ACEOF + This file contains any messages produced by compilers while + running configure, to aid debugging if configure makes a mistake. + +-It was created by LLVM $as_me 3.8.0, which was ++It was created by LLVM $as_me 3.8.1, which was + generated by GNU Autoconf 2.60. Invocation command line was + + $ $0 $@ +@@ -18279,7 +18279,7 @@ exec 6>&1 + # report actual input values of CONFIG_FILES etc. instead of their + # values after options handling. + ac_log=" +-This file was extended by LLVM $as_me 3.8.0, which was ++This file was extended by LLVM $as_me 3.8.1, which was + generated by GNU Autoconf 2.60. Invocation command line was + + CONFIG_FILES = $CONFIG_FILES +@@ -18332,7 +18332,7 @@ Report bugs to <bug-autoconf@gnu.org>." + _ACEOF + cat >>$CONFIG_STATUS <<_ACEOF + ac_cs_version="\\ +-LLVM config.status 3.8.0 ++LLVM config.status 3.8.1 + configured by $0, generated by GNU Autoconf 2.60, + with options \\"`echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" + From 7737cc1a60cdb63da411918d12ec4a371b6a22d4 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <Sacha0@users.noreply.github.com> Date: Thu, 21 Jul 2016 09:41:08 -0700 Subject: [PATCH 0569/1117] Revise convert methods involving SparseMatrixCSCs, leveraging multiple dispatch to displace some type computations. (#17515) --- base/sparse/sparsematrix.jl | 47 ++++++++++++++----------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 629e96a4a5e3a..dceca0b2657a1 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -253,39 +253,26 @@ similar(S::SparseMatrixCSC, Tv::Type=eltype(S)) = SparseMatrixCSC(S.m, S.n, copy similar{Tv,Ti,TvNew,TiNew}(S::SparseMatrixCSC{Tv,Ti}, ::Type{TvNew}, ::Type{TiNew}) = SparseMatrixCSC(S.m, S.n, convert(Array{TiNew},S.colptr), convert(Array{TiNew}, S.rowval), Array{TvNew}(length(S.nzval))) @inline similar{Tv}(S::SparseMatrixCSC, ::Type{Tv}, d::Dims) = spzeros(Tv, d...) -function convert{Tv,Ti,TvS,TiS}(::Type{SparseMatrixCSC{Tv,Ti}}, S::SparseMatrixCSC{TvS,TiS}) - if Tv == TvS && Ti == TiS - return S - else - return SparseMatrixCSC(S.m, S.n, - convert(Vector{Ti},S.colptr), - convert(Vector{Ti},S.rowval), - convert(Vector{Tv},S.nzval)) - end -end - -function convert{Tv,TvS,TiS}(::Type{SparseMatrixCSC{Tv}}, S::SparseMatrixCSC{TvS,TiS}) - if Tv == TvS - return S - else - return SparseMatrixCSC(S.m, S.n, - S.colptr, - S.rowval, - convert(Vector{Tv},S.nzval)) - end -end - +# convert'ing between SparseMatrixCSC types +convert{Tv}(::Type{AbstractMatrix{Tv}}, A::SparseMatrixCSC) = convert(SparseMatrixCSC{Tv}, A) +convert{Tv}(::Type{SparseMatrixCSC{Tv}}, S::SparseMatrixCSC) = convert(SparseMatrixCSC{Tv,eltype(S.colptr)}, S) +convert{Tv,Ti}(::Type{SparseMatrixCSC{Tv,Ti}}, S::SparseMatrixCSC{Tv,Ti}) = S +function convert{Tv,Ti}(::Type{SparseMatrixCSC{Tv,Ti}}, S::SparseMatrixCSC) + eltypeTicolptr = convert(Vector{Ti}, S.colptr) + eltypeTirowval = convert(Vector{Ti}, S.rowval) + eltypeTvnzval = convert(Vector{Tv}, S.nzval) + return SparseMatrixCSC(S.m, S.n, eltypeTicolptr, eltypeTirowval, eltypeTvnzval) +end +# convert'ing from other matrix types to SparseMatrixCSC (also see sparse()) +convert(::Type{SparseMatrixCSC}, M::Matrix) = sparse(M) function convert{Tv,Ti}(::Type{SparseMatrixCSC{Tv,Ti}}, M::AbstractMatrix) - m, n = size(M) (I, J, V) = findnz(M) - return sparse_IJ_sorted!(convert(Vector{Ti},I), - convert(Vector{Ti},J), - convert(Vector{Tv},V), - m, n) + eltypeTiI = convert(Vector{Ti}, I) + eltypeTiJ = convert(Vector{Ti}, J) + eltypeTvV = convert(Vector{Tv}, V) + return sparse_IJ_sorted!(eltypeTiI, eltypeTiJ, eltypeTvV, size(M)...) end -convert{T}(::Type{AbstractMatrix{T}}, A::SparseMatrixCSC) = convert(SparseMatrixCSC{T}, A) -convert(::Type{SparseMatrixCSC}, M::Matrix) = sparse(M) - +# convert'ing from SparseMatrixCSC to other matrix types function convert{Tv}(::Type{Matrix}, S::SparseMatrixCSC{Tv}) # Handle cases where zero(Tv) is not defined but the array is dense. A = length(S) == nnz(S) ? Array{Tv}(S.m, S.n) : zeros(Tv, S.m, S.n) From 49fe4b5c20ddc7d503ba94b5b6c92e7be45c3f14 Mon Sep 17 00:00:00 2001 From: Lyndon White <oxinabox@ucc.asn.au> Date: Fri, 22 Jul 2016 00:41:23 +0800 Subject: [PATCH 0570/1117] add size to Important optional methods (#17525) * add size to Important optional methods * More extensive tables explaining what methods are required by traits * fix typo [ci skip] * newlines cod reStructuredText likes them [ci skip] --- doc/manual/interfaces.rst | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 92af7faf5f5ea..22b1e808c946f 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -11,18 +11,35 @@ A lot of the power and extensibility in Julia comes from a collection of informa Iteration --------- -================================================== ======================== =========================================== +================================================== ======================== ===================================================================================== Required methods Brief description -================================================== ======================== =========================================== +================================================== ======================== ===================================================================================== :func:`start(iter) <start>` Returns the initial iteration state :func:`next(iter, state) <next>` Returns the current item and the next state :func:`done(iter, state) <done>` Tests if there are any items remaining **Important optional methods** **Default definition** **Brief description** +:func:`iteratorsize(IterType) <iteratorsize>` ``HasLength()`` One of `HasLength()`, `HasShape()`, `IsInfinite()`, or `SizeUnknown()` as appropriate +:func:`iteratoreltype(IterType) <iteratoreltype>` ``HasEltype()`` Either `EltypeUnknown()` or `HasEltype()` as appropriate :func:`eltype(IterType) <eltype>` ``Any`` The type the items returned by :func:`next` :func:`length(iter) <length>` (*undefined*) The number of items, if known -:func:`iteratorsize(IterType) <iteratorsize>` ``HasLength()`` -:func:`iteratoreltype(IterType) <iteratoreltype>` ``HasEltype()`` -================================================== ======================== =========================================== +:func:`size(iter, [dim...]) <size>` (*undefined*) The number of items in each dimension, if known +================================================== ======================== ===================================================================================== + +================================================================ ====================================================================== +Value returned by :func:`iteratorsize(IterType) <iteratorsize>` Required Methods +================================================================ ====================================================================== +`HasLength()` :func:`length(iter) <length>` +`HasShape()` :func:`length(iter) <length>` and :func:`size(iter, [dim...]) <size>` +`IsInfinite()` (*none*) +`SizeUnknown()` (*none*) +================================================================ ====================================================================== + +==================================================================== ================================== +Value returned by :func:`iteratoreltype(IterType) <iteratoreltype>` Required Methods +==================================================================== ================================== +`HasEltype()` :func:`eltype(IterType) <eltype>` +`EltypeUnknown()` (*none*) +==================================================================== ================================== Sequential iteration is implemented by the methods :func:`start`, :func:`done`, and :func:`next`. Instead of mutating objects as they are iterated over, Julia provides these three methods to keep track of the iteration state externally from the object. The :func:`start(iter) <start>` method returns the initial state for the iterable object ``iter``. That state gets passed along to :func:`done(iter, state) <done>`, which tests if there are any elements remaining, and :func:`next(iter, state) <next>`, which returns a tuple containing the current element and an updated ``state``. The ``state`` object can be anything, and is generally considered to be an implementation detail private to the iterable object. From 8293c299100760369f6202951128869e3ce756e8 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 21 Jul 2016 12:49:27 -0400 Subject: [PATCH 0571/1117] NEWS and doc updates for type declaration syntax deprecation --- NEWS.md | 8 ++++++++ doc/manual/types.rst | 16 ++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/NEWS.md b/NEWS.md index 95e5ce6a5ff93..0bf93945ccbc7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -91,6 +91,12 @@ Language changes computed, instead of using type inference ([#7258]). If the result is empty, then type inference is still used to determine the element type. + * Use of the syntax `x::T` to declare the type of a local variable is deprecated. + In the future this will always mean type assertion, and declarations should use + `local x::T` instead ([#16071]). + When `x` is global, `x::T = ...` and `global x::T` used to mean type assertion, + but this syntax is now reserved for type declaration ([#964]). + Command-line option changes --------------------------- @@ -280,6 +286,7 @@ Deprecated or removed [PkgDev]: https://github.com/JuliaLang/PkgDev.jl <!--- generated by NEWS-update.jl: --> [#550]: https://github.com/JuliaLang/julia/issues/550 +[#964]: https://github.com/JuliaLang/julia/issues/964 [#1090]: https://github.com/JuliaLang/julia/issues/1090 [#4163]: https://github.com/JuliaLang/julia/issues/4163 [#4211]: https://github.com/JuliaLang/julia/issues/4211 @@ -335,6 +342,7 @@ Deprecated or removed [#15763]: https://github.com/JuliaLang/julia/issues/15763 [#15975]: https://github.com/JuliaLang/julia/issues/15975 [#16058]: https://github.com/JuliaLang/julia/issues/16058 +[#16071]: https://github.com/JuliaLang/julia/issues/16071 [#16107]: https://github.com/JuliaLang/julia/issues/16107 [#16219]: https://github.com/JuliaLang/julia/issues/16219 [#16260]: https://github.com/JuliaLang/julia/issues/16260 diff --git a/doc/manual/types.rst b/doc/manual/types.rst index a193c1a7c3809..9cca9fd17e58a 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -87,7 +87,7 @@ do this: 2. To provide extra type information to the compiler, which can then improve performance in some cases -When appended to an expression computing a *value*, the ``::`` +When appended to an expression computing a value, the ``::`` operator is read as "is an instance of". It can be used anywhere to assert that the value of the expression on the left is an instance of the type on the right. When the type on the right is @@ -108,14 +108,11 @@ exception is thrown, otherwise, the left-hand value is returned: 3 This allows a type assertion to be attached to any expression -in-place. The most common usage of ``::`` as an assertion is in -function/methods signatures, such as ``f(x::Int8) = ...`` (see -:ref:`man-methods`). +in-place. - -When appended to a *variable* in a statement context, the ``::`` -operator means something a bit -different: it declares the variable to always have the specified type, +When appended to a variable on the left-hand side of an assignment, +or as part of a ``local`` declaration, the ``::`` operator means something +a bit different: it declares the variable to always have the specified type, like a type declaration in a statically-typed language such as C. Every value assigned to the variable will be converted to the declared type using :func:`convert`: @@ -138,9 +135,8 @@ This feature is useful for avoiding performance "gotchas" that could occur if one of the assignments to a variable changed its type unexpectedly. -The "declaration" behavior only occurs in specific contexts:: +This "declaration" behavior only occurs in specific contexts:: - x::Int8 # a variable by itself local x::Int8 # in a local declaration x::Int8 = 10 # as the left-hand side of an assignment From ac7a36e28247ccda9ae59adab8545a34f334e9a6 Mon Sep 17 00:00:00 2001 From: Pablo Zubieta <pabloferz@yahoo.com.mx> Date: Thu, 21 Jul 2016 19:15:33 +0200 Subject: [PATCH 0572/1117] Fix miscellaneous type instabilities (#17530) --- base/bool.jl | 14 ++++++-------- base/complex.jl | 6 +++--- base/irrationals.jl | 1 + base/rational.jl | 4 ++-- base/strings/io.jl | 2 ++ test/numbers.jl | 21 ++------------------- 6 files changed, 16 insertions(+), 32 deletions(-) diff --git a/base/bool.jl b/base/bool.jl index 90917d1d736d0..d43403140396d 100644 --- a/base/bool.jl +++ b/base/bool.jl @@ -51,18 +51,16 @@ abs2(x::Bool) = x ^(x::Bool, y::Bool) = x | !y ^(x::Integer, y::Bool) = ifelse(y, x, one(x)) -function +{T<:AbstractFloat}(x::Bool, y::T) - ifelse(x, one(promote_type(Bool,T)) + convert(promote_type(Bool,T),y), - convert(promote_type(Bool,T),y)) +function +{T<:AbstractFloat}(x::Bool, y::T)::promote_type(Bool,T) + return ifelse(x, one(y) + y, y) end +(y::AbstractFloat, x::Bool) = x + y -function *{T<:Number}(x::Bool, y::T) - ifelse(x, convert(promote_type(Bool,T),y), - ifelse(signbit(y), -zero(promote_type(Bool,T)), zero(promote_type(Bool,T)))) +function *{T<:Number}(x::Bool, y::T)::promote_type(Bool,T) + return ifelse(x, y, copysign(zero(y), y)) end -function *{T<:Unsigned}(x::Bool, y::T) - ifelse(x, convert(promote_type(Bool,T),y), zero(promote_type(Bool,T))) +function *{T<:Unsigned}(x::Bool, y::T)::promote_type(Bool,T) + return ifelse(x, y, zero(y)) end *(y::Number, x::Bool) = x * y diff --git a/base/complex.jl b/base/complex.jl index 038a1d10df91f..e1bf83a98830e 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -451,11 +451,11 @@ function log1p{T}(z::Complex{T}) end end -function ^{T<:AbstractFloat}(z::Complex{T}, p::Complex{T}) - if p==2 #square +function ^{T<:AbstractFloat}(z::Complex{T}, p::Complex{T})::Complex{T} + if p == 2 #square zr, zi = reim(z) x = (zr-zi)*(zr+zi) - y = T(2*zr*zi) + y = 2*zr*zi if isnan(x) if isinf(y) x = copysign(zero(T),zr) diff --git a/base/irrationals.jl b/base/irrationals.jl index c68947a938b44..e76f40760c0b9 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -94,6 +94,7 @@ hash(x::Irrational, h::UInt) = 3*object_id(x) - h for op in Symbol[:+, :-, :*, :/, :^] @eval $op(x::Irrational, y::Irrational) = $op(Float64(x),Float64(y)) end +*(x::Bool, y::Irrational) = ifelse(x, Float64(y), 0.0) macro irrational(sym, val, def) esym = esc(sym) diff --git a/base/rational.jl b/base/rational.jl index c79b0a7b9bf64..438e8e1724f4e 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -355,8 +355,8 @@ function ^(x::Rational, n::Integer) end ^(x::Number, y::Rational) = x^(y.num/y.den) -^{T<:AbstractFloat}(x::T, y::Rational) = x^(convert(T,y.num)/y.den) -^{T<:AbstractFloat}(x::Complex{T}, y::Rational) = x^(convert(T,y.num)/y.den) +^{T<:AbstractFloat}(x::T, y::Rational) = x^(convert(T, y.num / y.den)) +^{T<:AbstractFloat}(x::Complex{T}, y::Rational) = x^(convert(T, y.num / y.den)) ^{T<:Rational}(z::Complex{T}, n::Bool) = n ? z : one(z) # to resolve ambiguity function ^{T<:Rational}(z::Complex{T}, n::Integer) diff --git a/base/strings/io.jl b/base/strings/io.jl index 151909d6405e4..4dc87ae72c2b5 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -9,6 +9,7 @@ function print(io::IO, x) finally unlock(io) end + return nothing end function print(io::IO, xs...) @@ -20,6 +21,7 @@ function print(io::IO, xs...) finally unlock(io) end + return nothing end println(io::IO, xs...) = print(io, xs..., '\n') diff --git a/test/numbers.jl b/test/numbers.jl index f318a7c2f8e38..cab8bc626f208 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2790,7 +2790,7 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Complex{Int}, Complex{UInt}, Complex32, Complex64, Complex128) for S in types for op in (+, -) - if S === Float16 # type instability here? + if S === Float16 # broken, fixme then remove this branch @test_throws ErrorException @inferred(Base.promote_op(op, S)) T = Base.promote_op(op, S) @@ -2810,27 +2810,10 @@ let types = (Base.BitInteger_types..., BigInt, Bool, # broken, fixme then remove this branch @test_throws ErrorException @inferred(Base.promote_op(op, R, S)) T = Base.promote_op(op, R, S) - if ((R === Bool || S === Bool) && op in (+, *)) || - ((S in (Rational{Int}, Complex{Float16})) && op === ^) || - (R === Complex{Float16} && op === ^) - @test_throws ErrorException @inferred(op(one(R), one(S))) - t = op(one(R), one(S)) - else - t = @inferred op(one(R), one(S)) - end - elseif (R === Complex{Float16} || S === Complex{Float16}) && - ((R === Bool && op in (+, *, /, ^)) || - (S === Bool && op in (+, *)) || - (S in (R, Rational{Int}) && op === ^)) - # broken, fixme then remove this branch too - @test_throws ErrorException @inferred(Base.promote_op(op, R, S)) - T = Base.promote_op(op, R, S) - @test_throws ErrorException @inferred(op(one(R), one(S))) - t = op(one(R), one(S)) else T = @inferred Base.promote_op(op, R, S) - t = @inferred op(one(R), one(S)) end + t = @inferred op(one(R), one(S)) @test T === typeof(t) end end From d6f586479caf3776245231989cb3fc63ca6c7b92 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Thu, 21 Jul 2016 14:43:57 -0400 Subject: [PATCH 0573/1117] Add patch to fix libgit2 openssl hang Fixes #17423 --- deps/libgit2.mk | 6 ++--- deps/patches/libgit2-openssl-hang.patch | 32 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 deps/patches/libgit2-openssl-hang.patch diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 17e4517f36a89..b23daf7317a1b 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -33,12 +33,12 @@ $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied: $(S cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch echo 1 > $@ -$(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied | $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt - cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-agent-nonfatal.patch +$(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied: | $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-openssl-hang.patch echo 1 > $@ ifeq ($(OS),Linux) -$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied endif $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied $(LIBSSH2_OBJ_TARGET) mkdir -p $(dir $@) diff --git a/deps/patches/libgit2-openssl-hang.patch b/deps/patches/libgit2-openssl-hang.patch new file mode 100644 index 0000000000000..4681903018232 --- /dev/null +++ b/deps/patches/libgit2-openssl-hang.patch @@ -0,0 +1,32 @@ +From 4734c52ab2e5d8b77f2659282eefdbd7ebee4628 Mon Sep 17 00:00:00 2001 +From: Christian Schlack <christian@backhub.co> +Date: Tue, 26 Apr 2016 18:04:03 +0200 +Subject: [PATCH] Fix return value of openssl_read (infinite loop) + +openssl_read should return -1 in case of error. + +SSL_read returns values <= 0 in case of error. + +A return value of 0 can lead to an infinite loop, so the return value +of ssl_set_error will be returned if SSL_read is not successful (analog +to openssl_write). +--- + src/openssl_stream.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/src/openssl_stream.c b/src/openssl_stream.c +index edea8fe..dd4cd61 100644 +--- a/src/openssl_stream.c ++++ b/src/openssl_stream.c +@@ -522,8 +522,9 @@ ssize_t openssl_read(git_stream *stream, void *data, size_t len) + openssl_stream *st = (openssl_stream *) stream; + int ret; + +- if ((ret = SSL_read(st->ssl, data, len)) <= 0) +- ssl_set_error(st->ssl, ret); ++ if ((ret = SSL_read(st->ssl, data, len)) <= 0) { ++ return ssl_set_error(st->ssl, ret); ++ } + + return ret; + } From 12b97a79fc26ee6930dcb87db8cf56e8512c6dd2 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Wed, 20 Jul 2016 16:59:43 -0700 Subject: [PATCH 0574/1117] Fix pretty-printing of compact broadcast expressions and test. --- base/show.jl | 17 +++++++++++------ test/show.jl | 5 +++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/base/show.jl b/base/show.jl index 5d0bf1cf6e036..209d3bb154bbc 100644 --- a/base/show.jl +++ b/base/show.jl @@ -407,7 +407,8 @@ const expr_infix_wide = Set{Symbol}([ const expr_infix = Set{Symbol}([:(:), :(->), Symbol("::")]) const expr_infix_any = union(expr_infix, expr_infix_wide) const all_ops = union(quoted_syms, uni_ops, expr_infix_any) -const expr_calls = Dict(:call =>('(',')'), :calldecl =>('(',')'), :ref =>('[',']'), :curly =>('{','}')) +const expr_calls = Dict(:call => ('(',')'), :calldecl => ('(',')'), + :ref => ('[',']'), :curly => ('{','}'), :(.) => ('(',')')) const expr_parens = Dict(:tuple=>('(',')'), :vcat=>('[',']'), :hcat =>('[',']'), :row =>('[',']'), :vect=>('[',']')) @@ -538,6 +539,9 @@ function show_call(io::IO, head, func, func_args, indent) show_unquoted(io, func, indent) print(io, ')') end + if head == :(.) + print(io, '.') + end if !isempty(func_args) && isa(func_args[1], Expr) && func_args[1].head === :parameters print(io, op) show_list(io, func_args[2:end], ',', indent) @@ -648,8 +652,8 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) if !emphstate && ex.typ === Any show_type = false end - # dot (i.e. "x.y") - if is(head, :(.)) + # dot (i.e. "x.y"), but not compact broadcast exps + if is(head, :(.)) && !is_expr(args[2], :tuple) show_unquoted(io, args[1], indent + indent_width) print(io, '.') if is_quoted(args[2]) @@ -750,9 +754,10 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) show_call(io, head, func, func_args, indent) end - # other call-like expressions ("A[1,2]", "T{X,Y}") - elseif haskey(expr_calls, head) && nargs >= 1 # :ref/:curly/:calldecl - show_call(io, head, ex.args[1], ex.args[2:end], indent) + # other call-like expressions ("A[1,2]", "T{X,Y}", "f.(X,Y)") + elseif haskey(expr_calls, head) && nargs >= 1 # :ref/:curly/:calldecl/:(.) + funcargslike = head == :(.) ? ex.args[2].args : ex.args[2:end] + show_call(io, head, ex.args[1], funcargslike, indent) # comprehensions elseif (head === :typed_comprehension || head === :typed_dict_comprehension) && length(args) == 2 diff --git a/test/show.jl b/test/show.jl index 898b0ef625f36..c9d0369c39944 100644 --- a/test/show.jl +++ b/test/show.jl @@ -565,3 +565,8 @@ end for op in (:(.=), :(.+=), :(.&=)) @test repr(parse("x $op y")) == ":(x $op y)" end + +# pretty-printing of compact broadcast expressions (#17289) +@test repr(:(f.(X,Y))) == ":(f.(X,Y))" +@test repr(:(f.(X))) == ":(f.(X))" +@test repr(:(f.())) == ":(f.())" From dd22294bee758854c7201e18c6e938349c0a3c8d Mon Sep 17 00:00:00 2001 From: Keno Fischer <keno@juliacomputing.com> Date: Wed, 20 Jul 2016 02:21:27 +0000 Subject: [PATCH 0575/1117] WinAPI password prompts --- base/libgit2/callbacks.jl | 38 +++++++++++++----- base/libgit2/utils.jl | 3 ++ base/util.jl | 82 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index aa102546d067f..1c44530141d2f 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -146,7 +146,17 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, ENV["SSH_KEY_PASS"] else passdef = creds[:pass, credid] # check if credentials were already used - passdef !== nothing && !isusedcreds ? passdef : prompt("Passphrase for $privatekey", password=true) + if passdef === nothing || isusedcreds + if is_windows() + passdef = Base.winprompt( + "Your SSH Key requires a password, please enter it now:", + "Passphrase required", privatekey; prompt_username = false) + isnull(passdef) && return Cint(Error.EAUTH) + passdef = Base.get(passdef)[2] + else + passdef = prompt("Passphrase for $privatekey", password=true) + end + end end creds[:pass, credid] = passphrase # save credentials @@ -161,17 +171,27 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) creds == nothing && (creds = UserPasswordCredentials()) credid = "$schema$host" - username = creds[:user, credid] - if username === nothing || isusedcreds - username = prompt("Username for '$schema$host'") - creds[:user, credid] = username # save credentials - end + username = creds[:user, credid] userpass = creds[:pass, credid] - if userpass === nothing || isusedcreds - userpass = prompt("Password for '$schema$username@$host'", password=true) - creds[:pass, credid] = userpass # save credentials + if is_windows() + if username === nothing || userpass === nothing || isusedcreds + res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", + username === nothing ? "" : username; prompt_username = true) + isnull(res) && return Cint(Error.EAUTH) + username, userpass = Base.get(res) + end + else + if username === nothing || isusedcreds + username = prompt("Username for '$schema$host'") + end + + if userpass === nothing || isusedcreds + userpass = prompt("Password for '$schema$username@$host'", password=true) + end end + creds[:user, credid] = username # save credentials + creds[:pass, credid] = userpass # save credentials isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index 736a771c0bae6..ba2149dbda576 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -16,6 +16,9 @@ reset(val::Integer, flag::Integer) = (val &= ~flag) toggle(val::Integer, flag::Integer) = (val |= flag) function prompt(msg::AbstractString; default::AbstractString="", password::Bool=false) + if is_windows() && password + error("Command line prompt not supported for password entry on windows. Use winprompt instead") + end msg = !isempty(default) ? msg*" [$default]:" : msg*":" uinput = if password Base.getpass(msg) diff --git a/base/util.jl b/base/util.jl index 3bed25e62ee63..7fb266117db46 100644 --- a/base/util.jl +++ b/base/util.jl @@ -344,3 +344,85 @@ end else getpass(prompt::AbstractString) = unsafe_string(ccall(:getpass, Cstring, (Cstring,), prompt)) end + +# Windows authentication prompt +if is_windows() + immutable CREDUI_INFO + cbSize::UInt32 + parent::Ptr{Void} + pszMessageText::Ptr{UInt16} + pszCaptionText::Ptr{UInt16} + banner::Ptr{Void} + end + + const CREDUIWIN_GENERIC = 0x0001 + const CREDUIWIN_IN_CRED_ONLY = 0x0020 + const CREDUIWIN_ENUMERATE_CURRENT_USER = 0x0200 + + const CRED_PACK_GENERIC_CREDENTIALS = 0x0004 + + const ERROR_SUCCESS = 0x0000 + const ERROR_CANCELLED = 0x04c7 + + function winprompt(message, caption, default_username; prompt_username = true) + # Step 1: Create an encrypted username/password bundle that will be used to set + # the default username (in theory could also provide a default password) + credbuf = Array(UInt8, 1024) + credbufsize = Ref{UInt32}(sizeof(credbuf)) + succeeded = ccall((:CredPackAuthenticationBufferW, "credui.dll"), stdcall, Bool, + (UInt32, Cwstring, Cwstring, Ptr{UInt8}, Ptr{UInt32}), + CRED_PACK_GENERIC_CREDENTIALS, default_username, "", credbuf, credbufsize) + @assert succeeded + + # Step 2: Create the actual dialog + # 2.1: Set up the window + messageArr = Base.cwstring(message) + captionArr = Base.cwstring(caption) + pfSave = Ref{Bool}(false) + cred = Ref{CREDUI_INFO}(CREDUI_INFO(sizeof(CREDUI_INFO), C_NULL, pointer(messageArr), pointer(captionArr), C_NULL)) + dwflags = CREDUIWIN_GENERIC | CREDUIWIN_ENUMERATE_CURRENT_USER + if !prompt_username + # Disable setting anything other than default_username + dwflags |= CREDUIWIN_IN_CRED_ONLY + end + authPackage = Ref{Culong}(0) + outbuf_data = Ref{Ptr{Void}}(C_NULL) + outbuf_size = Ref{Culong}(0) + + # 2.2: Do the actual request + code = ccall((:CredUIPromptForWindowsCredentialsW, "credui.dll"), stdcall, UInt32, (Ptr{CREDUI_INFO}, UInt32, Ptr{Culong}, + Ptr{Void}, Culong, Ptr{Ptr{Void}}, Ptr{Culong}, Ptr{Bool}, UInt32), cred, 0, authPackage, credbuf, credbufsize[], + outbuf_data, outbuf_size, pfSave, dwflags) + + # 2.3: If that failed for any reason other than the user canceling, error out. + # If the user canceled, just return a nullable + if code == ERROR_CANCELLED + return Nullable{Tuple{String,String}}() + elseif code != ERROR_SUCCESS + error(Base.Libc.FormatMessage(code)) + end + + # Step 3: Convert encrypted credentials back to plain text + passbuf = Array(UInt16, 1024) + passlen = Ref{UInt32}(length(passbuf)) + usernamebuf = Array(UInt16, 1024) + usernamelen = Ref{UInt32}(length(usernamebuf)) + # Need valid buffers for domain, even though we don't care + dummybuf = Array(UInt16, 1024) + succeeded = ccall((:CredUnPackAuthenticationBufferW, "credui.dll"), Bool, + (UInt32, Ptr{Void}, UInt32, Ptr{UInt16}, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}), + 0, outbuf_data[], outbuf_size[], usernamebuf, usernamelen, dummybuf, Ref{UInt32}(1024), passbuf, passlen) + if !succeeded + error(Base.Libc.FormatMessage()) + end + + # Step 4: Free the encrypted buffer + # ccall(:SecureZeroMemory, Ptr{Void}, (Ptr{Void}, Csize_t), outbuf_data[], outbuf_size[]) - not an actual function + ccall((:CoTaskMemFree, "ole32.dll"), Void, (Ptr{Void},), outbuf_data[]) + + # Done + Nullable((String(transcode(UInt8, usernamebuf[1:usernamelen[]-1])), + String(transcode(UInt8, passbuf[1:passlen[]-1])))) + end + +end From 235f8eb523fc0c79045d3d8dc12c5eac5f13878c Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Wed, 20 Jul 2016 18:59:35 -0400 Subject: [PATCH 0576/1117] Add patch for libssh2 encrypted PEM support --- deps/libssh2.mk | 5 +- deps/patches/libssh2-encryptedpem.patch | 467 ++++++++++++++++++++++++ 2 files changed, 471 insertions(+), 1 deletion(-) create mode 100644 deps/patches/libssh2-encryptedpem.patch diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 41b1c82cd2eb2..be13f58c8653e 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -26,8 +26,11 @@ endif $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied: | $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-mbedtls.patch echo 1 > $@ +$(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-encryptedpem.patch-applied: | $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-encryptedpem.patch + echo 1 > $@ -$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied $(MBEDTLS_OBJ_TARGET) +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-encryptedpem.patch-applied $(MBEDTLS_OBJ_TARGET) mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBSSH2_OPTS) diff --git a/deps/patches/libssh2-encryptedpem.patch b/deps/patches/libssh2-encryptedpem.patch new file mode 100644 index 0000000000000..c621cc5ad0684 --- /dev/null +++ b/deps/patches/libssh2-encryptedpem.patch @@ -0,0 +1,467 @@ +commit 8304968ceb516ea21740f7c5383fa4a3a8d84e46 +Author: Keno Fischer <kfischer@college.harvard.edu> +Date: Wed Jul 20 18:49:21 2016 -0400 + + Add supported for passphrase-protected PEM files to WinCNG and GCrypt backends + + Since they use our own PEM parser which did not support encrypted PEM files, + trying to use such files on these backends failed. Fix that by augmenting + the PEM parser to support encrypted PEM files. + +diff --git a/src/crypt.c b/src/crypt.c +index 931ae8b..853682d 100644 +--- a/src/crypt.c ++++ b/src/crypt.c +@@ -53,6 +53,7 @@ crypt_none_crypt(LIBSSH2_SESSION * session, unsigned char *buf, + + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_none = { + "none", ++ "DEK-Info: NONE" + 8, /* blocksize (SSH2 defines minimum blocksize as 8) */ + 0, /* iv_len */ + 0, /* secret_len */ +@@ -119,6 +120,7 @@ crypt_dtor(LIBSSH2_SESSION * session, void **abstract) + #if LIBSSH2_AES_CTR + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes128_ctr = { + "aes128-ctr", ++ "", + 16, /* blocksize */ + 16, /* initial value length */ + 16, /* secret length -- 16*8 == 128bit */ +@@ -131,6 +133,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes128_ctr = { + + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes192_ctr = { + "aes192-ctr", ++ "", + 16, /* blocksize */ + 16, /* initial value length */ + 24, /* secret length -- 24*8 == 192bit */ +@@ -143,6 +146,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes192_ctr = { + + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes256_ctr = { + "aes256-ctr", ++ "", + 16, /* blocksize */ + 16, /* initial value length */ + 32, /* secret length -- 32*8 == 256bit */ +@@ -157,6 +161,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes256_ctr = { + #if LIBSSH2_AES + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes128_cbc = { + "aes128-cbc", ++ "DEK-Info: AES-128-CBC", + 16, /* blocksize */ + 16, /* initial value length */ + 16, /* secret length -- 16*8 == 128bit */ +@@ -169,6 +174,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes128_cbc = { + + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes192_cbc = { + "aes192-cbc", ++ "DEK-Info: AES-192-CBC", + 16, /* blocksize */ + 16, /* initial value length */ + 24, /* secret length -- 24*8 == 192bit */ +@@ -181,6 +187,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes192_cbc = { + + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes256_cbc = { + "aes256-cbc", ++ "DEK-Info: AES-256-CBC", + 16, /* blocksize */ + 16, /* initial value length */ + 32, /* secret length -- 32*8 == 256bit */ +@@ -195,6 +202,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_aes256_cbc = { + static const LIBSSH2_CRYPT_METHOD + libssh2_crypt_method_rijndael_cbc_lysator_liu_se = { + "rijndael-cbc@lysator.liu.se", ++ "DEK-Info: AES-256-CBC", + 16, /* blocksize */ + 16, /* initial value length */ + 32, /* secret length -- 32*8 == 256bit */ +@@ -209,6 +217,7 @@ static const LIBSSH2_CRYPT_METHOD + #if LIBSSH2_BLOWFISH + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_blowfish_cbc = { + "blowfish-cbc", ++ "", + 8, /* blocksize */ + 8, /* initial value length */ + 16, /* secret length */ +@@ -223,6 +232,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_blowfish_cbc = { + #if LIBSSH2_RC4 + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_arcfour = { + "arcfour", ++ "DEK-Info: RC4", + 8, /* blocksize */ + 8, /* initial value length */ + 16, /* secret length */ +@@ -258,6 +268,7 @@ crypt_init_arcfour128(LIBSSH2_SESSION * session, + + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_arcfour128 = { + "arcfour128", ++ "", + 8, /* blocksize */ + 8, /* initial value length */ + 16, /* secret length */ +@@ -272,6 +283,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_arcfour128 = { + #if LIBSSH2_CAST + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_cast128_cbc = { + "cast128-cbc", ++ "", + 8, /* blocksize */ + 8, /* initial value length */ + 16, /* secret length */ +@@ -286,6 +298,7 @@ static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_cast128_cbc = { + #if LIBSSH2_3DES + static const LIBSSH2_CRYPT_METHOD libssh2_crypt_method_3des_cbc = { + "3des-cbc", ++ "DEK-Info: DES-EDE3-CBC", + 8, /* blocksize */ + 8, /* initial value length */ + 24, /* secret length */ +diff --git a/src/libgcrypt.c b/src/libgcrypt.c +index 366d007..901be05 100644 +--- a/src/libgcrypt.c ++++ b/src/libgcrypt.c +@@ -172,8 +172,6 @@ _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa, + unsigned char *n, *e, *d, *p, *q, *e1, *e2, *coeff; + unsigned int nlen, elen, dlen, plen, qlen, e1len, e2len, coefflen; + +- (void) passphrase; +- + fp = fopen(filename, "r"); + if (!fp) { + return -1; +@@ -182,6 +180,7 @@ _libssh2_rsa_new_private(libssh2_rsa_ctx ** rsa, + ret = _libssh2_pem_parse(session, + "-----BEGIN RSA PRIVATE KEY-----", + "-----END RSA PRIVATE KEY-----", ++ passphrase, + fp, &data, &datalen); + fclose(fp); + if (ret) { +@@ -285,8 +284,6 @@ _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa, + unsigned char *p, *q, *g, *y, *x; + unsigned int plen, qlen, glen, ylen, xlen; + +- (void) passphrase; +- + fp = fopen(filename, "r"); + if (!fp) { + return -1; +@@ -295,6 +292,7 @@ _libssh2_dsa_new_private(libssh2_dsa_ctx ** dsa, + ret = _libssh2_pem_parse(session, + "-----BEGIN DSA PRIVATE KEY-----", + "-----END DSA PRIVATE KEY-----", ++ passphrase, + fp, &data, &datalen); + fclose(fp); + if (ret) { +diff --git a/src/libssh2_priv.h b/src/libssh2_priv.h +index b4296a2..77b51a9 100644 +--- a/src/libssh2_priv.h ++++ b/src/libssh2_priv.h +@@ -884,6 +884,7 @@ struct _LIBSSH2_HOSTKEY_METHOD + struct _LIBSSH2_CRYPT_METHOD + { + const char *name; ++ const char *pem_annotation; + + int blocksize; + +@@ -1041,6 +1042,7 @@ const LIBSSH2_HOSTKEY_METHOD **libssh2_hostkey_methods(void); + int _libssh2_pem_parse(LIBSSH2_SESSION * session, + const char *headerbegin, + const char *headerend, ++ const unsigned char *passphrase, + FILE * fp, unsigned char **data, unsigned int *datalen); + int _libssh2_pem_parse_memory(LIBSSH2_SESSION * session, + const char *headerbegin, +diff --git a/src/os400qc3.c b/src/os400qc3.c +index f8e46ab..a42b374 100644 +--- a/src/os400qc3.c ++++ b/src/os400qc3.c +@@ -2120,6 +2120,7 @@ try_pem_load(LIBSSH2_SESSION *session, FILE *fp, + fseek(fp, 0L, SEEK_SET); + for (;;) { + ret = _libssh2_pem_parse(session, header, trailer, ++ NULL, + fp, &data, &datalen); + + if (!ret) { +diff --git a/src/pem.c b/src/pem.c +index 9f51bba..523d9c5 100644 +--- a/src/pem.c ++++ b/src/pem.c +@@ -37,6 +37,7 @@ + */ + + #include "libssh2_priv.h" ++#include <assert.h> + + static int + readline(char *line, int line_size, FILE * fp) +@@ -96,16 +97,26 @@ readline_memory(char *line, size_t line_size, + + #define LINE_SIZE 128 + ++const char *crypt_annotation = "Proc-Type: 4,ENCRYPTED"; ++ ++static unsigned char hex_decode(char digit) ++{ ++ return (digit >= 'A') ? 0xA + (digit - 'A') : (digit - '0'); ++} ++ + int + _libssh2_pem_parse(LIBSSH2_SESSION * session, + const char *headerbegin, + const char *headerend, ++ const unsigned char *passphrase, + FILE * fp, unsigned char **data, unsigned int *datalen) + { + char line[LINE_SIZE]; ++ unsigned char iv[LINE_SIZE]; + char *b64data = NULL; + unsigned int b64datalen = 0; + int ret; ++ const LIBSSH2_CRYPT_METHOD *method = NULL; + + do { + *line = '\0'; +@@ -116,7 +127,45 @@ _libssh2_pem_parse(LIBSSH2_SESSION * session, + } + while (strcmp(line, headerbegin) != 0); + +- *line = '\0'; ++ if (readline(line, LINE_SIZE, fp)) { ++ return -1; ++ } ++ ++ if (passphrase && ++ memcmp(line, crypt_annotation, strlen(crypt_annotation)) == 0) { ++ const LIBSSH2_CRYPT_METHOD **all_methods, *cur_method; ++ int i; ++ ++ if (readline(line, LINE_SIZE, fp)) { ++ ret = -1; ++ goto out; ++ } ++ all_methods = libssh2_crypt_methods(); ++ while ((cur_method = *all_methods++)) { ++ if (*cur_method->pem_annotation && ++ memcmp(line, cur_method->pem_annotation, strlen(cur_method->pem_annotation)) == 0) { ++ method = cur_method; ++ memcpy(iv, line+strlen(method->pem_annotation)+1, 2*method->iv_len); ++ } ++ } ++ ++ /* None of the available crypt methods were able to decrypt this key */ ++ if (method == NULL) ++ return -1; ++ ++ /* Decode IV from hex */ ++ for (i = 0; i < method->iv_len; ++i) ++ { ++ iv[i] = hex_decode(iv[2*i]) << 4; ++ iv[i] |= hex_decode(iv[2*i+1]); ++ } ++ ++ /* skip to the next line */ ++ if (readline(line, LINE_SIZE, fp)) { ++ ret = -1; ++ goto out; ++ } ++ } + + do { + if (*line) { +@@ -152,6 +201,72 @@ _libssh2_pem_parse(LIBSSH2_SESSION * session, + goto out; + } + ++ if (method) { ++ /* Set up decryption */ ++ int free_iv = 0, free_secret = 0, len_decrypted = 0, padding = 0; ++ int blocksize = method->blocksize; ++ void *abstract; ++ unsigned char secret[2*MD5_DIGEST_LENGTH]; ++ libssh2_md5_ctx fingerprint_ctx; ++ ++ /* Perform key derivation (PBKDF1/MD5) */ ++ if (!libssh2_md5_init(&fingerprint_ctx)) { ++ ret = -1; ++ goto out; ++ } ++ libssh2_md5_update(fingerprint_ctx, passphrase, strlen((char*)passphrase)); ++ libssh2_md5_update(fingerprint_ctx, iv, 8); ++ libssh2_md5_final(fingerprint_ctx, secret); ++ if (method->secret_len > MD5_DIGEST_LENGTH) { ++ if (!libssh2_md5_init(&fingerprint_ctx)) { ++ ret = -1; ++ goto out; ++ } ++ libssh2_md5_update(fingerprint_ctx, secret, MD5_DIGEST_LENGTH); ++ libssh2_md5_update(fingerprint_ctx, passphrase, strlen((char*)passphrase)); ++ libssh2_md5_update(fingerprint_ctx, iv, 8); ++ libssh2_md5_final(fingerprint_ctx, secret + MD5_DIGEST_LENGTH); ++ } ++ ++ /* Initialize the decryption */ ++ if (method->init(session, method, iv, &free_iv, secret, ++ &free_secret, 0, &abstract)) { ++ memset((char*)secret, 0, sizeof(secret)); ++ LIBSSH2_FREE(session, data); ++ ret = -1; ++ goto out; ++ } ++ ++ if (free_secret) { ++ memset((char*)secret, 0, sizeof(secret)); ++ } ++ ++ /* Do the actual decryption */ ++ assert((*datalen % blocksize) == 0); ++ ++ while (len_decrypted <= *datalen - blocksize) { ++ if (method->crypt(session, *data + len_decrypted, blocksize, ++ &abstract)) { ++ ret = LIBSSH2_ERROR_DECRYPT; ++ method->dtor(session, &abstract); ++ memset(*data, 0, *datalen); ++ LIBSSH2_FREE(session, *data); ++ goto out; ++ } ++ ++ len_decrypted += blocksize; ++ } ++ ++ /* Account for padding */ ++ padding = (*data)[*datalen - 1]; ++ memset(&(*data)[*datalen-padding],0,padding); ++ *datalen -= padding; ++ ++ /* Clean up */ ++ memset((char*)secret, 0, sizeof(secret)); ++ method->dtor(session, &abstract); ++ } ++ + ret = 0; + out: + if (b64data) { +diff --git a/src/wincng.c b/src/wincng.c +index d3271b3..e6278c3 100755 +--- a/src/wincng.c ++++ b/src/wincng.c +@@ -537,14 +537,13 @@ _libssh2_wincng_load_pem(LIBSSH2_SESSION *session, + FILE *fp; + int ret; + +- (void)passphrase; +- + fp = fopen(filename, "r"); + if (!fp) { + return -1; + } + + ret = _libssh2_pem_parse(session, headerbegin, headerend, ++ passphrase, + fp, data, datalen); + + fclose(fp); +diff --git a/tests/etc/user b/tests/etc/user +index 01bf6ac..5f1783d 100644 +--- a/tests/etc/user ++++ b/tests/etc/user +@@ -1,27 +1,30 @@ + -----BEGIN RSA PRIVATE KEY----- +-MIIEoQIBAAKCAQEAxIgBuZS39D4bFnWminE7svGQLdVKx1aWKnEYEa+XtNU4DKZ/ +-pxUHg0zbEBya+IkX1yqQYWALoiOwI8XhdemLp8g03BX7o+DLSWisfiHpCDVGAuNq +-RDF7qnFyL/ZBH6e0XKMtsoB51TDuBc4Rxh6p1V2QL/fg8BoHcCrnKkoqN8PSoKUX +-2lPKJ3JIF/P8cDLbKYCvbSTFOdf56eqg0GJe7jFtSwweE9yz3IWZ3kSS1E/9E6sX +-aNCu/hUt1bvQthICQyBNoTtQP/igEUJ7n0GMetsnq9wiUSomLzWqIWNqmvOv62aC +-XRi5sYgpSAR4Zvnm3Cx/Wl0BEPz2rrFkG+G0SQIBIwKCAQEAgSYtBOyzZfztOqUV +-q277WFWZQrC8HJf8R8aparU3zpq+braOZnuImByP9KUVYX6pRECKw6WD/NWfonq4 +-uzMSoXTviVBGRx6xeWIK880kG1Y1UlruD447Ur+ULiV7QLAIzylnLCiKk2lL9S+l +-R63AD95mEOS4Y0ROB+Gt2fY5ABHRMqhGLvRKK8qwn35C1Z9qnTGhgiRbeoc373A5 +-ZAYyegyLnbvyV47UfPYS/TVzxZ9RCx3D3I/9fI7ZAFafkkIufQX3QPaVxf0zFUwW +-de/f+gTbySTL4RDF185Evunx+tYvzCyIimB0cTE5dfsCWcHDtO6DwehKiOgJsbeW +-IrpeTwKBgQDnEMfv7ORR35Ouj91iNCSfLU/v0TSzAJBfqYovByhXRsopgWAKXUmH +-mWpBBP5vcGu3NvKfiZcMMbBPfllvlxkafQwvCqrdn5mg01MdAHMWP/O9yfvuxMDE +-KycU2G2CT8j85mIPn19WeIgXC/kws+P0RAVNCBNeq89Gvp4IdLN67wKBgQDZvTYh +-TPCYG32jBK+CcWmOna2SLvBloDcNevhzfu3RGjLIXzUHGxLdx7slsP/tpndmXIAL +-CgV6GfrLxix5bO08203S2qnnwP8VrjjNIv1CyZIbbQFAFIeC3QxZMZHnXieLrO0z +-qF5CuUXcL3cMeGmF/0HN/rB+4sF0qfv8wD8kRwKBgFXTCG8O2HYueK6NNPiXBknA +-X4T17wCocCOIHWHstzZcHzP82oeBvDmuAuTzOe7gnQmJcA9e/ZbQoJKOA/Y/b7lh +-pXCO7wHcMb9kb1PqOWAJIASqG78V4TLrdOp8Re6Sqb0FHRu+2kSwbQ/f4DapN2lb +-F+lpZke8KGq71ExImm99AoGAN/10UbSy5UjlytVRs9QFM00eAQTBeTfTpGFzFmJ3 +-qsw48bIU8zLY9zNcAmC21rXG7m+Oo8C/lG0UmsyPF+jPSinDjf22qU7iger4qccr +-Lm5YxTlJduC1IaaOJZBnWMBwkaF+0sTlCdfew5ctPbiQKcVLb3wBf7amxjpWvVYB +-m50CgYAT6t2/Suav21J5zpzyrrt+oMZQ3MMzBnPHFRUQ1FdqZnE4eW5a10g0P+E+ +-YeTol+fYxL34+cI5PREK3dcnW1E8g8KOsOQqMgWdTfZEDHYRLqEyGIhu20aqfJCY +-qu9tBburQoSlym9aQp41CMxIyHrL4GnwRlJkTTEVhDuab1HmKw== ++Proc-Type: 4,ENCRYPTED ++DEK-Info: AES-128-CBC,452208EE925946C14BF29ABD1425A8DF ++ ++qTRxBL3/Ldh3NjoYUpZ+IsiQDqLrVG/rS20w1yQJMFXf+VPu+MjlWhGhrB6yscAH ++IyEje8X/YSB7tS3UNQuCLZVsPbDf5ZAUKz9EMERh3NmzxHFfOi9+7+enYJjaqyr9 ++cwYJ7OIk2stwMHlp71LMfSmmJOD+TA68j6PcOa0dsfJKZ6O/Pcc0o7qkWXUamybE ++YfvUoT9EigSqVukdEL2HqC4tlpUL5cJo8uteyYfnHP0b1weUy8i7utE5jPOt4sK2 ++3gUk4mZHHy78NgV8HLwKlDD12CmbotjzXXs9+Aqu6TUjqrzJkHy7z6JkXqMQLyfC ++wBehnpu4bCRTuEbV8wI/6cb6POjnSHBw9m+E7Uo8bPlQrFbtYrVh2gC1C4NuOZxr ++EgJS8JXFoodSH8tthZJa4Xs/IO7AGcz9W/hn6TZ5kBLrhfXBzy08PaMP6lr/icSS ++2YpImXnYAbqTGuaAPZwsKzNGew5iVds5QoOAzXEO+l5rSEp9L2OKjM/tli96a9T3 ++T3vOWnVv6IC6nAV2wFrreHUPJTINzJZ2cSD17uudX53QWBOXj7ym+CwODdE5OgsW ++5lysVuXHpMhV3CcTG7BihmmenP7dsyQ/nF/8MYAqY2PcQnQ2PQ8uXpPXTwkmDPnV +++jBpZc54EWk6OP99aUoqDk2tMMyi7FBZgtzY+yWqsvb8oB0RStmbqxxdy27EkHvD ++KNvrXVwkHxFSMduPwKr34x6Ecz7wwrcM/XlFD13/aZ46xhfXa3N1fQSLETf4l67e ++2/tuWHE7qMslJ86jKk0Pj2arT/tfuVufXvjokZjkOVez2GU9WnyP3CGBOVZfrrE9 ++Cub7ULRuG8gIKLCeaNcuuYen5R09oDpvgA1b9BnaF+x9mrrn3mPvpIl/+j6YeBpy ++PGV4mJuGkjphBWiwDelUG/HDqpnwXJ2fuIrIXqdwSsTV0HIholRWpxonpAepMCiZ ++arxAm1rIC8ku2ZyRG8lwd1o8VAmDbntbCE7JDcUpXI8/LeTFTVQ1Ewdy0LPE2/9o ++BBoCqzBygop+AJkm2Nb9i54dRbUqkl10wTf0M9THWzUZ4orFtSGObqEQHQekyoDP ++6/onV4M77Ptvz1oX/2x2CLUTDDS5RKHcqE8Z6br93W14Gq7pQgu0clPsFwaDeTPk ++NdEhX14R1MKUednIGSBTXt2DRych+OJFdHSG48kg90o0hHbD6dAkunTJJVmkIH6i ++qwaOLArr5+vQS2OEvr0I5UESdAqL5mXpz/ciheM//nUaQRqAa6eURTjJYEZjXCKv ++0UjqOo0ola7KzchUoH33k14BapcgcYWrBGkP27MEv4eVrNtRZtPUAFNN68esF0l5 ++d/sGtJAjcrpVjHnnd21XIuywP5VZsOzZZn1ea3obfVYzbuoVJ/L4zpmn+eFR2IIp ++SExrFWV+SCvzGvSDHDbg7y081qgaQgn/Xte3DwlCbBWCGSVS3TVSJSnYAXU5mq9z ++D6Zn1fE2foefWrlXYeDfZLY1M463Tufag55uI16RrSzb/nfSBFTqi4TJAzVkn+gi ++jy3LwEdOKAk/d/etr8wL/WWZ+w334TG7pMoAKErVJ/NexG4vzzHedExvTaEMD/eS + -----END RSA PRIVATE KEY----- +diff --git a/tests/etc/user-decrypted b/tests/etc/user-decrypted +new file mode 100644 +index 0000000..f99cf65 +--- /dev/null ++++ b/tests/etc/user-decrypted +@@ -0,0 +1,27 @@ ++*data[*datalen - 1]-----BEGIN RSA PRIVATE KEY----- ++MIIEogIBAAKCAQEAl46YP4uEaIStJEUuGEa6/dmtykdWrZCTqM9iOnfVNawlM8tE ++6yHppI8c6FppPRErhlFfzEUqqeB4o19vAyTNYB3WyfNUbtSVvWbNz5+j4jvllC/8 ++7TD5rGp9JzAjFtmzJcT5GplvHiBV9aqAnfvplA9C4cHmdTChSPg8C4tPXoMNhTl0 ++GmNVEhTdiBzQdAsPPJGn2pQzFcF2fnyHcPUD9Shx4P48dacmL4/T9mwSZ0IrBkuQ ++qcu0mE4RSd6mSuXRl4SiFv4vNmIGlt0NIBgytZbdi/bbkJ79iNiyCaFxjWDh+fOP ++AqJAQCSjTnjvXpe3mxDe2M58F6qETHGu3ssVOQIDAQABAoIBADTFrjXSOtS5/h/3 ++BsnlNw9VULJyjLf686+AJs/9EfySaJHUYbcOI/Ds9D/j22ksJV+rzhOuseBdDSS2 ++Ak2x7uFdAoFnfSrqUwmUVm3JRKNt/87NtuBQgV+L5PBUTqOSQlMQdzzP2GVBFS1p ++pUddhEPN3JgHkFuWB1MFPMM4Sva5WjruIA5QFQb1a/yU+RPlSqzlK7POp/jHrJwy ++zjhqvVn4eZ4/qQBAd1zGtHf4LaEaN0VBHcJv2M/+olR4p1fmOBxisIeoauTmGQCU ++nEYMAFydKW581S3qm2TLUudp3eeHvgFlfAbb0Mn88OlTu4+A7nTCsPe5og6ni4Wy ++c9TaKAECgYEAxbL33HMq5U8EcRM8fKFKzOPaN+jy0bgmXC8HzXwKh4XRp3AL1U08 ++MKxwBhtyPqZiasWRw2RFzyMZ+0xsdK41xLysY64STuaqbaFLFSSclJdt1OiTGFWG ++cULADzmTZIe7pWyJjtyvpUvzXtVSgB0arOkJFiOSA94qcqUAY/MajnkCgYEAxEAw ++3b1i2VBXbKiPIjqa8OCu55vli3Z4LRGVL++zLsv7Kjbd8wcbC3DtcVegJzPikuRq +++J+zF34vKy4ZhujGTpv9v02sQz3NP/I7Q74u8ptLNDa6GhLv4N2w+6Fg3xn0VIvw ++YQZnXlrumUHzABG89rU/lidomsYGNu3163vYDMECgYBHWglU+gfgTP14Zg3tmTTq ++781hYmTIxiKqIJs7UWpncSeyu6C5si9oDcQ8oAlCjUB18qpiD72WPb/xz6pRDlL/ ++l2zMaRXYfi3CdXUow1bz1hdgwTPdk3h7RjEVDZ2hqc9Sq/KVqhaGnPvQHUOyU06+ ++WzjZUaswVC2TxPN4H1RVWQKBgDpPGDXJ32TSCieurVhuMTsAhvcpyrn1KR+66aik ++f4iX8rhINQnQQqjTlLjoYB1FvTwHwgC88mk4hKs/tzXuU31nu6zrVxNjeWhWgGVQ ++TIKowV9yIzk8Bym4tK0O9clT3NSB/5nUXUIbomjGv3x1yL9EevASJXJXA+GuMjMV ++SMwBAoGADB7N+svIEmbxIOUURDJEIIDpmU4P1760ikLdzXHJFtkzeLaVmORex9OD ++dihk39bZctvgBvtSFFXxdImr7rzakFVwT0oQkKGfTgYpei9nK3VSOBeivAjuway6 ++aK73cbiiU7Mcuppn7ml8TqxVrGy//g4cJCoscPU9PG9KdoDNYvM= ++-----END RSA PRIVATE KEY----- +diff --git a/tests/etc/user.pub b/tests/etc/user.pub +index c60e90d..2fc7b4b 100644 +--- a/tests/etc/user.pub ++++ b/tests/etc/user.pub +@@ -1 +1 @@ +-ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxIgBuZS39D4bFnWminE7svGQLdVKx1aWKnEYEa+XtNU4DKZ/pxUHg0zbEBya+IkX1yqQYWALoiOwI8XhdemLp8g03BX7o+DLSWisfiHpCDVGAuNqRDF7qnFyL/ZBH6e0XKMtsoB51TDuBc4Rxh6p1V2QL/fg8BoHcCrnKkoqN8PSoKUX2lPKJ3JIF/P8cDLbKYCvbSTFOdf56eqg0GJe7jFtSwweE9yz3IWZ3kSS1E/9E6sXaNCu/hUt1bvQthICQyBNoTtQP/igEUJ7n0GMetsnq9wiUSomLzWqIWNqmvOv62aCXRi5sYgpSAR4Zvnm3Cx/Wl0BEPz2rrFkG+G0SQ== jas@mocca ++ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXjpg/i4RohK0kRS4YRrr92a3KR1atkJOoz2I6d9U1rCUzy0TrIemkjxzoWmk9ESuGUV/MRSqp4HijX28DJM1gHdbJ81Ru1JW9Zs3Pn6PiO+WUL/ztMPmsan0nMCMW2bMlxPkamW8eIFX1qoCd++mUD0LhweZ1MKFI+DwLi09egw2FOXQaY1USFN2IHNB0Cw88kafalDMVwXZ+fIdw9QP1KHHg/jx1pyYvj9P2bBJnQisGS5Cpy7SYThFJ3qZK5dGXhKIW/i82YgaW3Q0gGDK1lt2L9tuQnv2I2LIJoXGNYOH5848CokBAJKNOeO9el7ebEN7YznwXqoRMca7eyxU5 libssh2test@example.com From 23564cf00451e81837c3214e97246085574b6e95 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 21 Jul 2016 16:31:39 -0400 Subject: [PATCH 0577/1117] remove need for dump_lock state needed in MODE_AST is now contained on the stack the other modes need the toplevel lock (nonexistant) --- base/loading.jl | 2 +- doc/devdocs/locks.rst | 33 ++- src/dump.c | 654 +++++++++++++++++++++--------------------- 3 files changed, 352 insertions(+), 337 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index bf96a9d351436..6bda601ef549f 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -509,7 +509,7 @@ end module_uuid(m::Module) = ccall(:jl_module_uuid, UInt64, (Any,), m) -isvalid_cache_header(f::IOStream) = 0 != ccall(:jl_deserialize_verify_header, Cint, (Ptr{Void},), f.ios) +isvalid_cache_header(f::IOStream) = 0 != ccall(:jl_read_verify_header, Cint, (Ptr{Void},), f.ios) function cache_dependencies(f::IO) modules = Tuple{Symbol,UInt64}[] diff --git a/doc/devdocs/locks.rst b/doc/devdocs/locks.rst index 653ef43115941..191312d811323 100644 --- a/doc/devdocs/locks.rst +++ b/doc/devdocs/locks.rst @@ -13,6 +13,10 @@ The following strategies are used to ensure that the code is dead-lock free 3. avoid constructs that expect to need unrestricted recursion + +Locks +----- + Below are all of the locks that exist in the system and the mechanisms for using them that avoid the potential for deadlocks (no Ostrich algorithm allowed here): @@ -38,10 +42,9 @@ The following are definitely leaf locks (level 1), and must not try to acquire a The following is a leaf lock (level 2), and only acquires level 1 locks (safepoint) internally: - * dump (this lock may be possible to eliminate by storing the state on the stack) * typecache -The following are level 3 locks, which can only acquire level 1 or level 2 locks internally: +The following is a level 3 lock, which can only acquire level 1 or level 2 locks internally: * Method->writelock @@ -81,9 +84,10 @@ The following is the root lock, meaning no other lock shall be held when trying additionally, it's unclear if *any* code can safely run in parallel with an arbitrary toplevel expression, so it may require all threads to get to a safepoint first +Broken Locks +------------ The following locks are broken: -------------------------------- * toplevel @@ -121,8 +125,25 @@ The following locks are broken: fix: lock for ``apply_type`` / global (level 2?) -* dump +Shared Global Data Structures +----------------------------- + +These data structures each need locks due to being shared mutable global state. +It is the inverse list for the above lock priority list. +This list does not include level 1 leaf resources due to their simplicity. + +MethodTable modifications (def, cache, kwsorter type) : MethodTable->writelock + +Type declarations : toplevel lock + +Type application : typecache lock + +Module serializer : toplevel lock + +JIT : codegen lock - may be able to eliminate this by making the state thread-local +LLVMContext : codegen lock - might also need to ensure that incremental deserialize has a toplevel expression lock +Method : Method->writelock + - roots array (serializer and codegen) + - invoke / specializations / tfunc modifications diff --git a/src/dump.c b/src/dump.c index 96a7ec73d8218..9477aafc73922 100644 --- a/src/dump.c +++ b/src/dump.c @@ -25,8 +25,6 @@ extern "C" { #endif -static jl_mutex_t dump_lock; - // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers @@ -43,6 +41,7 @@ static jl_value_t *deser_symbols[256]; // (the order in the serializer stream) in MODE_MODULE, the low // bit is reserved for flagging certain entries and pos is // left shift by 1 +// (not used in MODE_AST) static htable_t backref_table; int backref_table_numel; static arraylist_t backref_list; @@ -50,17 +49,23 @@ static arraylist_t backref_list; // list of (jl_value_t **loc, size_t pos) entries // for anything that was flagged by the deserializer for later // type-rewriting of some sort +// (not used in MODE_AST) static arraylist_t flagref_list; // list of (size_t pos, (void *f)(jl_value_t*)) entries // for the serializer to mark values in need of rework by function f // during deserialization later +// (not used in MODE_AST) static arraylist_t reinit_list; // list of stuff that is being serialized // (only used by the incremental serializer in MODE_MODULE) static jl_array_t *serializer_worklist; +// list of modules being deserialized with __init__ methods +// (not used in MODE_AST) +jl_array_t *jl_module_init_order; + // hash of definitions for predefined function pointers static htable_t fptr_to_id; // array of definitions for the predefined function pointers @@ -75,10 +80,6 @@ static const jl_fptr_t id_to_fptrs[] = { jl_f_intrinsic_call, NULL }; -// pointers to non-AST-ish objects in a compressed tree -static jl_array_t *tree_literal_values=NULL; // (only used in MODE_AST) -static jl_module_t *tree_enclosing_module=NULL; // (only used in MODE_AST) - static const intptr_t LongSymbol_tag = 23; static const intptr_t LongSvec_tag = 24; static const intptr_t LongExpr_tag = 25; @@ -115,10 +116,17 @@ typedef enum _DUMP_MODES { MODE_MODULE, // first-stage (pre type-uid assignment) MODE_MODULE_POSTWORK, // second-stage (post type-uid assignment) } DUMP_MODES; -static DUMP_MODES mode = (DUMP_MODES) 0; - -static jl_value_t *jl_idtable_type=NULL; +typedef struct { + ios_t *s; + DUMP_MODES mode; + // pointers to non-AST-ish objects in a compressed tree + // (only used in MODE_AST) + jl_array_t *tree_literal_values; + jl_module_t *tree_enclosing_module; +} jl_serializer_state; + +static jl_value_t *jl_idtable_type = NULL; static arraylist_t builtin_types; #define write_uint8(s, n) ios_putc((n), (s)) @@ -192,8 +200,8 @@ static void write_float64(ios_t *s, double x) // --- Static Compile --- #define jl_serialize_value(s, v) jl_serialize_value_(s,(jl_value_t*)(v)) -static void jl_serialize_value_(ios_t *s, jl_value_t *v); -static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc); +static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v); +static jl_value_t *jl_deserialize_value(jl_serializer_state *s, jl_value_t **loc); static jl_value_t ***sysimg_gvars = NULL; static void **sysimg_fvars = NULL; @@ -273,23 +281,23 @@ static int jl_load_sysimg_so(void) return -1; } -static jl_value_t *jl_deserialize_gv(ios_t *s, jl_value_t *v) +static jl_value_t *jl_deserialize_gv(jl_serializer_state *s, jl_value_t *v) { // Restore the GlobalVariable reference to this jl_value_t via the sysimg_gvars table - int32_t gvname_index = read_int32(s)-1; - if (sysimg_gvars != NULL && gvname_index >= 0 && mode == MODE_SYSTEM_IMAGE) { + int32_t gvname_index = read_int32(s->s)-1; + if (sysimg_gvars != NULL && gvname_index >= 0 && s->mode == MODE_SYSTEM_IMAGE) { *sysimg_gvars[gvname_index] = v; } return v; } -static void jl_serialize_gv(ios_t *s, jl_value_t *v) +static void jl_serialize_gv(jl_serializer_state *s, jl_value_t *v) { // write the index of the literal_pointer_val into the system image - write_int32(s, jl_get_llvm_gv(v)); + write_int32(s->s, jl_get_llvm_gv(v)); } -static void jl_serialize_globalvals(ios_t *s) +static void jl_serialize_globalvals(jl_serializer_state *s) { size_t i, len = backref_table.size; void **p = backref_table.table; @@ -299,24 +307,24 @@ static void jl_serialize_globalvals(ios_t *s) uintptr_t pos = offs - (char*)HT_NOTFOUND - 1; int32_t gv = jl_get_llvm_gv((jl_value_t*)p[i]); if (gv != 0) { - write_int32(s, pos + 1); - write_int32(s, gv); + write_int32(s->s, pos + 1); + write_int32(s->s, gv); } } } - write_int32(s, 0); + write_int32(s->s, 0); } -static void jl_deserialize_globalvals(ios_t *s) +static void jl_deserialize_globalvals(jl_serializer_state *s) { while (1) { - intptr_t key = read_int32(s); + intptr_t key = read_int32(s->s); if (key == 0) break; jl_deserialize_gv(s, (jl_value_t*)backref_list.items[key - 1]); } } -static void jl_serialize_gv_syms(ios_t *s, jl_sym_t *v) +static void jl_serialize_gv_syms(jl_serializer_state *s, jl_sym_t *v) { // since symbols are static, they might not have had a // reference anywhere in the code image other than here @@ -325,14 +333,14 @@ static void jl_serialize_gv_syms(ios_t *s, jl_sym_t *v) int32_t gv = jl_get_llvm_gv((jl_value_t*)v); if (gv != 0) { jl_serialize_value(s, v); - write_int32(s, gv); + write_int32(s->s, gv); } } if (v->left) jl_serialize_gv_syms(s, v->left); if (v->right) jl_serialize_gv_syms(s, v->right); } -static void jl_serialize_gv_others(ios_t *s) +static void jl_serialize_gv_others(jl_serializer_state *s) { // ensures all objects referenced in the code have // references in the system image to their global variable @@ -347,7 +355,7 @@ static void jl_serialize_gv_others(ios_t *s) int32_t gv32 = jl_get_llvm_gv(v32); if (gv32 != 0) { jl_serialize_value(s, v32); - write_int32(s, gv32); + write_int32(s->s, gv32); } } } @@ -358,7 +366,7 @@ static void jl_serialize_gv_others(ios_t *s) int32_t gv64 = jl_get_llvm_gv(v64); if (gv64 != 0) { jl_serialize_value(s, v64); - write_int32(s, gv64); + write_int32(s->s, gv64); } } } @@ -366,7 +374,7 @@ static void jl_serialize_gv_others(ios_t *s) jl_serialize_value(s, NULL); // signal the end of this list } -static void jl_deserialize_gv_others(ios_t *s) +static void jl_deserialize_gv_others(jl_serializer_state *s) { while (1) { jl_value_t *v = jl_deserialize_value(s, NULL); @@ -444,13 +452,13 @@ static void jl_update_all_fptrs(void) // --- serialize --- -static void jl_serialize_fptr(ios_t *s, void *fptr) +static void jl_serialize_fptr(jl_serializer_state *s, void *fptr) { void **pbp = ptrhash_bp(&fptr_to_id, fptr); if (*pbp == HT_NOTFOUND || fptr == NULL) - write_uint16(s, 1); + write_uint16(s->s, 1); else - write_uint16(s, *(intptr_t*)pbp); + write_uint16(s->s, *(intptr_t*)pbp); } static int module_in_worklist(jl_module_t *mod) @@ -477,16 +485,16 @@ static int jl_prune_tcache(jl_typemap_entry_t *ml, void *closure) } -static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) +static void jl_serialize_datatype(jl_serializer_state *s, jl_datatype_t *dt) { int tag = 0; - if (mode == MODE_MODULE_POSTWORK) { + if (s->mode == MODE_MODULE_POSTWORK) { if (dt->name->primary == (jl_value_t*)dt) tag = 6; // primary type else if (dt->uid != 0) tag = 7; // must use apply_type } - else if (mode == MODE_MODULE) { + else if (s->mode == MODE_MODULE) { int internal = module_in_worklist(dt->name->module); int i, l = jl_array_len(serializer_worklist); for (i = 0; i < l; i++) { @@ -537,10 +545,10 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) } } - writetag(s, (jl_value_t*)SmallDataType_tag); - write_uint8(s, 0); // virtual size + writetag(s->s, (jl_value_t*)SmallDataType_tag); + write_uint8(s->s, 0); // virtual size jl_serialize_value(s, (jl_value_t*)jl_datatype_type); - write_uint8(s, tag); + write_uint8(s->s, tag); if (tag == 6) { jl_serialize_value(s, dt->name); return; @@ -555,16 +563,16 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) return; } - write_int32(s, dt->size); + write_int32(s->s, dt->size); int has_instance = (dt->instance != NULL); int has_layout = (dt->layout != NULL); - write_uint8(s, dt->abstract | (dt->mutabl<<1) | (has_layout<<2) | (has_instance<<3) | + write_uint8(s->s, dt->abstract | (dt->mutabl<<1) | (has_layout<<2) | (has_instance<<3) | (dt->hastypevars<<4) | (dt->haswildcard<<5) | (dt->isleaftype<<6)); - write_int32(s, dt->depth); + write_int32(s->s, dt->depth); if (!dt->abstract) { - write_uint16(s, dt->ninitialized); - if (mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK) { - write_int32(s, dt->uid); + write_uint16(s->s, dt->ninitialized); + if (s->mode != MODE_MODULE && s->mode != MODE_MODULE_POSTWORK) { + write_int32(s->s, dt->uid); } } @@ -579,16 +587,16 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) else if (dt->layout == jl_pointer_type->layout) { layout = 3; } - write_uint8(s, layout); + write_uint8(s->s, layout); if (layout == 0) { size_t nf = dt->layout->nfields; - write_uint16(s, nf); - write_int8(s, dt->layout->fielddesc_type); - write_int32(s, dt->layout->alignment); - write_int8(s, dt->layout->haspadding); - write_int8(s, dt->layout->pointerfree); + write_uint16(s->s, nf); + write_int8(s->s, dt->layout->fielddesc_type); + write_int32(s->s, dt->layout->alignment); + write_int8(s->s, dt->layout->haspadding); + write_int8(s->s, dt->layout->pointerfree); size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type); - ios_write(s, (char*)(&dt->layout[1]), nf * fieldsize); + ios_write(s->s, (char*)(&dt->layout[1]), nf * fieldsize); } } @@ -600,19 +608,19 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) jl_serialize_value(s, dt->types); } -static void jl_serialize_module(ios_t *s, jl_module_t *m) +static void jl_serialize_module(jl_serializer_state *s, jl_module_t *m) { - writetag(s, jl_module_type); + writetag(s->s, jl_module_type); jl_serialize_value(s, m->name); int ref_only = 0; - if (mode == MODE_MODULE_POSTWORK) { + if (s->mode == MODE_MODULE_POSTWORK) { assert(!module_in_worklist(m)); ref_only = 1; } - if (mode == MODE_MODULE) { + if (s->mode == MODE_MODULE) { if (!module_in_worklist(m)) ref_only = 1; - write_int8(s, ref_only); + write_int8(s->s, ref_only); } jl_serialize_value(s, m->parent); if (ref_only) { @@ -629,25 +637,25 @@ static void jl_serialize_module(ios_t *s, jl_module_t *m) jl_serialize_value(s, b->value); jl_serialize_value(s, b->globalref); jl_serialize_value(s, b->owner); - write_int8(s, (b->deprecated<<3) | (b->constp<<2) | (b->exportp<<1) | (b->imported)); + write_int8(s->s, (b->deprecated<<3) | (b->constp<<2) | (b->exportp<<1) | (b->imported)); jl_serialize_gv(s, (jl_value_t*)b); } } } jl_serialize_value(s, NULL); if (m == jl_main_module) { - write_int32(s, 1); + write_int32(s->s, 1); jl_serialize_value(s, (jl_value_t*)jl_core_module); } else { - write_int32(s, m->usings.len); + write_int32(s->s, m->usings.len); for(i=0; i < m->usings.len; i++) { jl_serialize_value(s, (jl_value_t*)m->usings.items[i]); } } - write_uint8(s, m->istopmod); - write_uint64(s, m->uuid); - write_int32(s, m->counter); + write_uint8(s->s, m->istopmod); + write_uint64(s->s, m->uuid); + write_int32(s->s, m->counter); } static int is_ast_node(jl_value_t *v) @@ -660,44 +668,45 @@ static int is_ast_node(jl_value_t *v) jl_is_labelnode(v) || jl_is_linenode(v) || jl_is_globalref(v); } -static int literal_val_id(jl_value_t *v) +static int literal_val_id(jl_serializer_state *s, jl_value_t *v) { - for(int i=0; i < jl_array_len(tree_literal_values); i++) { - if (jl_egal(jl_array_ptr_ref(tree_literal_values,i), v)) + int i, l = jl_array_len(s->tree_literal_values); + for (i = 0; i < l; i++) { + if (jl_egal(jl_array_ptr_ref(s->tree_literal_values, i), v)) return i; } - jl_array_ptr_1d_push(tree_literal_values, v); - return jl_array_len(tree_literal_values)-1; + jl_array_ptr_1d_push(s->tree_literal_values, v); + return jl_array_len(s->tree_literal_values) - 1; } -static void jl_serialize_value_(ios_t *s, jl_value_t *v) +static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) { if (v == NULL) { - write_uint8(s, Null_tag); + write_uint8(s->s, Null_tag); return; } void **bp = ptrhash_bp(&ser_tag, v); if (*bp != HT_NOTFOUND) { - write_as_tag(s, (uint8_t)(intptr_t)*bp); + write_as_tag(s->s, (uint8_t)(intptr_t)*bp); return; } if (jl_is_symbol(v)) { void *idx = ptrhash_get(&common_symbol_tag, v); if (idx != HT_NOTFOUND) { - writetag(s, (jl_value_t*)CommonSym_tag); - write_uint8(s, (uint8_t)(size_t)idx); + writetag(s->s, (jl_value_t*)CommonSym_tag); + write_uint8(s->s, (uint8_t)(size_t)idx); return; } } - if (mode == MODE_AST) { + if (s->mode == MODE_AST) { // compressing tree if (!is_ast_node(v)) { - writetag(s, (jl_value_t*)LiteralVal_tag); - int id = literal_val_id(v); + writetag(s->s, (jl_value_t*)LiteralVal_tag); + int id = literal_val_id(s, v); assert(id >= 0 && id < UINT16_MAX); - write_uint16(s, id); + write_uint16(s->s, id); return; } } @@ -706,12 +715,12 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) if (*bp != HT_NOTFOUND) { uintptr_t pos = (char*)*bp - (char*)HT_NOTFOUND - 1; if (pos < 65536) { - write_uint8(s, ShortBackRef_tag); - write_uint16(s, pos); + write_uint8(s->s, ShortBackRef_tag); + write_uint16(s->s, pos); } else { - write_uint8(s, BackRef_tag); - write_int32(s, pos); + write_uint8(s->s, BackRef_tag); + write_int32(s->s, pos); } return; } @@ -721,7 +730,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) arraylist_push(&reinit_list, (void*)pos); arraylist_push(&reinit_list, (void*)1); } - if (mode == MODE_MODULE && jl_is_module(v)) { + if (s->mode == MODE_MODULE && jl_is_module(v)) { jl_module_t *m = (jl_module_t*)v; if (module_in_worklist(m) && !module_in_worklist(m->parent)) { // will need to reinsert this into parent bindings, later (in case of any errors during reinsert) @@ -729,7 +738,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) arraylist_push(&reinit_list, (void*)2); } } - if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) { + if (s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK) { // TypeMapLevels need to be rehashed if (jl_is_mtable(v)) { arraylist_push(&reinit_list, (void*)pos); @@ -740,7 +749,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) arraylist_push(&reinit_list, (void*)4); } } - if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) + if (s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK) pos <<= 1; ptrhash_put(&backref_table, v, (char*)HT_NOTFOUND + pos + 1); } @@ -749,12 +758,12 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) if (jl_is_svec(v)) { size_t l = jl_svec_len(v); if (l <= 255) { - writetag(s, jl_simplevector_type); - write_uint8(s, (uint8_t)l); + writetag(s->s, jl_simplevector_type); + write_uint8(s->s, (uint8_t)l); } else { - writetag(s, (jl_value_t*)LongSvec_tag); - write_int32(s, l); + writetag(s->s, (jl_value_t*)LongSvec_tag); + write_int32(s->s, l); } for(i=0; i < l; i++) { jl_serialize_value(s, jl_svecref(v, i)); @@ -763,51 +772,51 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) else if (jl_is_symbol(v)) { size_t l = strlen(jl_symbol_name((jl_sym_t*)v)); if (l <= 255) { - writetag(s, jl_symbol_type); - write_uint8(s, (uint8_t)l); + writetag(s->s, jl_symbol_type); + write_uint8(s->s, (uint8_t)l); } else { - writetag(s, (jl_value_t*)LongSymbol_tag); - write_int32(s, l); + writetag(s->s, (jl_value_t*)LongSymbol_tag); + write_int32(s->s, l); } - ios_write(s, jl_symbol_name((jl_sym_t*)v), l); + ios_write(s->s, jl_symbol_name((jl_sym_t*)v), l); } else if (jl_is_globalref(v)) { - if (mode == MODE_AST && jl_globalref_mod(v) == tree_enclosing_module) { - writetag(s, (jl_value_t*)NearbyGlobal_tag); + if (s->mode == MODE_AST && jl_globalref_mod(v) == s->tree_enclosing_module) { + writetag(s->s, (jl_value_t*)NearbyGlobal_tag); jl_serialize_value(s, jl_globalref_name(v)); } else { - writetag(s, (jl_value_t*)jl_globalref_type); + writetag(s->s, (jl_value_t*)jl_globalref_type); jl_serialize_value(s, jl_globalref_mod(v)); jl_serialize_value(s, jl_globalref_name(v)); } } else if (jl_is_ssavalue(v) && ((jl_ssavalue_t*)v)->id < 65536) { - writetag(s, (jl_value_t*)jl_ssavalue_type); - write_uint16(s, ((jl_ssavalue_t*)v)->id); + writetag(s->s, (jl_value_t*)jl_ssavalue_type); + write_uint16(s->s, ((jl_ssavalue_t*)v)->id); } else if (jl_typeis(v,jl_slotnumber_type) && jl_slot_number(v) < 65536) { - writetag(s, (jl_value_t*)jl_slotnumber_type); - write_uint16(s, jl_slot_number(v)); + writetag(s->s, (jl_value_t*)jl_slotnumber_type); + write_uint16(s->s, jl_slot_number(v)); } else if (jl_is_array(v)) { jl_array_t *ar = (jl_array_t*)v; if (ar->flags.ndims == 1 && ar->elsize < 128) { - writetag(s, (jl_value_t*)Array1d_tag); - write_uint8(s, (ar->flags.ptrarray<<7) | (ar->elsize & 0x7f)); + writetag(s->s, (jl_value_t*)Array1d_tag); + write_uint8(s->s, (ar->flags.ptrarray<<7) | (ar->elsize & 0x7f)); } else { - writetag(s, (jl_value_t*)jl_array_type); - write_uint16(s, ar->flags.ndims); - write_uint16(s, (ar->flags.ptrarray<<15) | (ar->elsize & 0x7fff)); + writetag(s->s, (jl_value_t*)jl_array_type); + write_uint16(s->s, ar->flags.ndims); + write_uint16(s->s, (ar->flags.ptrarray<<15) | (ar->elsize & 0x7fff)); } for (i=0; i < ar->flags.ndims; i++) jl_serialize_value(s, jl_box_long(jl_array_dim(ar,i))); jl_serialize_value(s, jl_typeof(ar)); if (!ar->flags.ptrarray) { size_t tot = jl_array_len(ar) * ar->elsize; - ios_write(s, (char*)jl_array_data(ar), tot); + ios_write(s->s, (char*)jl_array_data(ar), tot); } else { for(i=0; i < jl_array_len(ar); i++) { @@ -819,12 +828,12 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_expr_t *e = (jl_expr_t*)v; size_t l = jl_array_len(e->args); if (l <= 255) { - writetag(s, jl_expr_type); - write_uint8(s, (uint8_t)l); + writetag(s->s, jl_expr_type); + write_uint8(s->s, (uint8_t)l); } else { - writetag(s, (jl_value_t*)LongExpr_tag); - write_int32(s, l); + writetag(s->s, (jl_value_t*)LongExpr_tag); + write_int32(s->s, l); } jl_serialize_value(s, e->head); jl_serialize_value(s, e->etype); @@ -836,14 +845,14 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_serialize_datatype(s, (jl_datatype_t*)v); } else if (jl_is_typevar(v)) { - writetag(s, jl_tvar_type); + writetag(s->s, jl_tvar_type); jl_serialize_value(s, ((jl_tvar_t*)v)->name); jl_serialize_value(s, ((jl_tvar_t*)v)->lb); jl_serialize_value(s, ((jl_tvar_t*)v)->ub); - write_int8(s, ((jl_tvar_t*)v)->bound); + write_int8(s->s, ((jl_tvar_t*)v)->bound); } else if (jl_is_method(v)) { - writetag(s, jl_method_type); + writetag(s->s, jl_method_type); jl_method_t *m = (jl_method_t*)v; union jl_typemap_t *tf = &m->specializations; if (tf->unknown && tf->unknown != jl_nothing) { @@ -854,22 +863,22 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } jl_serialize_value(s, tf->unknown); jl_serialize_value(s, (jl_value_t*)m->name); - write_int8(s, m->isstaged); + write_int8(s->s, m->isstaged); jl_serialize_value(s, (jl_value_t*)m->file); - write_int32(s, m->line); + write_int32(s->s, m->line); jl_serialize_value(s, (jl_value_t*)m->sig); jl_serialize_value(s, (jl_value_t*)m->tvars); - if (mode != MODE_MODULE_POSTWORK) + if (s->mode != MODE_MODULE_POSTWORK) jl_serialize_value(s, (jl_value_t*)m->ambig); - write_int8(s, m->called); + write_int8(s->s, m->called); jl_serialize_value(s, (jl_value_t*)m->module); jl_serialize_value(s, (jl_value_t*)m->roots); jl_serialize_value(s, (jl_value_t*)m->lambda_template); jl_serialize_value(s, (jl_value_t*)m->invokes.unknown); - write_int8(s, m->needs_sparam_vals_ducttape); + write_int8(s->s, m->needs_sparam_vals_ducttape); } else if (jl_is_lambda_info(v)) { - writetag(s, jl_lambda_info_type); + writetag(s->s, jl_lambda_info_type); jl_lambda_info_t *li = (jl_lambda_info_t*)v; if (li->jlcall_api == 2) jl_serialize_value(s, jl_nothing); @@ -883,18 +892,18 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_serialize_value(s, (jl_value_t*)li->sparam_syms); jl_serialize_value(s, (jl_value_t*)li->sparam_vals); jl_serialize_value(s, (jl_value_t*)li->specTypes); - write_int8(s, li->inferred); - write_int8(s, li->pure); - write_int8(s, li->inlineable); - write_int8(s, li->isva); - write_int32(s, li->nargs); + write_int8(s->s, li->inferred); + write_int8(s->s, li->pure); + write_int8(s->s, li->inlineable); + write_int8(s->s, li->isva); + write_int32(s->s, li->nargs); jl_serialize_value(s, (jl_value_t*)li->def); jl_serialize_value(s, li->constval); jl_serialize_fptr(s, (void*)(uintptr_t)li->fptr); // save functionObject pointers - write_int32(s, jl_assign_functionID(li->functionObjectsDecls.functionObject)); - write_int32(s, jl_assign_functionID(li->functionObjectsDecls.specFunctionObject)); - write_int8(s, li->jlcall_api); + write_int32(s->s, jl_assign_functionID(li->functionObjectsDecls.functionObject)); + write_int32(s->s, jl_assign_functionID(li->functionObjectsDecls.specFunctionObject)); + write_int8(s->s, li->jlcall_api); } else if (jl_typeis(v, jl_module_type)) { jl_serialize_module(s, (jl_module_t*)v); @@ -907,41 +916,41 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) void *data = jl_data_ptr(v); if (t == jl_int64_type && *(int64_t*)data >= S32_MIN && *(int64_t*)data <= S32_MAX) { - writetag(s, (jl_value_t*)SmallInt64_tag); - write_int32(s, (int32_t)*(int64_t*)data); + writetag(s->s, (jl_value_t*)SmallInt64_tag); + write_int32(s->s, (int32_t)*(int64_t*)data); } else if (t == jl_int32_type) { - writetag(s, (jl_value_t*)Int32_tag); - write_int32(s, (int32_t)*(int32_t*)data); + writetag(s->s, (jl_value_t*)Int32_tag); + write_int32(s->s, (int32_t)*(int32_t*)data); } else { if (v == t->instance) { - if (mode == MODE_MODULE) { + if (s->mode == MODE_MODULE) { // also flag this in the backref table as special uintptr_t *bp = (uintptr_t*)ptrhash_bp(&backref_table, v); assert(*bp != (uintptr_t)HT_NOTFOUND); *bp |= 1; assert(((uintptr_t)HT_NOTFOUND)|1); } - writetag(s, (jl_value_t*)Singleton_tag); + writetag(s->s, (jl_value_t*)Singleton_tag); jl_serialize_value(s, t); return; } if (t->size <= 255) { - writetag(s, (jl_value_t*)SmallDataType_tag); - write_uint8(s, t->size); + writetag(s->s, (jl_value_t*)SmallDataType_tag); + write_uint8(s->s, t->size); } else { - writetag(s, (jl_value_t*)jl_datatype_type); - write_int32(s, t->size); + writetag(s->s, (jl_value_t*)jl_datatype_type); + write_int32(s->s, t->size); } jl_serialize_value(s, t); - if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK)) { + if ((s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK)) { if (t == jl_typename_type) { if (module_in_worklist(((jl_typename_t*)v)->module)) { - write_uint8(s, 0); + write_uint8(s->s, 0); } else { - write_uint8(s, 1); + write_uint8(s->s, 1); jl_typename_t *tn = (jl_typename_t*)v; jl_serialize_value(s, tn->module); jl_serialize_value(s, tn->name); @@ -972,13 +981,13 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) size_t nf = jl_datatype_nfields(t); if (nf == 0 && jl_datatype_size(t)>0) { if (t->name == jl_pointer_type->name) { - write_int32(s, 0); + write_int32(s->s, 0); #ifdef _P64 - write_int32(s, 0); + write_int32(s->s, 0); #endif } else { - ios_write(s, (char*)data, jl_datatype_size(t)); + ios_write(s->s, (char*)data, jl_datatype_size(t)); } } else { @@ -993,7 +1002,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } struct jl_serialize_methcache_from_mod_env { - ios_t *s; + jl_serializer_state *s; jl_sym_t *name; jl_module_t *mod; }; @@ -1008,7 +1017,7 @@ static int jl_serialize_methcache_from_mod(jl_typemap_entry_t *ml, void *closure return 1; } -static void jl_serialize_methtable_from_mod(ios_t *s, jl_typename_t *tn) +static void jl_serialize_methtable_from_mod(jl_serializer_state *s, jl_typename_t *tn) { struct jl_serialize_methcache_from_mod_env env; env.s = s; @@ -1018,7 +1027,7 @@ static void jl_serialize_methtable_from_mod(ios_t *s, jl_typename_t *tn) jl_typemap_visitor(tn->mt->defs, jl_serialize_methcache_from_mod, &env); } -static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m) +static void jl_serialize_lambdas_from_mod(jl_serializer_state *s, jl_module_t *m) { if (module_in_worklist(m)) return; size_t i; @@ -1049,7 +1058,7 @@ static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m) } // serialize information about all of the modules accessible directly from Main -static void jl_serialize_mod_list(ios_t *s) +static void write_mod_list(ios_t *s) { jl_module_t *m = jl_main_module; size_t i; @@ -1079,7 +1088,7 @@ static void jl_serialize_mod_list(ios_t *s) static const int JI_FORMAT_VERSION = 2; static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature static const uint16_t BOM = 0xFEFF; // byte-order marker -static void jl_serialize_header(ios_t *s) +static void write_header(ios_t *s) { ios_write(s, JI_MAGIC, strlen(JI_MAGIC)); write_uint16(s, JI_FORMAT_VERSION); @@ -1095,7 +1104,7 @@ static void jl_serialize_header(ios_t *s) // serialize the global _require_dependencies array of pathnames that // are include depenencies -static void jl_serialize_dependency_list(ios_t *s) +static void write_dependency_list(ios_t *s) { size_t total_size = 0; static jl_array_t *deps = NULL; @@ -1140,9 +1149,9 @@ static void jl_serialize_dependency_list(ios_t *s) // --- deserialize --- -static jl_fptr_t jl_deserialize_fptr(ios_t *s) +static jl_fptr_t jl_deserialize_fptr(jl_serializer_state *s) { - int fptr = read_uint16(s); + int fptr = read_uint16(s->s); if (fptr < 2) return NULL; @@ -1151,9 +1160,9 @@ static jl_fptr_t jl_deserialize_fptr(ios_t *s) return id_to_fptrs[fptr]; } -static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) +static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_value_t **loc) { - int tag = read_uint8(s); + int tag = read_uint8(s->s); if (tag == 6 || tag == 7) { jl_typename_t *name = (jl_typename_t*)jl_deserialize_value(s, NULL); jl_value_t *dtv = name->primary; @@ -1170,9 +1179,9 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) backref_list.items[pos] = dtv; return dtv; } - size_t size = read_int32(s); - uint8_t flags = read_uint8(s); - uint8_t depth = read_int32(s); + size_t size = read_int32(s->s); + uint8_t flags = read_uint8(s->s); + uint8_t depth = read_int32(s->s); jl_datatype_t *dt = NULL; if (tag == 2) dt = jl_int32_type; @@ -1186,7 +1195,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt = jl_new_uninitialized_datatype(); else assert(0); - assert(tree_literal_values==NULL && mode != MODE_AST); + assert(s->tree_literal_values==NULL && s->mode != MODE_AST); backref_list.items[pos] = dt; dt->size = size; dt->struct_decl = NULL; @@ -1206,8 +1215,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->super = NULL; dt->layout = NULL; if (!dt->abstract) { - dt->ninitialized = read_uint16(s); - dt->uid = mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK ? read_int32(s) : 0; + dt->ninitialized = read_uint16(s->s); + dt->uid = s->mode != MODE_MODULE && s->mode != MODE_MODULE_POSTWORK ? read_int32(s->s) : 0; } else { dt->ninitialized = 0; @@ -1215,7 +1224,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) } if (has_layout) { - uint8_t layout = read_uint8(s); + uint8_t layout = read_uint8(s->s); if (layout == 1) { dt->layout = jl_array_type->layout; } @@ -1227,31 +1236,31 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) } else { assert(layout == 0); - uint16_t nf = read_uint16(s); - uint8_t fielddesc_type = read_int8(s); + uint16_t nf = read_uint16(s->s); + uint8_t fielddesc_type = read_int8(s->s); size_t fielddesc_size = nf > 0 ? jl_fielddesc_size(fielddesc_type) : 0; jl_datatype_layout_t *layout = (jl_datatype_layout_t*)jl_gc_perm_alloc( sizeof(jl_datatype_layout_t) + nf * fielddesc_size); layout->nfields = nf; layout->fielddesc_type = fielddesc_type; - layout->alignment = read_int32(s); - layout->haspadding = read_int8(s); - layout->pointerfree = read_int8(s); - ios_read(s, (char*)&layout[1], nf * fielddesc_size); + layout->alignment = read_int32(s->s); + layout->haspadding = read_int8(s->s); + layout->pointerfree = read_int8(s->s); + ios_read(s->s, (char*)&layout[1], nf * fielddesc_size); dt->layout = layout; } } if (tag == 5) { assert(pos > 0); - assert(mode != MODE_MODULE_POSTWORK); + assert(s->mode != MODE_MODULE_POSTWORK); arraylist_push(&flagref_list, loc); arraylist_push(&flagref_list, (void*)(uintptr_t)pos); dt->uid = -1; // mark that this type needs a new uid } if (has_instance) { - assert(mode != MODE_MODULE_POSTWORK); // there shouldn't be an instance on a type with uid = 0 + assert(s->mode != MODE_MODULE_POSTWORK); // there shouldn't be an instance on a type with uid = 0 dt->instance = jl_deserialize_value(s, &dt->instance); jl_gc_wb(dt, dt->instance); } @@ -1267,28 +1276,28 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) return (jl_value_t*)dt; } -static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t **loc); -static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc) +static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vtag, jl_value_t **loc); +static jl_value_t *jl_deserialize_value(jl_serializer_state *s, jl_value_t **loc) { - assert(!ios_eof(s)); - uint8_t tag = read_uint8(s); + assert(!ios_eof(s->s)); + uint8_t tag = read_uint8(s->s); if (tag == Null_tag) return NULL; if (tag == 0) { - tag = read_uint8(s); + tag = read_uint8(s->s); jl_value_t *v = deser_tag[tag]; assert(v != NULL); return v; } if (tag == BackRef_tag || tag == ShortBackRef_tag) { - assert(tree_literal_values == NULL && mode != MODE_AST); - uintptr_t offs = (tag == BackRef_tag) ? read_int32(s) : read_uint16(s); + assert(s->tree_literal_values == NULL && s->mode != MODE_AST); + uintptr_t offs = (tag == BackRef_tag) ? read_int32(s->s) : read_uint16(s->s); int isdatatype = 0; - if (mode == MODE_MODULE) { + if (s->mode == MODE_MODULE) { isdatatype = !!(offs & 1); offs >>= 1; } - else if (mode == MODE_MODULE_POSTWORK) { + else if (s->mode == MODE_MODULE_POSTWORK) { offs >>= 1; } // assert(offs >= 0); // offs is unsigned so this is always true @@ -1307,25 +1316,25 @@ static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc) return vtag; } else if (vtag == (jl_value_t*)LiteralVal_tag) { - return jl_array_ptr_ref(tree_literal_values, read_uint16(s)); + return jl_array_ptr_ref(s->tree_literal_values, read_uint16(s->s)); } jl_value_t *v = jl_deserialize_value_(s, vtag, loc); return v; } -static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t **loc) +static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vtag, jl_value_t **loc) { jl_ptls_t ptls = jl_get_ptls_states(); - int usetable = (mode != MODE_AST); + int usetable = (s->mode != MODE_AST); size_t i; if (vtag == (jl_value_t*)jl_simplevector_type || vtag == (jl_value_t*)LongSvec_tag) { size_t len; if (vtag == (jl_value_t*)jl_simplevector_type) - len = read_uint8(s); + len = read_uint8(s->s); else - len = read_int32(s); + len = read_int32(s->s); jl_svec_t *sv = jl_alloc_svec_uninit(len); if (usetable) arraylist_push(&backref_list, (jl_value_t*)sv); @@ -1336,18 +1345,18 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return (jl_value_t*)sv; } else if (vtag == (jl_value_t*)CommonSym_tag) { - int tag = read_uint8(s); + int tag = read_uint8(s->s); return deser_symbols[tag]; } else if (vtag == (jl_value_t*)jl_symbol_type || vtag == (jl_value_t*)LongSymbol_tag) { size_t len; if (vtag == (jl_value_t*)jl_symbol_type) - len = read_uint8(s); + len = read_uint8(s->s); else - len = read_int32(s); + len = read_int32(s->s); char *name = (char*) (len >= 256 ? malloc(len+1) : alloca(len+1)); - ios_read(s, name, len); + ios_read(s->s, name, len); name[len] = '\0'; jl_value_t *sym = (jl_value_t*)jl_symbol(name); if (len >= 256) free(name); @@ -1356,12 +1365,12 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return sym; } else if (vtag == (jl_value_t*)jl_ssavalue_type) { - jl_value_t *v = jl_box_ssavalue(read_uint16(s)); + jl_value_t *v = jl_box_ssavalue(read_uint16(s->s)); if (usetable) arraylist_push(&backref_list, v); return v; } else if (vtag == (jl_value_t*)jl_slotnumber_type) { - jl_value_t *v = jl_box_slotnumber(read_uint16(s)); + jl_value_t *v = jl_box_slotnumber(read_uint16(s->s)); if (usetable) arraylist_push(&backref_list, v); return v; } @@ -1371,13 +1380,13 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t int isunboxed, elsize; if (vtag == (jl_value_t*)Array1d_tag) { ndims = 1; - elsize = read_uint8(s); + elsize = read_uint8(s->s); isunboxed = !(elsize>>7); elsize = elsize&0x7f; } else { - ndims = read_uint16(s); - elsize = read_uint16(s); + ndims = read_uint16(s->s); + elsize = read_uint16(s->s); isunboxed = !(elsize>>15); elsize = elsize&0x7fff; } @@ -1394,7 +1403,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_set_typeof(a, aty); if (!a->flags.ptrarray) { size_t tot = jl_array_len(a) * a->elsize; - ios_read(s, (char*)jl_array_data(a), tot); + ios_read(s->s, (char*)jl_array_data(a), tot); } else { jl_value_t **data = (jl_value_t**)jl_array_data(a); @@ -1409,9 +1418,9 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t vtag == (jl_value_t*)LongExpr_tag) { size_t len; if (vtag == (jl_value_t*)jl_expr_type) - len = read_uint8(s); + len = read_uint8(s->s); else - len = read_int32(s); + len = read_int32(s->s); int pos = backref_list.len; if (usetable) arraylist_push(&backref_list, NULL); @@ -1437,7 +1446,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_gc_wb(tv, tv->lb); tv->ub = jl_deserialize_value(s, &tv->ub); jl_gc_wb(tv, tv->ub); - tv->bound = read_int8(s); + tv->bound = read_int8(s->s); return (jl_value_t*)tv; } else if (vtag == (jl_value_t*)jl_method_type) { @@ -1450,21 +1459,21 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_gc_wb(m, m->specializations.unknown); m->name = (jl_sym_t*)jl_deserialize_value(s, NULL); jl_gc_wb(m, m->name); - m->isstaged = read_int8(s); + m->isstaged = read_int8(s->s); m->file = (jl_sym_t*)jl_deserialize_value(s, NULL); - m->line = read_int32(s); + m->line = read_int32(s->s); m->sig = (jl_tupletype_t*)jl_deserialize_value(s, (jl_value_t**)&m->sig); jl_gc_wb(m, m->sig); m->tvars = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&m->tvars); jl_gc_wb(m, m->tvars); - if (mode != MODE_MODULE_POSTWORK) { + if (s->mode != MODE_MODULE_POSTWORK) { m->ambig = jl_deserialize_value(s, (jl_value_t**)&m->ambig); jl_gc_wb(m, m->ambig); } else { m->ambig = jl_nothing; } - m->called = read_int8(s); + m->called = read_int8(s->s); m->module = (jl_module_t*)jl_deserialize_value(s, (jl_value_t**)&m->module); jl_gc_wb(m, m->module); m->roots = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&m->roots); @@ -1473,7 +1482,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t if (m->lambda_template) jl_gc_wb(m, m->lambda_template); m->invokes.unknown = jl_deserialize_value(s, (jl_value_t**)&m->invokes); jl_gc_wb(m, m->invokes.unknown); - m->needs_sparam_vals_ducttape = read_int8(s); + m->needs_sparam_vals_ducttape = read_int8(s->s); m->traced = 0; JL_MUTEX_INIT(&m->writelock); return (jl_value_t*)m; @@ -1498,11 +1507,11 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t li->specTypes = (jl_tupletype_t*)jl_deserialize_value(s, (jl_value_t**)&li->specTypes); if (li->specTypes) jl_gc_wb(li, li->specTypes); li->unspecialized_ducttape = NULL; - li->inferred = read_int8(s); - li->pure = read_int8(s); - li->inlineable = read_int8(s); - li->isva = read_int8(s); - li->nargs = read_int32(s); + li->inferred = read_int8(s->s); + li->pure = read_int8(s->s); + li->inlineable = read_int8(s->s); + li->isva = read_int8(s->s); + li->nargs = read_int32(s->s); li->def = (jl_method_t*)jl_deserialize_value(s, (jl_value_t**)&li->def); if (li->def) jl_gc_wb(li, li->def); li->constval = jl_deserialize_value(s, &li->constval); @@ -1514,10 +1523,10 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t li->inCompile = 0; li->fptr = jl_deserialize_fptr(s); int32_t cfunc_llvm, func_llvm; - func_llvm = read_int32(s); - cfunc_llvm = read_int32(s); + func_llvm = read_int32(s->s); + cfunc_llvm = read_int32(s->s); jl_delayed_fptrs(li, func_llvm, cfunc_llvm); - li->jlcall_api = read_int8(s); + li->jlcall_api = read_int8(s->s); li->compile_traced = 0; return (jl_value_t*)li; } @@ -1527,11 +1536,11 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t arraylist_push(&backref_list, NULL); jl_sym_t *mname = (jl_sym_t*)jl_deserialize_value(s, NULL); int ref_only = 0; - if (mode == MODE_MODULE_POSTWORK) { + if (s->mode == MODE_MODULE_POSTWORK) { ref_only = 1; } - else if (mode == MODE_MODULE) { - ref_only = read_uint8(s); + else if (s->mode == MODE_MODULE) { + ref_only = read_uint8(s->s); } if (ref_only) { jl_value_t *m_ref = jl_get_global((jl_module_t*)jl_deserialize_value(s, NULL), mname); @@ -1557,7 +1566,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t if (b->globalref != NULL) jl_gc_wb(m, b->globalref); b->owner = (jl_module_t*)jl_deserialize_value(s, (jl_value_t**)&b->owner); if (b->owner != NULL) jl_gc_wb(m, b->owner); - int8_t flags = read_int8(s); + int8_t flags = read_int8(s->s); b->deprecated = (flags>>3) & 1; b->constp = (flags>>2) & 1; b->exportp = (flags>>1) & 1; @@ -1565,34 +1574,34 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_deserialize_gv(s, (jl_value_t*)b); } size_t i = m->usings.len; - size_t ni = read_int32(s); + size_t ni = read_int32(s->s); arraylist_grow(&m->usings, ni); ni += i; while (i < ni) { m->usings.items[i] = jl_deserialize_value(s, (jl_value_t**)&m->usings.items[i]); i++; } - m->istopmod = read_uint8(s); - m->uuid = read_uint64(s); - m->counter = read_int32(s); + m->istopmod = read_uint8(s->s); + m->uuid = read_uint64(s->s); + m->counter = read_int32(s->s); return (jl_value_t*)m; } else if (vtag == (jl_value_t*)SmallInt64_tag) { - jl_value_t *v = jl_box_int64(read_int32(s)); + jl_value_t *v = jl_box_int64(read_int32(s->s)); if (usetable) arraylist_push(&backref_list, v); return v; } else if (vtag == (jl_value_t*)Int32_tag) { - jl_value_t *v = jl_box_int32(read_int32(s)); + jl_value_t *v = jl_box_int32(read_int32(s->s)); if (usetable) arraylist_push(&backref_list, v); return v; } else if (vtag == (jl_value_t*)NearbyGlobal_tag) { - assert(tree_enclosing_module != NULL); + assert(s->tree_enclosing_module != NULL); jl_value_t *sym = jl_deserialize_value(s, NULL); - return jl_module_globalref(tree_enclosing_module, (jl_sym_t*)sym); + return jl_module_globalref(s->tree_enclosing_module, (jl_sym_t*)sym); } else if (vtag == (jl_value_t*)jl_globalref_type) { if (usetable) { @@ -1610,7 +1619,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } } else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)SmallDataType_tag) { - int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s) : read_int32(s)); + int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s->s) : read_int32(s->s)); jl_value_t *v = jl_gc_alloc(ptls, sz, NULL); int pos = backref_list.len; if (usetable) @@ -1619,9 +1628,9 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_set_typeof(v, dt); if (dt == jl_datatype_type) return jl_deserialize_datatype(s, pos, loc); - assert(mode==MODE_AST || sz!=0 || loc); - if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && dt == jl_typename_type) { - int ref_only = read_uint8(s); + assert(s->mode == MODE_AST || sz != 0 || loc); + if ((s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK) && dt == jl_typename_type) { + int ref_only = read_uint8(s->s); if (ref_only) { jl_module_t *m = (jl_module_t*)jl_deserialize_value(s, NULL); jl_sym_t *sym = (jl_sym_t*)jl_deserialize_value(s, NULL); @@ -1632,12 +1641,12 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t backref_list.items[pos] = v; return v; } - assert(mode != MODE_MODULE_POSTWORK); + assert(s->mode != MODE_MODULE_POSTWORK); } size_t nf = jl_datatype_nfields(dt); if (nf == 0 && jl_datatype_size(dt)>0) { int nby = jl_datatype_size(dt); - ios_read(s, (char*)jl_data_ptr(v), nby); + ios_read(s->s, (char*)jl_data_ptr(v), nby); } else { char *data = (char*)jl_data_ptr(v); @@ -1652,7 +1661,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } } } - if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK)) { + if ((s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK)) { if (dt == jl_typename_type) { jl_typename_t *tn = (jl_typename_t*)v; tn->uid = jl_assign_type_uid(); // make sure this has a new uid @@ -1664,7 +1673,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return v; } else if (vtag == (jl_value_t*)Singleton_tag) { - if (mode == MODE_MODULE_POSTWORK) { + if (s->mode == MODE_MODULE_POSTWORK) { uintptr_t pos = backref_list.len; arraylist_push(&backref_list, NULL); jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, NULL); @@ -1675,7 +1684,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t if (usetable) { uintptr_t pos = backref_list.len; arraylist_push(&backref_list, (void*)v); - if (mode == MODE_MODULE) { + if (s->mode == MODE_MODULE) { // TODO: optimize the case where the value can easily be obtained // from an external module (tag == 6) as dt->instance assert(loc != NULL); @@ -1691,7 +1700,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return NULL; } -static void jl_deserialize_lambdas_from_mod(ios_t *s) +static void jl_deserialize_lambdas_from_mod(jl_serializer_state *s) { while (1) { jl_method_t *meth = (jl_method_t*)jl_deserialize_value(s, NULL); @@ -1704,7 +1713,7 @@ static void jl_deserialize_lambdas_from_mod(ios_t *s) } } -static int jl_deserialize_verify_mod_list(ios_t *s) +static int read_verify_mod_list(ios_t *s) { if (!jl_main_module->uuid) { jl_printf(JL_STDERR, "ERROR: Main module uuid state is invalid for module deserialization.\n"); @@ -1755,14 +1764,14 @@ static int jl_deserialize_verify_mod_list(ios_t *s) static int readstr_verify(ios_t *s, const char *str) { - size_t len = strlen(str); - for (size_t i=0; i < len; ++i) - if ((char) read_uint8(s) != str[i]) + size_t i, len = strlen(str); + for (i = 0; i < len; ++i) + if ((char)read_uint8(s) != str[i]) return 0; return 1; } -JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s) +JL_DLLEXPORT int jl_read_verify_header(ios_t *s) { uint16_t bom; return (readstr_verify(s, JI_MAGIC) && @@ -1776,32 +1785,30 @@ JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s) readstr_verify(s, jl_git_commit()) && !read_uint8(s)); } -jl_array_t *jl_module_init_order; - -static void jl_finalize_serializer(ios_t *f) { +static void jl_finalize_serializer(jl_serializer_state *s) { size_t i, l; // save module initialization order if (jl_module_init_order != NULL) { l = jl_array_len(jl_module_init_order); - for(i=0; i < l; i++) { + for (i = 0; i < l; i++) { // verify that all these modules were saved assert(ptrhash_get(&backref_table, jl_array_ptr_ref(jl_module_init_order, i)) != HT_NOTFOUND); } } - if (mode != MODE_MODULE) - jl_serialize_value(f, jl_module_init_order); + if (s->mode != MODE_MODULE) + jl_serialize_value(s, jl_module_init_order); // record list of reinitialization functions l = reinit_list.len; for (i = 0; i < l; i += 2) { - write_int32(f, (int)((uintptr_t) reinit_list.items[i])); - write_int32(f, (int)((uintptr_t) reinit_list.items[i+1])); + write_int32(s->s, (int)((uintptr_t) reinit_list.items[i])); + write_int32(s->s, (int)((uintptr_t) reinit_list.items[i+1])); } - write_int32(f, -1); + write_int32(s->s, -1); } void jl_typemap_rehash(union jl_typemap_t ml, int8_t offs); -static void jl_reinit_item(ios_t *f, jl_value_t *v, int how, arraylist_t *tracee_list) +static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) { jl_ptls_t ptls = jl_get_ptls_states(); JL_TRY { @@ -1858,17 +1865,17 @@ static void jl_reinit_item(ios_t *f, jl_value_t *v, int how, arraylist_t *tracee } } -static jl_array_t *jl_finalize_deserializer(ios_t *f, arraylist_t *tracee_list) +static jl_array_t *jl_finalize_deserializer(jl_serializer_state *s, arraylist_t *tracee_list) { jl_array_t *init_order = NULL; - if (mode != MODE_MODULE) - init_order = (jl_array_t*)jl_deserialize_value(f, NULL); + if (s->mode != MODE_MODULE) + init_order = (jl_array_t*)jl_deserialize_value(s, NULL); // run reinitialization functions - int pos = read_int32(f); + int pos = read_int32(s->s); while (pos != -1) { - jl_reinit_item(f, (jl_value_t*)backref_list.items[pos], read_int32(f), tracee_list); - pos = read_int32(f); + jl_reinit_item((jl_value_t*)backref_list.items[pos], read_int32(s->s), tracee_list); + pos = read_int32(s->s); } return init_order; } @@ -1899,11 +1906,14 @@ static void jl_save_system_image_to_stream(ios_t *f) jl_gc_collect(1); // full jl_gc_collect(0); // incremental (sweep finalizers) JL_TIMING(SYSIMG_DUMP); - JL_LOCK(&dump_lock); // Might GC int en = jl_gc_enable(0); htable_reset(&backref_table, 250000); arraylist_new(&reinit_list, 0); backref_table_numel = 0; + jl_serializer_state s = { + f, MODE_SYSTEM_IMAGE, + NULL, NULL + }; // orphan old Base module if present jl_base_module = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol("Base")); @@ -1918,33 +1928,33 @@ static void jl_save_system_image_to_stream(ios_t *f) jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("ObjectIdDict")) : NULL; - jl_serialize_value(f, jl_main_module); - jl_serialize_value(f, jl_top_module); - jl_serialize_value(f, jl_typeinf_func); - jl_serialize_value(f, jl_type_type->name->mt); + jl_serialize_value(&s, jl_main_module); + jl_serialize_value(&s, jl_top_module); + jl_serialize_value(&s, jl_typeinf_func); + jl_serialize_value(&s, jl_type_type->name->mt); - intptr_t i=2; - for(i=0; i < builtin_types.len; i++) { - jl_serialize_value(f, ((jl_datatype_t*)builtin_types.items[i])->name->cache); - jl_serialize_value(f, ((jl_datatype_t*)builtin_types.items[i])->name->linearcache); + intptr_t i; + for (i = 0; i < builtin_types.len; i++) { + jl_serialize_value(&s, ((jl_datatype_t*)builtin_types.items[i])->name->cache); + jl_serialize_value(&s, ((jl_datatype_t*)builtin_types.items[i])->name->linearcache); } // ensure everything in deser_tag is reassociated with its GlobalValue - for (i=2; i < 255; i++) { - jl_serialize_gv(f, deser_tag[i]); + for (i = 2; i < 255; i++) { + jl_serialize_gv(&s, deser_tag[i]); } - jl_serialize_globalvals(f); - jl_serialize_gv_others(f); // serialize things that might not have visible gc roots roots with GlobalValue references + + jl_serialize_globalvals(&s); + jl_serialize_gv_others(&s); // serialize things that might not have visible gc roots roots with GlobalValue references write_int32(f, jl_get_t_uid_ctr()); write_int32(f, jl_get_gs_ctr()); - jl_finalize_serializer(f); // done with f + jl_finalize_serializer(&s); // done with f and s htable_reset(&backref_table, 0); arraylist_free(&reinit_list); jl_gc_enable(en); - JL_UNLOCK(&dump_lock); // Might GC } JL_DLLEXPORT void jl_save_system_image(const char *fname) @@ -2003,25 +2013,26 @@ static void jl_restore_system_image_from_stream(ios_t *f) { JL_TIMING(SYSIMG_LOAD); jl_ptls_t ptls = jl_get_ptls_states(); - JL_LOCK(&dump_lock); // Might GC int en = jl_gc_enable(0); - DUMP_MODES last_mode = mode; - mode = MODE_SYSTEM_IMAGE; arraylist_new(&backref_list, 250000); + jl_serializer_state s = { + f, MODE_SYSTEM_IMAGE, + NULL, NULL + }; - jl_main_module = (jl_module_t*)jl_deserialize_value(f, NULL); - jl_top_module = (jl_module_t*)jl_deserialize_value(f, NULL); + jl_main_module = (jl_module_t*)jl_deserialize_value(&s, NULL); + jl_top_module = (jl_module_t*)jl_deserialize_value(&s, NULL); jl_internal_main_module = jl_main_module; - jl_typeinf_func = (jl_function_t*)jl_deserialize_value(f, NULL); - jl_type_type_mt = (jl_methtable_t*)jl_deserialize_value(f, NULL); + jl_typeinf_func = (jl_function_t*)jl_deserialize_value(&s, NULL); + jl_type_type_mt = (jl_methtable_t*)jl_deserialize_value(&s, NULL); jl_type_type->name->mt = jl_typector_type->name->mt = jl_uniontype_type->name->mt = jl_datatype_type->name->mt = jl_type_type_mt; intptr_t i; for(i=0; i < builtin_types.len; i++) { jl_typename_t *tn = ((jl_datatype_t*)builtin_types.items[i])->name; - tn->cache = (jl_svec_t*)jl_deserialize_value(f, NULL); jl_gc_wb(tn, tn->cache); - tn->linearcache = (jl_svec_t*)jl_deserialize_value(f, NULL); jl_gc_wb(tn, tn->linearcache); + tn->cache = (jl_svec_t*)jl_deserialize_value(&s, NULL); jl_gc_wb(tn, tn->cache); + tn->linearcache = (jl_svec_t*)jl_deserialize_value(&s, NULL); jl_gc_wb(tn, tn->linearcache); jl_resort_type_cache(tn->cache); jl_resort_type_cache(tn->linearcache); } @@ -2034,14 +2045,14 @@ static void jl_restore_system_image_from_stream(ios_t *f) // ensure everything in deser_tag is reassociated with its GlobalValue for (i = 2; i < 255; i++) { - jl_deserialize_gv(f, deser_tag[i]); + jl_deserialize_gv(&s, deser_tag[i]); } - jl_deserialize_globalvals(f); - jl_deserialize_gv_others(f); + jl_deserialize_globalvals(&s); + jl_deserialize_gv_others(&s); int uid_ctr = read_int32(f); int gs_ctr = read_int32(f); - jl_module_init_order = jl_finalize_deserializer(f, NULL); // done with f + jl_module_init_order = jl_finalize_deserializer(&s, NULL); // done with s and f jl_set_t_uid_ctr(uid_ctr); jl_set_gs_ctr(gs_ctr); @@ -2059,9 +2070,7 @@ static void jl_restore_system_image_from_stream(ios_t *f) jl_gc_reset_alloc_count(); jl_gc_enable(en); - mode = last_mode; jl_update_all_fptrs(); - JL_UNLOCK(&dump_lock); // Might GC } JL_DLLEXPORT void jl_restore_system_image(const char *fname) @@ -2102,37 +2111,30 @@ JL_DLLEXPORT jl_array_t *jl_compress_ast(jl_lambda_info_t *li, jl_array_t *ast) { JL_TIMING(AST_COMPRESS); JL_LOCK(&li->def->writelock); // protect the roots array (Might GC) - JL_LOCK(&dump_lock); // protect global structures in this file (Might GC) assert(jl_is_lambda_info(li)); assert(jl_is_array(ast)); - DUMP_MODES last_mode = mode; - mode = MODE_AST; ios_t dest; ios_mem(&dest, 0); - jl_array_t *last_tlv = tree_literal_values; - jl_module_t *last_tem = tree_enclosing_module; int en = jl_gc_enable(0); // Might GC if (li->def->roots == NULL) { li->def->roots = jl_alloc_vec_any(0); jl_gc_wb(li->def, li->def->roots); } - tree_literal_values = li->def->roots; - tree_enclosing_module = li->def->module; - jl_serialize_value(&dest, ast); + jl_serializer_state s = { + &dest, MODE_AST, + li->def->roots, li->def->module + }; + jl_serialize_value(&s, ast); //jl_printf(JL_STDERR, "%d bytes, %d values\n", dest.size, vals->length); jl_array_t *v = jl_takebuf_array(&dest); - if (jl_array_len(tree_literal_values) == 0 && last_tlv == NULL) { + if (jl_array_len(li->def->roots) == 0) { li->def->roots = NULL; } - tree_literal_values = last_tlv; - tree_enclosing_module = last_tem; JL_GC_PUSH1(&v); jl_gc_enable(en); - mode = last_mode; - JL_UNLOCK(&dump_lock); // Might GC JL_UNLOCK(&li->def->writelock); // Might GC JL_GC_POP(); return v; @@ -2142,27 +2144,22 @@ JL_DLLEXPORT jl_array_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_array_t *dat { JL_TIMING(AST_UNCOMPRESS); JL_LOCK(&li->def->writelock); // protect the roots array (Might GC) - JL_LOCK(&dump_lock); // Might GC assert(jl_is_lambda_info(li)); assert(jl_is_array(data)); - DUMP_MODES last_mode = mode; - mode = MODE_AST; jl_array_t *bytes = (jl_array_t*)data; - tree_literal_values = li->def->roots; - tree_enclosing_module = li->def->module; ios_t src; ios_mem(&src, 0); ios_setbuf(&src, (char*)bytes->data, jl_array_len(bytes), 0); src.size = jl_array_len(bytes); int en = jl_gc_enable(0); // Might GC + jl_serializer_state s = { + &src, MODE_AST, + li->def->roots, li->def->module + }; - jl_array_t *v = (jl_array_t*)jl_deserialize_value(&src, NULL); + jl_array_t *v = (jl_array_t*)jl_deserialize_value(&s, NULL); JL_GC_PUSH1(&v); jl_gc_enable(en); - tree_literal_values = NULL; - tree_enclosing_module = NULL; - mode = last_mode; - JL_UNLOCK(&dump_lock); // Might GC JL_UNLOCK(&li->def->writelock); // Might GC JL_GC_POP(); return v; @@ -2177,11 +2174,10 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) return 1; } serializer_worklist = worklist; - jl_serialize_header(&f); - jl_serialize_mod_list(&f); // this can throw, keep it early (before any actual initialization) - jl_serialize_dependency_list(&f); + write_header(&f); + write_mod_list(&f); // this can throw, keep it early (before any actual initialization) + write_dependency_list(&f); - JL_LOCK(&dump_lock); // Might GC arraylist_new(&reinit_list, 0); htable_new(&backref_table, 5000); ptrhash_put(&backref_table, jl_main_module, (char*)HT_NOTFOUND + 1); @@ -2189,24 +2185,23 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("ObjectIdDict")) : NULL; int en = jl_gc_enable(0); - DUMP_MODES last_mode = mode; - mode = MODE_MODULE; - jl_serialize_value(&f, worklist); - jl_finalize_serializer(&f); // done with MODE_MODULE + jl_serializer_state s = { + &f, MODE_MODULE, + NULL, NULL + }; + jl_serialize_value(&s, worklist); + jl_finalize_serializer(&s); // done with MODE_MODULE reinit_list.len = 0; - mode = MODE_MODULE_POSTWORK; - jl_serialize_lambdas_from_mod(&f, jl_main_module); - jl_serialize_value(&f, NULL); // signal end of lambdas - jl_finalize_serializer(&f); // done with f + s.mode = MODE_MODULE_POSTWORK; + jl_serialize_lambdas_from_mod(&s, jl_main_module); + jl_serialize_value(&s, NULL); // signal end of lambdas + jl_finalize_serializer(&s); // done with f - mode = last_mode; jl_gc_enable(en); - htable_reset(&backref_table, 0); arraylist_free(&reinit_list); ios_close(&f); - JL_UNLOCK(&dump_lock); // Might GC if (jl_fs_rename(tmpfname, fname) < 0) { jl_printf(JL_STDERR, "Cannot write cache file \"%s\".\n", fname); @@ -2338,45 +2333,44 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) ios_close(f); return NULL; } - if (!jl_deserialize_verify_header(f) || - !jl_deserialize_verify_mod_list(f)) { + if (!jl_read_verify_header(f) || + !read_verify_mod_list(f)) { ios_close(f); return NULL; } size_t deplen = read_uint64(f); ios_skip(f, deplen); // skip past the dependency list - JL_LOCK(&dump_lock); // Might GC arraylist_new(&backref_list, 4000); arraylist_push(&backref_list, jl_main_module); arraylist_new(&flagref_list, 0); int en = jl_gc_enable(0); - DUMP_MODES last_mode = mode; - mode = MODE_MODULE; + jl_serializer_state s = { + f, MODE_MODULE, + NULL, NULL + }; jl_array_t *restored = NULL; jl_array_t *init_order = NULL; - restored = (jl_array_t*)jl_deserialize_value(f, (jl_value_t**)&restored); + restored = (jl_array_t*)jl_deserialize_value(&s, (jl_value_t**)&restored); arraylist_t *tracee_list = NULL; if (jl_newmeth_tracer) tracee_list = arraylist_new((arraylist_t*)malloc(sizeof(arraylist_t)), 0); jl_recache_types(); - jl_finalize_deserializer(f, tracee_list); // done with MODE_MODULE + jl_finalize_deserializer(&s, tracee_list); // done with MODE_MODULE // at this point, the AST is fully reconstructed, but still completely disconnected // in postwork mode, all of the interconnects will be created - mode = MODE_MODULE_POSTWORK; - jl_deserialize_lambdas_from_mod(f); // hook up methods of external generic functions - init_order = jl_finalize_deserializer(f, tracee_list); // done with f + s.mode = MODE_MODULE_POSTWORK; + jl_deserialize_lambdas_from_mod(&s); // hook up methods of external generic functions + init_order = jl_finalize_deserializer(&s, tracee_list); // done with f and s - mode = last_mode; jl_gc_enable(en); arraylist_free(&flagref_list); arraylist_free(&backref_list); ios_close(f); - JL_GC_PUSH2(&init_order,&restored); - JL_UNLOCK(&dump_lock); // Might GC + JL_GC_PUSH2(&init_order, &restored); if (tracee_list) { jl_methtable_t *mt; From 37317d242c931aaa71a1c2317bf0f82c9d5b8192 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 21 Jul 2016 16:33:17 -0400 Subject: [PATCH 0578/1117] dump: jl_array_any_type isn't an ast node type --- src/dump.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/dump.c b/src/dump.c index 9477aafc73922..41cdefd129ff5 100644 --- a/src/dump.c +++ b/src/dump.c @@ -661,8 +661,7 @@ static void jl_serialize_module(jl_serializer_state *s, jl_module_t *m) static int is_ast_node(jl_value_t *v) { return jl_is_symbol(v) || jl_is_slot(v) || jl_is_ssavalue(v) || - jl_is_expr(v) || jl_is_newvarnode(v) || jl_is_svec(v) || - jl_typeis(v, jl_array_any_type) || jl_is_tuple(v) || + jl_is_expr(v) || jl_is_newvarnode(v) || jl_is_svec(v) || jl_is_tuple(v) || jl_is_uniontype(v) || jl_is_int32(v) || jl_is_int64(v) || jl_is_bool(v) || jl_is_quotenode(v) || jl_is_gotonode(v) || jl_is_labelnode(v) || jl_is_linenode(v) || jl_is_globalref(v); @@ -837,7 +836,7 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) } jl_serialize_value(s, e->head); jl_serialize_value(s, e->etype); - for(i=0; i < l; i++) { + for (i = 0; i < l; i++) { jl_serialize_value(s, jl_exprarg(e, i)); } } @@ -2125,7 +2124,12 @@ JL_DLLEXPORT jl_array_t *jl_compress_ast(jl_lambda_info_t *li, jl_array_t *ast) &dest, MODE_AST, li->def->roots, li->def->module }; - jl_serialize_value(&s, ast); + size_t i, nstmts = jl_array_len(ast); + assert(nstmts < INT32_MAX); + write_int32(&dest, nstmts); + for (i = 0; i < nstmts; i++) { + jl_serialize_value(&s, jl_array_ptr_ref(ast, i)); + } //jl_printf(JL_STDERR, "%d bytes, %d values\n", dest.size, vals->length); @@ -2157,12 +2161,16 @@ JL_DLLEXPORT jl_array_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_array_t *dat li->def->roots, li->def->module }; - jl_array_t *v = (jl_array_t*)jl_deserialize_value(&s, NULL); - JL_GC_PUSH1(&v); + size_t i, nstmts = read_int32(&src); + jl_array_t *ast = jl_alloc_vec_any(nstmts); + JL_GC_PUSH1(&ast); + for (i = 0; i < nstmts; i++) { + jl_array_ptr_set(ast, i, jl_deserialize_value(&s, NULL)); + } jl_gc_enable(en); JL_UNLOCK(&li->def->writelock); // Might GC JL_GC_POP(); - return v; + return ast; } JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) From 585c0b9b8aa0d18ab100bb7157d5fe7d89d6a493 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat <nalimilan@club.fr> Date: Fri, 22 Jul 2016 07:36:56 +0200 Subject: [PATCH 0579/1117] Restore support for libgit2 0.23 (#17540) Testing for >= 0.24 when code base still contains branches to support 0.23 does not make much sense. --- README.md | 2 +- test/libgit2.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 09e65ff030d82..87aa47027db59 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,7 @@ Julia uses the following external libraries, which are automatically downloaded - **[PCRE]** (>= 10.00) — Perl-compatible regular expressions library. - **[GMP]** (>= 5.0) — GNU multiple precision arithmetic library, needed for `BigInt` support. - **[MPFR]** (>= 3.0) — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support. -- **[libgit2]** (>= 0.24) — Git linkable library, used by Julia's package manager +- **[libgit2]** (>= 0.23) — Git linkable library, used by Julia's package manager - **[libssh2]** (>= 1.7) — library for SSH transport, used by libgit2 for packages with SSH remotes - **[mbedtls]** (>= 2.2) — library used for cryptography and transport layer security, used by libssh2 - **[utf8proc]** (>= 2.0) — a library for processing UTF-8 encoded Unicode strings diff --git a/test/libgit2.jl b/test/libgit2.jl index 8202fc4e9d39d..970ab90f92b0c 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,15 +2,15 @@ #@testset "libgit2" begin -const LIBGIT2_VER = v"0.24.0" +const LIBGIT2_MIN_VER = v"0.23.0" ######### # TESTS # ######### -#@testset "Check library verison" begin +#@testset "Check library version" begin v = LibGit2.version() - @test v.major == LIBGIT2_VER.major && v.minor >= LIBGIT2_VER.minor + @test v.major == LIBGIT2_MIN_VER.major && v.minor >= LIBGIT2_MIN_VER.minor #end #@testset "Check library features" begin From 1295be56316133f7a26bed7c980b8ba059a47dd6 Mon Sep 17 00:00:00 2001 From: Mauro Werder <mauro_lc@runbox.com> Date: Fri, 15 Jul 2016 09:25:53 +0200 Subject: [PATCH 0580/1117] Added definition function_module(f) Also added equivalent function datatype_module(dt) --- base/docs/helpdb/Base.jl | 14 -------------- base/reflection.jl | 31 ++++++++++++++++++++++++++++--- doc/stdlib/base.rst | 17 ++++++++++++++--- test/reflection.jl | 18 ++++++++++++++---- 4 files changed, 56 insertions(+), 24 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 7f9125fb3e03d..c2bc9ea919a48 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2294,13 +2294,6 @@ Gives the number of columns needed to print a string. """ strwidth -""" - function_module(f::Function, types) -> Module - -Determine the module containing a given definition of a generic function. -""" -function_module - """ hex(n, [pad]) @@ -2950,13 +2943,6 @@ See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. """ Float64 -""" - function_name(f::Function) -> Symbol - -Get the name of a generic `Function` as a symbol, or `:anonymous`. -""" -function_name - """ ``` addprocs(n::Integer; exeflags=``) -> List of process identifiers diff --git a/base/reflection.jl b/base/reflection.jl index f311a3b109a01..c917acb2e8c90 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -70,6 +70,13 @@ end fieldnames(t::DataType) = Symbol[fieldname(t, n) for n in 1:nfields(t)] fieldnames{T<:Tuple}(t::Type{T}) = Int[n for n in 1:nfields(t)] +""" + Base.datatype_module(t::DataType) -> Module + +Determine the module containing the definition of a `DataType`. +""" +datatype_module(t::DataType) = t.name.module + isconst(s::Symbol) = ccall(:jl_is_const, Int32, (Ptr{Void}, Any), C_NULL, s) != 0 isconst(m::Module, s::Symbol) = @@ -170,9 +177,6 @@ end subtypes(m::Module, x::DataType) = sort(collect(_subtypes(m, x)), by=string) subtypes(x::DataType) = subtypes(Main, x) -# function reflection -function_name(f::Function) = typeof(f).name.mt.name - function to_tuple_type(t::ANY) @_pure_meta if isa(t,Tuple) || isa(t,AbstractArray) || isa(t,SimpleVector) @@ -429,6 +433,14 @@ function which_module(m::Module, s::Symbol) binding_module(m, s) end +# function reflection +""" + Base.function_name(f::Function) -> Symbol + +Get the name of a generic `Function` as a symbol, or `:anonymous`. +""" +function_name(f::Function) = typeof(f).name.mt.name + functionloc(m::LambdaInfo) = functionloc(m.def) function functionloc(m::Method) ln = m.line @@ -455,6 +467,19 @@ function functionloc(f) functionloc(first(mt)) end +""" + Base.function_module(f::Function) -> Module + +Determine the module containing the (first) definition of a generic +function. +""" +function_module(f::Function) = datatype_module(typeof(f)) + +""" + Base.function_module(f::Function, types) -> Module + +Determine the module containing a given definition of a generic function. +""" function function_module(f, types::ANY) m = methods(f, types) if isempty(m) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index bb8b0580aa7a0..61d522f00660a 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -1337,19 +1337,31 @@ Reflection Get the name of field ``i`` of a ``DataType``\ . +.. function:: Base.datatype_module(t::DataType) -> Module + + .. Docstring generated from Julia source + + Determine the module containing the definition of a ``DataType``\ . + .. function:: isconst([m::Module], s::Symbol) -> Bool .. Docstring generated from Julia source Determine whether a global is declared ``const`` in a given ``Module``\ . The default ``Module`` argument is ``current_module()``\ . -.. function:: function_name(f::Function) -> Symbol +.. function:: Base.function_name(f::Function) -> Symbol .. Docstring generated from Julia source Get the name of a generic ``Function`` as a symbol, or ``:anonymous``\ . -.. function:: function_module(f::Function, types) -> Module +.. function:: Base.function_module(f::Function) -> Module + + .. Docstring generated from Julia source + + Determine the module containing the (first) definition of a generic function. + +.. function:: Base.function_module(f::Function, types) -> Module .. Docstring generated from Julia source @@ -1467,4 +1479,3 @@ Internals .. Docstring generated from Julia source Compile the given function ``f`` for the argument tuple (of types) ``args``\ , but do not execute it. - diff --git a/test/reflection.jl b/test/reflection.jl index 773e734ab49cd..352ff8db8d9ea 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -163,19 +163,22 @@ not_const = 1 module TestMod7648 using Base.Test -export a9475, c7648, foo7648 +export a9475, foo9475, c7648, foo7648, foo7648_nomethods, Foo7648 const c7648 = 8 d7648 = 9 const f7648 = 10 foo7648(x) = x +function foo7648_nomethods end +type Foo7648 end module TestModSub9475 using Base.Test using ..TestMod7648 - export a9475 + export a9475, foo9475 a9475 = 5 b9475 = 7 + foo9475(x) = x let @test Base.binding_module(:a9475) == current_module() @test Base.binding_module(:c7648) == TestMod7648 @@ -199,8 +202,10 @@ let @test Base.binding_module(TestMod7648, :d7648) == TestMod7648 @test Base.binding_module(TestMod7648, :a9475) == TestMod7648.TestModSub9475 @test Base.binding_module(TestMod7648.TestModSub9475, :b9475) == TestMod7648.TestModSub9475 - @test Set(names(TestMod7648)) == Set([:TestMod7648, :a9475, :c7648, :foo7648]) - @test Set(names(TestMod7648, true)) == Set([:TestMod7648, :TestModSub9475, :a9475, :c7648, :d7648, :f7648, :foo7648, Symbol("#foo7648"), :eval, Symbol("#eval")]) + @test Set(names(TestMod7648))==Set([:TestMod7648, :a9475, :foo9475, :c7648, :foo7648, :foo7648_nomethods, :Foo7648]) + @test Set(names(TestMod7648, true)) == Set([:TestMod7648, :TestModSub9475, :a9475, :foo9475, :c7648, :d7648, :f7648, + :foo7648, Symbol("#foo7648"), :foo7648_nomethods, Symbol("#foo7648_nomethods"), + :Foo7648, :eval, Symbol("#eval")]) @test isconst(TestMod7648, :c7648) @test !isconst(TestMod7648, :d7648) end @@ -211,6 +216,11 @@ let @test Base.binding_module(:c7648) == TestMod7648 @test Base.function_name(foo7648) == :foo7648 @test Base.function_module(foo7648, (Any,)) == TestMod7648 + @test Base.function_module(foo7648) == TestMod7648 + @test Base.function_module(foo7648_nomethods) == TestMod7648 + @test Base.function_module(foo9475, (Any,)) == TestMod7648.TestModSub9475 + @test Base.function_module(foo9475) == TestMod7648.TestModSub9475 + @test Base.datatype_module(Foo7648) == TestMod7648 @test basename(functionloc(foo7648, (Any,))[1]) == "reflection.jl" @test first(methods(TestMod7648.TestModSub9475.foo7648)) == @which foo7648(5) @test TestMod7648 == @which foo7648 From 7857675b64037c6786eb6230352f4322271e7523 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 21 Jul 2016 20:11:35 -0400 Subject: [PATCH 0581/1117] Workaround LLVM bug Ref https://llvm.org/bugs/show_bug.cgi?id=27190 Workaround #17288 --- src/cgutils.cpp | 4 ++-- src/codegen.cpp | 2 +- src/gc.c | 6 +++++- src/julia_internal.h | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index c1d7e52b05c89..5aef0dfefa08a 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1560,8 +1560,8 @@ static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt) ArrayRef<Value*>(args, 2)); } else { - Value *pool_ptr = builder.CreateConstGEP1_32(ptls_ptr, offset); - Value *args[] = {ptls_ptr, pool_ptr, ConstantInt::get(T_int32, osize), + Value *pool_offs = ConstantInt::get(T_int32, offset); + Value *args[] = {ptls_ptr, pool_offs, ConstantInt::get(T_int32, osize), ConstantInt::get(T_int32, end_offset)}; v = builder.CreateCall(prepare_call(jlalloc_pool_func), ArrayRef<Value*>(args, 4)); diff --git a/src/codegen.cpp b/src/codegen.cpp index 1c2ab5200bbbf..abadcf918ece1 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5426,7 +5426,7 @@ static void init_julia_llvm_env(Module *m) std::vector<Type*> alloc_pool_args(0); alloc_pool_args.push_back(T_pint8); - alloc_pool_args.push_back(T_pint8); + alloc_pool_args.push_back(T_int32); alloc_pool_args.push_back(T_int32); alloc_pool_args.push_back(T_int32); jlalloc_pool_func = diff --git a/src/gc.c b/src/gc.c index 9245c27b299d0..7acc604c7478d 100644 --- a/src/gc.c +++ b/src/gc.c @@ -786,9 +786,13 @@ static NOINLINE jl_taggedvalue_t *add_page(jl_gc_pool_t *p) } // Size includes the tag and the tag is not cleared!! -JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, +JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, int pool_offset, int osize, int end_offset) { + // Use the pool offset instead of the pool address as the argument + // to workaround a llvm bug. + // Ref https://llvm.org/bugs/show_bug.cgi?id=27190 + jl_gc_pool_t *p = (jl_gc_pool_t*)((char*)ptls + pool_offset); assert(ptls->gc_state == 0); #ifdef MEMDEBUG return jl_gc_big_alloc(ptls, osize); diff --git a/src/julia_internal.h b/src/julia_internal.h index 4c077739659dc..757a8e2a5b057 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -42,7 +42,7 @@ extern unsigned sig_stack_size; JL_DLLEXPORT extern int jl_lineno; JL_DLLEXPORT extern const char *jl_filename; -JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, +JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, int pool_offset, int osize, int end_offset); JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz); int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset); @@ -133,7 +133,7 @@ STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) osize = p->osize; endoff = p->end_offset; } - v = jl_gc_pool_alloc(ptls, p, osize, endoff); + v = jl_gc_pool_alloc(ptls, (char*)p - (char*)ptls, osize, endoff); } else { v = jl_gc_big_alloc(ptls, allocsz); From 77f77298e154872191f3d01a0a68d90e1583914d Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Thu, 21 Jul 2016 19:02:25 +0200 Subject: [PATCH 0582/1117] Don't keep the global variables around. Save them in a list instead. --- src/cgutils.cpp | 22 ++++------------------ src/codegen.cpp | 47 ++++++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 41 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 5aef0dfefa08a..f841bdfa09465 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -208,6 +208,7 @@ static Value *julia_gv(const char *prefix, jl_sym_t *name, jl_module_t *mod, voi return julia_gv(fullname, addr); } +static GlobalVariable *julia_const_gv(jl_value_t *val); static Value *literal_pointer_val(jl_value_t *p) { // emit a pointer to any jl_value_t which will be valid across reloading code @@ -216,24 +217,9 @@ static Value *literal_pointer_val(jl_value_t *p) return ConstantPointerNull::get((PointerType*)T_pjlvalue); if (!imaging_mode) return literal_static_pointer_val(p, T_pjlvalue); - // some common constant values - if (p == jl_false) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlfalse_var))); - if (p == jl_true) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jltrue_var))); - if (p == (jl_value_t*)jl_emptysvec) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlemptysvec_var))); - // exceptions - if (p == (jl_value_t*)jl_diverror_exception) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jldiverr_var))); - if (p == (jl_value_t*)jl_undefref_exception) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlundeferr_var))); - if (p == (jl_value_t*)jl_domain_exception) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jldomerr_var))); - if (p == (jl_value_t*)jl_overflow_exception) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlovferr_var))); - if (p == (jl_value_t*)jl_inexact_exception) - return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jlinexacterr_var))); + if (auto gv = julia_const_gv(p)) { + return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(gv))); + } if (jl_is_datatype(p)) { jl_datatype_t *addr = (jl_datatype_t*)p; // DataTypes are prefixed with a + diff --git a/src/codegen.cpp b/src/codegen.cpp index abadcf918ece1..519118f37ff58 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -288,15 +288,6 @@ static bool type_is_ghost(Type *ty) } // global vars -static GlobalVariable *jltrue_var; -static GlobalVariable *jlfalse_var; -static GlobalVariable *jlemptysvec_var; -static GlobalVariable *jlemptytuple_var; -static GlobalVariable *jldiverr_var; -static GlobalVariable *jlundeferr_var; -static GlobalVariable *jldomerr_var; -static GlobalVariable *jlovferr_var; -static GlobalVariable *jlinexacterr_var; static GlobalVariable *jlRTLD_DEFAULT_var; #ifdef _OS_WINDOWS_ static GlobalVariable *jlexe_var; @@ -4807,6 +4798,21 @@ static GlobalVariable *global_to_llvm(const std::string &cname, void *addr, Modu add_named_global(gv, addr); return gv; } +llvm::SmallVector<std::pair<jl_value_t**, GlobalVariable*>, 16> gv_for_global; +static GlobalVariable *global_jlvalue_to_llvm(const std::string &cname, jl_value_t **addr, Module *m) +{ + GlobalVariable *gv = global_to_llvm(cname, (void*)addr, m); + gv_for_global.push_back(std::make_pair(addr, gv)); + return gv; +} +static GlobalVariable *julia_const_gv(jl_value_t *val) +{ + for (auto& kv : gv_for_global) { + if (*kv.first == val) + return kv.second; + } + return nullptr; +} static Function *jlcall_func_to_llvm(const std::string &cname, jl_fptr_t addr, Module *m) { @@ -5048,20 +5054,15 @@ static void init_julia_llvm_env(Module *m) jl__stack_chk_fail->setDoesNotReturn(); add_named_global(jl__stack_chk_fail, &__stack_chk_fail); - jltrue_var = global_to_llvm("jl_true", (void*)&jl_true, m); - jlfalse_var = global_to_llvm("jl_false", (void*)&jl_false, m); - jlemptysvec_var = global_to_llvm("jl_emptysvec", (void*)&jl_emptysvec, m); - jlemptytuple_var = global_to_llvm("jl_emptytuple", (void*)&jl_emptytuple, m); - jldiverr_var = global_to_llvm("jl_diverror_exception", - (void*)&jl_diverror_exception, m); - jlundeferr_var = global_to_llvm("jl_undefref_exception", - (void*)&jl_undefref_exception, m); - jldomerr_var = global_to_llvm("jl_domain_exception", - (void*)&jl_domain_exception, m); - jlovferr_var = global_to_llvm("jl_overflow_exception", - (void*)&jl_overflow_exception, m); - jlinexacterr_var = global_to_llvm("jl_inexact_exception", - (void*)&jl_inexact_exception, m); + global_jlvalue_to_llvm("jl_true", &jl_true, m); + global_jlvalue_to_llvm("jl_false", &jl_false, m); + global_jlvalue_to_llvm("jl_emptysvec", (jl_value_t**)&jl_emptysvec, m); + global_jlvalue_to_llvm("jl_emptytuple", &jl_emptytuple, m); + global_jlvalue_to_llvm("jl_diverror_exception", &jl_diverror_exception, m); + global_jlvalue_to_llvm("jl_undefref_exception", &jl_undefref_exception, m); + global_jlvalue_to_llvm("jl_domain_exception", &jl_domain_exception, m); + global_jlvalue_to_llvm("jl_overflow_exception", &jl_overflow_exception, m); + global_jlvalue_to_llvm("jl_inexact_exception", &jl_inexact_exception, m); jlRTLD_DEFAULT_var = new GlobalVariable(*m, T_pint8, From 5aba4e79c79dbfb9ace62c2943963e40a3052a5d Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 21 Jul 2016 17:10:39 -0400 Subject: [PATCH 0583/1117] Kill `end_offset` We can easily compute if we hits the end of the page in the allocator. --- src/cgutils.cpp | 8 +++--- src/codegen.cpp | 1 - src/gc.c | 63 ++++++++++++++++++++++++-------------------- src/gc.h | 7 +++-- src/julia_internal.h | 13 +++------ src/julia_threads.h | 1 - 6 files changed, 45 insertions(+), 48 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index f841bdfa09465..c58f60e271990 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1535,8 +1535,7 @@ static void emit_cpointercheck(const jl_cgval_t &x, const std::string &msg, jl_c static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt) { int osize; - int end_offset; - int offset = jl_gc_classify_pools(static_size, &osize, &end_offset); + int offset = jl_gc_classify_pools(static_size, &osize); Value *ptls_ptr = emit_bitcast(ctx->ptlsStates, T_pint8); Value *v; if (offset < 0) { @@ -1547,10 +1546,9 @@ static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt) } else { Value *pool_offs = ConstantInt::get(T_int32, offset); - Value *args[] = {ptls_ptr, pool_offs, ConstantInt::get(T_int32, osize), - ConstantInt::get(T_int32, end_offset)}; + Value *args[] = {ptls_ptr, pool_offs, ConstantInt::get(T_int32, osize)}; v = builder.CreateCall(prepare_call(jlalloc_pool_func), - ArrayRef<Value*>(args, 4)); + ArrayRef<Value*>(args, 3)); } tbaa_decorate(tbaa_tag, builder.CreateStore(jt, emit_typeptr_addr(v))); return v; diff --git a/src/codegen.cpp b/src/codegen.cpp index 519118f37ff58..74303cefb818a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5429,7 +5429,6 @@ static void init_julia_llvm_env(Module *m) alloc_pool_args.push_back(T_pint8); alloc_pool_args.push_back(T_int32); alloc_pool_args.push_back(T_int32); - alloc_pool_args.push_back(T_int32); jlalloc_pool_func = Function::Create(FunctionType::get(T_pjlvalue, alloc_pool_args, false), Function::ExternalLinkage, diff --git a/src/gc.c b/src/gc.c index 7acc604c7478d..7ceafddb07f85 100644 --- a/src/gc.c +++ b/src/gc.c @@ -751,22 +751,24 @@ static void sweep_malloced_arrays(void) } // pool allocation -static inline jl_taggedvalue_t *reset_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_taggedvalue_t *fl) +static inline jl_taggedvalue_t *reset_page(const jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_taggedvalue_t *fl) { + assert(GC_PAGE_OFFSET >= sizeof(void*)); pg->nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / p->osize; jl_ptls_t ptls2 = jl_all_tls_states[pg->thread_n]; pg->pool_n = p - ptls2->heap.norm_pools; memset(pg->ages, 0, GC_PAGE_SZ / 8 / p->osize + 1); jl_taggedvalue_t *beg = (jl_taggedvalue_t*)(pg->data + GC_PAGE_OFFSET); - jl_taggedvalue_t *end = (jl_taggedvalue_t*)((char*)beg + (pg->nfree - 1)*p->osize); - end->next = fl; + jl_taggedvalue_t *next = (jl_taggedvalue_t*)pg->data; + next->next = fl; pg->has_young = 0; pg->has_marked = 0; - pg->fl_begin_offset = GC_PAGE_OFFSET; - pg->fl_end_offset = (char*)end - (char*)beg + GC_PAGE_OFFSET; + pg->fl_begin_offset = -1; + pg->fl_end_offset = -1; return beg; } +// Add a new page to the pool. Discards any pages in `p->newpages` before. static NOINLINE jl_taggedvalue_t *add_page(jl_gc_pool_t *p) { // Do not pass in `ptls` as argument. This slows down the fast path @@ -780,14 +782,14 @@ static NOINLINE jl_taggedvalue_t *add_page(jl_gc_pool_t *p) pg->osize = p->osize; pg->ages = (uint8_t*)malloc(GC_PAGE_SZ / 8 / p->osize + 1); pg->thread_n = ptls->tid; - jl_taggedvalue_t *fl = reset_page(p, pg, p->newpages); + jl_taggedvalue_t *fl = reset_page(p, pg, NULL); p->newpages = fl; return fl; } // Size includes the tag and the tag is not cleared!! JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, int pool_offset, - int osize, int end_offset) + int osize) { // Use the pool offset instead of the pool address as the argument // to workaround a llvm bug. @@ -824,31 +826,36 @@ JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, int pool_offset, } // if the freelist is empty we reuse empty but not freed pages v = p->newpages; - if (__unlikely(!v)) - v = add_page(p); - jl_taggedvalue_t *end = (jl_taggedvalue_t*)&(gc_page_data(v)[end_offset]); - if (__likely(v != end)) { - p->newpages = (jl_taggedvalue_t*)((char*)v + osize); - } - else { - // like the freelist case, but only update the page metadata when it is full - jl_gc_pagemeta_t *pg = page_metadata(v); - assert(pg->osize == p->osize); - pg->nfree = 0; - pg->has_young = 1; - p->newpages = v->next; + jl_taggedvalue_t *next = (jl_taggedvalue_t*)((char*)v + osize); + // If there's no pages left or the current page is used up, + // we need to use the slow path. + char *cur_page = gc_page_data((char*)v - 1); + if (__unlikely(!v || cur_page + GC_PAGE_SZ < (char*)next)) { + if (v) { + // like the freelist case, + // but only update the page metadata when it is full + jl_gc_pagemeta_t *pg = page_metadata((char*)v - 1); + assert(pg->osize == p->osize); + pg->nfree = 0; + pg->has_young = 1; + v = *(jl_taggedvalue_t**)cur_page; + } + // Not an else!! + if (!v) + v = add_page(p); + next = (jl_taggedvalue_t*)((char*)v + osize); } + p->newpages = next; return jl_valueof(v); } -int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset) +int jl_gc_classify_pools(size_t sz, int *osize) { if (sz > GC_MAX_SZCLASS) return -1; size_t allocsz = sz + sizeof(jl_taggedvalue_t); int klass = jl_gc_szclass(allocsz); *osize = jl_gc_sizeclasses[klass]; - *end_offset = GC_POOL_END_OFS(*osize); return (int)(intptr_t)(&((jl_ptls_t)0)->heap.norm_pools[klass]); } @@ -875,10 +882,7 @@ static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_t // FIXME - need to do accounting on a per-thread basis // on quick sweeps, keep a few pages empty but allocated for performance if (!sweep_full && lazy_freed_pages <= default_collect_interval / GC_PAGE_SZ) { - jl_taggedvalue_t *begin = reset_page(p, pg, 0); - jl_taggedvalue_t **pend = (jl_taggedvalue_t**)((char*)begin + ((int)pg->nfree - 1)*osize); - jl_taggedvalue_t *npg = p->newpages; - *pend = npg; + jl_taggedvalue_t *begin = reset_page(p, pg, p->newpages); p->newpages = begin; begin->next = (jl_taggedvalue_t*)0; lazy_freed_pages++; @@ -1046,8 +1050,10 @@ static void gc_sweep_pool(int sweep_full) last = p->newpages; if (last) { - jl_gc_pagemeta_t *pg = page_metadata(last); - pg->nfree = (GC_PAGE_SZ - ((char*)last - gc_page_data(last))) / p->osize; + char *last_p = (char*)last; + jl_gc_pagemeta_t *pg = page_metadata(last_p - 1); + assert(last_p - gc_page_data(last_p - 1) >= GC_PAGE_OFFSET); + pg->nfree = (GC_PAGE_SZ - (last_p - gc_page_data(last_p - 1))) / p->osize; pg->has_young = 1; } p->newpages = NULL; @@ -1865,7 +1871,6 @@ void jl_mk_thread_heap(jl_ptls_t ptls) p[i].osize = jl_gc_sizeclasses[i]; p[i].freelist = NULL; p[i].newpages = NULL; - p[i].end_offset = GC_POOL_END_OFS(jl_gc_sizeclasses[i]); } arraylist_new(&heap->weak_refs, 0); heap->mallocarrays = NULL; diff --git a/src/gc.h b/src/gc.h index d59c7db7069a4..47eeb594df0ef 100644 --- a/src/gc.h +++ b/src/gc.h @@ -30,7 +30,9 @@ extern "C" { #endif -// manipulating mark bits +#define GC_PAGE_LG2 14 // log2(size of a page) +#define GC_PAGE_SZ (1 << GC_PAGE_LG2) // 16k +#define GC_PAGE_OFFSET (JL_SMALL_BYTE_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_SMALL_BYTE_ALIGNMENT)) // 8G * 32768 = 2^48 // It's really unlikely that we'll actually allocate that much though... @@ -157,7 +159,8 @@ __attribute__((aligned(GC_PAGE_SZ))) typedef struct { // Page layout: - // Padding: GC_PAGE_OFFSET + // Newpage freelist: sizeof(void*) + // Padding: GC_PAGE_OFFSET - sizeof(void*) // Blocks: osize * n // Tag: sizeof(jl_taggedvalue_t) // Data: <= osize - sizeof(jl_taggedvalue_t) diff --git a/src/julia_internal.h b/src/julia_internal.h index 757a8e2a5b057..fcc29f42b1791 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -43,9 +43,9 @@ JL_DLLEXPORT extern int jl_lineno; JL_DLLEXPORT extern const char *jl_filename; JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, int pool_offset, - int osize, int end_offset); + int osize); JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz); -int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset); +int jl_gc_classify_pools(size_t sz, int *osize); extern jl_mutex_t gc_perm_lock; void *jl_gc_perm_alloc_nolock(size_t sz); void *jl_gc_perm_alloc(size_t sz); @@ -108,10 +108,6 @@ STATIC_INLINE int JL_CONST_FUNC jl_gc_szclass(size_t sz) #endif #define JL_SMALL_BYTE_ALIGNMENT 16 #define JL_CACHE_BYTE_ALIGNMENT 64 -#define GC_POOL_END_OFS(osize) ((((GC_PAGE_SZ - GC_PAGE_OFFSET)/(osize)) - 1)*(osize) + GC_PAGE_OFFSET) -#define GC_PAGE_LG2 14 // log2(size of a page) -#define GC_PAGE_SZ (1 << GC_PAGE_LG2) // 16k -#define GC_PAGE_OFFSET (JL_SMALL_BYTE_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_SMALL_BYTE_ALIGNMENT)) #define GC_MAX_SZCLASS (2032-sizeof(void*)) STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) @@ -124,16 +120,13 @@ STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) int pool_id = jl_gc_szclass(allocsz); jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id]; int osize; - int endoff; if (jl_is_constexpr(allocsz)) { osize = jl_gc_sizeclasses[pool_id]; - endoff = GC_POOL_END_OFS(osize); } else { osize = p->osize; - endoff = p->end_offset; } - v = jl_gc_pool_alloc(ptls, (char*)p - (char*)ptls, osize, endoff); + v = jl_gc_pool_alloc(ptls, (char*)p - (char*)ptls, osize); } else { v = jl_gc_big_alloc(ptls, allocsz); diff --git a/src/julia_threads.h b/src/julia_threads.h index 6b8c9ea9a306a..50a9bc36c2820 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -31,7 +31,6 @@ typedef struct { jl_taggedvalue_t *freelist; // root of list of free objects jl_taggedvalue_t *newpages; // root of list of chunks of free objects - uint16_t end_offset; // stored to avoid computing it at each allocation uint16_t osize; // size of objects in this pool } jl_gc_pool_t; From 53b9c0379fb4fbdc3daf111e3ba5c1c5298232d4 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Fri, 22 Jul 2016 10:55:49 -0400 Subject: [PATCH 0584/1117] make x[...] .= ... assign in-place (#17546) * make x[...] .= ... assign in-place (fixes bug in #17510) * doc fix * You're the top! You're the Coliseum. * better var name --- doc/manual/functions.rst | 4 ++++ src/julia-syntax.scm | 18 ++++++++++++++---- test/broadcast.jl | 8 ++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index e898f663201ef..6f43aec2fecad 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -660,6 +660,10 @@ calls do not allocate new arrays over and over again for the results except that, as above, the ``broadcast!`` loop is fused with any nested "dot" calls. For example, ``X .= sin.(Y)`` is equivalent to ``broadcast!(sin, X, Y)``, overwriting ``X`` with ``sin.(Y)`` in-place. +If the left-hand side is a ``getindex`` expression, e.g. +``X[2:end] .= sin.(Y)``, then it translates to ``broadcast!`` on a ``view``, +e.g. ``broadcast!(sin, view(X, 2:endof(X)), Y)``, so that the left-hand +side is updated in-place. (In future versions of Julia, operators like ``.*`` will also be handled with the same mechanism: they will be equivalent to ``broadcast`` calls and diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index c08d48843e963..878105a0700bd 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1542,6 +1542,15 @@ (cadr expr) ;; eta reduce `x->f(x)` => `f` `(-> ,argname (block ,@splat ,expr))))) +(define (ref-to-view expr) + (if (and (pair? expr) (eq? (car expr) 'ref)) + (let* ((ex (partially-expand-ref expr)) + (stmts (butlast (cdr ex))) + (refex (last (cdr ex))) + (nuref `(call (top view) ,(caddr refex) ,@(cdddr refex)))) + `(block ,@stmts ,nuref)) + expr)) + ; fuse nested calls to expr == f.(args...) into a single broadcast call, ; or a broadcast! call if lhs is non-null. (define (expand-fuse-broadcast lhs rhs) @@ -1657,14 +1666,15 @@ (cons farg new-fargs) (cons arg new-args) renames varfarg vararg)))))) (cf (cdadr f) args '() '() '() '() '())) e)) ; (not (fuse? e)) - (let ((e (compress-fuse (dot-to-fuse rhs)))) ; an expression '(fuse func args) if expr is a dot call + (let ((e (compress-fuse (dot-to-fuse rhs))) ; an expression '(fuse func args) if expr is a dot call + (lhs-view (ref-to-view lhs))) ; x[...] expressions on lhs turn in to view(x, ...) to update x in-place (if (fuse? e) (if (null? lhs) - (expand-forms `(call broadcast ,(from-lambda (cadr e)) ,@(caddr e))) - (expand-forms `(call broadcast! ,(from-lambda (cadr e)) ,lhs ,@(caddr e)))) + (expand-forms `(call (top broadcast) ,(from-lambda (cadr e)) ,@(caddr e))) + (expand-forms `(call (top broadcast!) ,(from-lambda (cadr e)) ,lhs-view ,@(caddr e)))) (if (null? lhs) (expand-forms e) - (expand-forms `(call broadcast! identity ,lhs ,e)))))) + (expand-forms `(call (top broadcast!) identity ,lhs-view ,e)))))) ;; table mapping expression head to a function expanding that form (define expand-table diff --git a/test/broadcast.jl b/test/broadcast.jl index 31a2166a4ee14..52aeacc859041 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -265,6 +265,14 @@ let x = [1:4;], y = x @test y === x == [8,8,8,8] y .-= 1:4 @test y === x == [7,6,5,4] + x[1:2] .= 1 + @test y === x == [1,1,5,4] + x[1:2] .+= [2,3] + @test y === x == [3,4,5,4] + x[:] .= 0 + @test y === x == [0,0,0,0] + x[2:end] .= 1:3 + @test y === x == [0,1,2,3] end # PR 16988 From 610e45baab7dff84763a51f7701d1be7492709e9 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 22 Jul 2016 10:57:46 -0400 Subject: [PATCH 0585/1117] fix #9948, scoping bug in keyword arg defaults --- src/julia-syntax.scm | 4 +++- test/keywordargs.jl | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 878105a0700bd..e98029ebe1487 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -424,7 +424,8 @@ ,@lno ,@(if (not ordered-defaults) '() - (map make-assignment keynames vals)) + (append! (map (lambda (kwname) `(local ,kwname)) keynames) + (map make-assignment keynames vals))) ;; call mangled(vals..., [rest_kw ,]pargs..., [vararg]...) (return (call ,mangled ,@(if ordered-defaults keynames vals) @@ -466,6 +467,7 @@ `(block ;; initialize keyword args to their defaults, or set a flag telling ;; whether this keyword needs to be set. + ,@(map (lambda (kwname) `(local ,kwname)) keynames) ,@(map (lambda (name dflt flag) (if (const-default? dflt) `(= ,name ,dflt) diff --git a/test/keywordargs.jl b/test/keywordargs.jl index eff8844ec7fa4..bf6ac51b5e762 100644 --- a/test/keywordargs.jl +++ b/test/keywordargs.jl @@ -198,3 +198,16 @@ end @test ((a=1,b=2)->(a,b))() == (1,2) @test ((a=1,b=2)->(a,b))(5) == (5,2) @test ((a=1,b=2)->(a,b))(5,6) == (5,6) + +# issue #9948 +f9948, getx9948 = let + x = 3 + h(;x=x) = x + getx() = x + h, getx +end +@test_throws UndefVarError f9948() +@test getx9948() == 3 +@test f9948(x=5) == 5 +@test_throws UndefVarError f9948() +@test getx9948() == 3 From b96d800d9df19e610886ac7ca7b34948ac89892a Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Fri, 22 Jul 2016 11:04:18 -0400 Subject: [PATCH 0586/1117] note breaking changes to .+= etc. in #17510, #17546 --- NEWS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.md b/NEWS.md index 0bf93945ccbc7..750ee8df44937 100644 --- a/NEWS.md +++ b/NEWS.md @@ -108,6 +108,12 @@ Compiler/Runtime improvements Breaking changes ---------------- + * The assignment operations `.+=`, `.*=` and so on now generate calls + to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side + if the latter is a `a[...]` expression. This means that they will fail + if the left-hand side is immutable (or does not support `view`), and will + otherwise change the left-hand side in-place ([#17510], [#17546]). + * Method ambiguities no longer generate warnings when files are loaded, nor do they dispatch to an arbitrarily-chosen method; instead, a call that cannot be resolved to a single method results @@ -369,3 +375,4 @@ Deprecated or removed [#17402]: https://github.com/JuliaLang/julia/issues/17402 [#17404]: https://github.com/JuliaLang/julia/issues/17404 [#17510]: https://github.com/JuliaLang/julia/issues/17510 +[#17546]: https://github.com/JuliaLang/julia/issues/17546 From 0796df918d9e74166918e93fcb126836a925fdbf Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 13 Jul 2016 17:24:45 -0400 Subject: [PATCH 0587/1117] make object_ids and hashes in precompiled modules more stable fixes #17043, also helps #5849 --- src/alloc.c | 2 +- src/builtins.c | 25 +++++++------------------ src/dump.c | 1 - src/jltypes.c | 2 +- src/julia.h | 2 +- src/module.c | 2 +- src/support/hashing.h | 6 ++++++ 7 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index fc2da73d86992..898e2c2f6f419 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -818,7 +818,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu tn->cache = jl_emptysvec; tn->linearcache = jl_emptysvec; tn->names = NULL; - tn->uid = jl_assign_type_uid(); + tn->hash = bitmix(bitmix(module ? module->uuid : 0, name->hash), 0xa1ada1da); tn->mt = NULL; JL_GC_PUSH1(&tn); tn->mt = NULL; diff --git a/src/builtins.c b/src/builtins.c index 4af852b119bb4..2026934123e1a 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1012,21 +1012,17 @@ JL_CALLABLE(jl_f_invoke) // hashing -------------------------------------------------------------------- -#ifdef _P64 -#define bitmix(a,b) int64hash((a)^bswap_64(b)) -#define hash64(a) int64hash(a) -#else -#define bitmix(a,b) int64to32hash((((uint64_t)a)<<32)|((uint64_t)b)) -#define hash64(a) int64to32hash(a) -#endif - static uintptr_t bits_hash(void *b, size_t sz) { switch (sz) { case 1: return int32hash(*(int8_t*)b); case 2: return int32hash(*(int16_t*)b); case 4: return int32hash(*(int32_t*)b); - case 8: return hash64(*(int64_t*)b); +#ifdef _P64 + case 8: return int64hash(*(int64_t*)b); +#else + case 8: return int64to32hash(*(int64_t*)b); +#endif default: #ifdef _P64 return memhash((char*)b, sz); @@ -1057,17 +1053,10 @@ static uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v) jl_datatype_t *dt = (jl_datatype_t*)tv; if (dt == jl_datatype_type) { jl_datatype_t *dtv = (jl_datatype_t*)v; - uintptr_t h = 0xda1ada1a; - // has_typevars always returns 0 on name->primary, so that type - // can exist in the cache. however, interpreter.c mutates its - // typevars' `bound` fields to 0, corrupting the cache. this is - // avoided simply by hashing name->primary specially here. - if (jl_egal(dtv->name->primary, v)) - return bitmix(bitmix(h, dtv->name->uid), 0xaa5566aa); - return bitmix(bitmix(h, dtv->name->uid), hash_svec(dtv->parameters)); + return bitmix(~dtv->name->hash, hash_svec(dtv->parameters)); } if (dt == jl_typename_type) - return bitmix(((jl_typename_t*)v)->uid, 0xa1ada1ad); + return ((jl_typename_t*)v)->hash; if (dt->mutabl) return inthash((uintptr_t)v); size_t sz = jl_datatype_size(tv); uintptr_t h = jl_object_id(tv); diff --git a/src/dump.c b/src/dump.c index 41cdefd129ff5..b10909d27c4d8 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1663,7 +1663,6 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta if ((s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK)) { if (dt == jl_typename_type) { jl_typename_t *tn = (jl_typename_t*)v; - tn->uid = jl_assign_type_uid(); // make sure this has a new uid tn->cache = jl_emptysvec; // the cache is refilled later (tag 5) tn->linearcache = jl_emptysvec; // the cache is refilled later (tag 5) } diff --git a/src/jltypes.c b/src/jltypes.c index a2f54d88b929d..cc2d1acfa33b2 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3524,7 +3524,7 @@ void jl_init_types(void) jl_typename_type->name->names = jl_svec(8, jl_symbol("name"), jl_symbol("module"), jl_symbol("names"), jl_symbol("primary"), jl_symbol("cache"), jl_symbol("linearcache"), - jl_symbol("uid"), jl_symbol("mt")); + jl_symbol("hash"), jl_symbol("mt")); jl_typename_type->types = jl_svec(8, jl_sym_type, jl_any_type, jl_simplevector_type, jl_type_type, jl_simplevector_type, jl_simplevector_type, jl_any_type, jl_any_type); diff --git a/src/julia.h b/src/julia.h index 7bcce4bd07cb4..24f5d4f5cf7f0 100644 --- a/src/julia.h +++ b/src/julia.h @@ -300,7 +300,7 @@ typedef struct { jl_value_t *primary; jl_svec_t *cache; // sorted array jl_svec_t *linearcache; // unsorted array - intptr_t uid; + intptr_t hash; struct _jl_methtable_t *mt; } jl_typename_t; diff --git a/src/module.c b/src/module.c index 28ca7f88bd1dc..6fbee0f969345 100644 --- a/src/module.c +++ b/src/module.c @@ -26,7 +26,7 @@ JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name) m->name = name; m->parent = NULL; m->istopmod = 0; - m->uuid = uv_now(uv_default_loop()); + m->uuid = jl_hrtime(); m->counter = 0; htable_new(&m->bindings, 0); arraylist_new(&m->usings, 0); diff --git a/src/support/hashing.h b/src/support/hashing.h index 99ff397b91c18..1be2a9e7b5ec0 100644 --- a/src/support/hashing.h +++ b/src/support/hashing.h @@ -21,6 +21,12 @@ JL_DLLEXPORT uint64_t memhash_seed(const char *buf, size_t n, uint32_t seed); JL_DLLEXPORT uint32_t memhash32(const char *buf, size_t n); JL_DLLEXPORT uint32_t memhash32_seed(const char *buf, size_t n, uint32_t seed); +#ifdef _P64 +#define bitmix(a,b) int64hash((a)^bswap_64(b)) +#else +#define bitmix(a,b) int64to32hash((((uint64_t)a)<<32)|((uint64_t)b)) +#endif + #ifdef __cplusplus } #endif From 1e06b87498fcddf9bb2aa3766b023e8474f161b8 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 22 Jul 2016 11:14:02 -0700 Subject: [PATCH 0588/1117] create BUILDDIR before cd'ing to it in deps/blas.mk for lapack [av skip] --- deps/blas.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/deps/blas.mk b/deps/blas.mk index 19d29d6d5d97a..fcd6cd8074bb0 100644 --- a/deps/blas.mk +++ b/deps/blas.mk @@ -209,6 +209,7 @@ $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.netlib.org/lapack/$(notdir $@) $(BUILDDIR)/lapack-$(LAPACK_VER)/make.inc: $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz $(JLCHECKSUM) $< + mkdir -p $(BUILDDIR) cd $(BUILDDIR) && $(TAR) zxf $< cp $(dir $@)INSTALL/make.inc.gfortran $(dir $@)make.inc touch -c $@ From 7e51540d98f6f1642f11ac8c80df8991812c9353 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 21 Jul 2016 18:50:21 -0400 Subject: [PATCH 0589/1117] improve docs for `IOContext`. fixes #16763 also remove some references to the removed `multiline` property --- base/docs/helpdb/Base.jl | 3 ++ base/multimedia.jl | 2 +- base/show.jl | 57 +++++++++++++++++++++++++------------- doc/stdlib/io-network.rst | 32 +++++++++++++++------ doc/stdlib/stacktraces.rst | 11 +++----- 5 files changed, 68 insertions(+), 37 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index c2bc9ea919a48..c1478f81559f9 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -6280,6 +6280,9 @@ defining a 2-argument `show(stream::IO, x::MyType)` method. Technically, the `MIME"mime"` macro defines a singleton type for the given `mime` string, which allows us to exploit Julia's dispatch mechanisms in determining how to display objects of any given type. + +The first argument to `show` can be an `IOContext` specifying output format properties. +See `IOContext` for details. """ show(stream, mime, x) diff --git a/base/multimedia.jl b/base/multimedia.jl index 45b827e66d859..de2dd2492b11f 100644 --- a/base/multimedia.jl +++ b/base/multimedia.jl @@ -47,7 +47,7 @@ mimewritable(m::AbstractString, x) = mimewritable(MIME(m), x) # format and is returned unmodified. This is useful so that raw data can be # passed to display(m::MIME, x). -verbose_show(io, m, x) = show(IOContext(io,multiline=true,limit=false), m, x) +verbose_show(io, m, x) = show(IOContext(io,limit=false), m, x) macro textmime(mime) quote diff --git a/base/show.jl b/base/show.jl index 209d3bb154bbc..99ae006e4063e 100644 --- a/base/show.jl +++ b/base/show.jl @@ -2,6 +2,14 @@ print(io::IO, s::Symbol) = (write(io,s); nothing) +""" + IOContext + +IOContext provides a mechanism for passing output configuration settings among `show` methods. + +In short, it is an immutable dictionary that is a subclass of IO. It supports standard +dictionary operations such as `getindex`, and can also be used as an I/O stream. +""" immutable IOContext{IO_t <: IO} <: AbstractPipe io::IO_t dict::ImmutableDict{Symbol, Any} @@ -12,27 +20,10 @@ immutable IOContext{IO_t <: IO} <: AbstractPipe end """ - IOContext{<:IO} <: IO - -IOContext provides a mechanism for passing output-configuration keyword arguments through arbitrary show methods. - -In short, it is an immutable Dictionary that is a subclass of IO. - - IOContext(io::IO, KV::Pair) - -Create a new entry in the IO Dictionary for the key => value pair + IOContext(io::IO; properties...) - - use `(key => value) in dict` to see if this particular combination is in the properties set - - use `get(dict, key, default)` to retrieve the most recent value for a particular key - -``` -IOContext(io::IO, context::IOContext) -``` - -Create a IOContext that wraps an alternate IO but inherits the keyword arguments from the context +The same as `IOContext(io::IO, KV::Pair)`, but accepting properties as keyword arguments. """ -IOContext - IOContext(io::IO; kws...) = IOContext(IOContext(io, ImmutableDict{Symbol,Any}()); kws...) function IOContext(io::IOContext; kws...) for (k, v) in kws @@ -48,7 +39,34 @@ IOContext(io::IO, key, value) = IOContext(io, ImmutableDict{Symbol, Any}(key, va IOContext(io::IOContext, key, value) = IOContext(io, ImmutableDict{Symbol, Any}(io.dict, key, value)) IOContext(io::IO, context::IO) = IOContext(io) + +""" + IOContext(io::IO, context::IOContext) + +Create a IOContext that wraps an alternate IO but inherits the properties of `context`. +""" IOContext(io::IO, context::IOContext) = IOContext(io, context.dict) + +""" + IOContext(io::IO, KV::Pair) + +Create an `IOContext` that wraps a given stream, adding the specified key=>value pair to +the properties of that stream (note that `io` can itself be an `IOContext`). + + - use `(key => value) in dict` to see if this particular combination is in the properties set + - use `get(dict, key, default)` to retrieve the most recent value for a particular key + +The following properties are in common use: + + - `:compact`: Boolean specifying that small values should be printed more compactly, e.g. + that numbers should be printed with fewer digits. This is set when printing array + elements. + - `:limit`: Boolean specifying that containers should be truncated, e.g. showing `…` in + place of most elements. + - `:displaysize`: A `Tuple{Int,Int}` giving the size in rows and columns to use for text + output. This can be used to override the display size for called functions, but to + get the size of the screen use the `displaysize` function. +""" IOContext(io::IO, KV::Pair) = IOContext(io, KV[1], KV[2]) show(io::IO, ctx::IOContext) = (print(io, "IOContext("); show(io, ctx.io); print(io, ")")) @@ -1566,7 +1584,6 @@ function showarray(io::IO, X::AbstractArray, repr::Bool = true; header = true) if repr && ndims(X) == 1 return show_vector(io, X, "[", "]") end - io = IOContext(io, multiline=false) if !haskey(io, :compact) io = IOContext(io, compact=true) end diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index edfea05d67835..2ddf77b578d4a 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -398,28 +398,40 @@ General I/O Read all available data on the stream, blocking the task only if no data is available. The result is a ``Vector{UInt8,1}``\ . -.. function:: IOContext{<:IO} <: IO +.. type:: IOContext .. Docstring generated from Julia source - IOContext provides a mechanism for passing output-configuration keyword arguments through arbitrary show methods. + IOContext provides a mechanism for passing output configuration settings among ``show`` methods. - In short, it is an immutable Dictionary that is a subclass of IO. + In short, it is an immutable dictionary that is a subclass of IO. It supports standard dictionary operations such as ``getindex``\ , and can also be used as an I/O stream. - .. code-block:: julia +.. function:: IOContext(io::IO, KV::Pair) - IOContext(io::IO, KV::Pair) + .. Docstring generated from Julia source - Create a new entry in the IO Dictionary for the key => value pair + Create an ``IOContext`` that wraps a given stream, adding the specified key=>value pair to the properties of that stream (note that ``io`` can itself be an ``IOContext``\ ). * use ``(key => value) in dict`` to see if this particular combination is in the properties set * use ``get(dict, key, default)`` to retrieve the most recent value for a particular key - .. code-block:: julia + The following properties are in common use: - IOContext(io::IO, context::IOContext) + * ``:compact``\ : Boolean specifying that small values should be printed more compactly, e.g. that numbers should be printed with fewer digits. This is set when printing array elements. + * ``:limit``\ : Boolean specifying that containers should be truncated, e.g. showing ``…`` in place of most elements. + * ``:displaysize``\ : A ``Tuple{Int,Int}`` giving the size in rows and columns to use for text output. This can be used to override the display size for called functions, but to get the size of the screen use the ``displaysize`` function. + +.. function:: IOContext(io::IO, context::IOContext) + + .. Docstring generated from Julia source - Create a IOContext that wraps an alternate IO but inherits the keyword arguments from the context + Create a IOContext that wraps an alternate IO but inherits the properties of ``context``\ . + +.. function:: IOContext(io::IO; properties...) + + .. Docstring generated from Julia source + + The same as ``IOContext(io::IO, KV::Pair)``\ , but accepting properties as keyword arguments. Text I/O -------- @@ -717,6 +729,8 @@ Julia environments (such as the IPython-based IJulia notebook). Technically, the ``MIME"mime"`` macro defines a singleton type for the given ``mime`` string, which allows us to exploit Julia's dispatch mechanisms in determining how to display objects of any given type. + The first argument to ``show`` can be an ``IOContext`` specifying output format properties. See ``IOContext`` for details. + .. function:: mimewritable(mime, x) .. Docstring generated from Julia source diff --git a/doc/stdlib/stacktraces.rst b/doc/stdlib/stacktraces.rst index 46a5f0b857d48..1cc87e89aac3d 100644 --- a/doc/stdlib/stacktraces.rst +++ b/doc/stdlib/stacktraces.rst @@ -17,7 +17,7 @@ * ``func::Symbol`` The name of the function containing the execution context. - * ``outer_linfo::Nullable{LambdaInfo}`` + * ``linfo::Nullable{LambdaInfo}`` The LambdaInfo containing the execution context (if it could be found). * ``file::Symbol`` @@ -26,15 +26,12 @@ * ``line::Int`` The line number in the file containing the execution context. - * ``inlined_file::Symbol`` - - The path to the file containing the context for inlined code. - * ``inlined_line::Int`` - - The line number in the file containing the context for inlined code. * ``from_c::Bool`` True if the code is from C. + * ``inlined::Bool`` + + True if the code is from an inlined frame. * ``pointer::Int64`` Representation of the pointer to the execution context as returned by ``backtrace``\ . From 1a774b5ccff8f1d1b9c4a61c6493fbf9c21a547a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 22 Jul 2016 15:40:46 -0400 Subject: [PATCH 0590/1117] some doc and deprecation fixes - minor fixes to DocCheck.jl, but it needs much more work - add docs for isimag, clamp!, and isassigned - fix deprecation for pmap keyword arguments --- base/complex.jl | 5 +++++ base/deprecated.jl | 32 ++------------------------------ base/essentials.jl | 8 ++++++++ base/exports.jl | 1 - base/math.jl | 5 +++++ base/pmap.jl | 28 ++++++++++++++++++++++++---- base/sparse/sparse.jl | 2 +- doc/DocCheck.jl | 10 +++------- doc/stdlib/arrays.rst | 6 ++++++ doc/stdlib/math.rst | 6 ++++++ doc/stdlib/numbers.rst | 6 ++++++ 11 files changed, 66 insertions(+), 43 deletions(-) diff --git a/base/complex.jl b/base/complex.jl index e1bf83a98830e..da644f9412331 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -47,6 +47,11 @@ complex{T<:Real}(::Type{Complex{T}}) = Complex{T} isreal(x::Real) = true isreal(z::Complex) = imag(z) == 0 +""" + isimag(z) -> Bool + +Test whether `z` is purely imaginary, i.e. has a real part equal to 0. +""" isimag(z::Number) = real(z) == 0 isinteger(z::Complex) = isreal(z) & isinteger(real(z)) isfinite(z::Complex) = isfinite(real(z)) & isfinite(imag(z)) diff --git a/base/deprecated.jl b/base/deprecated.jl index 816bcea7f0b66..0b6d69eedaf05 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -292,36 +292,7 @@ export call #15409 # Deprecated definition of pmap with keyword arguments. -# When this is removed the following definition needs to be uncommented -# and added to pmap.jl -# pmap(f, c...) = pmap(default_worker_pool(), f, c...) - -function pmap(f, c...; err_retry=nothing, err_stop=nothing, pids=nothing, kwargs...) - kwargs = Dict{Symbol, Any}(kwargs) - - if err_retry != nothing - depwarn("err_retry is deprecated, use pmap(retry(f), c...).", :pmap) - if err_retry == true - f = retry(f) - end - end - - if pids == nothing - p = default_worker_pool() - else - depwarn("pids is deprecated, use pmap(::WorkerPool, f, c...).", :pmap) - p = WorkerPool(pids) - end - - if err_stop != nothing - depwarn("err_stop is deprecated, use pmap(f, c...; on_error = error_handling_func).", :pmap) - if err_stop == false - kwargs[:on_error] = e->e - end - end - - pmap(p, f, c...; kwargs...) -end +# deprecation warnings are in pmap.jl # 15692 typealias Func{N} Function @@ -780,6 +751,7 @@ function ereach{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, k::Integer, parent::Vector{Ti} error(string("ereach(A, k, parent) now lives in package SuiteSparse.jl. Run", "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) end +export etree function etree{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, postorder::Bool) error(string("etree(A[, post]) now lives in package SuiteSparse.jl. Run", "Pkg.add(\"SuiteSparse\") to install SuiteSparse on Julia v0.5.")) diff --git a/base/essentials.jl b/base/essentials.jl index 5756a4571c81b..fed6c191ea4de 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -187,6 +187,14 @@ map(f, v::SimpleVector) = Any[ f(v[i]) for i = 1:length(v) ] getindex(v::SimpleVector, I::AbstractArray) = Core.svec(Any[ v[i] for i in I ]...) +""" + isassigned(array, i) -> Bool + +Tests whether the given array has a value associated with index `i`. Returns `false` +if the index is out of bounds, or has an undefined reference. +""" +function isassigned end + function isassigned(v::SimpleVector, i::Int) 1 <= i <= length(v) || return false x = unsafe_load(convert(Ptr{Ptr{Void}},data_pointer_from_objref(v)) + i*sizeof(Ptr)) diff --git a/base/exports.jl b/base/exports.jl index 2e83da356f2b2..12204a129e6e3 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1420,7 +1420,6 @@ export AbstractSparseVector, SparseMatrixCSC, SparseVector, - etree, issparse, sparse, sparsevec, diff --git a/base/math.jl b/base/math.jl index a26f9ac853863..41ad90358a211 100644 --- a/base/math.jl +++ b/base/math.jl @@ -46,6 +46,11 @@ clamp{T}(x::AbstractArray{T,2}, lo, hi) = clamp{T}(x::AbstractArray{T}, lo, hi) = reshape([clamp(xx, lo, hi) for xx in x], size(x)) +""" + clamp!(array::AbstractArray, lo, hi) + +Restrict values in `array` to the specified range, in-place. +""" function clamp!{T}(x::AbstractArray{T}, lo, hi) @inbounds for i in eachindex(x) x[i] = clamp(x[i], lo, hi) diff --git a/base/pmap.jl b/base/pmap.jl index ca48804867a42..9ea825067df32 100644 --- a/base/pmap.jl +++ b/base/pmap.jl @@ -70,10 +70,30 @@ The following are equivalent: * `pmap(f, c; retry_n=1)` and `asyncmap(retry(remote(f)),c)` * `pmap(f, c; retry_n=1, on_error=e->e)` and `asyncmap(x->try retry(remote(f))(x) catch e; e end, c)` """ -function pmap(p::AbstractWorkerPool, f, c; distributed=true, batch_size=1, on_error=nothing, - retry_n=0, - retry_max_delay=DEFAULT_RETRY_MAX_DELAY, - retry_on=DEFAULT_RETRY_ON) +function pmap(p::AbstractWorkerPool, f, c; distributed=true, batch_size=1, on_error=nothing, + retry_n=0, + retry_max_delay=DEFAULT_RETRY_MAX_DELAY, + retry_on=DEFAULT_RETRY_ON, + # deprecated keyword args: + err_retry=nothing, err_stop=nothing, pids=nothing) + #15409 + if err_retry !== nothing + depwarn("err_retry is deprecated, use pmap(retry(f), c...).", :pmap) + if err_retry == true + f = retry(f) + end + end + if pids !== nothing + depwarn("pids is deprecated, use pmap(::WorkerPool, f, c...).", :pmap) + p = WorkerPool(pids) + end + if err_stop !== nothing + depwarn("err_stop is deprecated, use pmap(f, c...; on_error = error_handling_func).", :pmap) + if err_stop === false + on_error = e->e + end + end + f_orig = f # Don't do remote calls if there are no workers. if (length(p) == 0) || (length(p) == 1 && fetch(p.channel) == myid()) diff --git a/base/sparse/sparse.jl b/base/sparse/sparse.jl index 6409eb18c9a5b..e303c0e0b8969 100644 --- a/base/sparse/sparse.jl +++ b/base/sparse/sparse.jl @@ -29,7 +29,7 @@ import Base: @get!, acos, acosd, acot, acotd, acsch, asech, asin, asind, asinh, import Base.Broadcast: broadcast_shape export AbstractSparseArray, AbstractSparseMatrix, AbstractSparseVector, - SparseMatrixCSC, SparseVector, blkdiag, dense, droptol!, dropzeros!, dropzeros, etree, + SparseMatrixCSC, SparseVector, blkdiag, dense, droptol!, dropzeros!, dropzeros, issparse, nonzeros, nzrange, rowvals, sparse, sparsevec, spdiagm, speye, spones, sprand, sprandn, spzeros, symperm, nnz, permute diff --git a/doc/DocCheck.jl b/doc/DocCheck.jl index 07881743215bc..525d047069f35 100644 --- a/doc/DocCheck.jl +++ b/doc/DocCheck.jl @@ -21,7 +21,7 @@ export isdeprecated, isdocumented, undefined_exports, undocumented, undocumented isdeprecated(m::Module, v) = try endswith(functionloc(eval(m, v))[1], "deprecated.jl") catch return false end isdeprecated(v) = try endswith(functionloc(eval(v))[1], "deprecated.jl") catch return false end -isdocumented(v) = (s=string(v); haskey(FUNCTION_DICT, s) || haskey(MODULE_DICT, s)) +isdocumented(v) = !contains(string(Base.doc(v)), "No documentation found") modfuncjoin(m::AbstractString, f::AbstractString) = startswith(f, '@') ? "@$m.$(f[2:end])" : "$m.$f" @@ -34,10 +34,9 @@ undefined_exports() = undefined(Base) # Check for exported names that aren't documented, # and return a Dict with (fn::Symbol, fullname::AbstractString) pairs function undocumented(m::Module) - init_help() undoc = Dict{Symbol, Array}() for v in sort(names(m)) - if isdefined(m,v) && !isdocumented(v) && !isdeprecated(m,v) + if isdefined(m,v) && !isdocumented(eval(m,v)) && !isdeprecated(m,v) ms = modfuncjoin(m,v) haskey(undoc, v) ? push!(undoc[v], ms) : (undoc[v] = [ms]) end @@ -49,7 +48,6 @@ undocumented() = undocumented(Base) # Check for exported names that aren't documented, and # return the file, function names, and line numbers, if available function undocumented_by_file(m::Module) - init_help() undocf = Dict{AbstractString, Dict}() for (f,_) in undocumented(m) s = string(f) @@ -87,7 +85,6 @@ undocumented_by_file() = undocumented_by_file(Base) # Based on code by @jihao function _undocumented_rst() - init_help() depdoc = havecount = total = 0 out = AbstractString["The following exports are not documented:"] undoc_exports = Set() @@ -138,7 +135,6 @@ undocumented_rst() = println(_undocumented_rst()[1]) function gen_undocumented_template(outfile = "$JULIA_HOME/../../doc/UNDOCUMENTED.rst") out = open(outfile, "w") - init_help() println(out, ".. currentmodule:: Base") println(out) exports=[strip(x) for x in split(replace(readstring("$JULIA_HOME/../../base/exports.jl"),",",""),"\n")] @@ -168,7 +164,7 @@ function gen_undocumented_template(outfile = "$JULIA_HOME/../../doc/UNDOCUMENTED li = m.func.code e = uncompressed_ast(li) argnames = e.args[1] - decls = map(argtype_decl, argnames, {m.sig...}) + decls = map(argtype_decl, argnames, Any[m.sig...]) args = join(decls, ",") line = line * "($args)" else diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index b62866876cba1..448dd55c0caa9 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -419,6 +419,12 @@ Indexing, Assignment, and Concatenation Broadcasts the ``X`` and ``inds`` arrays to a common size and stores the value from each position in ``X`` at the indices given by the same positions in ``inds``\ . +.. function:: isassigned(array, i) -> Bool + + .. Docstring generated from Julia source + + Tests whether the given array has a value associated with index ``i``\ . Returns ``false`` if the index is out of bounds, or has an undefined reference. + .. function:: cat(dims, A...) .. Docstring generated from Julia source diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 0b97384ccdf09..0849b3c96e0a9 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1012,6 +1012,12 @@ Mathematical Functions Return ``x`` if ``lo <= x <= hi``\ . If ``x < lo``\ , return ``lo``\ . If ``x > hi``\ , return ``hi``\ . Arguments are promoted to a common type. Operates elementwise over ``x`` if it is an array. +.. function:: clamp!(array::AbstractArray, lo, hi) + + .. Docstring generated from Julia source + + Restrict values in ``array`` to the specified range, in-place. + .. function:: abs(x) .. Docstring generated from Julia source diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index 0217ee5125257..f9fce9e0b22e7 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -297,6 +297,12 @@ General Number Functions and Constants Test whether ``x`` or all its elements are numerically equal to some real number. +.. function:: isimag(z) -> Bool + + .. Docstring generated from Julia source + + Test whether ``z`` is purely imaginary, i.e. has a real part equal to 0. + .. function:: Float32(x [, mode::RoundingMode]) .. Docstring generated from Julia source From bfeb2e10ea831c0e0f05dd1d360482da2f2380ae Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 22 Jul 2016 16:40:59 -0400 Subject: [PATCH 0591/1117] fix #17439, update docs for `collect` to reflect dim-preserving behavior --- base/array.jl | 7 +++++-- doc/stdlib/base.rst | 1 + doc/stdlib/collections.rst | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/base/array.jl b/base/array.jl index 0381f26b63829..2efa5cc199e81 100644 --- a/base/array.jl +++ b/base/array.jl @@ -202,7 +202,8 @@ promote_rule{T,n,S}(::Type{Array{T,n}}, ::Type{Array{S,n}}) = Array{promote_type """ collect(element_type, collection) -Return an array of type `Array{element_type,1}` of all items in a collection. +Return an Array with the given element type of all items in a collection or iterable. +The result has the same shape and number of dimensions as `collection`. """ collect{T}(::Type{T}, itr) = _collect(T, itr, iteratorsize(itr)) @@ -225,7 +226,9 @@ _similar_for(c, T, itr, isz) = similar(c, T) """ collect(collection) -Return an array of all items in a collection. For associative collections, returns Pair{KeyType, ValType}. +Return an Array of all items in a collection or iterator. For associative collections, returns +Pair{KeyType, ValType}. If the argument is array-like or is an iterator with the `HasShape()` +trait, the result will have the same shape and number of dimensions as the argument. """ collect(itr) = _collect(1:1 #= Array =#, itr, iteratoreltype(itr), iteratorsize(itr)) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 61d522f00660a..fe227b14f303a 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -1479,3 +1479,4 @@ Internals .. Docstring generated from Julia source Compile the given function ``f`` for the argument tuple (of types) ``args``\ , but do not execute it. + diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 23c9aafca8b65..169edd46d6e40 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -676,13 +676,13 @@ Iterable Collections .. Docstring generated from Julia source - Return an array of all items in a collection. For associative collections, returns Pair{KeyType, ValType}. + Return an Array of all items in a collection or iterator. For associative collections, returns Pair{KeyType, ValType}. If the argument is array-like or is an iterator with the ``HasShape()`` trait, the result will have the same shape and number of dimensions as the argument. .. function:: collect(element_type, collection) .. Docstring generated from Julia source - Return an array of type ``Array{element_type,1}`` of all items in a collection. + Return an Array with the given element type of all items in a collection or iterable. The result has the same shape and number of dimensions as ``collection``\ . .. function:: issubset(a, b) ⊆(a,b) -> Bool From 8f03766c790786b02b82655296aa93577a9fc13e Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer+github@college.harvard.edu> Date: Fri, 22 Jul 2016 20:45:42 +0000 Subject: [PATCH 0592/1117] Reverse order of keydefpath checks (#17566) If keydefpath is nothing, we can't call `isempty` on it, so we should check for that first. Fixes #17553 --- base/libgit2/callbacks.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 1c44530141d2f..36db11bf2973b 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -114,7 +114,7 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, if keydefpath !== nothing && !isusedcreds keydefpath # use cached value else - if isempty(keydefpath) || keydefpath === nothing + if keydefpath === nothing || isempty(keydefpath) keydefpath = joinpath(homedir(),".ssh","id_rsa") end prompt("Private key location for '$schema$username@$host'", default=keydefpath) @@ -130,7 +130,7 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, if keydefpath !== nothing && !isusedcreds keydefpath # use cached value else - if isempty(keydefpath) || keydefpath === nothing + if keydefpath === nothing || isempty(keydefpath) keydefpath = privatekey*".pub" end if isfile(keydefpath) From f4e7a4c37056c637aad423f860a5f67f938a6310 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 22 Jul 2016 14:06:42 -0700 Subject: [PATCH 0593/1117] Revert "Towards #12251 part 2, replace all full(X) calls with convert(Array, X) and migrate tests" (#17564) --- base/abstractarray.jl | 10 +- base/linalg/bidiag.jl | 16 +- base/linalg/dense.jl | 18 +- base/linalg/diagonal.jl | 2 +- base/linalg/hessenberg.jl | 4 +- base/linalg/lq.jl | 7 +- base/linalg/lu.jl | 4 +- base/linalg/qr.jl | 7 +- base/linalg/special.jl | 20 +- base/linalg/symmetric.jl | 4 +- base/linalg/triangular.jl | 14 +- base/linalg/tridiag.jl | 4 +- base/linalg/uniformscaling.jl | 4 +- base/sparse/linalg.jl | 10 +- base/sparse/sparsematrix.jl | 48 +-- base/sparse/umfpack.jl | 2 +- base/test.jl | 2 +- doc/manual/arrays.rst | 2 +- doc/stdlib/linalg.rst | 12 +- test/hashing.jl | 2 +- test/linalg/arnoldi.jl | 4 +- test/linalg/bidiag.jl | 24 +- test/linalg/cholesky.jl | 22 +- test/linalg/dense.jl | 4 +- test/linalg/diagonal.jl | 32 +- test/linalg/hessenberg.jl | 4 +- test/linalg/lapack.jl | 2 +- test/linalg/lq.jl | 28 +- test/linalg/lu.jl | 22 +- test/linalg/matmul.jl | 2 +- test/linalg/qr.jl | 50 ++-- test/linalg/schur.jl | 2 +- test/linalg/special.jl | 66 ++--- test/linalg/svd.jl | 2 +- test/linalg/symmetric.jl | 8 +- test/linalg/triangular.jl | 170 +++++------ test/linalg/tridiag.jl | 45 +-- test/linalg/uniformscaling.jl | 32 +- test/perf/threads/stockcorr/pstockcorr.jl | 2 +- test/sparsedir/cholmod.jl | 16 +- test/sparsedir/sparse.jl | 340 +++++++++++----------- test/sparsedir/sparsevector.jl | 130 ++++----- test/sparsedir/spqr.jl | 4 +- test/sparsedir/umfpack.jl | 2 +- 44 files changed, 601 insertions(+), 604 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index cb6ecc9e68f8f..e990bf138d757 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -890,7 +890,7 @@ function typed_vcat{T}(::Type{T}, V::AbstractVector...) for Vk in V n += length(Vk) end - a = similar(convert(Array, V[1]), T, n) + a = similar(full(V[1]), T, n) pos = 1 for k=1:length(V) Vk = V[k] @@ -918,7 +918,7 @@ function typed_hcat{T}(::Type{T}, A::AbstractVecOrMat...) nd = ndims(Aj) ncols += (nd==2 ? size(Aj,2) : 1) end - B = similar(convert(Array, A[1]), T, nrows, ncols) + B = similar(full(A[1]), T, nrows, ncols) pos = 1 if dense for k=1:nargs @@ -950,7 +950,7 @@ function typed_vcat{T}(::Type{T}, A::AbstractMatrix...) throw(ArgumentError("number of columns of each array must match (got $(map(x->size(x,2), A)))")) end end - B = similar(convert(Array, A[1]), T, nrows, ncols) + B = similar(full(A[1]), T, nrows, ncols) pos = 1 for k=1:nargs Ak = A[k] @@ -997,7 +997,7 @@ function cat_t(catdims, typeC::Type, X...) end end - C = similar(isa(X[1],AbstractArray) ? convert(Array, X[1]) : [X[1]], typeC, tuple(dimsC...)) + C = similar(isa(X[1],AbstractArray) ? full(X[1]) : [X[1]], typeC, tuple(dimsC...)) if length(catdims)>1 fill!(C,0) end @@ -1069,7 +1069,7 @@ function typed_hvcat{T}(::Type{T}, rows::Tuple{Vararg{Int}}, as::AbstractMatrix. a += rows[i] end - out = similar(convert(Array, as[1]), T, nr, nc) + out = similar(full(as[1]), T, nr, nc) a = 1 r = 1 diff --git a/base/linalg/bidiag.jl b/base/linalg/bidiag.jl index 1eaff9c3ff533..15913677df72a 100644 --- a/base/linalg/bidiag.jl +++ b/base/linalg/bidiag.jl @@ -18,7 +18,7 @@ end Constructs an upper (`isupper=true`) or lower (`isupper=false`) bidiagonal matrix using the given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` and provides efficient specialized linear solvers, but may be converted into a regular -matrix with [`convert(Array, _)`](:func:`convert`). `ev`'s length must be one less than the length of `dv`. +matrix with [`full`](:func:`full`). `ev`'s length must be one less than the length of `dv`. **Example** @@ -38,7 +38,7 @@ Bidiagonal(dv::AbstractVector, ev::AbstractVector) = throw(ArgumentError("did yo Constructs an upper (`uplo='U'`) or lower (`uplo='L'`) bidiagonal matrix using the given diagonal (`dv`) and off-diagonal (`ev`) vectors. The result is of type `Bidiagonal` and provides efficient specialized linear solvers, but may be converted into a regular -matrix with [`convert(Array, _)`](:func:`convert`). `ev`'s length must be one less than the length of `dv`. +matrix with [`full`](:func:`full`). `ev`'s length must be one less than the length of `dv`. **Example** @@ -302,7 +302,7 @@ end function A_mul_B_td!(C::AbstractMatrix, A::BiTriSym, B::BiTriSym) check_A_mul_B!_sizes(C, A, B) n = size(A,1) - n <= 3 && return A_mul_B!(C, convert(Array, A), convert(Array, B)) + n <= 3 && return A_mul_B!(C, full(A), full(B)) fill!(C, zero(eltype(C))) Al = diag(A, -1) Ad = diag(A, 0) @@ -361,7 +361,7 @@ function A_mul_B_td!(C::AbstractVecOrMat, A::BiTriSym, B::AbstractVecOrMat) if size(C,2) != nB throw(DimensionMismatch("A has second dimension $nA, B has $(size(B,2)), C has $(size(C,2)) but all must match")) end - nA <= 3 && return A_mul_B!(C, convert(Array, A), convert(Array, B)) + nA <= 3 && return A_mul_B!(C, full(A), full(B)) l = diag(A, -1) d = diag(A, 0) u = diag(A, 1) @@ -382,7 +382,7 @@ end function A_mul_B_td!(C::AbstractMatrix, A::AbstractMatrix, B::BiTriSym) check_A_mul_B!_sizes(C, A, B) n = size(A,1) - n <= 3 && return A_mul_B!(C, convert(Array, A), convert(Array, B)) + n <= 3 && return A_mul_B!(C, full(A), full(B)) m = size(B,2) Bl = diag(B, -1) Bd = diag(B, 0) @@ -412,12 +412,12 @@ end SpecialMatrix = Union{Bidiagonal, SymTridiagonal, Tridiagonal} # to avoid ambiguity warning, but shouldn't be necessary -*(A::AbstractTriangular, B::SpecialMatrix) = convert(Array, A) * convert(Array, B) -*(A::SpecialMatrix, B::SpecialMatrix) = convert(Array, A) * convert(Array, B) +*(A::AbstractTriangular, B::SpecialMatrix) = full(A) * full(B) +*(A::SpecialMatrix, B::SpecialMatrix) = full(A) * full(B) #Generic multiplication for func in (:*, :Ac_mul_B, :A_mul_Bc, :/, :A_rdiv_Bc) - @eval ($func){T}(A::Bidiagonal{T}, B::AbstractVector{T}) = ($func)(convert(Array, A), B) + @eval ($func){T}(A::Bidiagonal{T}, B::AbstractVector{T}) = ($func)(full(A), B) end #Linear solvers diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index ffb5930e600f5..2a68cc7473c6d 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -206,7 +206,7 @@ expm(x::Number) = exp(x) function expm!{T<:BlasFloat}(A::StridedMatrix{T}) n = checksquare(A) if ishermitian(A) - return convert(Array, expm(Hermitian(A))) + return full(expm(Hermitian(A))) end ilo, ihi, scale = LAPACK.gebal!('B', A) # modifies A nA = norm(A, 1) @@ -321,13 +321,13 @@ function logm{T}(A::StridedMatrix{T}) return full(logm(Symmetric(A))) end if ishermitian(A) - return convert(Array, logm(Hermitian(A))) + return full(logm(Hermitian(A))) end # Use Schur decomposition n = checksquare(A) if istriu(A) - retmat = convert(Array, logm(UpperTriangular(complex(A)))) + retmat = full(logm(UpperTriangular(complex(A)))) d = diag(A) else S,Q,d = schur(complex(A)) @@ -358,27 +358,27 @@ logm(a::Complex) = log(a) function sqrtm{T<:Real}(A::StridedMatrix{T}) if issymmetric(A) - return convert(Array, sqrtm(Symmetric(A))) + return full(sqrtm(Symmetric(A))) end n = checksquare(A) if istriu(A) - return convert(Array, sqrtm(UpperTriangular(A))) + return full(sqrtm(UpperTriangular(A))) else SchurF = schurfact(complex(A)) - R = convert(Array, sqrtm(UpperTriangular(SchurF[:T]))) + R = full(sqrtm(UpperTriangular(SchurF[:T]))) return SchurF[:vectors] * R * SchurF[:vectors]' end end function sqrtm{T<:Complex}(A::StridedMatrix{T}) if ishermitian(A) - return convert(Array, sqrtm(Hermitian(A))) + return full(sqrtm(Hermitian(A))) end n = checksquare(A) if istriu(A) - return convert(Array, sqrtm(UpperTriangular(A))) + return full(sqrtm(UpperTriangular(A))) else SchurF = schurfact(A) - R = convert(Array, sqrtm(UpperTriangular(SchurF[:T]))) + R = full(sqrtm(UpperTriangular(SchurF[:T]))) return SchurF[:vectors] * R * SchurF[:vectors]' end end diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index d26f4739482c2..0d377c4b9308f 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -314,7 +314,7 @@ svdvals(D::Diagonal) = [svdvals(v) for v in D.diag] function svd{T<:Number}(D::Diagonal{T}) S = abs(D.diag) piv = sortperm(S, rev = true) - U = convert(Array, Diagonal(D.diag ./ S)) + U = full(Diagonal(D.diag ./ S)) Up = hcat([U[:,i] for i = 1:length(D.diag)][piv]...) V = eye(D) Vp = hcat([V[:,i] for i = 1:length(D.diag)][piv]...) diff --git a/base/linalg/hessenberg.jl b/base/linalg/hessenberg.jl index 2b9f2466b7984..83088cda465eb 100644 --- a/base/linalg/hessenberg.jl +++ b/base/linalg/hessenberg.jl @@ -23,7 +23,7 @@ end Compute the Hessenberg decomposition of `A` and return a `Hessenberg` object. If `F` is the factorization object, the unitary matrix can be accessed with `F[:Q]` and the Hessenberg matrix with `F[:H]`. When `Q` is extracted, the resulting type is the `HessenbergQ` object, -and may be converted to a regular matrix with [`convert(Array, _)`](:func:`convert`). +and may be converted to a regular matrix with [`full`](:func:`full`). """ hessfact @@ -56,7 +56,7 @@ end convert{T<:BlasFloat}(::Type{Matrix}, A::HessenbergQ{T}) = LAPACK.orghr!(1, size(A.factors, 1), copy(A.factors), A.τ) convert(::Type{Array}, A::HessenbergQ) = convert(Matrix, A) full(A::HessenbergQ) = convert(Array, A) -convert(::Type{AbstractMatrix}, F::Hessenberg) = (fq = convert(Array, F[:Q]); (fq * F[:H]) * fq') +convert(::Type{AbstractMatrix}, F::Hessenberg) = (fq = full(F[:Q]); (fq * F[:H]) * fq') convert(::Type{AbstractArray}, F::Hessenberg) = convert(AbstractMatrix, F) convert(::Type{Matrix}, F::Hessenberg) = convert(Array, convert(AbstractArray, F)) convert(::Type{Array}, F::Hessenberg) = convert(Matrix, F) diff --git a/base/linalg/lq.jl b/base/linalg/lq.jl index a0806370b1f1b..cb10bb81bac5d 100644 --- a/base/linalg/lq.jl +++ b/base/linalg/lq.jl @@ -42,9 +42,8 @@ zeros if the full `Q` is requested. """ function lq(A::Union{Number, AbstractMatrix}; thin::Bool=true) F = lqfact(A) - F[:L], thin ? convert(Array, F[:Q]) : thickQ(F[:Q]) + F[:L], full(F[:Q], thin=thin) end -thickQ{T}(Q::LQPackedQ{T}) = A_mul_B!(Q, eye(T, size(Q.factors,2), size(Q.factors,1))) copy(A::LQ) = LQ(copy(A.factors), copy(A.τ)) @@ -113,8 +112,8 @@ size(A::LQPackedQ) = size(A.factors) ## Multiplication by LQ A_mul_B!{T<:BlasFloat}(A::LQ{T}, B::StridedVecOrMat{T}) = A[:L]*LAPACK.ormlq!('L','N',A.factors,A.τ,B) -A_mul_B!{T<:BlasFloat}(A::LQ{T}, B::QR{T}) = A[:L]*LAPACK.ormlq!('L','N',A.factors,A.τ,convert(Array, B)) -A_mul_B!{T<:BlasFloat}(A::QR{T}, B::LQ{T}) = A_mul_B!(zeros(convert(Array, A)), convert(Array, A), convert(Array, B)) +A_mul_B!{T<:BlasFloat}(A::LQ{T}, B::QR{T}) = A[:L]*LAPACK.ormlq!('L','N',A.factors,A.τ,full(B)) +A_mul_B!{T<:BlasFloat}(A::QR{T}, B::LQ{T}) = A_mul_B!(zeros(full(A)), full(A), full(B)) function *{TA,TB}(A::LQ{TA},B::StridedVecOrMat{TB}) TAB = promote_type(TA, TB) A_mul_B!(convert(Factorization{TAB},A), copy_oftype(B, TAB)) diff --git a/base/linalg/lu.jl b/base/linalg/lu.jl index 49ebea091c2f8..5201decd46fc6 100644 --- a/base/linalg/lu.jl +++ b/base/linalg/lu.jl @@ -308,7 +308,7 @@ factorize(A::Tridiagonal) = lufact(A) function getindex{T}(F::Base.LinAlg.LU{T,Tridiagonal{T}}, d::Symbol) m, n = size(F) if d == :L - L = convert(Array, Bidiagonal(ones(T, n), F.factors.dl, false)) + L = full(Bidiagonal(ones(T, n), F.factors.dl, false)) for i = 2:n tmp = L[F.ipiv[i], 1:i - 1] L[F.ipiv[i], 1:i - 1] = L[i, 1:i - 1] @@ -316,7 +316,7 @@ function getindex{T}(F::Base.LinAlg.LU{T,Tridiagonal{T}}, d::Symbol) end return L elseif d == :U - U = convert(Array, Bidiagonal(F.factors.d, F.factors.du, true)) + U = full(Bidiagonal(F.factors.d, F.factors.du, true)) for i = 1:n - 2 U[i,i + 2] = F.factors.du2[i] end diff --git a/base/linalg/qr.jl b/base/linalg/qr.jl index 82009e25252c2..ba1797bc6105e 100644 --- a/base/linalg/qr.jl +++ b/base/linalg/qr.jl @@ -174,14 +174,13 @@ qr(A::Union{Number, AbstractMatrix}, pivot::Union{Type{Val{false}}, Type{Val{tru _qr(A, pivot, thin=thin) function _qr(A::Union{Number, AbstractMatrix}, ::Type{Val{false}}; thin::Bool=true) F = qrfact(A, Val{false}) - (thin ? convert(Array, getq(F)) : thickQ(getq(F))), F[:R]::Matrix{eltype(F)} + full(getq(F), thin=thin), F[:R]::Matrix{eltype(F)} end function _qr(A::Union{Number, AbstractMatrix}, ::Type{Val{true}}; thin::Bool=true) F = qrfact(A, Val{true}) - (thin ? convert(Array, getq(F)) : thickQ(getq(F))), F[:R]::Matrix{eltype(F)}, F[:p]::Vector{BlasInt} + full(getq(F), thin=thin), F[:R]::Matrix{eltype(F)}, F[:p]::Vector{BlasInt} end - """ qr(v::AbstractVector) @@ -326,8 +325,6 @@ function full{T}(A::Union{QRPackedQ{T},QRCompactWYQ{T}}; thin::Bool = true) end end -thickQ{T}(Q::Union{QRPackedQ{T},QRCompactWYQ{T}}) = A_mul_B!(Q, eye(T, size(Q.factors, 1))) - size(A::Union{QR,QRCompactWY,QRPivoted}, dim::Integer) = size(A.factors, dim) size(A::Union{QR,QRCompactWY,QRPivoted}) = size(A.factors) size(A::Union{QRPackedQ,QRCompactWYQ}, dim::Integer) = 0 < dim ? (dim <= 2 ? size(A.factors, 1) : 1) : throw(BoundsError()) diff --git a/base/linalg/special.jl b/base/linalg/special.jl index c8d86d559b689..f70d67130807b 100644 --- a/base/linalg/special.jl +++ b/base/linalg/special.jl @@ -6,21 +6,21 @@ convert{T}(::Type{Bidiagonal}, A::Diagonal{T})=Bidiagonal(A.diag, zeros(T, size(A.diag,1)-1), true) convert{T}(::Type{SymTridiagonal}, A::Diagonal{T})=SymTridiagonal(A.diag, zeros(T, size(A.diag,1)-1)) convert{T}(::Type{Tridiagonal}, A::Diagonal{T})=Tridiagonal(zeros(T, size(A.diag,1)-1), A.diag, zeros(T, size(A.diag,1)-1)) -convert(::Type{LowerTriangular}, A::Bidiagonal) = !A.isupper ? LowerTriangular(convert(Array, A)) : throw(ArgumentError("Bidiagonal matrix must have lower off diagonal to be converted to LowerTriangular")) -convert(::Type{UpperTriangular}, A::Bidiagonal) = A.isupper ? UpperTriangular(convert(Array, A)) : throw(ArgumentError("Bidiagonal matrix must have upper off diagonal to be converted to UpperTriangular")) +convert(::Type{LowerTriangular}, A::Bidiagonal) = !A.isupper ? LowerTriangular(full(A)) : throw(ArgumentError("Bidiagonal matrix must have lower off diagonal to be converted to LowerTriangular")) +convert(::Type{UpperTriangular}, A::Bidiagonal) = A.isupper ? UpperTriangular(full(A)) : throw(ArgumentError("Bidiagonal matrix must have upper off diagonal to be converted to UpperTriangular")) function convert(::Type{UnitUpperTriangular}, A::Diagonal) if !all(A.diag .== one(eltype(A))) throw(ArgumentError("matrix cannot be represented as UnitUpperTriangular")) end - UnitUpperTriangular(convert(Array, A)) + UnitUpperTriangular(full(A)) end function convert(::Type{UnitLowerTriangular}, A::Diagonal) if !all(A.diag .== one(eltype(A))) throw(ArgumentError("matrix cannot be represented as UnitLowerTriangular")) end - UnitLowerTriangular(convert(Array, A)) + UnitLowerTriangular(full(A)) end function convert(::Type{Diagonal}, A::Union{Bidiagonal, SymTridiagonal}) @@ -72,14 +72,14 @@ function convert(::Type{Tridiagonal}, A::SymTridiagonal) end function convert(::Type{Diagonal}, A::AbstractTriangular) - if convert(Array, A) != diagm(diag(A)) + if full(A) != diagm(diag(A)) throw(ArgumentError("matrix cannot be represented as Diagonal")) end Diagonal(diag(A)) end function convert(::Type{Bidiagonal}, A::AbstractTriangular) - fA = convert(Array, A) + fA = full(A) if fA == diagm(diag(A)) + diagm(diag(fA, 1), 1) return Bidiagonal(diag(A), diag(fA,1), true) elseif fA == diagm(diag(A)) + diagm(diag(fA, -1), -1) @@ -92,7 +92,7 @@ end convert(::Type{SymTridiagonal}, A::AbstractTriangular) = convert(SymTridiagonal, convert(Tridiagonal, A)) function convert(::Type{Tridiagonal}, A::AbstractTriangular) - fA = convert(Array, A) + fA = full(A) if fA == diagm(diag(A)) + diagm(diag(fA, 1), 1) + diagm(diag(fA, -1), -1) return Tridiagonal(diag(fA, -1), diag(A), diag(fA,1)) else @@ -154,12 +154,12 @@ for op in (:+, :-) end for matrixtype in (:SymTridiagonal,:Tridiagonal,:Bidiagonal,:Matrix) @eval begin - ($op)(A::AbstractTriangular, B::($matrixtype)) = ($op)(convert(Array, A), B) - ($op)(A::($matrixtype), B::AbstractTriangular) = ($op)(A, convert(Array, B)) + ($op)(A::AbstractTriangular, B::($matrixtype)) = ($op)(full(A), B) + ($op)(A::($matrixtype), B::AbstractTriangular) = ($op)(A, full(B)) end end end A_mul_Bc!(A::AbstractTriangular, B::QRCompactWYQ) = A_mul_Bc!(full!(A),B) A_mul_Bc!(A::AbstractTriangular, B::QRPackedQ) = A_mul_Bc!(full!(A),B) -A_mul_Bc(A::AbstractTriangular, B::Union{QRCompactWYQ,QRPackedQ}) = A_mul_Bc(convert(Array, A), B) +A_mul_Bc(A::AbstractTriangular, B::Union{QRCompactWYQ,QRPackedQ}) = A_mul_Bc(full(A), B) diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index 02a0904c33cc0..40898a9aa2789 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -174,8 +174,8 @@ A_mul_B!{T<:BlasFloat,S<:StridedMatrix}(C::StridedMatrix{T}, A::StridedMatrix{T} A_mul_B!{T<:BlasComplex,S<:StridedMatrix}(C::StridedMatrix{T}, A::Hermitian{T,S}, B::StridedMatrix{T}) = BLAS.hemm!('L', A.uplo, one(T), A.data, B, zero(T), C) A_mul_B!{T<:BlasComplex,S<:StridedMatrix}(C::StridedMatrix{T}, A::StridedMatrix{T}, B::Hermitian{T,S}) = BLAS.hemm!('R', B.uplo, one(T), B.data, A, zero(T), C) -*(A::HermOrSym, B::HermOrSym) = convert(Array, A)*convert(Array, B) -*(A::StridedMatrix, B::HermOrSym) = A*convert(Array, B) +*(A::HermOrSym, B::HermOrSym) = full(A)*full(B) +*(A::StridedMatrix, B::HermOrSym) = A*full(B) bkfact(A::HermOrSym) = bkfact(A.data, Symbol(A.uplo), issymmetric(A)) factorize(A::HermOrSym) = bkfact(A) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 37b39d132da91..5dd80ac8f8cd5 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -362,7 +362,7 @@ scale!(c::Number, A::Union{UpperTriangular,LowerTriangular}) = scale!(A,c) +(A::UnitLowerTriangular, B::LowerTriangular) = LowerTriangular(tril(A.data, -1) + B.data + I) +(A::UnitUpperTriangular, B::UnitUpperTriangular) = UpperTriangular(triu(A.data, 1) + triu(B.data, 1) + 2I) +(A::UnitLowerTriangular, B::UnitLowerTriangular) = LowerTriangular(tril(A.data, -1) + tril(B.data, -1) + 2I) -+(A::AbstractTriangular, B::AbstractTriangular) = convert(Array, A) + convert(Array, B) ++(A::AbstractTriangular, B::AbstractTriangular) = full(A) + full(B) -(A::UpperTriangular, B::UpperTriangular) = UpperTriangular(A.data - B.data) -(A::LowerTriangular, B::LowerTriangular) = LowerTriangular(A.data - B.data) @@ -372,15 +372,15 @@ scale!(c::Number, A::Union{UpperTriangular,LowerTriangular}) = scale!(A,c) -(A::UnitLowerTriangular, B::LowerTriangular) = LowerTriangular(tril(A.data, -1) - B.data + I) -(A::UnitUpperTriangular, B::UnitUpperTriangular) = UpperTriangular(triu(A.data, 1) - triu(B.data, 1)) -(A::UnitLowerTriangular, B::UnitLowerTriangular) = LowerTriangular(tril(A.data, -1) - tril(B.data, -1)) --(A::AbstractTriangular, B::AbstractTriangular) = convert(Array, A) - convert(Array, B) +-(A::AbstractTriangular, B::AbstractTriangular) = full(A) - full(B) ###################### # BlasFloat routines # ###################### A_mul_B!(A::Tridiagonal, B::AbstractTriangular) = A*full!(B) -A_mul_B!(C::AbstractMatrix, A::AbstractTriangular, B::Tridiagonal) = A_mul_B!(C, convert(Array, A), B) -A_mul_B!(C::AbstractMatrix, A::Tridiagonal, B::AbstractTriangular) = A_mul_B!(C, A, convert(Array, B)) +A_mul_B!(C::AbstractMatrix, A::AbstractTriangular, B::Tridiagonal) = A_mul_B!(C, full(A), B) +A_mul_B!(C::AbstractMatrix, A::Tridiagonal, B::AbstractTriangular) = A_mul_B!(C, A, full(B)) A_mul_B!(C::AbstractVector, A::AbstractTriangular, B::AbstractVector) = A_mul_B!(A, copy!(C, B)) A_mul_B!(C::AbstractMatrix, A::AbstractTriangular, B::AbstractVecOrMat) = A_mul_B!(A, copy!(C, B)) A_mul_B!(C::AbstractVecOrMat, A::AbstractTriangular, B::AbstractVecOrMat) = A_mul_B!(A, copy!(C, B)) @@ -437,7 +437,7 @@ for (t, uploc, isunitc) in ((:LowerTriangular, 'L', 'N'), elseif p == Inf return inv(LAPACK.trcon!('I', $uploc, $isunitc, A.data)) else #use fallback - return cond(convert(Array, A), p) + return cond(full(A), p) end end end @@ -1321,7 +1321,7 @@ end ## Some Triangular-Triangular cases. We might want to write taylored methods for these cases, but I'm not sure it is worth it. for t in (UpperTriangular, UnitUpperTriangular, LowerTriangular, UnitLowerTriangular) @eval begin - (*)(A::Tridiagonal, B::$t) = A_mul_B!(convert(Array, A), B) + (*)(A::Tridiagonal, B::$t) = A_mul_B!(full(A), B) end end @@ -1987,7 +1987,7 @@ eigfact(A::AbstractTriangular) = Eigen(eigvals(A), eigvecs(A)) #Generic singular systems for func in (:svd, :svdfact, :svdfact!, :svdvals) @eval begin - ($func)(A::AbstractTriangular) = ($func)(convert(Array, A)) + ($func)(A::AbstractTriangular) = ($func)(full(A)) end end diff --git a/base/linalg/tridiag.jl b/base/linalg/tridiag.jl index 0e79204b5069e..aee5e05483456 100644 --- a/base/linalg/tridiag.jl +++ b/base/linalg/tridiag.jl @@ -19,7 +19,7 @@ end Construct a symmetric tridiagonal matrix from the diagonal and first sub/super-diagonal, respectively. The result is of type `SymTridiagonal` and provides efficient specialized -eigensolvers, but may be converted into a regular matrix with [`convert(Array, _)`](:func:`convert`). +eigensolvers, but may be converted into a regular matrix with [`full`](:func:`full`). """ SymTridiagonal{T}(dv::Vector{T}, ev::Vector{T}) = SymTridiagonal{T}(dv, ev) @@ -327,7 +327,7 @@ end Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type `Tridiagonal` and provides efficient specialized linear -solvers, but may be converted into a regular matrix with [`convert(Array, _)`](:func:`convert`). +solvers, but may be converted into a regular matrix with [`full`](:func:`full`). The lengths of `dl` and `du` must be one less than the length of `d`. """ # Basic constructor takes in three dense vectors of same type diff --git a/base/linalg/uniformscaling.jl b/base/linalg/uniformscaling.jl index db26156fef0a2..227e41ac5dc0c 100644 --- a/base/linalg/uniformscaling.jl +++ b/base/linalg/uniformscaling.jl @@ -61,7 +61,7 @@ for (t1, t2) in ((:UnitUpperTriangular, :UpperTriangular), end function (-)(J::UniformScaling, UL::Union{UpperTriangular,UnitUpperTriangular}) - ULnew = similar(convert(Array, UL), promote_type(eltype(J), eltype(UL))) + ULnew = similar(full(UL), promote_type(eltype(J), eltype(UL))) n = size(ULnew, 1) ULold = UL.data for j = 1:n @@ -77,7 +77,7 @@ function (-)(J::UniformScaling, UL::Union{UpperTriangular,UnitUpperTriangular}) return UpperTriangular(ULnew) end function (-)(J::UniformScaling, UL::Union{LowerTriangular,UnitLowerTriangular}) - ULnew = similar(convert(Array, UL), promote_type(eltype(J), eltype(UL))) + ULnew = similar(full(UL), promote_type(eltype(J), eltype(UL))) n = size(ULnew, 1) ULold = UL.data for j = 1:n diff --git a/base/sparse/linalg.jl b/base/sparse/linalg.jl index d32f62977ecd9..a80e5f4a6cb51 100644 --- a/base/sparse/linalg.jl +++ b/base/sparse/linalg.jl @@ -292,8 +292,8 @@ end A_ldiv_B!{T,Ti}(L::LowerTriangular{T,SparseMatrixCSC{T,Ti}}, B::StridedVecOrMat) = fwdTriSolve!(L.data, B) A_ldiv_B!{T,Ti}(U::UpperTriangular{T,SparseMatrixCSC{T,Ti}}, B::StridedVecOrMat) = bwdTriSolve!(U.data, B) -(\){T,Ti}(L::LowerTriangular{T,SparseMatrixCSC{T,Ti}}, B::SparseMatrixCSC) = A_ldiv_B!(L, convert(Array, B)) -(\){T,Ti}(U::UpperTriangular{T,SparseMatrixCSC{T,Ti}}, B::SparseMatrixCSC) = A_ldiv_B!(U, convert(Array, B)) +(\){T,Ti}(L::LowerTriangular{T,SparseMatrixCSC{T,Ti}}, B::SparseMatrixCSC) = A_ldiv_B!(L, full(B)) +(\){T,Ti}(U::UpperTriangular{T,SparseMatrixCSC{T,Ti}}, B::SparseMatrixCSC) = A_ldiv_B!(U, full(B)) ## triu, tril @@ -504,7 +504,7 @@ function norm(A::SparseMatrixCSC,p::Real=2) return float(real(zero(eltype(A)))) elseif m == 1 || n == 1 # TODO: compute more efficiently using A.nzval directly - return norm(convert(Array, A), p) + return norm(full(A), p) else Tnorm = typeof(float(real(zero(eltype(A))))) Tsum = promote_type(Float64,Tnorm) @@ -519,7 +519,7 @@ function norm(A::SparseMatrixCSC,p::Real=2) end return convert(Tnorm, nA) elseif p==2 - throw(ArgumentError("2-norm not yet implemented for sparse matrices. Try norm(convert(Array, A)) or norm(A, p) where p=1 or Inf.")) + throw(ArgumentError("2-norm not yet implemented for sparse matrices. Try norm(full(A)) or norm(A, p) where p=1 or Inf.")) elseif p==Inf rowSum = zeros(Tsum,m) for i=1:length(A.nzval) @@ -544,7 +544,7 @@ function cond(A::SparseMatrixCSC, p::Real=2) normA = norm(A, Inf) return normA * normAinv elseif p == 2 - throw(ArgumentError("2-norm condition number is not implemented for sparse matrices, try cond(convert(Array, A), 2) instead")) + throw(ArgumentError("2-norm condition number is not implemented for sparse matrices, try cond(full(A), 2) instead")) else throw(ArgumentError("second argument must be either 1 or Inf, got $p")) end diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index dceca0b2657a1..876f1e74b0cae 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1669,39 +1669,39 @@ for op in (+, -, min, max, &, |, $) end end # macro -(.+)(A::SparseMatrixCSC, B::Number) = convert(Array, A) .+ B -( +)(A::SparseMatrixCSC, B::Array ) = convert(Array, A) + B -(.+)(A::Number, B::SparseMatrixCSC) = A .+ convert(Array, B) -( +)(A::Array , B::SparseMatrixCSC) = A + convert(Array, B) +(.+)(A::SparseMatrixCSC, B::Number) = full(A) .+ B +( +)(A::SparseMatrixCSC, B::Array ) = full(A) + B +(.+)(A::Number, B::SparseMatrixCSC) = A .+ full(B) +( +)(A::Array , B::SparseMatrixCSC) = A + full(B) -(.-)(A::SparseMatrixCSC, B::Number) = convert(Array, A) .- B -( -)(A::SparseMatrixCSC, B::Array ) = convert(Array, A) - B -(.-)(A::Number, B::SparseMatrixCSC) = A .- convert(Array, B) -( -)(A::Array , B::SparseMatrixCSC) = A - convert(Array, B) +(.-)(A::SparseMatrixCSC, B::Number) = full(A) .- B +( -)(A::SparseMatrixCSC, B::Array ) = full(A) - B +(.-)(A::Number, B::SparseMatrixCSC) = A .- full(B) +( -)(A::Array , B::SparseMatrixCSC) = A - full(B) (.*)(A::AbstractArray, B::AbstractArray) = broadcast_zpreserving(*, A, B) (.*)(A::SparseMatrixCSC, B::Number) = SparseMatrixCSC(A.m, A.n, copy(A.colptr), copy(A.rowval), A.nzval .* B) (.*)(A::Number, B::SparseMatrixCSC) = SparseMatrixCSC(B.m, B.n, copy(B.colptr), copy(B.rowval), A .* B.nzval) (./)(A::SparseMatrixCSC, B::Number) = SparseMatrixCSC(A.m, A.n, copy(A.colptr), copy(A.rowval), A.nzval ./ B) -(./)(A::Number, B::SparseMatrixCSC) = (./)(A, convert(Array, B)) -(./)(A::SparseMatrixCSC, B::Array) = (./)(convert(Array, A), B) -(./)(A::Array, B::SparseMatrixCSC) = (./)(A, convert(Array, B)) -(./)(A::SparseMatrixCSC, B::SparseMatrixCSC) = (./)(convert(Array, A), convert(Array, B)) +(./)(A::Number, B::SparseMatrixCSC) = (./)(A, full(B)) +(./)(A::SparseMatrixCSC, B::Array) = (./)(full(A), B) +(./)(A::Array, B::SparseMatrixCSC) = (./)(A, full(B)) +(./)(A::SparseMatrixCSC, B::SparseMatrixCSC) = (./)(full(A), full(B)) -(.\)(A::SparseMatrixCSC, B::Number) = (.\)(convert(Array, A), B) +(.\)(A::SparseMatrixCSC, B::Number) = (.\)(full(A), B) (.\)(A::Number, B::SparseMatrixCSC) = SparseMatrixCSC(B.m, B.n, copy(B.colptr), copy(B.rowval), A .\ B.nzval ) -(.\)(A::SparseMatrixCSC, B::Array) = (.\)(convert(Array, A), B) -(.\)(A::Array, B::SparseMatrixCSC) = (.\)(A, convert(Array, B)) -(.\)(A::SparseMatrixCSC, B::SparseMatrixCSC) = (.\)(convert(Array, A), convert(Array, B)) +(.\)(A::SparseMatrixCSC, B::Array) = (.\)(full(A), B) +(.\)(A::Array, B::SparseMatrixCSC) = (.\)(A, full(B)) +(.\)(A::SparseMatrixCSC, B::SparseMatrixCSC) = (.\)(full(A), full(B)) (.^)(A::SparseMatrixCSC, B::Number) = B==0 ? sparse(ones(typeof(one(eltype(A)).^B), A.m, A.n)) : SparseMatrixCSC(A.m, A.n, copy(A.colptr), copy(A.rowval), A.nzval .^ B) (.^)(::Irrational{:e}, B::SparseMatrixCSC) = exp(B) -(.^)(A::Number, B::SparseMatrixCSC) = (.^)(A, convert(Array, B)) -(.^)(A::SparseMatrixCSC, B::Array) = (.^)(convert(Array, A), B) -(.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, convert(Array, B)) +(.^)(A::Number, B::SparseMatrixCSC) = (.^)(A, full(B)) +(.^)(A::SparseMatrixCSC, B::Array) = (.^)(full(A), B) +(.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, full(B)) .+{Tv1,Ti1,Tv2,Ti2}(A_1::SparseMatrixCSC{Tv1,Ti1}, A_2::SparseMatrixCSC{Tv2,Ti2}) = broadcast!(+, spzeros(promote_op(+, Tv1, Tv2), promote_type(Ti1, Ti2), to_shape(broadcast_shape(A_1, A_2))), A_1, A_2) @@ -2809,11 +2809,11 @@ setindex!(A::SparseMatrixCSC, x::Matrix, I::AbstractVector{Bool}, J::AbstractVec setindex!{T<:Integer}(A::SparseMatrixCSC, x::Matrix, I::AbstractVector{T}, J::AbstractVector{Bool}) = setindex!(A, sparse(x), I, find(J)) setindex!{T<:Integer}(A::SparseMatrixCSC, x::Matrix, I::AbstractVector{Bool}, J::AbstractVector{T}) = setindex!(A, sparse(x), find(I),J) -setindex!(A::Matrix, x::SparseMatrixCSC, I::Integer, J::AbstractVector{Bool}) = setindex!(A, convert(Array, x), I, find(J)) -setindex!(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::Integer) = setindex!(A, convert(Array, x), find(I), J) -setindex!(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{Bool}) = setindex!(A, convert(Array, x), find(I), find(J)) -setindex!{T<:Integer}(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{T}, J::AbstractVector{Bool}) = setindex!(A, convert(Array, x), I, find(J)) -setindex!{T<:Integer}(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{T}) = setindex!(A, convert(Array, x), find(I), J) +setindex!(A::Matrix, x::SparseMatrixCSC, I::Integer, J::AbstractVector{Bool}) = setindex!(A, full(x), I, find(J)) +setindex!(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::Integer) = setindex!(A, full(x), find(I), J) +setindex!(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{Bool}) = setindex!(A, full(x), find(I), find(J)) +setindex!{T<:Integer}(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{T}, J::AbstractVector{Bool}) = setindex!(A, full(x), I, find(J)) +setindex!{T<:Integer}(A::Matrix, x::SparseMatrixCSC, I::AbstractVector{Bool}, J::AbstractVector{T}) = setindex!(A, full(x), find(I), J) setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractVector{Bool}) = throw(BoundsError()) function setindex!{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, x, I::AbstractMatrix{Bool}) diff --git a/base/sparse/umfpack.jl b/base/sparse/umfpack.jl index 0120fd2d4fbcd..3b426bb9f234d 100644 --- a/base/sparse/umfpack.jl +++ b/base/sparse/umfpack.jl @@ -154,7 +154,7 @@ lufact{Tv<:Union{Complex32,Complex64}, Ti<:UMFITypes}(A::SparseMatrixCSC{Tv,Ti}) lufact{T<:AbstractFloat}(A::Union{SparseMatrixCSC{T},SparseMatrixCSC{Complex{T}}}) = throw(ArgumentError(string("matrix type ", typeof(A), "not supported. ", "Try lufact(convert(SparseMatrixCSC{Float64/Complex128,Int}, A)) for ", - "sparse floating point LU using UMFPACK or lufact(convert(Array, A)) for generic ", + "sparse floating point LU using UMFPACK or lufact(full(A)) for generic ", "dense LU."))) lufact(A::SparseMatrixCSC) = lufact(float(A)) lufact(A::SparseMatrixCSC, pivot::Type{Val{false}}) = lufact(A) diff --git a/base/test.jl b/base/test.jl index 3752ea63f5539..2436a0800f5b9 100644 --- a/base/test.jl +++ b/base/test.jl @@ -817,7 +817,7 @@ end approx_full(x::AbstractArray) = x approx_full(x::Number) = x -approx_full(x) = convert(Array, x) +approx_full(x) = full(x) function test_approx_eq(va, vb, Eps, astr, bstr) va = approx_full(va) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 6a718926416ea..20a5e0342cf4f 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -829,7 +829,7 @@ reference. +----------------------------------------+----------------------------------+--------------------------------------------+ | :func:`speye(n) <speye>` | :func:`eye(n) <eye>` | Creates a *n*-by-*n* identity matrix. | +----------------------------------------+----------------------------------+--------------------------------------------+ -| :func:`convert(Array, S) <convert>` | :func:`sparse(A) <sparse>` | Interconverts between dense | +| :func:`full(S) <full>` | :func:`sparse(A) <sparse>` | Interconverts between dense | | | | and sparse formats. | +----------------------------------------+----------------------------------+--------------------------------------------+ | :func:`sprand(m,n,d) <sprand>` | :func:`rand(m,n) <rand>` | Creates a *m*-by-*n* random matrix (of | diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index dc9ea8a81f0bf..713d6c55912b6 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -110,7 +110,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`convert`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . + Constructs an upper (``isupper=true``\ ) or lower (``isupper=false``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . **Example** @@ -125,7 +125,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Constructs an upper (``uplo='U'``\ ) or lower (``uplo='L'``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`convert`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . + Constructs an upper (``uplo='U'``\ ) or lower (``uplo='L'``\ ) bidiagonal matrix using the given diagonal (``dv``\ ) and off-diagonal (``ev``\ ) vectors. The result is of type ``Bidiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . ``ev``\ 's length must be one less than the length of ``dv``\ . **Example** @@ -154,13 +154,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Construct a symmetric tridiagonal matrix from the diagonal and first sub/super-diagonal, respectively. The result is of type ``SymTridiagonal`` and provides efficient specialized eigensolvers, but may be converted into a regular matrix with :func:`convert`\ . + Construct a symmetric tridiagonal matrix from the diagonal and first sub/super-diagonal, respectively. The result is of type ``SymTridiagonal`` and provides efficient specialized eigensolvers, but may be converted into a regular matrix with :func:`full`\ . .. function:: Tridiagonal(dl, d, du) .. Docstring generated from Julia source - Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`convert`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . + Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . .. function:: Symmetric(A, uplo=:U) @@ -658,7 +658,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the Hessenberg decomposition of ``A`` and return a ``Hessenberg`` object. If ``F`` is the factorization object, the unitary matrix can be accessed with ``F[:Q]`` and the Hessenberg matrix with ``F[:H]``\ . When ``Q`` is extracted, the resulting type is the ``HessenbergQ`` object, and may be converted to a regular matrix with :func:`convert`\ . + Compute the Hessenberg decomposition of ``A`` and return a ``Hessenberg`` object. If ``F`` is the factorization object, the unitary matrix can be accessed with ``F[:Q]`` and the Hessenberg matrix with ``F[:H]``\ . When ``Q`` is extracted, the resulting type is the ``HessenbergQ`` object, and may be converted to a regular matrix with :func:`full`\ . .. function:: hessfact!(A) @@ -974,7 +974,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`convert`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . + Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . .. function:: rank(M) diff --git a/test/hashing.jl b/test/hashing.jl index b143db1b3c387..1919226f41d2a 100644 --- a/test/hashing.jl +++ b/test/hashing.jl @@ -89,7 +89,7 @@ end x = sprand(10, 10, 0.5) x[1] = 1 x.nzval[1] = 0 -@test hash(x) == hash(convert(Array, x)) +@test hash(x) == hash(full(x)) let a = QuoteNode(1), b = QuoteNode(1.0) @test (hash(a)==hash(b)) == (a==b) diff --git a/test/linalg/arnoldi.jl b/test/linalg/arnoldi.jl index 5c781a062e20b..81dc9217972ef 100644 --- a/test/linalg/arnoldi.jl +++ b/test/linalg/arnoldi.jl @@ -155,7 +155,7 @@ debug && println("real svds") let # svds test A = sparse([1, 1, 2, 3, 4], [2, 1, 1, 3, 1], [2.0, -1.0, 6.1, 7.0, 1.5]) S1 = svds(A, nsv = 2) - S2 = svd(convert(Array, A)) + S2 = svd(full(A)) ## singular values match: @test S1[1][:S] ≈ S2[2][1:2] @@ -196,7 +196,7 @@ debug && println("complex svds") let # complex svds test A = sparse([1, 1, 2, 3, 4], [2, 1, 1, 3, 1], exp(im*[2.0:2:10;])) S1 = svds(A, nsv = 2) - S2 = svd(convert(Array, A)) + S2 = svd(full(A)) ## singular values match: @test S1[1][:S] ≈ S2[2][1:2] diff --git a/test/linalg/bidiag.jl b/test/linalg/bidiag.jl index f96677b4038eb..d4d7f397f555f 100644 --- a/test/linalg/bidiag.jl +++ b/test/linalg/bidiag.jl @@ -62,12 +62,12 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test size(T, 1) == size(T, 2) == n @test size(T) == (n, n) - @test convert(Array, T) == diagm(dv) + diagm(ev, isupper?1:-1) - @test Bidiagonal(convert(Array, T), isupper) == T + @test full(T) == diagm(dv) + diagm(ev, isupper?1:-1) + @test Bidiagonal(full(T), isupper) == T @test big(T) == T - @test convert(Array, abs(T)) == abs(diagm(dv)) + abs(diagm(ev, isupper?1:-1)) - @test convert(Array, real(T)) == real(diagm(dv)) + real(diagm(ev, isupper?1:-1)) - @test convert(Array, imag(T)) == imag(diagm(dv)) + imag(diagm(ev, isupper?1:-1)) + @test full(abs(T)) == abs(diagm(dv)) + abs(diagm(ev, isupper?1:-1)) + @test full(real(T)) == real(diagm(dv)) + real(diagm(ev, isupper?1:-1)) + @test full(imag(T)) == imag(diagm(dv)) + imag(diagm(ev, isupper?1:-1)) z = zeros(elty, n) debug && println("Idempotent tests") @@ -113,7 +113,7 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) b += im*convert(Matrix{elty}, rand(1:10, n, 2)) end end - Tfull = convert(Array, T) + Tfull = full(T) condT = cond(map(Complex128,Tfull)) promty = typeof((zero(relty)*zero(relty) + zero(relty)*zero(relty))/one(relty)) if relty != BigFloat @@ -184,7 +184,7 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) debug && println("Singular systems") if (elty <: BlasReal) - @test convert(Array, svdfact(T)) ≈ convert(Array, svdfact!(copy(Tfull))) + @test full(svdfact(T)) ≈ full(svdfact!(copy(Tfull))) @test svdvals(Tfull) ≈ svdvals(T) u1, d1, v1 = svd(Tfull) u2, d2, v2 = svd(T) @@ -206,9 +206,9 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) dv = convert(Vector{elty}, relty <: AbstractFloat ? randn(n) : rand(1:10, n)) ev = convert(Vector{elty}, relty <: AbstractFloat ? randn(n-1) : rand(1:10, n-1)) T2 = Bidiagonal(dv, ev, isupper2) - Tfull2 = convert(Array, T2) + Tfull2 = full(T2) for op in (+, -, *) - @test convert(Array, op(T, T2)) ≈ op(Tfull, Tfull2) + @test full(op(T, T2)) ≈ op(Tfull, Tfull2) end end @@ -234,7 +234,7 @@ A = Bidiagonal(ones(Float32,10),ones(Float32,9),true) B = rand(Float64,10,10) C = Tridiagonal(rand(Float64,9),rand(Float64,10),rand(Float64,9)) @test promote_rule(Matrix{Float64}, Bidiagonal{Float64}) == Matrix{Float64} -@test promote(B,A) == (B,convert(Matrix{Float64},convert(Array, A))) +@test promote(B,A) == (B,convert(Matrix{Float64},full(A))) @test promote(C,A) == (C,Tridiagonal(zeros(Float64,9),convert(Vector{Float64},A.dv),convert(Vector{Float64},A.ev))) import Base.LinAlg: fillslots!, UnitLowerTriangular @@ -271,14 +271,14 @@ let #fill! b = Bidiagonal(randn(1,1), true) st = SymTridiagonal(randn(1,1)) for x in (b, st) - @test convert(Array, fill!(x, val)) == fill!(convert(Array, x), val) + @test full(fill!(x, val)) == fill!(full(x), val) end b = Bidiagonal(randn(2,2), true) st = SymTridiagonal(randn(3), randn(2)) t = Tridiagonal(randn(3,3)) for x in (b, t, st) @test_throws ArgumentError fill!(x, val) - @test convert(Array, fill!(x, 0)) == fill!(convert(Array, x), 0) + @test full(fill!(x, 0)) == fill!(full(x), 0) end end end diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index 29fafb69e6bd6..00e4696b4f12b 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -46,7 +46,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) for i=1:n, j=1:n @test E[i,j] <= (n+1)ε/(1-(n+1)ε)*real(sqrt(apd[i,i]*apd[j,j])) end - E = abs(apd - convert(Array, capd)) + E = abs(apd - full(capd)) for i=1:n, j=1:n @test E[i,j] <= (n+1)ε/(1-(n+1)ε)*real(sqrt(apd[i,i]*apd[j,j])) end @@ -65,13 +65,13 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) end # test chol of 2x2 Strang matrix - S = convert(AbstractMatrix{eltya},convert(Array, SymTridiagonal([2,2],[-1]))) + S = convert(AbstractMatrix{eltya},full(SymTridiagonal([2,2],[-1]))) U = Bidiagonal([2,sqrt(eltya(3))],[-1],true) / sqrt(eltya(2)) - @test convert(Array, chol(S)) ≈ convert(Array, U) + @test full(chol(S)) ≈ full(U) #lower Cholesky factor lapd = cholfact(apd, :L) - @test convert(Array, lapd) ≈ apd + @test full(lapd) ≈ apd l = lapd[:L] @test l*l' ≈ apd @test triu(capd.factors) ≈ lapd[:U] @@ -95,12 +95,12 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) if isreal(apd) @test apd*inv(cpapd) ≈ eye(n) end - @test convert(Array, cpapd) ≈ apd + @test full(cpapd) ≈ apd #getindex @test_throws KeyError cpapd[:Z] @test size(cpapd) == size(apd) - @test convert(Array, copy(cpapd)) ≈ apd + @test full(copy(cpapd)) ≈ apd @test det(cpapd) ≈ det(apd) @test cpapd[:P]*cpapd[:L]*cpapd[:U]*cpapd[:P]' ≈ apd end @@ -151,9 +151,9 @@ begin # Cholesky factor of Matrix with non-commutative elements, here 2x2-matrices X = Matrix{Float64}[0.1*rand(2,2) for i in 1:3, j = 1:3] - L = convert(Array, Base.LinAlg._chol!(X*X', LowerTriangular)) - U = convert(Array, Base.LinAlg._chol!(X*X', UpperTriangular)) - XX = convert(Array, X*X') + L = full(Base.LinAlg._chol!(X*X', LowerTriangular)) + U = full(Base.LinAlg._chol!(X*X', UpperTriangular)) + XX = full(X*X') @test sum(sum(norm, L*L' - XX)) < eps() @test sum(sum(norm, U'*U - XX)) < eps() @@ -167,8 +167,8 @@ for elty in (Float32, Float64, Complex{Float32}, Complex{Float64}) A = randn(5,5) end A = convert(Matrix{elty}, A'A) - @test convert(Array, cholfact(A)[:L]) ≈ convert(Array, invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{LowerTriangular}}, copy(A), LowerTriangular)) - @test convert(Array, cholfact(A)[:U]) ≈ convert(Array, invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{UpperTriangular}}, copy(A), UpperTriangular)) + @test full(cholfact(A)[:L]) ≈ full(invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{LowerTriangular}}, copy(A), LowerTriangular)) + @test full(cholfact(A)[:U]) ≈ full(invoke(Base.LinAlg._chol!, Tuple{AbstractMatrix, Type{UpperTriangular}}, copy(A), UpperTriangular)) end # Test up- and downdates diff --git a/test/linalg/dense.jl b/test/linalg/dense.jl index 8f640a864d57e..3f1c7a3c45995 100644 --- a/test/linalg/dense.jl +++ b/test/linalg/dense.jl @@ -120,12 +120,12 @@ debug && println("Factorize") @test factorize(A) == Bidiagonal(d,e,true) if eltya <: Real A = diagm(d) + diagm(e,1) + diagm(e,-1) - @test convert(Array, factorize(A)) ≈ convert(Array, factorize(SymTridiagonal(d,e))) + @test full(factorize(A)) ≈ full(factorize(SymTridiagonal(d,e))) A = diagm(d) + diagm(e,1) + diagm(e,-1) + diagm(f,2) + diagm(f,-2) @test inv(factorize(A)) ≈ inv(factorize(Symmetric(A))) end A = diagm(d) + diagm(e,1) + diagm(e2,-1) - @test convert(Array, factorize(A)) ≈ convert(Array, factorize(Tridiagonal(e2,d,e))) + @test full(factorize(A)) ≈ full(factorize(Tridiagonal(e2,d,e))) A = diagm(d) + diagm(e,1) + diagm(f,2) @test factorize(A) == UpperTriangular(A) end # for eltya diff --git a/test/linalg/diagonal.jl b/test/linalg/diagonal.jl index 22597c55f4f32..be856257cef9d 100644 --- a/test/linalg/diagonal.jl +++ b/test/linalg/diagonal.jl @@ -27,9 +27,9 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test typeof(convert(Diagonal{Complex64},D)) == Diagonal{Complex64} @test typeof(convert(AbstractMatrix{Complex64},D)) == Diagonal{Complex64} - @test convert(Array, real(D)) == real(DM) - @test convert(Array, abs(D)) == abs(DM) - @test convert(Array, imag(D)) == imag(DM) + @test full(real(D)) == real(DM) + @test full(abs(D)) == abs(DM) + @test full(imag(D)) == imag(DM) @test parent(D) == d @test diag(D) == d @@ -79,12 +79,12 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test_throws SingularException A_ldiv_B!(Diagonal(zeros(relty,n)),copy(v)) b = rand(elty,n,n) b = sparse(b) - @test A_ldiv_B!(D,copy(b)) ≈ convert(Array, D)\convert(Array, b) + @test A_ldiv_B!(D,copy(b)) ≈ full(D)\full(b) @test_throws SingularException A_ldiv_B!(Diagonal(zeros(elty,n)),copy(b)) b = view(rand(elty,n),collect(1:n)) b2 = copy(b) c = A_ldiv_B!(D,b) - d = convert(Array, D)\b2 + d = full(D)\b2 for i in 1:n @test c[i] ≈ d[i] end @@ -101,23 +101,23 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) D2 = Diagonal(d) DM2= diagm(d) for op in (+, -, *) - @test convert(Array, op(D, D2)) ≈ op(DM, DM2) + @test full(op(D, D2)) ≈ op(DM, DM2) end # binary ops with plain numbers a = rand() - @test convert(Array, a*D) ≈ a*DM - @test convert(Array, D*a) ≈ DM*a - @test convert(Array, D/a) ≈ DM/a + @test full(a*D) ≈ a*DM + @test full(D*a) ≈ DM*a + @test full(D/a) ≈ DM/a if relty <: BlasFloat b = rand(elty,n,n) b = sparse(b) - @test A_mul_B!(copy(D), copy(b)) ≈ convert(Array, D)*convert(Array, b) - @test At_mul_B!(copy(D), copy(b)) ≈ convert(Array, D).'*convert(Array, b) - @test Ac_mul_B!(copy(D), copy(b)) ≈ convert(Array, D)'*convert(Array, b) + @test A_mul_B!(copy(D), copy(b)) ≈ full(D)*full(b) + @test At_mul_B!(copy(D), copy(b)) ≈ full(D).'*full(b) + @test Ac_mul_B!(copy(D), copy(b)) ≈ full(D)'*full(b) end - @test U.'*D ≈ U.'*convert(Array, D) - @test U'*D ≈ U'*convert(Array, D) + @test U.'*D ≈ U.'*full(D) + @test U'*D ≈ U'*full(D) #division of two Diagonals @test D/D2 ≈ Diagonal(D.diag./D2.diag) @@ -155,7 +155,7 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) debug && println("conj and transpose") @test transpose(D) == D if elty <: BlasComplex - @test convert(Array, conj(D)) ≈ conj(DM) + @test full(conj(D)) ≈ conj(DM) @test ctranspose(D) == conj(D) end @@ -233,7 +233,7 @@ end # inv for d in (randn(n), [1, 2, 3], [1im, 2im, 3im]) D = Diagonal(d) - @test inv(D) ≈ inv(convert(Array, D)) + @test inv(D) ≈ inv(full(D)) end @test_throws SingularException inv(Diagonal(zeros(n))) @test_throws SingularException inv(Diagonal([0, 1, 2])) diff --git a/test/linalg/hessenberg.jl b/test/linalg/hessenberg.jl index 163f05570ac4b..8ea3b290056bc 100644 --- a/test/linalg/hessenberg.jl +++ b/test/linalg/hessenberg.jl @@ -26,11 +26,11 @@ debug && println("\ntype of a: ", eltya, " type of b: ", eltyb, "\n") @test size(H[:Q], 2) == size(A, 2) @test size(H[:Q]) == size(A) @test_throws KeyError H[:Z] - @test convert(Array, H) ≈ A + @test full(H) ≈ A @test (H[:Q] * H[:H]) * H[:Q]' ≈ A @test (H[:Q]' *A) * H[:Q] ≈ H[:H] #getindex for HessenbergQ - @test H[:Q][1,1] ≈ convert(Array, H[:Q])[1,1] + @test H[:Q][1,1] ≈ full(H[:Q])[1,1] end end end diff --git a/test/linalg/lapack.jl b/test/linalg/lapack.jl index 42a0b6f6c4554..51a47eace3a4a 100644 --- a/test/linalg/lapack.jl +++ b/test/linalg/lapack.jl @@ -46,7 +46,7 @@ let # gebrd, bdsqr & throw for bdsdc d, e = convert(Vector{elty}, randn(n)), convert(Vector{elty}, randn(n - 1)) U, Vt, C = eye(elty, n), eye(elty, n), eye(elty, n) s, _ = LAPACK.bdsqr!('U', copy(d), copy(e), Vt, U, C) - @test convert(Array, Bidiagonal(d, e, true)) ≈ U*Diagonal(s)*Vt + @test full(Bidiagonal(d, e, true)) ≈ U*Diagonal(s)*Vt @test_throws ArgumentError LAPACK.bdsqr!('A', d, e, Vt, U, C) @test_throws DimensionMismatch LAPACK.bdsqr!('U', d, [e; 1], Vt, U, C) diff --git a/test/linalg/lq.jl b/test/linalg/lq.jl index aaaa960a8ef20..e898dbcfdaed8 100644 --- a/test/linalg/lq.jl +++ b/test/linalg/lq.jl @@ -52,25 +52,25 @@ debug && println("LQ decomposition") @test size(lqa[:Q],3) == 1 @test Base.LinAlg.getq(lqa) == lqa[:Q] @test_throws KeyError lqa[:Z] - @test convert(Array, lqa') ≈ a' + @test full(lqa') ≈ a' @test lqa * lqa' ≈ a * a' @test lqa' * lqa ≈ a' * a - @test q*Base.LinAlg.thickQ(q)' ≈ eye(eltya,n) + @test q*full(q, thin = false)' ≈ eye(eltya,n) @test l*q ≈ a - @test convert(Array, lqa) ≈ a - @test convert(Array, copy(lqa)) ≈ a + @test full(lqa) ≈ a + @test full(copy(lqa)) ≈ a @test_approx_eq_eps a*(lqa\b) b 3000ε @test_approx_eq_eps lqa*b qra[:Q]*qra[:R]*b 3000ε - @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*Base.LinAlg.thickQ(q) eye(n) 5000ε + @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*full(q, thin=false) eye(n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) end - @test_approx_eq_eps q*b Base.LinAlg.thickQ(q)*b 100ε - @test_approx_eq_eps q.'*b Base.LinAlg.thickQ(q).'*b 100ε - @test_approx_eq_eps q'*b Base.LinAlg.thickQ(q)'*b 100ε - @test_approx_eq_eps a*q a*Base.LinAlg.thickQ(q) 100ε - @test_approx_eq_eps a*q.' a*Base.LinAlg.thickQ(q).' 100ε - @test_approx_eq_eps a*q' a*Base.LinAlg.thickQ(q)' 100ε + @test_approx_eq_eps q*b full(q, thin=false)*b 100ε + @test_approx_eq_eps q.'*b full(q, thin=false).'*b 100ε + @test_approx_eq_eps q'*b full(q, thin=false)'*b 100ε + @test_approx_eq_eps a*q a*full(q, thin=false) 100ε + @test_approx_eq_eps a*q.' a*full(q, thin=false).' 100ε + @test_approx_eq_eps a*q' a*full(q, thin=false)' 100ε @test_throws DimensionMismatch q*b[1:n1 + 1] @test_throws DimensionMismatch Ac_mul_B(q,ones(eltya,n+2,n+2)) @test_throws DimensionMismatch ones(eltyb,n+2,n+2)*q @@ -80,10 +80,10 @@ debug && println("LQ decomposition") debug && println("Matmul with LQ factorizations") lqa = lqfact(a[:,1:n1]) l,q = lqa[:L], lqa[:Q] - @test convert(Array, q)*convert(Array, q)' ≈ eye(eltya,n1) - @test (Base.LinAlg.thickQ(q)'*Base.LinAlg.thickQ(q))[1:n1,:] ≈ eye(eltya,n1,n) + @test full(q)*full(q)' ≈ eye(eltya,n1) + @test (full(q,thin=false)'*full(q,thin=false))[1:n1,:] ≈ eye(eltya,n1,n) @test_throws DimensionMismatch A_mul_B!(eye(eltya,n+1),q) - @test Ac_mul_B!(q,convert(Array, q)) ≈ eye(eltya,n1) + @test Ac_mul_B!(q,full(q)) ≈ eye(eltya,n1) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) end diff --git a/test/linalg/lu.jl b/test/linalg/lu.jl index a0e291c0d4c9d..ca5831029d204 100644 --- a/test/linalg/lu.jl +++ b/test/linalg/lu.jl @@ -36,7 +36,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) if eltya <: BlasFloat num = rand(eltya) @test lu(num) == (one(eltya),num,1) - @test convert(Array, lufact(num)) ≈ eltya[num] + @test full(lufact(num)) ≈ eltya[num] end for eltyb in (Float32, Float64, Complex64, Complex128, Int) b = eltyb == Int ? rand(1:5, n, 2) : convert(Matrix{eltyb}, eltyb <: Complex ? complex(breal, bimg) : breal) @@ -68,7 +68,7 @@ debug && println("(Automatic) Square LU decomposition") @test norm(a'*(lua'\a') - a', 1) < ε*κ*n^2 @test norm(a*(lua\c) - c, 1) < ε*κ*n # c is a vector @test norm(a'*(lua'\c) - c, 1) < ε*κ*n # c is a vector - @test convert(Array, lua) ≈ a + @test full(lua) ≈ a if eltya <: Real && eltyb <: Real @test norm(a.'*(lua.'\b) - b,1) < ε*κ*n*2 # Two because the right hand side has two columns @test norm(a.'*(lua.'\c) - c,1) < ε*κ*n @@ -81,13 +81,13 @@ debug && println("(Automatic) Square LU decomposition") end debug && println("Tridiagonal LU") - κd = cond(convert(Array, d),1) + κd = cond(full(d),1) lud = lufact(d) @test lufact(lud) == lud @test_throws KeyError lud[:Z] - @test lud[:L]*lud[:U] ≈ lud[:P]*convert(Array, d) - @test lud[:L]*lud[:U] ≈ convert(Array, d)[lud[:p],:] - @test convert(Array, lud) ≈ d + @test lud[:L]*lud[:U] ≈ lud[:P]*full(d) + @test lud[:L]*lud[:U] ≈ full(d)[lud[:p],:] + @test full(lud) ≈ d f = zeros(eltyb, n+1) @test_throws DimensionMismatch lud\f @test_throws DimensionMismatch lud.'\f @@ -102,17 +102,17 @@ debug && println("Tridiagonal LU") @test norm(d*(lud\b) - b, 1) < ε*κd*n*2 # Two because the right hand side has two columns if eltya <: Real - @test norm((lud.'\b) - convert(Array, d.')\b, 1) < ε*κd*n*2 # Two because the right hand side has two columns + @test norm((lud.'\b) - full(d.')\b, 1) < ε*κd*n*2 # Two because the right hand side has two columns end if eltya <: Complex - @test norm((lud'\b) - convert(Array, d')\b, 1) < ε*κd*n*2 # Two because the right hand side has two columns + @test norm((lud'\b) - full(d')\b, 1) < ε*κd*n*2 # Two because the right hand side has two columns end end end if eltya <: BlasFloat && eltyb <: BlasFloat e = rand(eltyb,n,n) @test norm(e/lud - e/d,1) < ε*κ*n^2 - @test norm((lud.'\e') - convert(Array, d.')\e',1) < ε*κd*n^2 + @test norm((lud.'\e') - full(d.')\e',1) < ε*κd*n^2 #test singular du = rand(eltya,n-1) dl = rand(eltya,n-1) @@ -136,11 +136,11 @@ end # test conversion routine a = Tridiagonal(rand(9),rand(10),rand(9)) -fa = convert(Array, a) +fa = full(a) falu = lufact(fa) alu = lufact(a) falu = convert(typeof(falu),alu) -@test convert(Array, alu) == fa +@test full(alu) == fa # Test rational matrices ## Integrate in general tests when more linear algebra is implemented in julia diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index b8f72bca058c4..44bb9a6afa85b 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -336,7 +336,7 @@ A = [RootInt(3) RootInt(5)] function test_mul(C, A, B) A_mul_B!(C, A, B) - @test convert(Array, A) * convert(Array, B) ≈ C + @test full(A) * full(B) ≈ C @test A*B ≈ C end diff --git a/test/linalg/qr.jl b/test/linalg/qr.jl index 78e64b7744acb..8df1097b926d6 100644 --- a/test/linalg/qr.jl +++ b/test/linalg/qr.jl @@ -46,15 +46,15 @@ debug && println("QR decomposition (without pivoting)") @inferred qr(a) q, r = qra[:Q], qra[:R] @test_throws KeyError qra[:Z] - @test q'*Base.LinAlg.thickQ(q) ≈ eye(n) - @test q*Base.LinAlg.thickQ(q)' ≈ eye(n) - @test q'*eye(n)' ≈ Base.LinAlg.thickQ(q)' - @test Base.LinAlg.thickQ(q)'q ≈ eye(n) - @test eye(n)'q' ≈ Base.LinAlg.thickQ(q)' + @test q'*full(q, thin=false) ≈ eye(n) + @test q*full(q, thin=false)' ≈ eye(n) + @test q'*eye(n)' ≈ full(q, thin=false)' + @test full(q, thin=false)'q ≈ eye(n) + @test eye(n)'q' ≈ full(q, thin=false)' @test q*r ≈ a @test_approx_eq_eps a*(qra\b) b 3000ε - @test convert(Array, qra) ≈ a - @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*Base.LinAlg.thickQ(q) eye(n) 5000ε + @test full(qra) ≈ a + @test_approx_eq_eps A_mul_Bc(eye(eltyb,size(q.factors,2)),q)*full(q, thin=false) eye(n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) ac = copy(a) @@ -66,14 +66,14 @@ debug && println("Thin QR decomposition (without pivoting)") @inferred qr(a[:,1:n1], Val{false}) q,r = qra[:Q], qra[:R] @test_throws KeyError qra[:Z] - @test q'*Base.LinAlg.thickQ(q) ≈ eye(n) - @test q'*convert(Array, q) ≈ eye(n,n1) + @test q'*full(q, thin=false) ≈ eye(n) + @test q'*full(q) ≈ eye(n,n1) @test q*r ≈ a[:,1:n1] - @test_approx_eq_eps q*b[1:n1] convert(Array, q)*b[1:n1] 100ε - @test_approx_eq_eps q*b Base.LinAlg.thickQ(q)*b 100ε + @test_approx_eq_eps q*b[1:n1] full(q)*b[1:n1] 100ε + @test_approx_eq_eps q*b full(q, thin=false)*b 100ε @test_throws DimensionMismatch q*b[1:n1 + 1] @test_throws DimensionMismatch b[1:n1 + 1]*q' - @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*Base.LinAlg.thickQ(q) eye(n1,n) 5000ε + @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*full(q, thin=false) eye(n1,n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) end @@ -86,14 +86,14 @@ debug && println("(Automatic) Fat (pivoted) QR decomposition") q,r = qrpa[:Q], qrpa[:R] @test_throws KeyError qrpa[:Z] p = qrpa[:p] - @test q'*Base.LinAlg.thickQ(q) ≈ eye(n1) - @test q*Base.LinAlg.thickQ(q)' ≈ eye(n1) - @test (UpperTriangular(eye(eltya,size(q,2)))*q')*Base.LinAlg.thickQ(q) ≈ eye(n1) + @test q'*full(q, thin=false) ≈ eye(n1) + @test q*full(q, thin=false)' ≈ eye(n1) + @test (UpperTriangular(eye(eltya,size(q,2)))*q')*full(q, thin=false) ≈ eye(n1) @test q*r ≈ (isa(qrpa,QRPivoted) ? a[1:n1,p] : a[1:n1,:]) @test q*r[:,invperm(p)] ≈ a[1:n1,:] @test q*r*qrpa[:P].' ≈ a[1:n1,:] @test_approx_eq_eps a[1:n1,:]*(qrpa\b[1:n1]) b[1:n1] 5000ε - @test convert(Array, qrpa) ≈ a[1:5,:] + @test full(qrpa) ≈ a[1:5,:] @test_throws DimensionMismatch q*b[1:n1+1] @test_throws DimensionMismatch b[1:n1+1]*q' if eltya != Int @@ -105,14 +105,14 @@ debug && println("(Automatic) Thin (pivoted) QR decomposition") q,r = qrpa[:Q], qrpa[:R] @test_throws KeyError qrpa[:Z] p = qrpa[:p] - @test q'*Base.LinAlg.thickQ(q) ≈ eye(n) - @test q*Base.LinAlg.thickQ(q)' ≈ eye(n) + @test q'*full(q, thin=false) ≈ eye(n) + @test q*full(q, thin=false)' ≈ eye(n) @test q*r ≈ a[:,p] @test q*r[:,invperm(p)] ≈ a[:,1:n1] - @test convert(Array, qrpa) ≈ a[:,1:5] + @test full(qrpa) ≈ a[:,1:5] @test_throws DimensionMismatch q*b[1:n1+1] @test_throws DimensionMismatch b[1:n1+1]*q' - @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*Base.LinAlg.thickQ(q) eye(n1,n) 5000ε + @test_approx_eq_eps A_mul_Bc(UpperTriangular(eye(eltyb,size(q.factors,2))),q)*full(q, thin=false) eye(n1,n) 5000ε if eltya != Int @test eye(eltyb,n)*q ≈ convert(AbstractMatrix{tab},q) end @@ -123,9 +123,9 @@ debug && println("Matmul with QR factorizations") if eltya != Int qrpa = factorize(a[:,1:n1]) q, r = qrpa[:Q], qrpa[:R] - @test A_mul_B!(Base.LinAlg.thickQ(q)',q) ≈ eye(n) + @test A_mul_B!(full(q, thin=false)',q) ≈ eye(n) @test_throws DimensionMismatch A_mul_B!(eye(eltya,n+1),q) - @test A_mul_Bc!(Base.LinAlg.thickQ(q),q) ≈ eye(n) + @test A_mul_Bc!(full(q, thin=false),q) ≈ eye(n) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) @test_throws DimensionMismatch Base.LinAlg.A_mul_B!(q,zeros(eltya,n1+1)) @@ -133,9 +133,9 @@ debug && println("Matmul with QR factorizations") qra = qrfact(a[:,1:n1], Val{false}) q, r = qra[:Q], qra[:R] - @test A_mul_B!(Base.LinAlg.thickQ(q)',q) ≈ eye(n) + @test A_mul_B!(full(q, thin=false)',q) ≈ eye(n) @test_throws DimensionMismatch A_mul_B!(eye(eltya,n+1),q) - @test A_mul_Bc!(Base.LinAlg.thickQ(q),q) ≈ eye(n) + @test A_mul_Bc!(full(q, thin=false),q) ≈ eye(n) @test_throws DimensionMismatch A_mul_Bc!(eye(eltya,n+1),q) @test_throws BoundsError size(q,-1) @test_throws DimensionMismatch q * eye(Int8,n+4) @@ -154,7 +154,7 @@ end # Issue 7304 let A = [-√.5 -√.5; -√.5 √.5] - Q = convert(Array, qrfact(A)[:Q]) + Q = full(qrfact(A)[:Q]) @test vecnorm(A-Q) < eps() end diff --git a/test/linalg/schur.jl b/test/linalg/schur.jl index 2f195f087808a..5370afd3f82f5 100644 --- a/test/linalg/schur.jl +++ b/test/linalg/schur.jl @@ -38,7 +38,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) @test sort(real(f[:values])) ≈ sort(real(d)) @test sort(imag(f[:values])) ≈ sort(imag(d)) @test istriu(f[:Schur]) || eltype(a)<:Real - @test convert(Array, f) ≈ a + @test full(f) ≈ a @test_throws KeyError f[:A] debug && println("Reorder Schur") diff --git a/test/linalg/special.jl b/test/linalg/special.jl index 2ccaf724c9dd5..89ddde5ee4b00 100644 --- a/test/linalg/special.jl +++ b/test/linalg/special.jl @@ -11,11 +11,11 @@ let a=[1.0:n;] A=Diagonal(a) for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Tridiagonal, LowerTriangular, UpperTriangular, Matrix] debug && println("newtype is $(newtype)") - @test convert(Array, convert(newtype, A)) == convert(Array, A) + @test full(convert(newtype, A)) == full(A) end for newtype in [Base.LinAlg.UnitUpperTriangular, Base.LinAlg.UnitLowerTriangular] @test_throws ArgumentError convert(newtype, A) - @test convert(Array, convert(newtype, Diagonal(ones(n)))) == eye(n) + @test full(convert(newtype, Diagonal(ones(n)))) == eye(n) end for isupper in (true, false) @@ -23,56 +23,56 @@ let a=[1.0:n;] A=Bidiagonal(a, [1.0:n-1;], isupper) for newtype in [Bidiagonal, Tridiagonal, isupper ? UpperTriangular : LowerTriangular, Matrix] debug && println("newtype is $(newtype)") - @test convert(Array, convert(newtype, A)) == convert(Array, A) - @test convert(Array, newtype(A)) == convert(Array, A) + @test full(convert(newtype, A)) == full(A) + @test full(newtype(A)) == full(A) end @test_throws ArgumentError convert(SymTridiagonal, A) A=Bidiagonal(a, zeros(n-1), isupper) #morally Diagonal for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Tridiagonal, isupper ? UpperTriangular : LowerTriangular, Matrix] debug && println("newtype is $(newtype)") - @test convert(Array, convert(newtype, A)) == convert(Array, A) - @test convert(Array, newtype(A)) == convert(Array, A) + @test full(convert(newtype, A)) == full(A) + @test full(newtype(A)) == full(A) end end A = SymTridiagonal(a, [1.0:n-1;]) for newtype in [Tridiagonal, Matrix] - @test convert(Array, convert(newtype, A)) == convert(Array, A) + @test full(convert(newtype, A)) == full(A) end for newtype in [Diagonal, Bidiagonal] @test_throws ArgumentError convert(newtype,A) end A = SymTridiagonal(a, zeros(n-1)) - @test convert(Array, convert(Bidiagonal,A)) == convert(Array, A) + @test full(convert(Bidiagonal,A)) == full(A) A = Tridiagonal(zeros(n-1), [1.0:n;], zeros(n-1)) #morally Diagonal for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Matrix] - @test convert(Array, convert(newtype, A)) == convert(Array, A) + @test full(convert(newtype, A)) == full(A) end A = Tridiagonal(ones(n-1), [1.0:n;], ones(n-1)) #not morally Diagonal for newtype in [SymTridiagonal, Matrix] - @test convert(Array, convert(newtype, A)) == convert(Array, A) + @test full(convert(newtype, A)) == full(A) end for newtype in [Diagonal, Bidiagonal] @test_throws ArgumentError convert(newtype,A) end A = Tridiagonal(zeros(n-1), [1.0:n;], ones(n-1)) #not morally Diagonal - @test convert(Array, convert(Bidiagonal, A)) == convert(Array, A) + @test full(convert(Bidiagonal, A)) == full(A) A = UpperTriangular(Tridiagonal(zeros(n-1), [1.0:n;], ones(n-1))) - @test convert(Array, convert(Bidiagonal, A)) == convert(Array, A) + @test full(convert(Bidiagonal, A)) == full(A) A = Tridiagonal(ones(n-1), [1.0:n;], zeros(n-1)) #not morally Diagonal - @test convert(Array, convert(Bidiagonal, A)) == convert(Array, A) + @test full(convert(Bidiagonal, A)) == full(A) A = LowerTriangular(Tridiagonal(ones(n-1), [1.0:n;], zeros(n-1))) - @test convert(Array, convert(Bidiagonal, A)) == convert(Array, A) + @test full(convert(Bidiagonal, A)) == full(A) @test_throws ArgumentError convert(SymTridiagonal,A) - A = LowerTriangular(convert(Array, Diagonal(a))) #morally Diagonal + A = LowerTriangular(full(Diagonal(a))) #morally Diagonal for newtype in [Diagonal, Bidiagonal, SymTridiagonal, LowerTriangular, Matrix] - @test convert(Array, convert(newtype, A)) == convert(Array, A) + @test full(convert(newtype, A)) == full(A) end - A = UpperTriangular(convert(Array, Diagonal(a))) #morally Diagonal + A = UpperTriangular(full(Diagonal(a))) #morally Diagonal for newtype in [Diagonal, Bidiagonal, SymTridiagonal, UpperTriangular, Matrix] - @test convert(Array, convert(newtype, A)) == convert(Array, A) + @test full(convert(newtype, A)) == full(A) end A = UpperTriangular(triu(rand(n,n))) for newtype in [Diagonal, Bidiagonal, Tridiagonal, SymTridiagonal] @@ -80,7 +80,7 @@ let a=[1.0:n;] end A = Diagonal(a) for newtype in [UpperTriangular, LowerTriangular] - @test convert(Array, convert(newtype,A)) == convert(Array, A) + @test full(convert(newtype,A)) == full(A) end end @@ -92,26 +92,26 @@ let a=[1.0:n;] for type2 in Spectypes B = convert(type1,A) C = convert(type2,A) - @test convert(Array, B + C) ≈ convert(Array, A + A) - @test convert(Array, B - C) ≈ convert(Array, A - A) + @test full(B + C) ≈ full(A + A) + @test full(B - C) ≈ full(A - A) end end B = SymTridiagonal(a, ones(n-1)) for Spectype in [Diagonal, Bidiagonal, Tridiagonal, Matrix] - @test convert(Array, B + convert(Spectype,A)) ≈ convert(Array, B + A) - @test convert(Array, convert(Spectype,A) + B) ≈ convert(Array, B + A) - @test convert(Array, B - convert(Spectype,A)) ≈ convert(Array, B - A) - @test convert(Array, convert(Spectype,A) - B) ≈ convert(Array, A - B) + @test full(B + convert(Spectype,A)) ≈ full(B + A) + @test full(convert(Spectype,A) + B) ≈ full(B + A) + @test full(B - convert(Spectype,A)) ≈ full(B - A) + @test full(convert(Spectype,A) - B) ≈ full(A - B) end C = rand(n,n) for TriType in [Base.LinAlg.UnitLowerTriangular, Base.LinAlg.UnitUpperTriangular, UpperTriangular, LowerTriangular] D = TriType(C) for Spectype in [Diagonal, Bidiagonal, Tridiagonal, Matrix] - @test convert(Array, D + convert(Spectype,A)) ≈ convert(Array, D + A) - @test convert(Array, convert(Spectype,A) + D) ≈ convert(Array, A + D) - @test convert(Array, D - convert(Spectype,A)) ≈ convert(Array, D - A) - @test convert(Array, convert(Spectype,A) - D) ≈ convert(Array, A - D) + @test full(D + convert(Spectype,A)) ≈ full(D + A) + @test full(convert(Spectype,A) + D) ≈ full(A + D) + @test full(D - convert(Spectype,A)) ≈ full(D - A) + @test full(convert(Spectype,A) - D) ≈ full(A - D) end end end @@ -122,9 +122,9 @@ for typ in [UpperTriangular,LowerTriangular,Base.LinAlg.UnitUpperTriangular,Base atri = typ(a) b = rand(n,n) qrb = qrfact(b,Val{true}) - @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ convert(Array, atri) * qrb[:Q]' - @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ convert(Array, atri) * qrb[:Q]' + @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ full(atri) * qrb[:Q]' + @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ full(atri) * qrb[:Q]' qrb = qrfact(b,Val{false}) - @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ convert(Array, atri) * qrb[:Q]' - @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ convert(Array, atri) * qrb[:Q]' + @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ full(atri) * qrb[:Q]' + @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ full(atri) * qrb[:Q]' end diff --git a/test/linalg/svd.jl b/test/linalg/svd.jl index f76ca1c3eea41..d98a63c1d7625 100644 --- a/test/linalg/svd.jl +++ b/test/linalg/svd.jl @@ -39,7 +39,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) usv = svdfact(a) @test usv[:S] === svdvals(usv) @test usv[:U] * (Diagonal(usv[:S]) * usv[:Vt]) ≈ a - @test convert(Array, usv) ≈ a + @test full(usv) ≈ a @test usv[:Vt]' ≈ usv[:V] @test_throws KeyError usv[:Z] b = rand(eltya,n) diff --git a/test/linalg/symmetric.jl b/test/linalg/symmetric.jl index e7a8e12f8c96f..3a33105b5372f 100644 --- a/test/linalg/symmetric.jl +++ b/test/linalg/symmetric.jl @@ -56,8 +56,8 @@ let n=10 @test isa(similar(Symmetric(asym), Int, (3,2)), Matrix{Int}) @test isa(similar(Hermitian(asym), Int, (3,2)), Matrix{Int}) - # convert(Array, _) - @test asym == convert(Array, Hermitian(asym)) + # full + @test asym == full(Hermitian(asym)) # parent @@ -111,7 +111,7 @@ let n=10 eig(Hermitian(asym), d[1] - 1, (d[2] + d[3])/2) # same result, but checks that method works @test eigvals(Hermitian(asym), 1:2) ≈ d[1:2] @test eigvals(Hermitian(asym), d[1] - 1, (d[2] + d[3])/2) ≈ d[1:2] - @test convert(Array, eigfact(asym)) ≈ asym + @test full(eigfact(asym)) ≈ asym # relation to svdvals @test sum(sort(abs(eigvals(Hermitian(asym))))) == sum(sort(svdvals(Hermitian(asym)))) @@ -213,7 +213,7 @@ end #Issue #7933 let A7933 = [1 2; 3 4] B7933 = copy(A7933) - C7933 = convert(Array, Symmetric(A7933)) + C7933 = full(Symmetric(A7933)) @test A7933 == B7933 end diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index e9a794d90000a..4c570babc80e9 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -11,7 +11,7 @@ srand(123) debug && println("Test basic type functionality") @test_throws DimensionMismatch LowerTriangular(randn(5, 4)) -@test LowerTriangular(randn(3, 3)) |> t -> [size(t, i) for i = 1:3] == [size(convert(Array, t), i) for i = 1:3] +@test LowerTriangular(randn(3, 3)) |> t -> [size(t, i) for i = 1:3] == [size(full(t), i) for i = 1:3] # The following test block tries to call all methods in base/linalg/triangular.jl in order for a combination of input element types. Keep the ordering when adding code. for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloat}, Int) @@ -29,13 +29,13 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa # Convert @test convert(AbstractMatrix{elty1}, A1) == A1 - @test convert(Matrix, A1) == convert(Array, A1) + @test convert(Matrix, A1) == full(A1) # full! - @test full!(copy(A1)) == convert(Array, A1) + @test full!(copy(A1)) == full(A1) # fill! - @test full!(fill!(copy(A1), 1)) == convert(Array, t1(ones(size(A1)...))) + @test full!(fill!(copy(A1), 1)) == full(t1(ones(size(A1)...))) # similar @test isa(similar(A1), t1) @@ -48,14 +48,14 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa # getindex ## Linear indexing for i = 1:length(A1) - @test A1[i] == convert(Array, A1)[i] + @test A1[i] == full(A1)[i] end @test isa(A1[2:4,1], Vector) ## Cartesian indexing for i = 1:size(A1, 1) for j = 1:size(A1, 2) - @test A1[i,j] == convert(Array, A1)[i,j] + @test A1[i,j] == full(A1)[i,j] end end @@ -101,8 +101,8 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa #tril/triu if uplo1 == :L @test tril(A1,0) == A1 - @test tril(A1,-1) == LowerTriangular(tril(convert(Array, A1),-1)) - @test tril(A1,1) == t1(tril(tril(convert(Array, A1),1))) + @test tril(A1,-1) == LowerTriangular(tril(full(A1),-1)) + @test tril(A1,1) == t1(tril(tril(full(A1),1))) @test_throws ArgumentError tril!(A1,n+1) @test triu(A1,0) == t1(diagm(diag(A1))) @test triu(A1,-1) == t1(tril(triu(A1.data,-1))) @@ -110,8 +110,8 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test_throws ArgumentError triu!(A1,n+1) else @test triu(A1,0) == A1 - @test triu(A1,1) == UpperTriangular(triu(convert(Array, A1),1)) - @test triu(A1,-1) == t1(triu(triu(convert(Array, A1),-1))) + @test triu(A1,1) == UpperTriangular(triu(full(A1),1)) + @test triu(A1,-1) == t1(triu(triu(full(A1),-1))) @test_throws ArgumentError triu!(A1,n+1) @test tril(A1,0) == t1(diagm(diag(A1))) @test tril(A1,1) == t1(triu(tril(A1.data,1))) @@ -125,11 +125,11 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa # [c]transpose[!] (test views as well, see issue #14317) let vrange = 1:n-1, viewA1 = t1(view(A1.data, vrange, vrange)) # transpose - @test convert(Array, A1.') == convert(Array, A1).' - @test convert(Array, viewA1.') == convert(Array, viewA1).' + @test full(A1.') == full(A1).' + @test full(viewA1.') == full(viewA1).' # ctranspose - @test convert(Array, A1') == convert(Array, A1)' - @test convert(Array, viewA1') == convert(Array, viewA1)' + @test full(A1') == full(A1)' + @test full(viewA1') == full(viewA1)' # transpose! @test transpose!(copy(A1)) == A1.' @test transpose!(t1(view(copy(A1).data, vrange, vrange))) == viewA1.' @@ -139,21 +139,21 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa end # diag - @test diag(A1) == diag(convert(Array, A1)) + @test diag(A1) == diag(full(A1)) # real - @test convert(Array, real(A1)) == real(convert(Array, A1)) - @test convert(Array, imag(A1)) == imag(convert(Array, A1)) - @test convert(Array, abs(A1)) == abs(convert(Array, A1)) + @test full(real(A1)) == real(full(A1)) + @test full(imag(A1)) == imag(full(A1)) + @test full(abs(A1)) == abs(full(A1)) # Unary operations - @test convert(Array, -A1) == -convert(Array, A1) + @test full(-A1) == -full(A1) # copy and copy! (test views as well, see issue #14317) let vrange = 1:n-1, viewA1 = t1(view(A1.data, vrange, vrange)) # copy - @test copy(A1) == copy(convert(Array, A1)) - @test copy(viewA1) == copy(convert(Array, viewA1)) + @test copy(A1) == copy(full(A1)) + @test copy(viewA1) == copy(full(viewA1)) # copy! B = similar(A1) copy!(B, A1) @@ -171,7 +171,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa #expm/logm if (elty1 == Float64 || elty1 == Complex128) && (t1 == UpperTriangular || t1 == LowerTriangular) - @test expm(convert(Array, logm(A1))) ≈ convert(Array, A1) + @test expm(full(logm(A1))) ≈ full(A1) end # scale @@ -209,23 +209,23 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa end # Binary operations - @test A1*0.5 == convert(Array, A1)*0.5 - @test 0.5*A1 == 0.5*convert(Array, A1) - @test A1/0.5 == convert(Array, A1)/0.5 - @test 0.5\A1 == 0.5\convert(Array, A1) + @test A1*0.5 == full(A1)*0.5 + @test 0.5*A1 == 0.5*full(A1) + @test A1/0.5 == full(A1)/0.5 + @test 0.5\A1 == 0.5\full(A1) # inversion - @test inv(A1) ≈ inv(lufact(convert(Array, A1))) - inv(convert(Array, A1)) # issue #11298 + @test inv(A1) ≈ inv(lufact(full(A1))) + inv(full(A1)) # issue #11298 @test isa(inv(A1), t1) # make sure the call to LAPACK works right if elty1 <: BlasFloat - @test Base.LinAlg.inv!(copy(A1)) ≈ inv(lufact(convert(Array, A1))) + @test Base.LinAlg.inv!(copy(A1)) ≈ inv(lufact(full(A1))) end # Determinant - @test_approx_eq_eps det(A1) det(lufact(convert(Array, A1))) sqrt(eps(real(float(one(elty1)))))*n*n - @test_approx_eq_eps logdet(A1) logdet(lufact(convert(Array, A1))) sqrt(eps(real(float(one(elty1)))))*n*n + @test_approx_eq_eps det(A1) det(lufact(full(A1))) sqrt(eps(real(float(one(elty1)))))*n*n + @test_approx_eq_eps logdet(A1) logdet(lufact(full(A1))) sqrt(eps(real(float(one(elty1)))))*n*n # Matrix square root @test sqrtm(A1) |> t -> t*t ≈ A1 @@ -237,7 +237,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa if !(elty1 in (BigFloat, Complex{BigFloat})) # Not handled yet vals, vecs = eig(A1) if (t1 == UpperTriangular || t1 == LowerTriangular) && elty1 != Int # Cannot really handle degenerate eigen space and Int matrices will probably have repeated eigenvalues. - @test_approx_eq_eps vecs*diagm(vals)/vecs convert(Array, A1) sqrt(eps(float(real(one(vals[1])))))*(norm(A1, Inf)*n)^2 + @test_approx_eq_eps vecs*diagm(vals)/vecs full(A1) sqrt(eps(float(real(one(vals[1])))))*(norm(A1, Inf)*n)^2 end end @@ -246,7 +246,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa for p in (1.0, Inf) @test_approx_eq_eps cond(A1, p) cond(A1, p) (cond(A1, p) + cond(A1, p)) end - @test cond(A1,2) == cond(convert(Array, A1),2) + @test cond(A1,2) == cond(full(A1),2) end if !(elty1 in (BigFloat, Complex{BigFloat})) # Not implemented yet @@ -275,19 +275,19 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa end # Binary operations - @test convert(Array, A1 + A2) == convert(Array, A1) + convert(Array, A2) - @test convert(Array, A1 - A2) == convert(Array, A1) - convert(Array, A2) + @test full(A1 + A2) == full(A1) + full(A2) + @test full(A1 - A2) == full(A1) - full(A2) # Triangular-Triangualar multiplication and division - @test convert(Array, A1*A2) ≈ convert(Array, A1)*convert(Array, A2) - @test convert(Array, A1.'A2) ≈ convert(Array, A1).'convert(Array, A2) - @test convert(Array, A1'A2) ≈ convert(Array, A1)'convert(Array, A2) - @test convert(Array, A1*A2.') ≈ convert(Array, A1)*convert(Array, A2).' - @test convert(Array, A1*A2') ≈ convert(Array, A1)*convert(Array, A2)' - @test convert(Array, A1.'A2.') ≈ convert(Array, A1).'convert(Array, A2).' - @test convert(Array, A1'A2') ≈ convert(Array, A1)'convert(Array, A2)' - @test convert(Array, A1/A2) ≈ convert(Array, A1)/convert(Array, A2) - @test convert(Array, A1\A2) ≈ convert(Array, A1)\convert(Array, A2) + @test full(A1*A2) ≈ full(A1)*full(A2) + @test full(A1.'A2) ≈ full(A1).'full(A2) + @test full(A1'A2) ≈ full(A1)'full(A2) + @test full(A1*A2.') ≈ full(A1)*full(A2).' + @test full(A1*A2') ≈ full(A1)*full(A2)' + @test full(A1.'A2.') ≈ full(A1).'full(A2).' + @test full(A1'A2') ≈ full(A1)'full(A2)' + @test full(A1/A2) ≈ full(A1)/full(A2) + @test full(A1\A2) ≈ full(A1)\full(A2) @test_throws DimensionMismatch eye(n+1)/A2 @test_throws DimensionMismatch eye(n+1)/A2.' @test_throws DimensionMismatch eye(n+1)/A2' @@ -308,29 +308,29 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa if !(eltyB in (BigFloat, Complex{BigFloat})) # rand does not support BigFloat and Complex{BigFloat} as of Dec 2015 Tri = Tridiagonal(rand(eltyB,n-1),rand(eltyB,n),rand(eltyB,n-1)) - @test Base.LinAlg.A_mul_B!(Tri,copy(A1)) ≈ Tri*convert(Array, A1) + @test Base.LinAlg.A_mul_B!(Tri,copy(A1)) ≈ Tri*full(A1) end # Triangular-dense Matrix/vector multiplication - @test A1*B[:,1] ≈ convert(Array, A1)*B[:,1] - @test A1*B ≈ convert(Array, A1)*B - @test A1.'B[:,1] ≈ convert(Array, A1).'B[:,1] - @test A1'B[:,1] ≈ convert(Array, A1)'B[:,1] - @test A1.'B ≈ convert(Array, A1).'B - @test A1'B ≈ convert(Array, A1)'B - @test A1*B.' ≈ convert(Array, A1)*B.' - @test A1*B' ≈ convert(Array, A1)*B' - @test B*A1 ≈ B*convert(Array, A1) - @test B[:,1].'A1 ≈ B[:,1].'convert(Array, A1) - @test B[:,1]'A1 ≈ B[:,1]'convert(Array, A1) - @test B.'A1 ≈ B.'convert(Array, A1) - @test B'A1 ≈ B'convert(Array, A1) - @test B*A1.' ≈ B*convert(Array, A1).' - @test B*A1' ≈ B*convert(Array, A1)' - @test B[:,1].'A1.' ≈ B[:,1].'convert(Array, A1).' - @test B[:,1]'A1' ≈ B[:,1]'convert(Array, A1)' - @test B.'A1.' ≈ B.'convert(Array, A1).' - @test B'A1' ≈ B'convert(Array, A1)' + @test A1*B[:,1] ≈ full(A1)*B[:,1] + @test A1*B ≈ full(A1)*B + @test A1.'B[:,1] ≈ full(A1).'B[:,1] + @test A1'B[:,1] ≈ full(A1)'B[:,1] + @test A1.'B ≈ full(A1).'B + @test A1'B ≈ full(A1)'B + @test A1*B.' ≈ full(A1)*B.' + @test A1*B' ≈ full(A1)*B' + @test B*A1 ≈ B*full(A1) + @test B[:,1].'A1 ≈ B[:,1].'full(A1) + @test B[:,1]'A1 ≈ B[:,1]'full(A1) + @test B.'A1 ≈ B.'full(A1) + @test B'A1 ≈ B'full(A1) + @test B*A1.' ≈ B*full(A1).' + @test B*A1' ≈ B*full(A1)' + @test B[:,1].'A1.' ≈ B[:,1].'full(A1).' + @test B[:,1]'A1' ≈ B[:,1]'full(A1)' + @test B.'A1.' ≈ B.'full(A1).' + @test B'A1' ≈ B'full(A1)' if eltyB == elty1 @test A_mul_B!(zeros(B),A1,B) ≈ A1*B @@ -346,29 +346,29 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa @test_throws DimensionMismatch Base.LinAlg.A_mul_Bt!(ones(eltyB,n+1,n+1),A1) # ... and division - @test A1\B[:,1] ≈ convert(Array, A1)\B[:,1] - @test A1\B ≈ convert(Array, A1)\B - @test A1.'\B[:,1] ≈ convert(Array, A1).'\B[:,1] - @test A1'\B[:,1] ≈ convert(Array, A1)'\B[:,1] - @test A1.'\B ≈ convert(Array, A1).'\B - @test A1'\B ≈ convert(Array, A1)'\B - @test A1\B.' ≈ convert(Array, A1)\B.' - @test A1\B' ≈ convert(Array, A1)\B' - @test A1.'\B.' ≈ convert(Array, A1).'\B.' - @test A1'\B' ≈ convert(Array, A1)'\B' + @test A1\B[:,1] ≈ full(A1)\B[:,1] + @test A1\B ≈ full(A1)\B + @test A1.'\B[:,1] ≈ full(A1).'\B[:,1] + @test A1'\B[:,1] ≈ full(A1)'\B[:,1] + @test A1.'\B ≈ full(A1).'\B + @test A1'\B ≈ full(A1)'\B + @test A1\B.' ≈ full(A1)\B.' + @test A1\B' ≈ full(A1)\B' + @test A1.'\B.' ≈ full(A1).'\B.' + @test A1'\B' ≈ full(A1)'\B' @test_throws DimensionMismatch A1\ones(elty1,n+2) @test_throws DimensionMismatch A1'\ones(elty1,n+2) @test_throws DimensionMismatch A1.'\ones(elty1,n+2) if t1 == UpperTriangular || t1 == LowerTriangular @test_throws Base.LinAlg.SingularException naivesub!(t1(zeros(elty1,n,n)),ones(eltyB,n)) end - @test B/A1 ≈ B/convert(Array, A1) - @test B/A1.' ≈ B/convert(Array, A1).' - @test B/A1' ≈ B/convert(Array, A1)' - @test B.'/A1 ≈ B.'/convert(Array, A1) - @test B'/A1 ≈ B'/convert(Array, A1) - @test B.'/A1.' ≈ B.'/convert(Array, A1).' - @test B'/A1' ≈ B'/convert(Array, A1)' + @test B/A1 ≈ B/full(A1) + @test B/A1.' ≈ B/full(A1).' + @test B/A1' ≈ B/full(A1)' + @test B.'/A1 ≈ B.'/full(A1) + @test B'/A1 ≈ B'/full(A1) + @test B.'/A1.' ≈ B.'/full(A1).' + @test B'/A1' ≈ B'/full(A1)' # Error bounds !(elty1 in (BigFloat, Complex{BigFloat})) && !(eltyB in (BigFloat, Complex{BigFloat})) && errorbounds(A1, A1\B, B) @@ -403,8 +403,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) debug && println("Solve upper triangular system") Atri = UpperTriangular(lufact(A)[:U]) |> t -> eltya <: Complex && eltyb <: Real ? real(t) : t # Here the triangular matrix can't be too badly conditioned - b = convert(Matrix{eltyb}, eltya <: Complex ? convert(Array, Atri)*ones(n, 2) : convert(Array, Atri)*ones(n, 2)) - x = convert(Array, Atri) \ b + b = convert(Matrix{eltyb}, eltya <: Complex ? full(Atri)*ones(n, 2) : full(Atri)*ones(n, 2)) + x = full(Atri) \ b debug && println("Test error estimates") if eltya != BigFloat && eltyb != BigFloat @@ -431,8 +431,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) debug && println("Solve lower triangular system") Atri = UpperTriangular(lufact(A)[:U]) |> t -> eltya <: Complex && eltyb <: Real ? real(t) : t # Here the triangular matrix can't be too badly conditioned - b = convert(Matrix{eltyb}, eltya <: Complex ? convert(Array, Atri)*ones(n, 2) : convert(Array, Atri)*ones(n, 2)) - x = convert(Array, Atri)\b + b = convert(Matrix{eltyb}, eltya <: Complex ? full(Atri)*ones(n, 2) : full(Atri)*ones(n, 2)) + x = full(Atri)\b debug && println("Test error estimates") if eltya != BigFloat && eltyb != BigFloat diff --git a/test/linalg/tridiag.jl b/test/linalg/tridiag.jl index 7436882ff2fbe..53fb78a416fb4 100644 --- a/test/linalg/tridiag.jl +++ b/test/linalg/tridiag.jl @@ -42,7 +42,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) F[i,i+1] = du[i] F[i+1,i] = dl[i] end - @test convert(Array, T) == F + @test full(T) == F # elementary operations on tridiagonals @test conj(T) == Tridiagonal(conj(dl), conj(d), conj(du)) @@ -62,7 +62,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) @test Tridiagonal(dl, d, du) + Tridiagonal(du, d, dl) == SymTridiagonal(2d, dl+du) @test SymTridiagonal(d, dl) + Tridiagonal(dl, d, du) == Tridiagonal(dl + dl, d+d, dl+du) @test convert(SymTridiagonal,Tridiagonal(Ts)) == Ts - @test convert(Array, convert(SymTridiagonal{Complex64},Tridiagonal(Ts))) == convert(Matrix{Complex64},convert(Array, Ts)) + @test full(convert(SymTridiagonal{Complex64},Tridiagonal(Ts))) == convert(Matrix{Complex64},full(Ts)) if elty == Int vv = rand(1:100, n) BB = rand(1:100, n, 2) @@ -101,7 +101,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) # symmetric tridiagonal if elty <: Real Ts = SymTridiagonal(d, dl) - Fs = convert(Array, Ts) + Fs = full(Ts) Tldlt = factorize(Ts) @test_throws DimensionMismatch Tldlt\rand(elty,n+1) @test size(Tldlt) == size(Ts) @@ -119,7 +119,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) invFsv = Fs\vv x = Ts\vv @test x ≈ invFsv - @test convert(Array, convert(Array, Tldlt)) ≈ Fs + @test full(full(Tldlt)) ≈ Fs end # similar @@ -237,7 +237,7 @@ let n = 12 #Size of matrix problem to test @test_throws ArgumentError SymTridiagonal(rand(n,n)) A = SymTridiagonal(a, b) - fA = map(elty <: Complex ? Complex128 : Float64, convert(Array, A)) + fA = map(elty <: Complex ? Complex128 : Float64, full(A)) debug && println("getindex") @test_throws BoundsError A[n+1,1] @@ -289,10 +289,10 @@ let n = 12 #Size of matrix problem to test @test B - A == A - B debug && println("Multiplication with strided vector") - @test A*ones(n) ≈ convert(Array, A)*ones(n) + @test A*ones(n) ≈ full(A)*ones(n) debug && println("Multiplication with strided matrix") - @test A*ones(n, 2) ≈ convert(Array, A)*ones(n, 2) + @test A*ones(n, 2) ≈ full(A)*ones(n, 2) debug && println("Eigensystems") if elty <: Real @@ -312,13 +312,13 @@ let n = 12 #Size of matrix problem to test debug && println("stegr! call with index range") F = eigfact(SymTridiagonal(a, b),1:2) - fF = eigfact(Symmetric(convert(Array, SymTridiagonal(a, b))),1:2) + fF = eigfact(Symmetric(full(SymTridiagonal(a, b))),1:2) Test.test_approx_eq_modphase(F[:vectors], fF[:vectors]) @test F[:values] ≈ fF[:values] debug && println("stegr! call with value range") F = eigfact(SymTridiagonal(a, b),0.0,1.0) - fF = eigfact(Symmetric(convert(Array, SymTridiagonal(a, b))),0.0,1.0) + fF = eigfact(Symmetric(full(SymTridiagonal(a, b))),0.0,1.0) Test.test_approx_eq_modphase(F[:vectors], fF[:vectors]) @test F[:values] ≈ fF[:values] end @@ -332,15 +332,15 @@ let n = 12 #Size of matrix problem to test end B = SymTridiagonal(a, b) - fB = map(elty <: Complex ? Complex128 : Float64, convert(Array, B)) + fB = map(elty <: Complex ? Complex128 : Float64, full(B)) for op in (+, -, *) - @test convert(Array, op(A, B)) ≈ op(fA, fB) + @test full(op(A, B)) ≈ op(fA, fB) end α = rand(elty) - @test convert(Array, α*A) ≈ α*convert(Array, A) - @test convert(Array, A*α) ≈ convert(Array, A)*α - @test convert(Array, A/α) ≈ convert(Array, A)/α + @test full(α*A) ≈ α*full(A) + @test full(A*α) ≈ full(A)*α + @test full(A/α) ≈ full(A)/α debug && println("A_mul_B!") @test_throws DimensionMismatch A_mul_B!(zeros(elty,n,n),B,ones(elty,n+1,n)) @@ -363,7 +363,7 @@ let n = 12 #Size of matrix problem to test @test_throws ArgumentError Tridiagonal(a,a,a) A = Tridiagonal(a, b, c) - fA = map(elty <: Complex ? Complex128 : Float64, convert(Array, A)) + fA = map(elty <: Complex ? Complex128 : Float64, full(A)) debug && println("Similar, size, and copy!") B = similar(A) @@ -411,21 +411,22 @@ let n = 12 #Size of matrix problem to test end debug && println("Multiplication with strided vector") - @test A*ones(n) ≈ convert(Array, A)*ones(n) + @test A*ones(n) ≈ full(A)*ones(n) debug && println("Multiplication with strided matrix") - @test A*ones(n, 2) ≈ convert(Array, A)*ones(n, 2) + @test A*ones(n, 2) ≈ full(A)*ones(n, 2) + B = Tridiagonal(a, b, c) - fB = map(elty <: Complex ? Complex128 : Float64, convert(Array, B)) + fB = map(elty <: Complex ? Complex128 : Float64, full(B)) for op in (+, -, *) - @test convert(Array, op(A, B)) ≈ op(fA, fB) + @test full(op(A, B)) ≈ op(fA, fB) end α = rand(elty) - @test convert(Array, α*A) ≈ α*convert(Array, A) - @test convert(Array, A*α) ≈ convert(Array, A)*α - @test convert(Array, A/α) ≈ convert(Array, A)/α + @test full(α*A) ≈ α*full(A) + @test full(A*α) ≈ full(A)*α + @test full(A/α) ≈ full(A)/α @test_throws ArgumentError convert(SymTridiagonal{elty},A) diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index a07a38056489a..4d1c3e20d0ad8 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -76,10 +76,10 @@ let AA = randn(2, 2) else T = LowerTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == convert(Array, T) + J - @test J + T == J + convert(Array, T) - @test T - J == convert(Array, T) - J - @test J - T == J - convert(Array, T) + @test T + J == full(T) + J + @test J + T == J + full(T) + @test T - J == full(T) - J + @test J - T == J - full(T) @test T\I == inv(T) if atype == "Array" @@ -87,10 +87,10 @@ let AA = randn(2, 2) else T = LinAlg.UnitLowerTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == convert(Array, T) + J - @test J + T == J + convert(Array, T) - @test T - J == convert(Array, T) - J - @test J - T == J - convert(Array, T) + @test T + J == full(T) + J + @test J + T == J + full(T) + @test T - J == full(T) - J + @test J - T == J - full(T) @test T\I == inv(T) if atype == "Array" @@ -98,10 +98,10 @@ let AA = randn(2, 2) else T = UpperTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == convert(Array, T) + J - @test J + T == J + convert(Array, T) - @test T - J == convert(Array, T) - J - @test J - T == J - convert(Array, T) + @test T + J == full(T) + J + @test J + T == J + full(T) + @test T - J == full(T) - J + @test J - T == J - full(T) @test T\I == inv(T) if atype == "Array" @@ -109,10 +109,10 @@ let AA = randn(2, 2) else T = LinAlg.UnitUpperTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == convert(Array, T) + J - @test J + T == J + convert(Array, T) - @test T - J == convert(Array, T) - J - @test J - T == J - convert(Array, T) + @test T + J == full(T) + J + @test J + T == J + full(T) + @test T - J == full(T) - J + @test J - T == J - full(T) @test T\I == inv(T) @test I\A == A diff --git a/test/perf/threads/stockcorr/pstockcorr.jl b/test/perf/threads/stockcorr/pstockcorr.jl index 93d4bcf0864c2..076feb7cc58b2 100644 --- a/test/perf/threads/stockcorr/pstockcorr.jl +++ b/test/perf/threads/stockcorr/pstockcorr.jl @@ -78,7 +78,7 @@ function pstockcorr(n) SimulPriceB[1,:] = CurrentPrice[2] ## Generating the paths of stock prices by Geometric Brownian Motion - const UpperTriangle = convert(Array, chol(Corr)) # UpperTriangle Matrix by Cholesky decomposition + const UpperTriangle = full(chol(Corr)) # UpperTriangle Matrix by Cholesky decomposition # Optimization: pre-allocate these for performance # NOTE: the new GC will hopefully fix this, but currently GC time diff --git a/test/sparsedir/cholmod.jl b/test/sparsedir/cholmod.jl index f42afcf6d252b..f7b4461911a70 100644 --- a/test/sparsedir/cholmod.jl +++ b/test/sparsedir/cholmod.jl @@ -356,8 +356,8 @@ for elty in (Float64, Complex{Float64}) # Factor @test_throws ArgumentError cholfact(A1) - @test_throws Base.LinAlg.PosDefException cholfact(A1 + A1' - 2eigmax(convert(Array, A1 + A1'))I) - @test_throws Base.LinAlg.PosDefException cholfact(A1 + A1', shift=-2eigmax(convert(Array, A1 + A1'))) + @test_throws Base.LinAlg.PosDefException cholfact(A1 + A1' - 2eigmax(full(A1 + A1'))I) + @test_throws Base.LinAlg.PosDefException cholfact(A1 + A1', shift=-2eigmax(full(A1 + A1'))) @test_throws ArgumentError ldltfact(A1 + A1' - 2real(A1[1,1])I) @test_throws ArgumentError ldltfact(A1 + A1', shift=-2real(A1[1,1])) @test_throws ArgumentError cholfact(A1) @@ -372,19 +372,19 @@ for elty in (Float64, Complex{Float64}) @test F\CHOLMOD.Sparse(sparse(ones(elty, 5))) ≈ A1pd\ones(5) @test_throws DimensionMismatch F\CHOLMOD.Dense(ones(elty, 4)) @test_throws DimensionMismatch F\CHOLMOD.Sparse(sparse(ones(elty, 4))) - @test F'\ones(elty, 5) ≈ convert(Array, A1pd)'\ones(5) - @test F'\sparse(ones(elty, 5)) ≈ convert(Array, A1pd)'\ones(5) - @test logdet(F) ≈ logdet(convert(Array, A1pd)) + @test F'\ones(elty, 5) ≈ full(A1pd)'\ones(5) + @test F'\sparse(ones(elty, 5)) ≈ full(A1pd)'\ones(5) + @test logdet(F) ≈ logdet(full(A1pd)) @test det(F) == exp(logdet(F)) let # to test supernodal, we must use a larger matrix Ftmp = sprandn(100,100,0.1) Ftmp = Ftmp'Ftmp + I - @test logdet(cholfact(Ftmp)) ≈ logdet(convert(Array, Ftmp)) + @test logdet(cholfact(Ftmp)) ≈ logdet(full(Ftmp)) end - @test logdet(ldltfact(A1pd)) ≈ logdet(convert(Array, A1pd)) + @test logdet(ldltfact(A1pd)) ≈ logdet(full(A1pd)) @test isposdef(A1pd) @test !isposdef(A1) - @test !isposdef(A1 + A1' |> t -> t - 2eigmax(convert(Array, t))*I) + @test !isposdef(A1 + A1' |> t -> t - 2eigmax(full(t))*I) if elty <: Real @test CHOLMOD.issymmetric(Sparse(A1pd, 0)) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index e70122b5851a2..3666e96e9bc89 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -7,7 +7,7 @@ using Base.Test @test Base.SparseArrays.indtype(sparse(ones(Int8,2),ones(Int8,2),rand(2))) == Int8 # check sparse matrix construction -@test isequal(convert(Array, sparse(complex(ones(5,5),ones(5,5)))), complex(ones(5,5),ones(5,5))) +@test isequal(full(sparse(complex(ones(5,5),ones(5,5)))), complex(ones(5,5),ones(5,5))) @test_throws ArgumentError sparse([1,2,3], [1,2], [1,2,3], 3, 3) @test_throws ArgumentError sparse([1,2,3], [1,2,3], [1,2], 3, 3) @test_throws ArgumentError sparse([1,2,3], [1,2,3], [1,2,3], 0, 1) @@ -22,8 +22,8 @@ do33 = ones(3) @test isequal(se33 * se33, se33) # check sparse binary op -@test all(convert(Array, se33 + convert(SparseMatrixCSC{Float32,Int32}, se33)) == 2*eye(3)) -@test all(convert(Array, se33 * convert(SparseMatrixCSC{Float32,Int32}, se33)) == eye(3)) +@test all(full(se33 + convert(SparseMatrixCSC{Float32,Int32}, se33)) == 2*eye(3)) +@test all(full(se33 * convert(SparseMatrixCSC{Float32,Int32}, se33)) == eye(3)) # check horiz concatenation @test all([se33 se33] == sparse([1, 2, 3, 1, 2, 3], [1, 2, 3, 4, 5, 6], ones(6))) @@ -52,7 +52,7 @@ se33_i32 = speye(Int32, 3, 3) # check mixed sparse-dense concatenation sz33 = spzeros(3, 3) de33 = eye(3) -@test all([se33 de33; sz33 se33] == convert(Array, [se33 se33; sz33 se33 ])) +@test all([se33 de33; sz33 se33] == full([se33 se33; sz33 se33 ])) # check splicing + concatenation on # random instances, with nested vcat @@ -66,9 +66,9 @@ end a116 = copy(reshape(1:16, 4, 4)) s116 = sparse(a116) p = [4, 1, 2, 3, 2] -@test convert(Array, s116[p,:]) == a116[p,:] -@test convert(Array, s116[:,p]) == a116[:,p] -@test convert(Array, s116[p,p]) == a116[p,p] +@test full(s116[p,:]) == a116[p,:] +@test full(s116[:,p]) == a116[:,p] +@test full(s116[p,p]) == a116[p,p] # sparse assign p = [4, 1, 3] @@ -97,16 +97,16 @@ end for i = 1:5 a = sprand(10, 5, 0.5) b = rand(5) - @test maximum(abs(a*b - convert(Array, a)*b)) < 100*eps() + @test maximum(abs(a*b - full(a)*b)) < 100*eps() end # sparse matrix * BitArray A = sprand(5,5,0.2) B = trues(5) -@test A*B ≈ convert(Array, A)*B +@test A*B ≈ full(A)*B B = trues(5,5) -@test A*B ≈ convert(Array, A)*B -@test B*A ≈ B*convert(Array, A) +@test A*B ≈ full(A)*B +@test B*A ≈ B*full(A) # complex matrix-vector multiplication and left-division if Base.USE_GPL_LIBS @@ -117,91 +117,91 @@ for i = 1:5 d = randn(5) + im*randn(5) α = rand(Complex128) β = rand(Complex128) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(A_mul_B!(similar(b), a, b) - convert(Array, a)*b)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. - @test (maximum(abs(A_mul_B!(similar(c), a, c) - convert(Array, a)*c)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) - @test (maximum(abs((a'*c + d) - (convert(Array, a)'*c + d))) < 1000*eps()) - @test (maximum(abs((α*a.'*c + β*d) - (α*convert(Array, a).'*c + β*d))) < 1000*eps()) - @test (maximum(abs((a.'*c + d) - (convert(Array, a).'*c + d))) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(A_mul_B!(similar(b), a, b) - full(a)*b)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. + @test (maximum(abs(A_mul_B!(similar(c), a, c) - full(a)*c)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs((a'*c + d) - (full(a)'*c + d))) < 1000*eps()) + @test (maximum(abs((α*a.'*c + β*d) - (α*full(a).'*c + β*d))) < 1000*eps()) + @test (maximum(abs((a.'*c + d) - (full(a).'*c + d))) < 1000*eps()) c = randn(6) + im*randn(6) @test_throws DimensionMismatch α*a.'*c + β*c @test_throws DimensionMismatch α*a.'*ones(5) + β*c a = speye(5) + 0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2) b = randn(5,3) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + tril(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + tril(0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2)) b = randn(5,3) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2)) b = randn(5,3) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) a = spdiagm(randn(5)) + im*spdiagm(randn(5)) b = randn(5,3) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - convert(Array, a)*b)) < 100*eps()) - @test (maximum(abs(a'b - convert(Array, a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - convert(Array, a).'b)) < 100*eps()) - @test (maximum(abs(a\b - convert(Array, a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - convert(Array, a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - convert(Array, a.')\b)) < 1000*eps()) + @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) end end @@ -209,24 +209,24 @@ end for i = 1:5 a = sprand(10, 5, 0.7) b = sprand(5, 15, 0.3) - @test maximum(abs(a*b - convert(Array, a)*convert(Array, b))) < 100*eps() - @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:sortcols) - convert(Array, a)*convert(Array, b))) < 100*eps() - @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:doubletranspose) - convert(Array, a)*convert(Array, b))) < 100*eps() - @test convert(Array, kron(a,b)) == kron(convert(Array, a), convert(Array, b)) - @test convert(Array, kron(convert(Array, a),b)) == kron(convert(Array, a), convert(Array, b)) - @test convert(Array, kron(a,convert(Array, b))) == kron(convert(Array, a), convert(Array, b)) + @test maximum(abs(a*b - full(a)*full(b))) < 100*eps() + @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:sortcols) - full(a)*full(b))) < 100*eps() + @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:doubletranspose) - full(a)*full(b))) < 100*eps() + @test full(kron(a,b)) == kron(full(a), full(b)) + @test full(kron(full(a),b)) == kron(full(a), full(b)) + @test full(kron(a,full(b))) == kron(full(a), full(b)) c = sparse(rand(Float32,5,5)) d = sparse(rand(Float64,5,5)) - @test convert(Array, kron(c,d)) == kron(convert(Array, c),convert(Array, d)) + @test full(kron(c,d)) == kron(full(c),full(d)) f = Diagonal(rand(5)) - @test convert(Array, a*f) == convert(Array, a)*f - @test convert(Array, f*b) == f*convert(Array, b) + @test full(a*f) == full(a)*f + @test full(f*b) == f*full(b) end # scale and scale! sA = sprandn(3, 7, 0.5) sC = similar(sA) -dA = convert(Array, sA) +dA = full(sA) b = randn(7) @test dA * Diagonal(b) == sA * Diagonal(b) @test dA * Diagonal(b) == scale!(sC, sA, b) @@ -286,7 +286,7 @@ end # conj cA = sprandn(5,5,0.2) + im*sprandn(5,5,0.2) -@test convert(Array, conj(cA)) == conj(convert(Array, cA)) +@test full(conj(cA)) == conj(full(cA)) # Test SparseMatrixCSC [c]transpose[!] and permute[!] methods let smalldim = 5, largedim = 10, nzprob = 0.4 @@ -326,19 +326,19 @@ let smalldim = 5, largedim = 10, nzprob = 0.4 A = sprand(m, n, nzprob) At = transpose(A) # transpose[!] - fullAt = transpose(convert(Array, A)) + fullAt = transpose(full(A)) @test transpose(A) == fullAt @test transpose!(similar(At), A) == fullAt # ctranspose[!] C = A + im*A/2 - fullCh = ctranspose(convert(Array, C)) + fullCh = ctranspose(full(C)) @test ctranspose(C) == fullCh @test ctranspose!(similar(sparse(fullCh)), C) == fullCh # permute[!] p = randperm(m) q = randperm(n) - fullPAQ = convert(Array, A)[p,q] - @test permute(A, p, q) == sparse(convert(Array, A[p,q])) + fullPAQ = full(A)[p,q] + @test permute(A, p, q) == sparse(full(A[p,q])) @test permute!(similar(A), A, p, q) == fullPAQ @test permute!(similar(A), A, p, q, similar(At)) == fullPAQ @test permute!(copy(A), p, q) == fullPAQ @@ -349,19 +349,19 @@ end # transpose of SubArrays A = view(sprandn(10, 10, 0.3), 1:4, 1:4) -@test transpose(convert(Array, A)) == convert(Array, transpose(A)) -@test ctranspose(convert(Array, A)) == convert(Array, ctranspose(A)) +@test transpose(full(A)) == full(transpose(A)) +@test ctranspose(full(A)) == full(ctranspose(A)) # exp A = sprandn(5,5,0.2) -@test e.^A ≈ e.^convert(Array, A) +@test e.^A ≈ e.^full(A) # reductions pA = sparse(rand(3, 7)) for arr in (se33, sA, pA) for f in (sum, prod, minimum, maximum, var) - farr = convert(Array, arr) + farr = full(arr) @test f(arr) ≈ f(farr) @test f(arr, 1) ≈ f(farr, 1) @test f(arr, 2) ≈ f(farr, 2) @@ -399,7 +399,7 @@ for f in (sum, prod, minimum, maximum, var) end # spdiagm -@test convert(Array, spdiagm((ones(2), ones(2)), (0, -1), 3, 3)) == +@test full(spdiagm((ones(2), ones(2)), (0, -1), 3, 3)) == [1.0 0.0 0.0; 1.0 1.0 0.0; 0.0 1.0 0.0] # issue #4986, reinterpret @@ -417,7 +417,7 @@ K,J,V = findnz(SparseMatrixCSC(2,1,[1,3],[1,2],[1.0,0.0])) # https://groups.google.com/d/msg/julia-users/Yq4dh8NOWBQ/GU57L90FZ3EJ A = speye(Bool, 5) -@test find(A) == find(x -> x == true, A) == find(convert(Array, A)) +@test find(A) == find(x -> x == true, A) == find(full(A)) # issue #5824 @@ -463,16 +463,16 @@ end # Unary functions a = sprand(5,15, 0.5) -afull = convert(Array, a) +afull = full(a) for op in (:sin, :cos, :tan, :ceil, :floor, :abs, :abs2) @eval begin - @test ($op)(afull) == convert(Array, $(op)(a)) + @test ($op)(afull) == full($(op)(a)) end end for op in (:ceil, :floor) @eval begin - @test ($op)(Int,afull) == convert(Array, $(op)(Int,a)) + @test ($op)(Int,afull) == full($(op)(Int,a)) end end @@ -499,42 +499,42 @@ for (aa116, ss116) in [(a116, s116), (ad116, sd116)] @test ss116[:,:] == copy(ss116) # range indexing - @test convert(Array, ss116[i,:]) == aa116[i,:] - @test convert(Array, ss116[:,j]) == aa116[:,j] - @test convert(Array, ss116[i,1:2:end]) == aa116[i,1:2:end] - @test convert(Array, ss116[1:2:end,j]) == aa116[1:2:end,j] - @test convert(Array, ss116[i,end:-2:1]) == aa116[i,end:-2:1] - @test convert(Array, ss116[end:-2:1,j]) == aa116[end:-2:1,j] + @test full(ss116[i,:]) == aa116[i,:] + @test full(ss116[:,j]) == aa116[:,j] + @test full(ss116[i,1:2:end]) == aa116[i,1:2:end] + @test full(ss116[1:2:end,j]) == aa116[1:2:end,j] + @test full(ss116[i,end:-2:1]) == aa116[i,end:-2:1] + @test full(ss116[end:-2:1,j]) == aa116[end:-2:1,j] # float-range indexing is not supported # sorted vector indexing - @test convert(Array, ss116[i,[3:2:end-3;]]) == aa116[i,[3:2:end-3;]] - @test convert(Array, ss116[[3:2:end-3;],j]) == aa116[[3:2:end-3;],j] - @test convert(Array, ss116[i,[end-3:-2:1;]]) == aa116[i,[end-3:-2:1;]] - @test convert(Array, ss116[[end-3:-2:1;],j]) == aa116[[end-3:-2:1;],j] + @test full(ss116[i,[3:2:end-3;]]) == aa116[i,[3:2:end-3;]] + @test full(ss116[[3:2:end-3;],j]) == aa116[[3:2:end-3;],j] + @test full(ss116[i,[end-3:-2:1;]]) == aa116[i,[end-3:-2:1;]] + @test full(ss116[[end-3:-2:1;],j]) == aa116[[end-3:-2:1;],j] # unsorted vector indexing with repetition p = [4, 1, 2, 3, 2, 6] - @test convert(Array, ss116[p,:]) == aa116[p,:] - @test convert(Array, ss116[:,p]) == aa116[:,p] - @test convert(Array, ss116[p,p]) == aa116[p,p] + @test full(ss116[p,:]) == aa116[p,:] + @test full(ss116[:,p]) == aa116[:,p] + @test full(ss116[p,p]) == aa116[p,p] # bool indexing li = bitrand(size(aa116,1)) lj = bitrand(size(aa116,2)) - @test convert(Array, ss116[li,j]) == aa116[li,j] - @test convert(Array, ss116[li,:]) == aa116[li,:] - @test convert(Array, ss116[i,lj]) == aa116[i,lj] - @test convert(Array, ss116[:,lj]) == aa116[:,lj] - @test convert(Array, ss116[li,lj]) == aa116[li,lj] + @test full(ss116[li,j]) == aa116[li,j] + @test full(ss116[li,:]) == aa116[li,:] + @test full(ss116[i,lj]) == aa116[i,lj] + @test full(ss116[:,lj]) == aa116[:,lj] + @test full(ss116[li,lj]) == aa116[li,lj] # empty indices for empty in (1:0, Int[]) - @test convert(Array, ss116[empty,:]) == aa116[empty,:] - @test convert(Array, ss116[:,empty]) == aa116[:,empty] - @test convert(Array, ss116[empty,lj]) == aa116[empty,lj] - @test convert(Array, ss116[li,empty]) == aa116[li,empty] - @test convert(Array, ss116[empty,empty]) == aa116[empty,empty] + @test full(ss116[empty,:]) == aa116[empty,:] + @test full(ss116[:,empty]) == aa116[:,empty] + @test full(ss116[empty,lj]) == aa116[empty,lj] + @test full(ss116[li,empty]) == aa116[li,empty] + @test full(ss116[empty,empty]) == aa116[empty,empty] end # out of bounds indexing @@ -564,7 +564,7 @@ S1290 = SparseMatrixCSC(3, 3, UInt8[1,1,1,1], UInt8[], Int64[]) S1290[end] = 3 @test S1290[end] == (S1290[1] + S1290[2,2]) @test 6 == sum(diag(S1290)) - @test convert(Array, S1290)[[3,1],1] == convert(Array, S1290[[3,1],1]) + @test full(S1290)[[3,1],1] == full(S1290[[3,1],1]) # end @@ -716,8 +716,8 @@ let A = speye(Int, 5), I=1:10, X=reshape([trues(10); falses(15)],5,5) end let S = sprand(50, 30, 0.5, x->round(Int,rand(x)*100)), I = sprand(Bool, 50, 30, 0.2) - FS = convert(Array, S) - FI = convert(Array, I) + FS = full(S) + FI = full(I) @test sparse(FS[FI]) == S[I] == S[FI] @test sum(S[FI]) + sum(S[!FI]) == sum(S) @@ -841,7 +841,7 @@ let A = sprand(5,5,0.5,(n)->rand(Float64,n)), ACPY = copy(A) end # indmax, indmin, findmax, findmin -let S = sprand(100,80, 0.5), A = convert(Array, S) +let S = sprand(100,80, 0.5), A = full(S) @test indmax(S) == indmax(A) @test indmin(S) == indmin(A) @test findmin(S) == findmin(A) @@ -851,7 +851,7 @@ let S = sprand(100,80, 0.5), A = convert(Array, S) end end -let S = spzeros(10,8), A = convert(Array, S) +let S = spzeros(10,8), A = full(S) @test indmax(S) == indmax(A) == 1 @test indmin(S) == indmin(A) == 1 end @@ -1032,7 +1032,7 @@ end # issue #9917 @test sparse([]') == reshape(sparse([]), 1, 0) -@test convert(Array, sparse([])) == zeros(0) +@test full(sparse([])) == zeros(0) @test_throws BoundsError sparse([])[1] @test_throws BoundsError sparse([])[1] = 1 x = speye(100) @@ -1066,22 +1066,22 @@ end # test sparse constructors from special matrices T = Tridiagonal(randn(4),randn(5),randn(4)) S = sparse(T) -@test norm(convert(Array, T) - convert(Array, S)) == 0.0 +@test norm(full(T) - full(S)) == 0.0 T = SymTridiagonal(randn(5),rand(4)) S = sparse(T) -@test norm(convert(Array, T) - convert(Array, S)) == 0.0 +@test norm(full(T) - full(S)) == 0.0 B = Bidiagonal(randn(5),randn(4),true) S = sparse(B) -@test norm(convert(Array, B) - convert(Array, S)) == 0.0 +@test norm(full(B) - full(S)) == 0.0 B = Bidiagonal(randn(5),randn(4),false) S = sparse(B) -@test norm(convert(Array, B) - convert(Array, S)) == 0.0 +@test norm(full(B) - full(S)) == 0.0 # promotion in spdiagm @test spdiagm(([1,2],[3.5],[4+5im]), (0,1,-1), 2,2) == [1 3.5; 4+5im 2] #Test broadcasting of sparse matrixes -let A = sprand(10,10,0.3), B = sprand(10,10,0.3), CF = rand(10,10), AF = convert(Array, A), BF = convert(Array, B), C = sparse(CF) +let A = sprand(10,10,0.3), B = sprand(10,10,0.3), CF = rand(10,10), AF = full(A), BF = full(B), C = sparse(CF) @test A .* B == AF .* BF @test A[1,:] .* B == AF[1,:] .* BF @test A[:,1] .* B == AF[:,1] .* BF @@ -1173,18 +1173,18 @@ A = speye(5) @test size(similar(A,Complex128,Int8)) == (5,5) @test typeof(similar(A,Complex128,Int8)) == SparseMatrixCSC{Complex128,Int8} @test similar(A,Complex128,(6,6)) == spzeros(Complex128,6,6) -@test convert(Matrix,A) == convert(Array, A) +@test convert(Matrix,A) == full(A) # test float A = sprand(Bool, 5,5,0.0) @test eltype(float(A)) == Float64 # issue #11658 A = sprand(Bool, 5,5,0.2) -@test float(A) == float(convert(Array, A)) +@test float(A) == float(full(A)) # test sparsevec A = sparse(ones(5,5)) -@test all(convert(Array, sparsevec(A)) .== ones(25)) -@test all(convert(Array, sparsevec([1:5;],1)) .== ones(5)) +@test all(full(sparsevec(A)) .== ones(25)) +@test all(full(sparsevec([1:5;],1)) .== ones(5)) @test_throws ArgumentError sparsevec([1:5;], [1:4;]) #test sparse @@ -1262,11 +1262,11 @@ end # triu/tril A = sprand(5,5,0.2) -AF = convert(Array, A) -@test convert(Array, triu(A,1)) == triu(AF,1) -@test convert(Array, tril(A,1)) == tril(AF,1) -@test convert(Array, triu!(copy(A), 2)) == triu(AF,2) -@test convert(Array, tril!(copy(A), 2)) == tril(AF,2) +AF = full(A) +@test full(triu(A,1)) == triu(AF,1) +@test full(tril(A,1)) == tril(AF,1) +@test full(triu!(copy(A), 2)) == triu(AF,2) +@test full(tril!(copy(A), 2)) == tril(AF,2) @test_throws BoundsError tril(A,6) @test_throws BoundsError tril(A,-6) @test_throws BoundsError triu(A,6) @@ -1374,10 +1374,10 @@ nonzeros(A1)[2:5]=0 # UniformScaling A = sprandn(10,10,0.5) -@test A + I == convert(Array, A) + I -@test I + A == I + convert(Array, A) -@test A - I == convert(Array, A) - I -@test I - A == I - convert(Array, A) +@test A + I == full(A) + I +@test I + A == I + full(A) +@test A - I == full(A) - I +@test I - A == I - full(A) # Test error path if triplet vectors are not all the same length (#12177) @test_throws ArgumentError sparse([1,2,3], [1,2], [1,2,3], 3, 3) @@ -1405,15 +1405,15 @@ end Ac = sprandn(10,10,.1) + im* sprandn(10,10,.1) Ar = sprandn(10,10,.1) Ai = ceil(Int,Ar*100) -@test norm(Ac,1) ≈ norm(convert(Array, Ac),1) -@test norm(Ac,Inf) ≈ norm(convert(Array, Ac),Inf) -@test vecnorm(Ac) ≈ vecnorm(convert(Array, Ac)) -@test norm(Ar,1) ≈ norm(convert(Array, Ar),1) -@test norm(Ar,Inf) ≈ norm(convert(Array, Ar),Inf) -@test vecnorm(Ar) ≈ vecnorm(convert(Array, Ar)) -@test norm(Ai,1) ≈ norm(convert(Array, Ai),1) -@test norm(Ai,Inf) ≈ norm(convert(Array, Ai),Inf) -@test vecnorm(Ai) ≈ vecnorm(convert(Array, Ai)) +@test norm(Ac,1) ≈ norm(full(Ac),1) +@test norm(Ac,Inf) ≈ norm(full(Ac),Inf) +@test vecnorm(Ac) ≈ vecnorm(full(Ac)) +@test norm(Ar,1) ≈ norm(full(Ar),1) +@test norm(Ar,Inf) ≈ norm(full(Ar),Inf) +@test vecnorm(Ar) ≈ vecnorm(full(Ar)) +@test norm(Ai,1) ≈ norm(full(Ai),1) +@test norm(Ai,Inf) ≈ norm(full(Ai),Inf) +@test vecnorm(Ai) ≈ vecnorm(full(Ai)) # test sparse matrix cond A = sparse(reshape([1.0],1,1)) @@ -1422,10 +1422,10 @@ Ar = sprandn(20,20,.5) @test cond(A,1) == 1.0 # For a discussion of the tolerance, see #14778 if Base.USE_GPL_LIBS - @test 0.99 <= cond(Ar, 1) \ norm(Ar, 1) * norm(inv(convert(Array, Ar)), 1) < 3 - @test 0.99 <= cond(Ac, 1) \ norm(Ac, 1) * norm(inv(convert(Array, Ac)), 1) < 3 - @test 0.99 <= cond(Ar, Inf) \ norm(Ar, Inf) * norm(inv(convert(Array, Ar)), Inf) < 3 - @test 0.99 <= cond(Ac, Inf) \ norm(Ac, Inf) * norm(inv(convert(Array, Ac)), Inf) < 3 + @test 0.99 <= cond(Ar, 1) \ norm(Ar, 1) * norm(inv(full(Ar)), 1) < 3 + @test 0.99 <= cond(Ac, 1) \ norm(Ac, 1) * norm(inv(full(Ac)), 1) < 3 + @test 0.99 <= cond(Ar, Inf) \ norm(Ar, Inf) * norm(inv(full(Ar)), Inf) < 3 + @test 0.99 <= cond(Ac, Inf) \ norm(Ac, Inf) * norm(inv(full(Ac)), Inf) < 3 end @test_throws ArgumentError cond(A,2) @test_throws ArgumentError cond(A,3) @@ -1441,9 +1441,9 @@ Aci = ceil(Int64,100*sprand(20,20,.5))+ im*ceil(Int64,sprand(20,20,.5)) Ar = sprandn(20,20,.5) Ari = ceil(Int64,100*Ar) if Base.USE_GPL_LIBS - @test_approx_eq_eps Base.SparseArrays.normestinv(Ac,3) norm(inv(convert(Array, Ac)),1) 1e-4 - @test_approx_eq_eps Base.SparseArrays.normestinv(Aci,3) norm(inv(convert(Array, Aci)),1) 1e-4 - @test_approx_eq_eps Base.SparseArrays.normestinv(Ar) norm(inv(convert(Array, Ar)),1) 1e-4 + @test_approx_eq_eps Base.SparseArrays.normestinv(Ac,3) norm(inv(full(Ac)),1) 1e-4 + @test_approx_eq_eps Base.SparseArrays.normestinv(Aci,3) norm(inv(full(Aci)),1) 1e-4 + @test_approx_eq_eps Base.SparseArrays.normestinv(Ar) norm(inv(full(Ar)),1) 1e-4 @test_throws ArgumentError Base.SparseArrays.normestinv(Ac,0) @test_throws ArgumentError Base.SparseArrays.normestinv(Ac,21) end @@ -1474,32 +1474,32 @@ let @test typeof(min(A13024, B13024)) == SparseMatrixCSC{Bool,Int} for op in (+, -, &, |, $, max, min) - @test op(A13024, B13024) == op(convert(Array, A13024), convert(Array, B13024)) + @test op(A13024, B13024) == op(full(A13024), full(B13024)) end end let A = 2. * speye(5,5) - @test convert(Array, spones(A)) == eye(convert(Array, A)) + @test full(spones(A)) == eye(full(A)) end let A = spdiagm(rand(5)) + sprandn(5,5,0.2) + im*sprandn(5,5,0.2) A = A + A' - @test !Base.USE_GPL_LIBS || abs(det(factorize(Hermitian(A)))) ≈ abs(det(factorize(convert(Array, A)))) + @test !Base.USE_GPL_LIBS || abs(det(factorize(Hermitian(A)))) ≈ abs(det(factorize(full(A)))) A = spdiagm(rand(5)) + sprandn(5,5,0.2) + im*sprandn(5,5,0.2) A = A*A' - @test !Base.USE_GPL_LIBS || abs(det(factorize(Hermitian(A)))) ≈ abs(det(factorize(convert(Array, A)))) + @test !Base.USE_GPL_LIBS || abs(det(factorize(Hermitian(A)))) ≈ abs(det(factorize(full(A)))) A = spdiagm(rand(5)) + sprandn(5,5,0.2) A = A + A.' - @test !Base.USE_GPL_LIBS || abs(det(factorize(Symmetric(A)))) ≈ abs(det(factorize(convert(Array, A)))) + @test !Base.USE_GPL_LIBS || abs(det(factorize(Symmetric(A)))) ≈ abs(det(factorize(full(A)))) A = spdiagm(rand(5)) + sprandn(5,5,0.2) A = A*A.' - @test !Base.USE_GPL_LIBS || abs(det(factorize(Symmetric(A)))) ≈ abs(det(factorize(convert(Array, A)))) + @test !Base.USE_GPL_LIBS || abs(det(factorize(Symmetric(A)))) ≈ abs(det(factorize(full(A)))) @test factorize(triu(A)) == triu(A) @test isa(factorize(triu(A)), UpperTriangular{Float64, SparseMatrixCSC{Float64, Int}}) @test factorize(tril(A)) == tril(A) @test isa(factorize(tril(A)), LowerTriangular{Float64, SparseMatrixCSC{Float64, Int}}) - @test !Base.USE_GPL_LIBS || factorize(A[:,1:4])\ones(size(A,1)) ≈ convert(Array, A[:,1:4])\ones(size(A,1)) + @test !Base.USE_GPL_LIBS || factorize(A[:,1:4])\ones(size(A,1)) ≈ full(A[:,1:4])\ones(size(A,1)) @test_throws ErrorException chol(A) @test_throws ErrorException lu(A) @test_throws ErrorException eig(A) @@ -1532,12 +1532,12 @@ let @test issparse(LinAlg.UnitLowerTriangular(m)) @test issparse(UpperTriangular(m)) @test issparse(LinAlg.UnitUpperTriangular(m)) - @test issparse(Symmetric(convert(Array, m))) == false - @test issparse(Hermitian(convert(Array, m))) == false - @test issparse(LowerTriangular(convert(Array, m))) == false - @test issparse(LinAlg.UnitLowerTriangular(convert(Array, m))) == false - @test issparse(UpperTriangular(convert(Array, m))) == false - @test issparse(LinAlg.UnitUpperTriangular(convert(Array, m))) == false + @test issparse(Symmetric(full(m))) == false + @test issparse(Hermitian(full(m))) == false + @test issparse(LowerTriangular(full(m))) == false + @test issparse(LinAlg.UnitLowerTriangular(full(m))) == false + @test issparse(UpperTriangular(full(m))) == false + @test issparse(LinAlg.UnitUpperTriangular(full(m))) == false end let diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index 2e05de46b80b7..62a7649cc1e4d 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -26,11 +26,11 @@ let x = spv_x1 @test nonzeros(x) == [1.25, -0.75, 3.5] end -# convert(Array, _) +# full for (x, xf) in [(spv_x1, x1_full)] - @test isa(convert(Array, x), Vector{Float64}) - @test convert(Array, x) == xf + @test isa(full(x), Vector{Float64}) + @test full(x) == xf end ### Show @@ -173,7 +173,7 @@ let x = sprand(100, 0.5) r = x[I] @test isa(r, SparseVector{Float64,Int}) @test all(nonzeros(r) .!= 0.0) - @test convert(Array, r) == convert(Array, x)[I] + @test full(r) == full(x)[I] end # setindex @@ -369,11 +369,11 @@ let x = spv_x1, xf = x1_full xm = convert(SparseMatrixCSC, x) @test isa(xm, SparseMatrixCSC{Float64,Int}) - @test convert(Array, xm) == reshape(xf, 8, 1) + @test full(xm) == reshape(xf, 8, 1) xm = convert(SparseMatrixCSC{Float32}, x) @test isa(xm, SparseMatrixCSC{Float32,Int}) - @test convert(Array, xm) == reshape(convert(Vector{Float32}, xf), 8, 1) + @test full(xm) == reshape(convert(Vector{Float32}, xf), 8, 1) end @@ -393,22 +393,22 @@ let m = 80, n = 100 @test nnz(H) == tnnz Hr = zeros(m, n) for j = 1:n - Hr[:,j] = convert(Array, A[j]) + Hr[:,j] = full(A[j]) end - @test convert(Array, H) == Hr + @test full(H) == Hr V = vcat(A...) @test isa(V, SparseVector{Float64,Int}) @test length(V) == m * n Vr = vec(Hr) - @test convert(Array, V) == Vr + @test full(V) == Vr end ## sparsemat: combinations with sparse matrix let S = sprand(4, 8, 0.5) - Sf = convert(Array, S) + Sf = full(S) @assert isa(Sf, Matrix{Float64}) # get a single column @@ -416,84 +416,84 @@ let S = sprand(4, 8, 0.5) col = S[:, j] @test isa(col, SparseVector{Float64,Int}) @test length(col) == size(S,1) - @test convert(Array, col) == Sf[:,j] + @test full(col) == Sf[:,j] end # Get a reshaped vector v = S[:] @test isa(v, SparseVector{Float64,Int}) @test length(v) == length(S) - @test convert(Array, v) == Sf[:] + @test full(v) == Sf[:] # Get a linear subset for i=0:length(S) v = S[1:i] @test isa(v, SparseVector{Float64,Int}) @test length(v) == i - @test convert(Array, v) == Sf[1:i] + @test full(v) == Sf[1:i] end for i=1:length(S)+1 v = S[i:end] @test isa(v, SparseVector{Float64,Int}) @test length(v) == length(S) - i + 1 - @test convert(Array, v) == Sf[i:end] + @test full(v) == Sf[i:end] end for i=0:div(length(S),2) v = S[1+i:end-i] @test isa(v, SparseVector{Float64,Int}) @test length(v) == length(S) - 2i - @test convert(Array, v) == Sf[1+i:end-i] + @test full(v) == Sf[1+i:end-i] end end let r = [1,10], S = sparse(r, r, r) - Sf = convert(Array, S) + Sf = full(S) @assert isa(Sf, Matrix{Int}) inds = [1,1,1,1,1,1] v = S[inds] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(inds) - @test convert(Array, v) == Sf[inds] + @test full(v) == Sf[inds] inds = [2,2,2,2,2,2] v = S[inds] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(inds) - @test convert(Array, v) == Sf[inds] + @test full(v) == Sf[inds] # get a single column for j = 1:size(S,2) col = S[:, j] @test isa(col, SparseVector{Int,Int}) @test length(col) == size(S,1) - @test convert(Array, col) == Sf[:,j] + @test full(col) == Sf[:,j] end # Get a reshaped vector v = S[:] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(S) - @test convert(Array, v) == Sf[:] + @test full(v) == Sf[:] # Get a linear subset for i=0:length(S) v = S[1:i] @test isa(v, SparseVector{Int,Int}) @test length(v) == i - @test convert(Array, v) == Sf[1:i] + @test full(v) == Sf[1:i] end for i=1:length(S)+1 v = S[i:end] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(S) - i + 1 - @test convert(Array, v) == Sf[i:end] + @test full(v) == Sf[i:end] end for i=0:div(length(S),2) v = S[1+i:end-i] @test isa(v, SparseVector{Int,Int}) @test length(v) == length(S) - 2i - @test convert(Array, v) == Sf[1+i:end-i] + @test full(v) == Sf[1+i:end-i] end end @@ -502,10 +502,10 @@ end ### Data rnd_x0 = sprand(50, 0.6) -rnd_x0f = convert(Array, rnd_x0) +rnd_x0f = full(rnd_x0) rnd_x1 = sprand(50, 0.7) * 4.0 -rnd_x1f = convert(Array, rnd_x1) +rnd_x1f = full(rnd_x1) spv_x1 = SparseVector(8, [2, 5, 6], [1.25, -0.75, 3.5]) spv_x2 = SparseVector(8, [1, 2, 6, 7], [3.25, 4.0, -5.5, -6.0]) @@ -532,10 +532,10 @@ let x = spv_x1, x2 = x2 = spv_x2 @test exact_equal(x - x2, xb) @test exact_equal(x2 - x, -xb) - @test convert(Array, x) + x2 == convert(Array, xa) - @test convert(Array, x) - x2 == convert(Array, xb) - @test x + convert(Array, x2) == convert(Array, xa) - @test x - convert(Array, x2) == convert(Array, xb) + @test full(x) + x2 == full(xa) + @test full(x) - x2 == full(xb) + @test x + full(x2) == full(xa) + @test x - full(x2) == full(xb) # multiplies xm = SparseVector(8, [2, 6], [5.0, -19.25]) @@ -543,8 +543,8 @@ let x = spv_x1, x2 = x2 = spv_x2 @test exact_equal(x .* x2, xm) @test exact_equal(x2 .* x, xm) - @test convert(Array, x) .* x2 == convert(Array, xm) - @test x .* convert(Array, x2) == convert(Array, xm) + @test full(x) .* x2 == full(xm) + @test x .* full(x2) == full(xm) # max & min @test exact_equal(max(x, x), x) @@ -584,7 +584,7 @@ function check_nz2z_z2z{T}(f::Function, x::SparseVector{T}, xf::Vector{T}) isa(r, AbstractSparseVector) || error("$f(x) is not a sparse vector.") eltype(r) == R || error("$f(x) results in eltype = $(eltype(r)), expect $R") all(r.nzval .!= 0) || error("$f(x) contains zeros in nzval.") - convert(Array, r) == f(xf) || error("Incorrect results found in $f(x).") + full(r) == f(xf) || error("Incorrect results found in $f(x).") end for f in [floor, ceil, trunc, round] @@ -671,14 +671,14 @@ end ### BLAS Level-1 let x = sprand(16, 0.5), x2 = sprand(16, 0.4) - xf = convert(Array, x) - xf2 = convert(Array, x2) + xf = full(x) + xf2 = full(x2) # axpy! for c in [1.0, -1.0, 2.0, -2.0] - y = convert(Array, x) + y = full(x) @test is(Base.axpy!(c, x2, y), y) - @test y == convert(Array, x2 * c + x) + @test y == full(x2 * c + x) end # scale @@ -704,15 +704,15 @@ let x = sprand(16, 0.5), x2 = sprand(16, 0.4) @test dot(x2, x2) == sumabs2(x2) @test dot(x, x2) ≈ dv @test dot(x2, x) ≈ dv - @test dot(convert(Array, x), x2) ≈ dv - @test dot(x, convert(Array, x2)) ≈ dv + @test dot(full(x), x2) ≈ dv + @test dot(x, full(x2)) ≈ dv end end let x = complex(sprand(32, 0.6), sprand(32, 0.6)), y = complex(sprand(32, 0.6), sprand(32, 0.6)) - xf = convert(Array, x)::Vector{Complex128} - yf = convert(Array, y)::Vector{Complex128} + xf = full(x)::Vector{Complex128} + yf = full(y)::Vector{Complex128} @test dot(x, x) ≈ dot(xf, xf) @test dot(x, y) ≈ dot(xf, yf) end @@ -723,7 +723,7 @@ end ## dense A * sparse x -> dense y let A = randn(9, 16), x = sprand(16, 0.7) - xf = convert(Array, x) + xf = full(x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) rr = α*A*xf + β*y @@ -736,7 +736,7 @@ let A = randn(9, 16), x = sprand(16, 0.7) end let A = randn(16, 9), x = sprand(16, 0.7) - xf = convert(Array, x) + xf = full(x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) rr = α*A'xf + β*y @@ -751,8 +751,8 @@ end ## sparse A * sparse x -> dense y let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7) - Af = convert(Array, A) - xf = convert(Array, x) + Af = full(A) + xf = full(x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) rr = α*Af*xf + β*y @@ -765,8 +765,8 @@ let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7) end let A = sprandn(16, 9, 0.5), x = sprand(16, 0.7) - Af = convert(Array, A) - xf = convert(Array, x) + Af = full(A) + xf = full(x) for α in [0.0, 1.0, 2.0], β in [0.0, 0.5, 1.0] y = rand(9) rr = α*Af'xf + β*y @@ -781,9 +781,9 @@ end let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), x = complex(sprandn(8, 0.6), sprandn(8, 0.6)), x2 = complex(sprandn(7, 0.75), sprandn(7, 0.75)) - Af = convert(Array, A) - xf = convert(Array, x) - x2f = convert(Array, x2) + Af = full(A) + xf = full(x) + x2f = full(x2) @test SparseArrays.densemv(A, x; trans='N') ≈ Af * xf @test SparseArrays.densemv(A, x2; trans='T') ≈ Af.' * x2f @test SparseArrays.densemv(A, x2; trans='C') ≈ Af'x2f @@ -792,39 +792,39 @@ end ## sparse A * sparse x -> sparse y let A = sprandn(9, 16, 0.5), x = sprand(16, 0.7), x2 = sprand(9, 0.7) - Af = convert(Array, A) - xf = convert(Array, x) - x2f = convert(Array, x2) + Af = full(A) + xf = full(x) + x2f = full(x2) y = A*x @test isa(y, SparseVector{Float64,Int}) @test all(nonzeros(y) .!= 0.0) - @test convert(Array, y) ≈ Af * xf + @test full(y) ≈ Af * xf y = At_mul_B(A, x2) @test isa(y, SparseVector{Float64,Int}) @test all(nonzeros(y) .!= 0.0) - @test convert(Array, y) ≈ Af'x2f + @test full(y) ≈ Af'x2f end let A = complex(sprandn(7, 8, 0.5), sprandn(7, 8, 0.5)), x = complex(sprandn(8, 0.6), sprandn(8, 0.6)), x2 = complex(sprandn(7, 0.75), sprandn(7, 0.75)) - Af = convert(Array, A) - xf = convert(Array, x) - x2f = convert(Array, x2) + Af = full(A) + xf = full(x) + x2f = full(x2) y = A*x @test isa(y, SparseVector{Complex128,Int}) - @test convert(Array, y) ≈ Af * xf + @test full(y) ≈ Af * xf y = At_mul_B(A, x2) @test isa(y, SparseVector{Complex128,Int}) - @test convert(Array, y) ≈ Af.' * x2f + @test full(y) ≈ Af.' * x2f y = Ac_mul_B(A, x2) @test isa(y, SparseVector{Complex128,Int}) - @test convert(Array, y) ≈ Af'x2f + @test full(y) ≈ Af'x2f end # left-division operations involving triangular matrices and sparse vectors (#14005) @@ -965,7 +965,7 @@ end sv = sparse(1:10) sm = convert(SparseMatrixCSC, sv) sv[1] = 0 -@test convert(Array, sm)[2:end] == collect(2:10) +@test full(sm)[2:end] == collect(2:10) # Ensure that sparsevec with all-zero values returns an array of zeros @test sparsevec([1,2,3],[0,0,0]) == [0,0,0] @@ -991,8 +991,8 @@ s14013 = sparse([10.0 0.0 30.0; 0.0 1.0 0.0]) a14013 = [10.0 0.0 30.0; 0.0 1.0 0.0] @test s14013 == a14013 @test vec(s14013) == s14013[:] == a14013[:] -@test convert(Array, s14013)[1,:] == s14013[1,:] == a14013[1,:] == [10.0, 0.0, 30.0] -@test convert(Array, s14013)[2,:] == s14013[2,:] == a14013[2,:] == [0.0, 1.0, 0.0] +@test full(s14013)[1,:] == s14013[1,:] == a14013[1,:] == [10.0, 0.0, 30.0] +@test full(s14013)[2,:] == s14013[2,:] == a14013[2,:] == [0.0, 1.0, 0.0] # Issue 14046 s14046 = sprand(5, 1.0) @@ -1024,9 +1024,9 @@ for Tv in [Float32, Float64, Int64, Int32, Complex128] sparr = Sp(arr) fillval = rand(Tv) fill!(sparr, fillval) - @test convert(Array, sparr) == fillval * ones(arr) + @test full(sparr) == fillval * ones(arr) fill!(sparr, 0) - @test convert(Array, sparr) == zeros(arr) + @test full(sparr) == zeros(arr) end end end diff --git a/test/sparsedir/spqr.jl b/test/sparsedir/spqr.jl index ef15ea35db0b8..5411dd503f7a7 100644 --- a/test/sparsedir/spqr.jl +++ b/test/sparsedir/spqr.jl @@ -25,8 +25,8 @@ for eltyA in (Float64, Complex{Float64}) end @inferred A\B - @test A\B[:,1] ≈ convert(Array, A)\B[:,1] - @test A\B ≈ convert(Array, A)\B + @test A\B[:,1] ≈ full(A)\B[:,1] + @test A\B ≈ full(A)\B @test_throws DimensionMismatch A\B[1:m-1,:] @test A[1:9,:]*(A[1:9,:]\ones(eltyB, 9)) ≈ ones(9) # Underdetermined system diff --git a/test/sparsedir/umfpack.jl b/test/sparsedir/umfpack.jl index 9c609ab7969c4..6db636a20353e 100644 --- a/test/sparsedir/umfpack.jl +++ b/test/sparsedir/umfpack.jl @@ -23,7 +23,7 @@ for Tv in (Float64, Complex128) L,U,p,q,Rs = lua[:(:)] @test (Diagonal(Rs) * A)[p,q] ≈ L * U - det(lua) ≈ det(convert(Array, A)) + det(lua) ≈ det(full(A)) b = [8., 45., -3., 3., 19.] x = lua\b From d1841c252f4b815a07abdba40788300270469105 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Fri, 22 Jul 2016 17:35:57 -0400 Subject: [PATCH 0594/1117] improve #17546 for a[...] .= handling of arrays of arrays and dictionaries of arrays --- base/broadcast.jl | 26 +++++++++++++++++++++++++- src/julia-syntax.scm | 2 +- test/broadcast.jl | 10 ++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index d577857b62937..4c2a881489dd1 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -5,7 +5,7 @@ module Broadcast using Base.Cartesian using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, tail, OneTo, to_shape import Base: .+, .-, .*, ./, .\, .//, .==, .<, .!=, .<=, .÷, .%, .<<, .>>, .^ -export broadcast, broadcast!, bitbroadcast +export broadcast, broadcast!, bitbroadcast, dotview export broadcast_getindex, broadcast_setindex! ## Broadcasting utilities ## @@ -437,4 +437,28 @@ for (sigA, sigB) in ((BitArray, BitArray), end end +############################################################ + +# x[...] .= f.(y...) ---> broadcast!(f, dotview(x, ...), y...). +# The dotview function defaults to view, but we override it in +# a few cases to get the expected in-place behavior without affecting +# explicit calls to view. (All of this can go away if slices +# are changed to generate views by default.) + +dotview(args...) = view(args...) +# avoid splatting penalty in common cases: +for nargs = 0:5 + args = Symbol[Symbol("x",i) for i = 1:nargs] + eval(Expr(:(=), Expr(:call, :dotview, args...), Expr(:call, :view, args...))) +end + +# for a[i...] .= ... where a is an array-of-arrays, just pass a[i...] directly +# to broadcast! +dotview{T<:AbstractArray,N,I<:Integer}(a::AbstractArray{T,N}, i::Vararg{I,N}) = + a[i...] + +# dict[k] .= ... should work if dict[k] is an array +dotview(a::Associative, k) = a[k] +dotview(a::Associative, k1, k2, ks...) = a[tuple(k1,k2,ks...)] + end # module diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index e98029ebe1487..a8e3e704d114b 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1549,7 +1549,7 @@ (let* ((ex (partially-expand-ref expr)) (stmts (butlast (cdr ex))) (refex (last (cdr ex))) - (nuref `(call (top view) ,(caddr refex) ,@(cdddr refex)))) + (nuref `(call (top dotview) ,(caddr refex) ,@(cdddr refex)))) `(block ,@stmts ,nuref)) expr)) diff --git a/test/broadcast.jl b/test/broadcast.jl index 52aeacc859041..70b2b39beb3fc 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -274,6 +274,16 @@ let x = [1:4;], y = x x[2:end] .= 1:3 @test y === x == [0,1,2,3] end +let a = [[4, 5], [6, 7]] + a[1] .= 3 + @test a == [[3, 3], [6, 7]] +end +let d = Dict(:foo => [1,3,7], (3,4) => [5,9]) + d[:foo] .+= 2 + @test d[:foo] == [3,5,9] + d[3,4] .-= 1 + @test d[3,4] == [4,8] +end # PR 16988 @test Base.promote_op(+, Bool) === Int From 76cca72512d838aec4a39ff8f3b3e5314a0ad78f Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 22 Jul 2016 18:25:29 -0400 Subject: [PATCH 0595/1117] remove REPL frames from interactive backtrace printing again We've had this behavior in previous versions. Also simplifies the backtrace processing code a bit. --- base/REPL.jl | 15 +++++++++++++++ base/replutil.jl | 25 ++++++------------------- base/task.jl | 2 +- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/base/REPL.jl b/base/REPL.jl index e7e29bb82a2c5..edc2109516dd5 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -95,9 +95,24 @@ function start_repl_backend(repl_channel::Channel, response_channel::Channel) backend end +function ip_matches_func(ip, func::Symbol) + for fr in StackTraces.lookup(ip) + if fr === StackTraces.UNKNOWN || fr.from_c + return false + end + fr.func === func && return true + end + return false +end + function display_error(io::IO, er, bt) Base.with_output_color(:red, io) do io print(io, "ERROR: ") + # remove REPL-related frames from interactive printing + eval_ind = findlast(addr->ip_matches_func(addr, :eval), bt) + if eval_ind != 0 + bt = bt[1:eval_ind-1] + end Base.showerror(io, er, bt) end end diff --git a/base/replutil.jl b/base/replutil.jl index b3be4ce49dc0d..54be916eb01e6 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -565,32 +565,20 @@ function show_trace_entry(io, frame, n) n > 1 && print(io, " (repeats ", n, " times)") end -function show_backtrace(io::IO, t::Vector, set=1:typemax(Int)) - # we may not declare :eval_user_input - # directly so that we get a compile error - # in case its name changes in the future - show_backtrace(io, - try - typeof(eval_user_input).name.mt.name - catch - :(:) #for when client.jl is not yet defined - end, t, set) -end - -function show_backtrace(io::IO, top_function::Symbol, t::Vector, set) +function show_backtrace(io::IO, t::Vector) process_entry(last_frame, n) = show_trace_entry(io, last_frame, n) - process_backtrace(process_entry, top_function, t, set) + process_backtrace(process_entry, t) end -function show_backtrace(io::IO, top_function::Symbol, t::Vector{Any}, set) +function show_backtrace(io::IO, t::Vector{Any}) for entry in t show_trace_entry(io, entry...) end end -# process the backtrace, up to (but not including) top_function -function process_backtrace(process_func::Function, top_function::Symbol, t::Vector, set; skipC = true) +# call process_func on each frame in a backtrace +function process_backtrace(process_func::Function, t::Vector, limit::Int=typemax(Int); skipC = true) n = 0 last_frame = StackTraces.UNKNOWN count = 0 @@ -603,9 +591,8 @@ function process_backtrace(process_func::Function, top_function::Symbol, t::Vect if lkup.from_c && skipC; continue; end if i == 1 && lkup.func == :error; continue; end - if lkup.func == top_function; break; end count += 1 - if !in(count, set); continue; end + if count > limit; break; end if lkup.file != last_frame.file || lkup.line != last_frame.line || lkup.func != last_frame.func if n > 0 diff --git a/base/task.jl b/base/task.jl index 00d546f705b04..148b77861cc58 100644 --- a/base/task.jl +++ b/base/task.jl @@ -14,7 +14,7 @@ type CapturedException <: Exception # Process bt_raw so that it can be safely serialized bt_lines = Any[] process_func(args...) = push!(bt_lines, args) - process_backtrace(process_func, :(:), bt_raw, 1:100) # Limiting this to 100 lines. + process_backtrace(process_func, bt_raw, 100) # Limiting this to 100 lines. new(ex, bt_lines) end From 2b2c217ec78f76aa01567cc5f1d9f2435f22aa1f Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Fri, 22 Jul 2016 19:45:06 -0400 Subject: [PATCH 0596/1117] documentation for besselhx --- base/docs/helpdb/Base.jl | 8 -------- base/special/bessel.jl | 24 ++++++++++++++++++++++++ doc/stdlib/math.rst | 12 ++++++++++-- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index c1478f81559f9..800c321e63aae 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4793,14 +4793,6 @@ Airy function derivative ``\\operatorname{Ai}'(x)``. """ airyaiprime -""" - besselh(nu, k, x) - -Bessel function of the third kind of order `nu` (Hankel function). `k` is either 1 or 2, -selecting `hankelh1` or `hankelh2`, respectively. -""" -besselh - """ prepend!(collection, items) -> collection diff --git a/base/special/bessel.jl b/base/special/bessel.jl index 3728244dd3f1b..286770488759e 100644 --- a/base/special/bessel.jl +++ b/base/special/bessel.jl @@ -190,6 +190,15 @@ function _bessely(nu::Float64, z::Complex128, kode::Int32) end end +""" + besselh(nu, [k=1,] x) + +Bessel function of the third kind of order `nu` (the Hankel function). `k` is either 1 or 2, +selecting `hankelh1` or `hankelh2`, respectively. `k` defaults to 1 if it is omitted. +(See also [`besselhx`](:func:`besselhx`) for an exponentially scaled variant.) +""" +function besselh end + function besselh(nu::Float64, k::Integer, z::Complex128) if nu < 0 s = (k == 1) ? 1 : -1 @@ -198,6 +207,21 @@ function besselh(nu::Float64, k::Integer, z::Complex128) return _besselh(nu,Int32(k),z,Int32(1)) end +""" + besselhx(nu, [k=1,] z) + +Compute the scaled Hankel function ``\\exp(∓iz) H_ν^{(k)}(z)``, where +``k`` is 1 or 2, ``H_ν^{(k)}(z)`` is `besselh(nu, k, z)`, and ``∓`` is +``-`` for ``k=1`` and ``+`` for ``k=2``. `k` defaults to 1 if it is omitted. + +The reason for this function is that ``H_ν^{(k)}(z)`` is asymptotically +proportional to ``\\exp(∓iz)/\\sqrt{z}`` for large ``|z|``, and so the +[`besselh`](:func:`besselh`) function is susceptible to overflow or underflow +when `z` has a large imaginary part. The `besselhx` function cancels this +exponential factor (analytically), so it avoids these problems. +""" +function besselhx end + function besselhx(nu::Float64, k::Integer, z::Complex128) if nu < 0 s = (k == 1) ? 1 : -1 diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 0849b3c96e0a9..b411d4e0da5ed 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1480,11 +1480,19 @@ Mathematical Functions Scaled Bessel function of the third kind of order ``nu``\ , :math:`H^{(2)}_\nu(x) e^{x i}`\ . -.. function:: besselh(nu, k, x) +.. function:: besselh(nu, [k=1,] x) .. Docstring generated from Julia source - Bessel function of the third kind of order ``nu`` (Hankel function). ``k`` is either 1 or 2, selecting ``hankelh1`` or ``hankelh2``\ , respectively. + Bessel function of the third kind of order ``nu`` (the Hankel function). ``k`` is either 1 or 2, selecting ``hankelh1`` or ``hankelh2``\ , respectively. ``k`` defaults to 1 if it is omitted. (See also :func:`besselhx` for an exponentially scaled variant.) + +.. function:: besselhx(nu, [k=1,] z) + + .. Docstring generated from Julia source + + Compute the scaled Hankel function :math:`\exp(∓iz) H_ν^{(k)}(z)`\ , where :math:`k` is 1 or 2, :math:`H_ν^{(k)}(z)` is ``besselh(nu, k, z)``\ , and :math:`∓` is :math:`-` for :math:`k=1` and :math:`+` for :math:`k=2`\ . ``k`` defaults to 1 if it is omitted. + + The reason for this function is that :math:`H_ν^{(k)}(z)` is asymptotically proportional to :math:`\exp(∓iz)/\sqrt{z}` for large :math:`|z|`\ , and so the :func:`besselh` function is susceptible to overflow or underflow when ``z`` has a large imaginary part. The ``besselhx`` function cancels this exponential factor (analytically), so it avoids these problems. .. function:: besseli(nu, x) From 85560b753e17c5aa391e56c6deeddc5f1c327181 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 22 Jul 2016 19:10:15 -0600 Subject: [PATCH 0597/1117] only accept `Int` in `reshape` (#17567) Before, the definition of `reshape` that returns a `ReshapedArray` accepted any integer, but the Array->Array definition only accepted `Int`, so passing a different type of integer strangely resulted in a different type of array. This behavior was observed as part of #17372. --- base/abstractarray.jl | 1 + base/reshapedarray.jl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index e990bf138d757..ca46ba05d13c1 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -7,6 +7,7 @@ typealias AbstractMatrix{T} AbstractArray{T,2} typealias AbstractVecOrMat{T} Union{AbstractVector{T}, AbstractMatrix{T}} typealias RangeIndex Union{Int, Range{Int}, AbstractUnitRange{Int}, Colon} typealias DimOrInd Union{Integer, AbstractUnitRange} +typealias IntOrInd Union{Int, AbstractUnitRange} typealias DimsOrInds{N} NTuple{N,DimOrInd} macro _inline_pure_meta() diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index d1a4b0961a5f6..0f82bcca73d64 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -37,7 +37,7 @@ end length(R::ReshapedArrayIterator) = length(R.iter) reshape(parent::AbstractArray, shp::Tuple) = _reshape(parent, to_shape(shp)) -reshape(parent::AbstractArray, dims::DimOrInd...) = reshape(parent, dims) +reshape(parent::AbstractArray, dims::IntOrInd...) = reshape(parent, dims) reshape{T,N}(parent::AbstractArray{T,N}, ndims::Type{Val{N}}) = parent function reshape{T,AN,N}(parent::AbstractArray{T,AN}, ndims::Type{Val{N}}) From d896e6ece0f41fbb832cc24c3899c5f464afb33e Mon Sep 17 00:00:00 2001 From: Nathan Daly <NHDaly@gmail.com> Date: Fri, 22 Jul 2016 22:34:02 -0500 Subject: [PATCH 0598/1117] Fixes capitalization for Array variable names in "Dot Syntax for Vectorizing Functions" Capitalizes `X` and `Y` in "Dot Syntax for Vectorizing Functions" section, because they represent Array, not scalar, variables. This is keeping consistent with the previous paragraphs. --- doc/manual/functions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 6f43aec2fecad..139a63323dd7d 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -667,8 +667,8 @@ side is updated in-place. (In future versions of Julia, operators like ``.*`` will also be handled with the same mechanism: they will be equivalent to ``broadcast`` calls and -will be fused with other nested "dot" calls. ``x .+= y`` is equivalent -to ``x .= x .+ y`` and will eventually result in a fused in-place assignment. +will be fused with other nested "dot" calls. ``X .+= Y`` is equivalent +to ``X .= X .+ Y`` and will eventually result in a fused in-place assignment. Similarly for ``.*=`` etcetera.) Further Reading From 0abe82e04d1950f75a431f6fddb8b53a0e996e26 Mon Sep 17 00:00:00 2001 From: cormullion <cormullion@mac.com> Date: Sat, 23 Jul 2016 11:04:29 +0100 Subject: [PATCH 0599/1117] Update NEWS.md (#17562) * Update NEWS.md A few changes to mollify the OCD of the one who opened issue #17478... * Update NEWS.md Added some spaces. Perhaps I shouldn't have edited it in a browser window... * Update NEWS.md finally found the space --- NEWS.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/NEWS.md b/NEWS.md index 750ee8df44937..746dc70713396 100644 --- a/NEWS.md +++ b/NEWS.md @@ -257,6 +257,22 @@ Library improvements * `Date` and `DateTime` values can now be rounded to a specified resolution (e.g., 1 month or 15 minutes) with `floor`, `ceil`, and `round` ([#17037]). + * File handling: + + * A new function `walkdir()` returns an iterator that walks the directory tree of a directory. ([#1765]) + + ``` + for (root, dirs, files) in walkdir(expanduser("~/.julia/v0.5/Plots/src")) + println("$(length(files)) \t files in $root") + end + 19 files in /Users/me/.julia/v0.5/Plots/src + 15 files in /Users/me/.julia/v0.5/Plots/src/backends + 4 files in /Users/me/.julia/v0.5/Plots/src/deprecated + ``` + + * A new function `chown()` changes the ownership of files. ([#15007]) + + Deprecated or removed --------------------- From 26bebe841b1bf172577cd6201367492ec85afa58 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Sat, 23 Jul 2016 19:31:40 +0200 Subject: [PATCH 0600/1117] update docs for code_... (#17578) * update docs for code_... * address comments * trying not be a bone head [ci skip] --- base/docs/helpdb/Base.jl | 46 ---------------------------------------- base/interactiveutil.jl | 11 ++++++++++ base/reflection.jl | 27 +++++++++++++++++++++++ doc/stdlib/base.rst | 12 +++++------ 4 files changed, 44 insertions(+), 52 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 800c321e63aae..267d4cd74ec20 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -877,15 +877,6 @@ Get the local machine's host name. """ gethostname -""" - code_typed(f, types; optimize=true) - -Returns an array of lowered and type-inferred ASTs for the methods matching the given -generic function and type signature. The keyword argument `optimize` controls whether -additional optimizations, such as inlining, are also applied. -""" -code_typed - """ hankelh1x(nu, x) @@ -4915,13 +4906,6 @@ Send a printed form of `x` to the operating system clipboard ("copy"). """ clipboard(x) -""" - code_lowered(f, types) - -Returns an array of lowered ASTs for the methods matching the given generic function and type signature. -""" -code_lowered - """ values(collection) @@ -5165,16 +5149,6 @@ Determine whether a stream is read-only. """ isreadonly -""" - code_llvm(f, types) - -Prints the LLVM bitcodes generated for running the method matching the given generic -function and type signature to [`STDOUT`](:const:`STDOUT`). - -All metadata and dbg.* calls are removed from the printed bitcode. Use code_llvm_raw for the full IR. -""" -code_llvm - """ notify(condition, val=nothing; all=true, error=false) @@ -5332,18 +5306,6 @@ The largest power of two not greater than `n`. Returns 0 for `n==0`, and returns """ prevpow2 -""" - code_warntype(f, types) - -Displays lowered and type-inferred ASTs for the methods matching the given generic function -and type signature. The ASTs are annotated in such a way as to cause "non-leaf" types to be -emphasized (if color is available, displayed in red). This serves as a warning of potential -type instability. Not all non-leaf types are particularly problematic for performance, so -the results need to be used judiciously. See [Manual](:ref:`man-code-warntype`) for more -information. -""" -code_warntype - """ Mmap.sync!(array) @@ -5543,14 +5505,6 @@ counters. """ :@timed -""" - code_native(f, types) - -Prints the native assembly instructions generated for running the method matching the given -generic function and type signature to `STDOUT`. -""" -code_native - """ symdiff(s1,s2...) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 9cfb4d6616cbb..61f6a111090f8 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -248,6 +248,17 @@ versioninfo(verbose::Bool) = versioninfo(STDOUT,verbose) # displaying type-ambiguity warnings + +""" + code_warntype([io], f, types) + +Prints lowered and type-inferred ASTs for the methods matching the given generic function +and type signature to `io` which defaults to `STDOUT`. The ASTs are annotated in such a way +as to cause "non-leaf" types to be emphasized (if color is available, displayed in red). +This serves as a warning of potential type instability. Not all non-leaf types are particularly +problematic for performance, so the results need to be used judiciously. +See [Manual](:ref:`man-code-warntype`) for more information. +""" function code_warntype(io::IO, f, t::ANY) emph_io = IOContext(io, :TYPEEMPHASIZE => true) for li in code_typed(f, t) diff --git a/base/reflection.jl b/base/reflection.jl index c917acb2e8c90..aa18e49cdcfc2 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -194,6 +194,11 @@ end tt_cons(t::ANY, tup::ANY) = (@_pure_meta; Tuple{t, (isa(tup, Type) ? tup.parameters : tup)...}) +""" + code_lowered(f, types) + +Returns an array of lowered ASTs for the methods matching the given generic function and type signature. +""" code_lowered(f, t::ANY=Tuple) = map(m -> (m::Method).lambda_template, methods(f, t)) # low-level method lookup functions used by the compiler @@ -350,11 +355,25 @@ function _dump_function(f, t::ANY, native, wrapper, strip_ir_metadata, dump_modu return str end +""" + code_llvm([io], f, types) + +Prints the LLVM bitcodes generated for running the method matching the given generic +function and type signature to `io` which defaults to `STDOUT`. + +All metadata and dbg.* calls are removed from the printed bitcode. Use code_llvm_raw for the full IR. +""" code_llvm(io::IO, f::ANY, types::ANY=Tuple, strip_ir_metadata=true, dump_module=false) = print(io, _dump_function(f, types, false, false, strip_ir_metadata, dump_module)) code_llvm(f::ANY, types::ANY=Tuple) = code_llvm(STDOUT, f, types) code_llvm_raw(f::ANY, types::ANY=Tuple) = code_llvm(STDOUT, f, types, false) +""" + code_native([io], f, types) + +Prints the native assembly instructions generated for running the method matching the given +generic function and type signature to `io` which defaults to `STDOUT`. +""" code_native(io::IO, f::ANY, types::ANY=Tuple) = print(io, _dump_function(f, types, true, false, false, false)) code_native(f::ANY, types::ANY=Tuple) = code_native(STDOUT, f, types) @@ -368,6 +387,14 @@ function func_for_method_checked(m::Method, types) return m end + +""" + code_typed(f, types; optimize=true) + +Returns an array of lowered and type-inferred ASTs for the methods matching the given +generic function and type signature. The keyword argument `optimize` controls whether +additional optimizations, such as inlining, are also applied. +""" function code_typed(f::ANY, types::ANY=Tuple; optimize=true) ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions") if isa(f, Core.Builtin) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index fe227b14f303a..40d0b40d307c3 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -1436,11 +1436,11 @@ Internals Evaluates the arguments to the function or macro call, determines their types, and calls :func:`code_typed` on the resulting expression. -.. function:: code_warntype(f, types) +.. function:: code_warntype([io], f, types) .. Docstring generated from Julia source - Displays lowered and type-inferred ASTs for the methods matching the given generic function and type signature. The ASTs are annotated in such a way as to cause "non-leaf" types to be emphasized (if color is available, displayed in red). This serves as a warning of potential type instability. Not all non-leaf types are particularly problematic for performance, so the results need to be used judiciously. See :ref:`man-code-warntype` for more information. + Prints lowered and type-inferred ASTs for the methods matching the given generic function and type signature to ``io`` which defaults to ``STDOUT``\ . The ASTs are annotated in such a way as to cause "non-leaf" types to be emphasized (if color is available, displayed in red). This serves as a warning of potential type instability. Not all non-leaf types are particularly problematic for performance, so the results need to be used judiciously. See :ref:`man-code-warntype` for more information. .. function:: @code_warntype @@ -1448,11 +1448,11 @@ Internals Evaluates the arguments to the function or macro call, determines their types, and calls :func:`code_warntype` on the resulting expression. -.. function:: code_llvm(f, types) +.. function:: code_llvm([io], f, types) .. Docstring generated from Julia source - Prints the LLVM bitcodes generated for running the method matching the given generic function and type signature to :const:`STDOUT`\ . + Prints the LLVM bitcodes generated for running the method matching the given generic function and type signature to ``io`` which defaults to ``STDOUT``\ . All metadata and dbg.* calls are removed from the printed bitcode. Use code_llvm_raw for the full IR. @@ -1462,11 +1462,11 @@ Internals Evaluates the arguments to the function or macro call, determines their types, and calls :func:`code_llvm` on the resulting expression. -.. function:: code_native(f, types) +.. function:: code_native([io], f, types) .. Docstring generated from Julia source - Prints the native assembly instructions generated for running the method matching the given generic function and type signature to ``STDOUT``\ . + Prints the native assembly instructions generated for running the method matching the given generic function and type signature to ``io`` which defaults to ``STDOUT``\ . .. function:: @code_native From e1c47e1a89313575f9ab435f8f25f2b7f4bdfd7d Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman <benditlikeranjan@gmail.com> Date: Sat, 23 Jul 2016 23:30:22 +0530 Subject: [PATCH 0601/1117] Add some docs to threading constructs (#17369) Add some docs for threading constructs [ci skip] --- base/threadingconstructs.jl | 18 +++++++ doc/manual/parallel-computing.rst | 80 +++++++++++++++++++++++++++++++ doc/stdlib/parallel.rst | 18 +++++++ 3 files changed, 116 insertions(+) diff --git a/base/threadingconstructs.jl b/base/threadingconstructs.jl index e2adacf9c88bd..89ee766ecbb6c 100644 --- a/base/threadingconstructs.jl +++ b/base/threadingconstructs.jl @@ -2,9 +2,20 @@ export threadid, nthreads, @threads +""" + Threads.threadid() + +Get the ID number of the current thread of execution. The master thread has ID `1`. +""" threadid() = Int(ccall(:jl_threadid, Int16, ())+1) # Inclusive upper bound on threadid() +""" + Threads.nthreads() + +Get the number of threads available to the Julia process. This is the inclusive upper bound +on `threadid()`. +""" nthreads() = Int(unsafe_load(cglobal(:jl_n_threads, Cint))) function _threadsfor(iter,lbody) @@ -46,7 +57,14 @@ function _threadsfor(iter,lbody) ccall(:jl_threading_run, Void, (Any,), Core.svec($fun)) end end +""" + Threads.@threads +A macro to parallelize a for-loop to run with multiple threads. This spawns `nthreads()` +number of threads, splits the iteration space amongst them, and iterates in parallel. +A barrier is placed at the end of the loop which waits for all the threads to finish +execution, and the loop returns. +""" macro threads(args...) na = length(args) if na != 1 diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index f36682002a570..057c3f3e4c719 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -997,6 +997,86 @@ Keyword argument ``topology`` to ``addprocs`` is used to specify how the workers Currently sending a message between unconnected workers results in an error. This behaviour, as also the functionality and interface should be considered experimental in nature and may change in future releases. +Multi-threading (Experimental) +------------------------------- +In addition to tasks, remote calls and remote references, Julia from v0.5 will natively support +multi-threading. Note that this section is experimental and the interfaces may change in the +future. + +Setup +===== + +By default, Julia starts up with a single thread of execution. This can be verified by +using the command :obj:`Threads.nthreads()`:: + + julia> Threads.nthreads() + 1 + +The number of threads Julia starts up with is controlled by an environment variable +called ``JULIA_NUM_THREADS``. Now, let's start up Julia with 4 threads:: + + export JULIA_NUM_THREADS=4 + +(The above command works on bourne shells on Linux and OSX. Note that if you're using +a C shell on these platforms, you should use the keyword ``set`` instead of ``export``. +If you're on Windows, start up the command line in the location of ``julia.exe`` and +use ``set`` instead of ``export``.) + +Let's verify there are 4 threads at our disposal. :: + + julia> Threads.nthreads() + 4 + +But we are currently on the master thread. To check, we use the command :obj:`Threads.threadid()` :: + + julia> Threads.threadid() + 1 + +The ``@threads`` Macro +======================= + +Let's work a simple example using our native threads. Let us create an array of zeros:: + + julia> a = zeros(10) + 10-element Array{Float64,1}: + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + 0.0 + +Let us operate on this array simultaneously using 4 threads. We'll have each thread write +its thread ID into each location. + +Julia supports parallel loops using the :obj:`Threads.@threads` macro. This macro is affixed in front +of a ``for`` loop to indicate to Julia that the loop is a multi-threaded region. :: + + Threads.@threads for i = 1:10 + a[i] = threadid() + end + +The iteration space is split amongst the threads, after which each thread writes its thread ID to its assigned locations.:: + + julia> a + 10-element Array{Float64,1}: + 1.0 + 1.0 + 1.0 + 2.0 + 2.0 + 2.0 + 3.0 + 3.0 + 4.0 + 4.0 + +Note that :obj:`Threads.@threads` does not have an optional reduction parameter like :obj:`@parallel`. + .. rubric:: Footnotes .. [#mpi2rma] In this context, MPI refers to the MPI-1 standard. Beginning with MPI-2, the MPI standards committee introduced a new set of communication mechanisms, collectively referred to as Remote Memory Access (RMA). The motivation for adding RMA to the MPI standard was to facilitate one-sided communication patterns. For additional information on the latest MPI standard, see http://www.mpi-forum.org/docs. diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 4fa5fd778760a..472a8744868a7 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -648,6 +648,24 @@ This experimental interface supports Julia's multi-threading capabilities. Types and function described here might (and likely will) change in the future. +.. function:: Threads.threadid() + + .. Docstring generated from Julia source + + Get the ID number of the current thread of execution. The master thread has ID ``1``\ . + +.. function:: Threads.nthreads() + + .. Docstring generated from Julia source + + Get the number of threads available to the Julia process. This is the inclusive upper bound on ``threadid()``\ . + +.. function:: Threads.@threads + + .. Docstring generated from Julia source + + A macro to parallelize a for-loop to run with multiple threads. This spawns ``nthreads()`` number of threads, splits the iteration space amongst them, and iterates in parallel. A barrier is placed at the end of the loop which waits for all the threads to finish execution, and the loop returns. + .. function:: Threads.Atomic{T} .. Docstring generated from Julia source From 5d9b90d0656cac996bba8ea2c0f42caaaf1a1249 Mon Sep 17 00:00:00 2001 From: Alex Arslan <ararslan@comcast.net> Date: Sat, 23 Jul 2016 12:06:56 -0700 Subject: [PATCH 0602/1117] Change Array(Type, ...) to Array{Type}(...) --- base/array.jl | 6 +++--- base/boot.jl | 2 +- base/deprecated.jl | 6 +++--- base/multi.jl | 4 ++-- base/statistics.jl | 2 +- base/sysimg.jl | 2 +- base/util.jl | 8 ++++---- test/bigint.jl | 2 +- test/priorityqueue.jl | 2 +- test/replutil.jl | 2 +- test/show.jl | 4 ++-- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/base/array.jl b/base/array.jl index 2efa5cc199e81..959e069360d73 100644 --- a/base/array.jl +++ b/base/array.jl @@ -255,14 +255,14 @@ else end _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T -_array_for(T, itr, ::HasLength) = Array(T, Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = Array(T, convert(Dims,size(itr))) +_array_for(T, itr, ::HasLength) = Array{T}(Int(length(itr)::Integer)) +_array_for(T, itr, ::HasShape) = Array{T}(convert(Dims,size(itr))) function collect(itr::Generator) isz = iteratorsize(itr.iter) et = _default_eltype(typeof(itr)) if isa(isz, SizeUnknown) - return grow_to!(Array(et, 0), itr) + return grow_to!(Array{et}(0), itr) else st = start(itr) if done(itr,st) diff --git a/base/boot.jl b/base/boot.jl index 88d7ed85972fe..4e75175aa2a89 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -326,7 +326,7 @@ typealias NTuple{N,T} Tuple{Vararg{T,N}} # TODO: possibly turn these into deprecations Array{T,N}(::Type{T}, d::NTuple{N,Int}) = Array{T,N}(d) -Array{T}(::Type{T}, d::Int...) = Array(T, d) +Array{T}(::Type{T}, d::Int...) = Array{T}(d) Array{T}(::Type{T}, m::Int) = Array{T,1}(m) Array{T}(::Type{T}, m::Int,n::Int) = Array{T,2}(m,n) Array{T}(::Type{T}, m::Int,n::Int,o::Int) = Array{T,3}(m,n,o) diff --git a/base/deprecated.jl b/base/deprecated.jl index 0b6d69eedaf05..fb05a35b67038 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -642,7 +642,7 @@ function hist!{HT}(h::AbstractArray{HT}, v::AbstractVector, edg::AbstractVector; edg, h end -hist(v::AbstractVector, edg::AbstractVector) = hist!(Array(Int, length(edg)-1), v, edg) +hist(v::AbstractVector, edg::AbstractVector) = hist!(Array{Int}(length(edg)-1), v, edg) hist(v::AbstractVector, n::Integer) = hist(v,histrange(v,n)) hist(v::AbstractVector) = hist(v,sturges(length(v))) @@ -662,7 +662,7 @@ function hist!{HT}(H::AbstractArray{HT,2}, A::AbstractMatrix, edg::AbstractVecto edg, H end -hist(A::AbstractMatrix, edg::AbstractVector) = hist!(Array(Int, length(edg)-1, size(A,2)), A, edg) +hist(A::AbstractMatrix, edg::AbstractVector) = hist!(Array{Int}(length(edg)-1, size(A,2)), A, edg) hist(A::AbstractMatrix, n::Integer) = hist(A,histrange(A,n)) hist(A::AbstractMatrix) = hist(A,sturges(size(A,1))) @@ -690,7 +690,7 @@ function hist2d!{HT}(H::AbstractArray{HT,2}, v::AbstractMatrix, end hist2d(v::AbstractMatrix, edg1::AbstractVector, edg2::AbstractVector) = - hist2d!(Array(Int, length(edg1)-1, length(edg2)-1), v, edg1, edg2) + hist2d!(Array{Int}(length(edg1)-1, length(edg2)-1), v, edg1, edg2) hist2d(v::AbstractMatrix, edg::AbstractVector) = hist2d(v, edg, edg) diff --git a/base/multi.jl b/base/multi.jl index 6d21904b24892..5c141658bdec3 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -115,7 +115,7 @@ for (idx, tname) in enumerate(msgtypes) end function deserialize_msg(s::AbstractSerializer, ::Type{$tname}) - data=Array(Any, $nflds) + data=Array{Any}($nflds) for i in 1:$nflds data[i] = deserialize(s) end @@ -306,7 +306,7 @@ function serialize_hdr_raw(io, hdr) end function deserialize_hdr_raw(io) - data = Array(Int, 4) + data = Array{Int}(4) read!(io, data) return MsgHeader(RRID(data[1], data[2]), RRID(data[3], data[4])) end diff --git a/base/statistics.jl b/base/statistics.jl index 1545ce0d28e2f..2181da0d80697 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -491,7 +491,7 @@ function median!{T}(v::AbstractVector{T}) end end median!{T}(v::AbstractArray{T}) = median!(vec(v)) -median{T}(v::AbstractArray{T}) = median!(copy!(Array(T, length(v)), v)) +median{T}(v::AbstractArray{T}) = median!(copy!(Array{T}(length(v)), v)) median{T}(v::AbstractArray{T}, region) = mapslices(median!, v, region) diff --git a/base/sysimg.jl b/base/sysimg.jl index 89217dbf71b8a..a2ec4b59a0cd5 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -85,7 +85,7 @@ include("array.jl") (::Type{Matrix})(m::Integer, n::Integer) = Matrix{Any}(Int(m), Int(n)) # TODO: possibly turn these into deprecations -Array{T}(::Type{T}, d::Integer...) = Array(T, convert(Tuple{Vararg{Int}}, d)) +Array{T}(::Type{T}, d::Integer...) = Array{T}(convert(Tuple{Vararg{Int}}, d)) Array{T}(::Type{T}, m::Integer) = Array{T,1}(Int(m)) Array{T}(::Type{T}, m::Integer,n::Integer) = Array{T,2}(Int(m),Int(n)) Array{T}(::Type{T}, m::Integer,n::Integer,o::Integer) = Array{T,3}(Int(m),Int(n),Int(o)) diff --git a/base/util.jl b/base/util.jl index 7fb266117db46..708a6122fa8cf 100644 --- a/base/util.jl +++ b/base/util.jl @@ -367,7 +367,7 @@ if is_windows() function winprompt(message, caption, default_username; prompt_username = true) # Step 1: Create an encrypted username/password bundle that will be used to set # the default username (in theory could also provide a default password) - credbuf = Array(UInt8, 1024) + credbuf = Array{UInt8}(1024) credbufsize = Ref{UInt32}(sizeof(credbuf)) succeeded = ccall((:CredPackAuthenticationBufferW, "credui.dll"), stdcall, Bool, (UInt32, Cwstring, Cwstring, Ptr{UInt8}, Ptr{UInt32}), @@ -403,12 +403,12 @@ if is_windows() end # Step 3: Convert encrypted credentials back to plain text - passbuf = Array(UInt16, 1024) + passbuf = Array{UInt16}(1024) passlen = Ref{UInt32}(length(passbuf)) - usernamebuf = Array(UInt16, 1024) + usernamebuf = Array{UInt16}(1024) usernamelen = Ref{UInt32}(length(usernamebuf)) # Need valid buffers for domain, even though we don't care - dummybuf = Array(UInt16, 1024) + dummybuf = Array{UInt16}(1024) succeeded = ccall((:CredUnPackAuthenticationBufferW, "credui.dll"), Bool, (UInt32, Ptr{Void}, UInt32, Ptr{UInt16}, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}), 0, outbuf_data[], outbuf_size[], usernamebuf, usernamelen, dummybuf, Ref{UInt32}(1024), passbuf, passlen) diff --git a/test/bigint.jl b/test/bigint.jl index 9fd05362dfcb6..dba405232c8c4 100644 --- a/test/bigint.jl +++ b/test/bigint.jl @@ -226,7 +226,7 @@ g = parse(BigInt,"-1") # from Bill Hart, https://groups.google.com/group/julia-dev/browse_frm/thread/798e2d1322daf633 function mul(a::Vector{BigInt}, b::Vector{BigInt}) x = a[2]*b[2] - c = Array(BigInt,3) + c = Array{BigInt}(3) c[1] = a[1]*b[1] + x c[2] = a[1]*b[2] + a[2]*b[3] c[3] = x + a[3]*b[3] diff --git a/test/priorityqueue.jl b/test/priorityqueue.jl index 4c97e8828e1a5..0b98cce53363e 100644 --- a/test/priorityqueue.jl +++ b/test/priorityqueue.jl @@ -102,7 +102,7 @@ xs = heapify!([v for v in values(priorities)]) xs = heapify(10:-1:1) @test issorted([heappop!(xs) for _ in 1:10]) -xs = Array(Int, 0) +xs = Array{Int}(0) for priority in values(priorities) heappush!(xs, priority) end diff --git a/test/replutil.jl b/test/replutil.jl index 08f7ccd27d4c3..d4646b396ccde 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -268,7 +268,7 @@ let undefvar err_str = @except_str mod(1,0) DivideError @test err_str == "DivideError: integer division error" - err_str = @except_str Array(Any,1)[1] UndefRefError + err_str = @except_str Array{Any}(1)[1] UndefRefError @test err_str == "UndefRefError: access to undefined reference" err_str = @except_str undefvar UndefVarError @test err_str == "UndefVarError: undefvar not defined" diff --git a/test/show.jl b/test/show.jl index c9d0369c39944..075987e3b7f6c 100644 --- a/test/show.jl +++ b/test/show.jl @@ -9,7 +9,7 @@ replstr(x) = sprint((io,x) -> show(IOContext(io, limit=true), MIME("text/plain") immutable T5589 names::Vector{String} end -@test replstr(T5589(Array(String,100))) == "T5589(String[#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef … #undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef])" +@test replstr(T5589(Array{String}(100))) == "T5589(String[#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef … #undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef])" @test replstr(parse("type X end")) == ":(type X # none, line 1:\n end)" @test replstr(parse("immutable X end")) == ":(immutable X # none, line 1:\n end)" @@ -105,7 +105,7 @@ end n *= d end nc = num_bit_chunks(n) - chunks = Array(UInt64, nc) + chunks = Array{UInt64}(nc) if nc > 0 chunks[end] = UInt64(0) end From e257317cc674217651646a6d4bce404873aade1d Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Sat, 23 Jul 2016 15:35:04 -0400 Subject: [PATCH 0603/1117] Add binaries and notes for Power8. --- README.power.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 README.power.md diff --git a/README.power.md b/README.power.md new file mode 100644 index 0000000000000..15ebab1a9f145 --- /dev/null +++ b/README.power.md @@ -0,0 +1,26 @@ +# Julia Binaries for Power8 + +Experimental support is now available for Julia on power8 little +endian architectures running linux - ppc64le. + +[Precompiled +binaries](https://drive.google.com/drive/u/0/folders/0B0rXlkvSbIfhR1RsbUV2VkpFMFk) +for Julia 0.5-pre are available. Make sure to download `libtatlas.so` +from there and put it on your `LD_LIBRARY_PATH`. + +The latest release of ATLAS on most linux distributions is buggy on +Power and using the provided binary is preferable. The next stable +release of ATLAS (3.10.3) should fix all known issues, but it may be a +while before an update is available on popular linux distros. + +# Building Julia on Power8 + +Since OpenBLAS power support is still in the works, we use ATLAS as a +system provided BLAS for now through a custom Make.user file: + +```` +override USE_SYSTEM_BLAS = 1 +override USE_BLAS64 = 0 +override LIBBLAS = -L/usr/lib64/atlas -ltatlas +override LIBBLASNAME = libtatlas +```` From 8eaaf33089bd5cc9311e10ba3ab2fd678fa3e05a Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Sat, 23 Jul 2016 15:42:51 -0400 Subject: [PATCH 0604/1117] Note that the power binaries are unofficial. [ci skip] --- README.power.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.power.md b/README.power.md index 15ebab1a9f145..d9a3dc305a7d1 100644 --- a/README.power.md +++ b/README.power.md @@ -13,6 +13,10 @@ Power and using the provided binary is preferable. The next stable release of ATLAS (3.10.3) should fix all known issues, but it may be a while before an update is available on popular linux distros. +Note that these are unofficial binaries and contributed strictly for +convenience. They may or may not work for you, and have not gone +through the same rigour as official binaries. + # Building Julia on Power8 Since OpenBLAS power support is still in the works, we use ATLAS as a From d96528d5a7c73cbc288bfc3e5e821aa2e9a8642d Mon Sep 17 00:00:00 2001 From: Alex Arslan <ararslan@comcast.net> Date: Sat, 23 Jul 2016 12:51:19 -0700 Subject: [PATCH 0605/1117] Add the N parameter to Array{T,N}(dims) --- base/array.jl | 6 +++--- base/boot.jl | 2 +- base/deprecated.jl | 6 +++--- base/multi.jl | 4 ++-- base/sysimg.jl | 2 +- base/util.jl | 8 ++++---- test/bigint.jl | 2 +- test/priorityqueue.jl | 2 +- test/replutil.jl | 2 +- test/show.jl | 4 ++-- 10 files changed, 19 insertions(+), 19 deletions(-) diff --git a/base/array.jl b/base/array.jl index 959e069360d73..2004de7094389 100644 --- a/base/array.jl +++ b/base/array.jl @@ -255,14 +255,14 @@ else end _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T -_array_for(T, itr, ::HasLength) = Array{T}(Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = Array{T}(convert(Dims,size(itr))) +_array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) +_array_for(T, itr, ::HasShape) = Array{T,1}(convert(Dims,size(itr))) function collect(itr::Generator) isz = iteratorsize(itr.iter) et = _default_eltype(typeof(itr)) if isa(isz, SizeUnknown) - return grow_to!(Array{et}(0), itr) + return grow_to!(Array{et,1}(0), itr) else st = start(itr) if done(itr,st) diff --git a/base/boot.jl b/base/boot.jl index 4e75175aa2a89..70bd6dfb95cd4 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -326,7 +326,7 @@ typealias NTuple{N,T} Tuple{Vararg{T,N}} # TODO: possibly turn these into deprecations Array{T,N}(::Type{T}, d::NTuple{N,Int}) = Array{T,N}(d) -Array{T}(::Type{T}, d::Int...) = Array{T}(d) +Array{T}(::Type{T}, d::Int...) = Array{T,length(d)}(d) Array{T}(::Type{T}, m::Int) = Array{T,1}(m) Array{T}(::Type{T}, m::Int,n::Int) = Array{T,2}(m,n) Array{T}(::Type{T}, m::Int,n::Int,o::Int) = Array{T,3}(m,n,o) diff --git a/base/deprecated.jl b/base/deprecated.jl index fb05a35b67038..ac4eb4e1c5347 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -642,7 +642,7 @@ function hist!{HT}(h::AbstractArray{HT}, v::AbstractVector, edg::AbstractVector; edg, h end -hist(v::AbstractVector, edg::AbstractVector) = hist!(Array{Int}(length(edg)-1), v, edg) +hist(v::AbstractVector, edg::AbstractVector) = hist!(Array{Int,1}(length(edg)-1), v, edg) hist(v::AbstractVector, n::Integer) = hist(v,histrange(v,n)) hist(v::AbstractVector) = hist(v,sturges(length(v))) @@ -662,7 +662,7 @@ function hist!{HT}(H::AbstractArray{HT,2}, A::AbstractMatrix, edg::AbstractVecto edg, H end -hist(A::AbstractMatrix, edg::AbstractVector) = hist!(Array{Int}(length(edg)-1, size(A,2)), A, edg) +hist(A::AbstractMatrix, edg::AbstractVector) = hist!(Array{Int,2}(length(edg)-1, size(A,2)), A, edg) hist(A::AbstractMatrix, n::Integer) = hist(A,histrange(A,n)) hist(A::AbstractMatrix) = hist(A,sturges(size(A,1))) @@ -690,7 +690,7 @@ function hist2d!{HT}(H::AbstractArray{HT,2}, v::AbstractMatrix, end hist2d(v::AbstractMatrix, edg1::AbstractVector, edg2::AbstractVector) = - hist2d!(Array{Int}(length(edg1)-1, length(edg2)-1), v, edg1, edg2) + hist2d!(Array{Int,2}(length(edg1)-1, length(edg2)-1), v, edg1, edg2) hist2d(v::AbstractMatrix, edg::AbstractVector) = hist2d(v, edg, edg) diff --git a/base/multi.jl b/base/multi.jl index 5c141658bdec3..2ccd2ac1a41d9 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -115,7 +115,7 @@ for (idx, tname) in enumerate(msgtypes) end function deserialize_msg(s::AbstractSerializer, ::Type{$tname}) - data=Array{Any}($nflds) + data=Array{Any,1}($nflds) for i in 1:$nflds data[i] = deserialize(s) end @@ -306,7 +306,7 @@ function serialize_hdr_raw(io, hdr) end function deserialize_hdr_raw(io) - data = Array{Int}(4) + data = Array{Int,1}(4) read!(io, data) return MsgHeader(RRID(data[1], data[2]), RRID(data[3], data[4])) end diff --git a/base/sysimg.jl b/base/sysimg.jl index a2ec4b59a0cd5..89217dbf71b8a 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -85,7 +85,7 @@ include("array.jl") (::Type{Matrix})(m::Integer, n::Integer) = Matrix{Any}(Int(m), Int(n)) # TODO: possibly turn these into deprecations -Array{T}(::Type{T}, d::Integer...) = Array{T}(convert(Tuple{Vararg{Int}}, d)) +Array{T}(::Type{T}, d::Integer...) = Array(T, convert(Tuple{Vararg{Int}}, d)) Array{T}(::Type{T}, m::Integer) = Array{T,1}(Int(m)) Array{T}(::Type{T}, m::Integer,n::Integer) = Array{T,2}(Int(m),Int(n)) Array{T}(::Type{T}, m::Integer,n::Integer,o::Integer) = Array{T,3}(Int(m),Int(n),Int(o)) diff --git a/base/util.jl b/base/util.jl index 708a6122fa8cf..c456472ee694a 100644 --- a/base/util.jl +++ b/base/util.jl @@ -367,7 +367,7 @@ if is_windows() function winprompt(message, caption, default_username; prompt_username = true) # Step 1: Create an encrypted username/password bundle that will be used to set # the default username (in theory could also provide a default password) - credbuf = Array{UInt8}(1024) + credbuf = Array{UInt8,1}(1024) credbufsize = Ref{UInt32}(sizeof(credbuf)) succeeded = ccall((:CredPackAuthenticationBufferW, "credui.dll"), stdcall, Bool, (UInt32, Cwstring, Cwstring, Ptr{UInt8}, Ptr{UInt32}), @@ -403,12 +403,12 @@ if is_windows() end # Step 3: Convert encrypted credentials back to plain text - passbuf = Array{UInt16}(1024) + passbuf = Array{UInt16,1}(1024) passlen = Ref{UInt32}(length(passbuf)) - usernamebuf = Array{UInt16}(1024) + usernamebuf = Array{UInt16,1}(1024) usernamelen = Ref{UInt32}(length(usernamebuf)) # Need valid buffers for domain, even though we don't care - dummybuf = Array{UInt16}(1024) + dummybuf = Array{UInt16,1}(1024) succeeded = ccall((:CredUnPackAuthenticationBufferW, "credui.dll"), Bool, (UInt32, Ptr{Void}, UInt32, Ptr{UInt16}, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}), 0, outbuf_data[], outbuf_size[], usernamebuf, usernamelen, dummybuf, Ref{UInt32}(1024), passbuf, passlen) diff --git a/test/bigint.jl b/test/bigint.jl index dba405232c8c4..999b0c42cf6dd 100644 --- a/test/bigint.jl +++ b/test/bigint.jl @@ -226,7 +226,7 @@ g = parse(BigInt,"-1") # from Bill Hart, https://groups.google.com/group/julia-dev/browse_frm/thread/798e2d1322daf633 function mul(a::Vector{BigInt}, b::Vector{BigInt}) x = a[2]*b[2] - c = Array{BigInt}(3) + c = Array{BigInt,1}(3) c[1] = a[1]*b[1] + x c[2] = a[1]*b[2] + a[2]*b[3] c[3] = x + a[3]*b[3] diff --git a/test/priorityqueue.jl b/test/priorityqueue.jl index 0b98cce53363e..326f1c686e287 100644 --- a/test/priorityqueue.jl +++ b/test/priorityqueue.jl @@ -102,7 +102,7 @@ xs = heapify!([v for v in values(priorities)]) xs = heapify(10:-1:1) @test issorted([heappop!(xs) for _ in 1:10]) -xs = Array{Int}(0) +xs = Array{Int,1}(0) for priority in values(priorities) heappush!(xs, priority) end diff --git a/test/replutil.jl b/test/replutil.jl index d4646b396ccde..11288aa33e19d 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -268,7 +268,7 @@ let undefvar err_str = @except_str mod(1,0) DivideError @test err_str == "DivideError: integer division error" - err_str = @except_str Array{Any}(1)[1] UndefRefError + err_str = @except_str Array{Any,1}(1)[1] UndefRefError @test err_str == "UndefRefError: access to undefined reference" err_str = @except_str undefvar UndefVarError @test err_str == "UndefVarError: undefvar not defined" diff --git a/test/show.jl b/test/show.jl index 075987e3b7f6c..7e2545c7683f1 100644 --- a/test/show.jl +++ b/test/show.jl @@ -9,7 +9,7 @@ replstr(x) = sprint((io,x) -> show(IOContext(io, limit=true), MIME("text/plain") immutable T5589 names::Vector{String} end -@test replstr(T5589(Array{String}(100))) == "T5589(String[#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef … #undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef])" +@test replstr(T5589(Array{String,1}(100))) == "T5589(String[#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef … #undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef])" @test replstr(parse("type X end")) == ":(type X # none, line 1:\n end)" @test replstr(parse("immutable X end")) == ":(immutable X # none, line 1:\n end)" @@ -105,7 +105,7 @@ end n *= d end nc = num_bit_chunks(n) - chunks = Array{UInt64}(nc) + chunks = Array{UInt64,1}(nc) if nc > 0 chunks[end] = UInt64(0) end From db832499688aa23c423c1b9747c3b2cab5263b9c Mon Sep 17 00:00:00 2001 From: Alex Arslan <ararslan@comcast.net> Date: Sat, 23 Jul 2016 14:32:12 -0700 Subject: [PATCH 0606/1117] Fixed per Jameson's comments --- base/array.jl | 4 ++-- base/boot.jl | 2 +- base/statistics.jl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/array.jl b/base/array.jl index 2004de7094389..15a6c9b5d66a0 100644 --- a/base/array.jl +++ b/base/array.jl @@ -255,8 +255,8 @@ else end _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T -_array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = Array{T,1}(convert(Dims,size(itr))) +_array_for(T, itr, ::HasLength) = Array{T,ndims(iter)}(Int(length(itr)::Integer)) +_array_for(T, itr, ::HasShape) = Array{T,ndims(iter)}(convert(Dims,size(itr))) function collect(itr::Generator) isz = iteratorsize(itr.iter) diff --git a/base/boot.jl b/base/boot.jl index 70bd6dfb95cd4..88d7ed85972fe 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -326,7 +326,7 @@ typealias NTuple{N,T} Tuple{Vararg{T,N}} # TODO: possibly turn these into deprecations Array{T,N}(::Type{T}, d::NTuple{N,Int}) = Array{T,N}(d) -Array{T}(::Type{T}, d::Int...) = Array{T,length(d)}(d) +Array{T}(::Type{T}, d::Int...) = Array(T, d) Array{T}(::Type{T}, m::Int) = Array{T,1}(m) Array{T}(::Type{T}, m::Int,n::Int) = Array{T,2}(m,n) Array{T}(::Type{T}, m::Int,n::Int,o::Int) = Array{T,3}(m,n,o) diff --git a/base/statistics.jl b/base/statistics.jl index 2181da0d80697..9629d105a93d7 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -491,7 +491,7 @@ function median!{T}(v::AbstractVector{T}) end end median!{T}(v::AbstractArray{T}) = median!(vec(v)) -median{T}(v::AbstractArray{T}) = median!(copy!(Array{T}(length(v)), v)) +median{T}(v::AbstractArray{T}) = median!(copy!(Array{T,1}(length(v)), v)) median{T}(v::AbstractArray{T}, region) = mapslices(median!, v, region) From 5bbab3e53115f77731ae334d0fdaa7316032a38e Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer+github@college.harvard.edu> Date: Sat, 23 Jul 2016 17:43:29 -0400 Subject: [PATCH 0607/1117] In objconv.mk, make sure to create build dir if doesn't exit --- deps/objconv.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/deps/objconv.mk b/deps/objconv.mk index 79f8f0725e187..41a9498f84295 100644 --- a/deps/objconv.mk +++ b/deps/objconv.mk @@ -7,6 +7,7 @@ $(SRCDIR)/srccache/objconv.zip: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.agner.org/optimize/objconv.zip $(BUILDDIR)/objconv/config.status: $(SRCDIR)/srccache/objconv.zip -rm -r $(dir $@) + mkdir -p $(BUILDDIR) unzip -d $(dir $@) $< cd $(dir $@) && unzip source.zip echo 1 > $@ From eff1b95c32f444a48007eca4c03f8400973592f5 Mon Sep 17 00:00:00 2001 From: Alex Arslan <ararslan@comcast.net> Date: Sat, 23 Jul 2016 16:34:50 -0700 Subject: [PATCH 0608/1117] Fixed typo --- base/array.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index 15a6c9b5d66a0..bbf0e4cb950b0 100644 --- a/base/array.jl +++ b/base/array.jl @@ -255,8 +255,8 @@ else end _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T -_array_for(T, itr, ::HasLength) = Array{T,ndims(iter)}(Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = Array{T,ndims(iter)}(convert(Dims,size(itr))) +_array_for(T, itr, ::HasLength) = Array{T,ndims(itr)}(Int(length(itr)::Integer)) +_array_for(T, itr, ::HasShape) = Array{T,ndims(itr)}(convert(Dims,size(itr))) function collect(itr::Generator) isz = iteratorsize(itr.iter) From 8a27b6d0e5a1106802ad543dbce79f033566e839 Mon Sep 17 00:00:00 2001 From: Alex Arslan <ararslan@comcast.net> Date: Sat, 23 Jul 2016 16:54:14 -0700 Subject: [PATCH 0609/1117] Fix per Pablo's comment --- base/array.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index bbf0e4cb950b0..bf637fc688a69 100644 --- a/base/array.jl +++ b/base/array.jl @@ -255,8 +255,8 @@ else end _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T -_array_for(T, itr, ::HasLength) = Array{T,ndims(itr)}(Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = Array{T,ndims(itr)}(convert(Dims,size(itr))) +_array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) +_array_for(T, itr, ::HasShape) = Array{T}(convert(Dims,size(itr))) function collect(itr::Generator) isz = iteratorsize(itr.iter) From 6ac81d2d9a57ba54acd7a2ace90c969ded5a05ef Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Fri, 22 Jul 2016 23:04:15 -0400 Subject: [PATCH 0610/1117] A few finalizer thread safety fixes. * Fix wrong address to `memset` ... * Clearing the memory in `finalize_object` should be unconditional and before the `cmpxchg`. Since even when the `cmpxchg` succeeds, the owning thread might have read the old length and set it to a different value based on the old length, leaving an uncleared hole. * List length needs to be read again after acquiring the finalizer lock. Someone might have changed it and the resizing might not have the expected effect. --- src/gc.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/gc.c b/src/gc.c index 7ceafddb07f85..f62f056b011d4 100644 --- a/src/gc.c +++ b/src/gc.c @@ -119,7 +119,13 @@ static void run_finalizer(jl_ptls_t ptls, jl_value_t *o, jl_value_t *ff) static void finalize_object(arraylist_t *list, jl_value_t *o, arraylist_t *copied_list, int need_sync) { - // The acquire load makes sure that the first `len` objects are valid + // The acquire load makes sure that the first `len` objects are valid. + // If `need_sync` is true, all mutations of the content should be limited + // to the first `oldlen` elements and no mutation is allowed after the + // new length is published with the `cmpxchg` at the end of the function. + // This way, the mutation should not conflict with the owning thread, + // which only writes to locations later than `len` + // and will not resize the buffer without acquiring the lock. size_t len = need_sync ? jl_atomic_load_acquire(&list->len) : list->len; size_t oldlen = len; void **items = list->items; @@ -149,10 +155,12 @@ static void finalize_object(arraylist_t *list, jl_value_t *o, if (oldlen == len) return; if (need_sync) { - if (__unlikely(jl_atomic_compare_exchange(&list->len, - oldlen, len) != oldlen)) { - memset(items[len], 0, (oldlen - len) * sizeof(void*)); - } + // The memset needs to be unconditional since the thread might have + // already read the length. + // The `memset` (like any other content mutation) has to be done + // **before** the `cmpxchg` which publishes the length. + memset(&items[len], 0, (oldlen - len) * sizeof(void*)); + jl_atomic_compare_exchange(&list->len, oldlen, len); } else { list->len = len; @@ -246,9 +254,20 @@ static void gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) { int8_t gc_state = jl_gc_unsafe_enter(ptls); arraylist_t *a = &ptls->finalizers; + // This acquire load and the release store at the end are used to + // synchronize with `finalize_object` on another thread. Apart from the GC, + // which is blocked by entering a unsafe region, there might be only + // one other thread accessing our list in `finalize_object` + // (only one thread since it needs to acquire the finalizer lock). + // Similar to `finalize_object`, all content mutation has to be done + // between the acquire and the release of the length. size_t oldlen = jl_atomic_load_acquire(&a->len); if (__unlikely(oldlen + 2 > a->max)) { JL_LOCK_NOGC(&finalizers_lock); + // `a->len` might have been modified. + // Another possiblility is to always grow the array to `oldlen + 2` but + // it's simpler this way and uses slightly less memory =) + oldlen = a->len; arraylist_grow(a, 2); a->len = oldlen; JL_UNLOCK_NOGC(&finalizers_lock); From 6227c16f054f4c5b331d30b8ca53c82169b6511a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Sun, 24 Jul 2016 11:08:28 +0200 Subject: [PATCH 0611/1117] Add more docs to repl completion. (#17580) * Add more doc to repl completion. * Fix comments [ci skip]. --- doc/manual/interacting-with-julia.rst | 47 ++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/doc/manual/interacting-with-julia.rst b/doc/manual/interacting-with-julia.rst index 56c746eedb0de..882d0c5a3cb3b 100644 --- a/doc/manual/interacting-with-julia.rst +++ b/doc/manual/interacting-with-julia.rst @@ -228,11 +228,56 @@ and get a list of LaTeX matches as well:: \hat \heartsuit \hksearow \hookleftarrow \hslash \hbar \hermitconjmatrix \hkswarow \hookrightarrow \hspace + julia> α="\alpha[TAB]" # LaTeX completion also works in strings + julia> α="α" + A full list of tab-completions can be found in the :ref:`man-unicode-input` section of the manual. +Completion of paths works for strings and julia's shell mode:: + + julia> path="/[TAB]" + .dockerenv .juliabox/ boot/ etc/ lib/ media/ opt/ root/ sbin/ sys/ usr/ + .dockerinit bin/ dev/ home/ lib64/ mnt/ proc/ run/ srv/ tmp/ var/ + shell> /[TAB] + .dockerenv .juliabox/ boot/ etc/ lib/ media/ opt/ root/ sbin/ sys/ usr/ + .dockerinit bin/ dev/ home/ lib64/ mnt/ proc/ run/ srv/ tmp/ var/ + +Tab completion can help with investigation of the available methods matching the input arguments:: + + julia> max([TAB] # All methods are displayed, not shown here due to size of the list + + julia> max([1,2],[TAB] # All methods where `Vector{Int}` matches as first argument + max{T1<:Real,T2<:Real}(x::AbstractArray{T1,N<:Any}, y::T2) at operators.jl:544 + max{Tx<:Real,Ty<:Real}(x::Union{Base.ReshapedArray{Tx,1,A<:DenseArray,MI<:Tuple{Vararg{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64},N<:Any}}},DenseArray{Tx,1},SubArray{Tx,1,A<:Union{Base.ReshapedArray{T<:Any,N<:Any,A<:DenseArray,MI<:Tuple{Vararg{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64},N<:Any}}},DenseArray},I<:Tuple{Vararg{Union{Base.AbstractCartesianIndex,Colon,Int64,Range{Int64}},N<:Any}},L<:Any}}, y::AbstractSparseArray{Ty,Ti<:Any,1}) at sparse\sparsevector.jl:1127 + max{T1<:Real,T2<:Real}(x::AbstractArray{T1,N<:Any}, y::AbstractArray{T2,N<:Any}) at operators.jl:548 + max(x, y) at operators.jl:78 + max(a, b, c, xs...) at operators.jl:119 + + julia> max([1,2], max(1,2),[TAB] # All methods matching the arguments. + max{T1<:Real,T2<:Real}(x::AbstractArray{T1,N<:Any}, y::T2) at operators.jl:544 + max(x, y) at operators.jl:78 + max(a, b, c, xs...) at operators.jl:119 + + julia> split("1 1 1", # Keywords are also displayed in the suggested methods, see second line after `;` where `limit` and `keep` are keyword arguments + split(str::AbstractString) at strings/util.jl:151 + split{T<:AbstractString}(str::T, splitter; limit, keep) at strings/util.jl:127 + +The completion of the methods uses type inference and can therefore see if the arguments match even if the arguments are output from functions. The function needs to be type stable for the completion to be able to remove non-matching methods. + +Tab completion can also help completing fields:: + + julia> Pkg.a + add available + +Fields for output from functions can also be completed:: + + julia> split("","")[1].[TAB] + endof offset string + +The completion of fields for output from functions uses type inference, and it can only suggest fields if the function is type stable. Customizing Colors -~~~~~~~~~~~~~~~~~~ +------------------ The colors used by Julia and the REPL can be customized, as well. To change the color of the Julia prompt you can add something like the following to your ``juliarc.jl`` file:: From 97cbcb70655ef3b5309daebf27302c8f16833e3e Mon Sep 17 00:00:00 2001 From: rfourquet <fourquet.rafael+github@gmail.com> Date: Sun, 24 Jul 2016 22:15:23 +0530 Subject: [PATCH 0612/1117] implement randn!(::Array{Float32}) etc. (fix #9836) (#14811) * implement randn!(::Array{Float32}) etc. (fix #9836) * re-enable tests for Char in rand API (disabled in 84349f2) --- base/docs/helpdb/Base.jl | 31 ---------------- base/random.jl | 79 +++++++++++++++++++++++++++++----------- doc/stdlib/numbers.rst | 14 +++---- test/random.jl | 59 ++++++++++++++++++------------ 4 files changed, 99 insertions(+), 84 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 267d4cd74ec20..28f47e6e8496b 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -896,14 +896,6 @@ references in `r` are replaced with the corresponding matched text. """ replace -""" - randexp([rng], [dims...]) - -Generate a random number according to the exponential distribution with scale 1. Optionally -generate an array of such random numbers. -""" -randexp - """ chop(string) @@ -1917,14 +1909,6 @@ Kronecker tensor product of two vectors or two matrices. """ kron -""" - randn([rng], [dims...]) - -Generate a normally-distributed random number with mean 0 and standard deviation 1. -Optionally generate an array of normally-distributed random numbers. -""" -randn - """ process_exited(p::Process) @@ -6040,13 +6024,6 @@ the topmost backend that does not throw a `MethodError`). """ pushdisplay -""" - randexp!([rng], A::Array{Float64,N}) - -Fill the array `A` with random numbers following the exponential distribution (with scale 1). -""" -randexp! - """ prevind(str, i) @@ -8996,14 +8973,6 @@ no effect outside of compilation. """ include_dependency -""" - randn!([rng], A::Array{Float64,N}) - -Fill the array `A` with normally-distributed (mean 0, standard deviation 1) random numbers. -Also see the rand function. -""" -randn! - """ ldexp(x, n) diff --git a/base/random.jl b/base/random.jl index 641721bfd1d31..2f9a6e8bd5e7f 100644 --- a/base/random.jl +++ b/base/random.jl @@ -1117,7 +1117,14 @@ const ziggurat_nor_r = 3.6541528853610087963519472518 const ziggurat_nor_inv_r = inv(ziggurat_nor_r) const ziggurat_exp_r = 7.6971174701310497140446280481 +""" + randn([rng], [T=Float64], [dims...]) +Generate a normally-distributed random number of type `T` with mean 0 and standard deviation 1. +Optionally generate an array of normally-distributed random numbers. +The `Base` module currently provides an implementation for the types +`Float16`, `Float32`, and `Float64` (the default). +""" @inline function randn(rng::AbstractRNG=GLOBAL_RNG) @inbounds begin r = rand_ui52(rng) @@ -1144,19 +1151,14 @@ function randn_unlikely(rng, idx, rabs, x) end end -function randn!(rng::AbstractRNG, A::AbstractArray{Float64}) - for i in eachindex(A) - @inbounds A[i] = randn(rng) - end - A -end - -randn!(A::AbstractArray{Float64}) = randn!(GLOBAL_RNG, A) -randn(dims::Dims) = randn!(Array{Float64}(dims)) -randn(dims::Integer...) = randn!(Array{Float64}(dims...)) -randn(rng::AbstractRNG, dims::Dims) = randn!(rng, Array{Float64}(dims)) -randn(rng::AbstractRNG, dims::Integer...) = randn!(rng, Array{Float64}(dims...)) +""" + randexp([rng], [T=Float64], [dims...]) +Generate a random number of type `T` according to the exponential distribution with scale 1. +Optionally generate an array of such random numbers. +The `Base` module currently provides an implementation for the types +`Float16`, `Float32`, and `Float64` (the default). +""" @inline function randexp(rng::AbstractRNG=GLOBAL_RNG) @inbounds begin ri = rand_ui52(rng) @@ -1177,19 +1179,52 @@ function randexp_unlikely(rng, idx, x) end end -function randexp!(rng::AbstractRNG, A::Array{Float64}) - for i in eachindex(A) - @inbounds A[i] = randexp(rng) +""" + randn!([rng], A::AbstractArray) -> A + +Fill the array `A` with normally-distributed (mean 0, standard deviation 1) random numbers. +Also see the `rand` function. +""" +function randn! end + +""" + randexp!([rng], A::AbstractArray) -> A + +Fill the array `A` with random numbers following the exponential distribution (with scale 1). +""" +function randexp! end + +let Floats = Union{Float16,Float32,Float64} + for randfun in [:randn, :randexp] + randfun! = Symbol(randfun, :!) + @eval begin + # scalars + $randfun{T<:$Floats}(rng::AbstractRNG, ::Type{T}) = convert(T, $randfun(rng)) + $randfun{T<:$Floats}(::Type{T}) = $randfun(GLOBAL_RNG, T) + + # filling arrays + function $randfun!{T}(rng::AbstractRNG, A::AbstractArray{T}) + for i in eachindex(A) + @inbounds A[i] = $randfun(rng, T) + end + A + end + + $randfun!(A::AbstractArray) = $randfun!(GLOBAL_RNG, A) + + # generating arrays + $randfun{T}(rng::AbstractRNG, ::Type{T}, dims::Dims) = $randfun!(rng, Array{T}(dims)) + $randfun{T}(rng::AbstractRNG, ::Type{T}, dims::Integer...) = $randfun!(rng, Array{T}(dims...)) + $randfun{T}( ::Type{T}, dims::Dims) = $randfun(GLOBAL_RNG, T, dims) + $randfun{T}( ::Type{T}, dims::Integer...) = $randfun(GLOBAL_RNG, T, dims...) + $randfun( rng::AbstractRNG, dims::Dims) = $randfun(rng, Float64, dims) + $randfun( rng::AbstractRNG, dims::Integer...) = $randfun(rng, Float64, dims...) + $randfun( dims::Dims) = $randfun(GLOBAL_RNG, Float64, dims) + $randfun( dims::Integer...) = $randfun(GLOBAL_RNG, Float64, dims...) + end end - A end -randexp!(A::Array{Float64}) = randexp!(GLOBAL_RNG, A) -randexp(dims::Dims) = randexp!(Array{Float64}(dims)) -randexp(dims::Int...) = randexp!(Array{Float64}(dims)) -randexp(rng::AbstractRNG, dims::Dims) = randexp!(rng, Array{Float64}(dims)) -randexp(rng::AbstractRNG, dims::Int...) = randexp!(rng, Array{Float64}(dims)) - ## random UUID generation immutable UUID diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index f9fce9e0b22e7..ae1d47fa34c5f 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -598,25 +598,25 @@ As ``BigInt`` represents unbounded integers, the interval must be specified (e.g Generate a ``BitArray`` of random boolean values. -.. function:: randn([rng], [dims...]) +.. function:: randn([rng], [T=Float64], [dims...]) .. Docstring generated from Julia source - Generate a normally-distributed random number with mean 0 and standard deviation 1. Optionally generate an array of normally-distributed random numbers. + Generate a normally-distributed random number of type ``T`` with mean 0 and standard deviation 1. Optionally generate an array of normally-distributed random numbers. The ``Base`` module currently provides an implementation for the types ``Float16``\ , ``Float32``\ , and ``Float64`` (the default). -.. function:: randn!([rng], A::Array{Float64,N}) +.. function:: randn!([rng], A::AbstractArray) -> A .. Docstring generated from Julia source - Fill the array ``A`` with normally-distributed (mean 0, standard deviation 1) random numbers. Also see the rand function. + Fill the array ``A`` with normally-distributed (mean 0, standard deviation 1) random numbers. Also see the ``rand`` function. -.. function:: randexp([rng], [dims...]) +.. function:: randexp([rng], [T=Float64], [dims...]) .. Docstring generated from Julia source - Generate a random number according to the exponential distribution with scale 1. Optionally generate an array of such random numbers. + Generate a random number of type ``T`` according to the exponential distribution with scale 1. Optionally generate an array of such random numbers. The ``Base`` module currently provides an implementation for the types ``Float16``\ , ``Float32``\ , and ``Float64`` (the default). -.. function:: randexp!([rng], A::Array{Float64,N}) +.. function:: randexp!([rng], A::AbstractArray) -> A .. Docstring generated from Julia source diff --git a/test/random.jl b/test/random.jl index 91358626f5601..4115a612e4b5f 100644 --- a/test/random.jl +++ b/test/random.jl @@ -300,38 +300,49 @@ end # test all rand APIs for rng in ([], [MersenneTwister()], [RandomDevice()]) + types = [Base.BitInteger_types..., Bool, Float16, Float32, Float64, Char] + ftypes = [Float16, Float32, Float64] + b2 = big(2) + u3 = UInt(3) for f in [rand, randn, randexp] - f(rng...) ::Float64 - f(rng..., 5) ::Vector{Float64} - f(rng..., 2, 3) ::Array{Float64, 2} + f(rng...) ::Float64 + f(rng..., 5) ::Vector{Float64} + f(rng..., 2, 3) ::Array{Float64, 2} + f(rng..., b2, u3) ::Array{Float64, 2} + f(rng..., (2, 3)) ::Array{Float64, 2} + for T in (f === rand ? types : ftypes) + a0 = f(rng..., T) ::T + a1 = f(rng..., T, 5) ::Vector{T} + a2 = f(rng..., T, 2, 3) ::Array{T, 2} + a3 = f(rng..., T, b2, u3) ::Array{T, 2} + a4 = f(rng..., T, (2, 3)) ::Array{T, 2} + if T <: AbstractFloat && f === rand + for a in [a0, a1..., a2..., a3..., a4...] + @test 0.0 <= a < 1.0 + end + end + end end - for f! in [randn!, randexp!] - f!(rng..., Array{Float64}(5)) ::Vector{Float64} - f!(rng..., Array{Float64}(2, 3)) ::Array{Float64, 2} + for f! in [rand!, randn!, randexp!] + for T in (f! === rand! ? types : ftypes) + X = T == Bool ? T[0,1] : T[0,1,2] + for A in (Array{T}(5), Array{T}(2, 3)) + f!(rng..., A) ::typeof(A) + if f! === rand! + f!(rng..., A, X) ::typeof(A) + if T !== Char # Char/Integer comparison + f!(rng..., sparse(A)) ::typeof(sparse(A)) + f!(rng..., sparse(A), X) ::typeof(sparse(A)) + end + end + end + end end bitrand(rng..., 5) ::BitArray{1} bitrand(rng..., 2, 3) ::BitArray{2} rand!(rng..., BitArray(5)) ::BitArray{1} rand!(rng..., BitArray(2, 3)) ::BitArray{2} - - for T in [Base.BitInteger_types..., Bool, Float16, Float32, Float64] - a0 = rand(rng..., T) ::T - a1 = rand(rng..., T, 5) ::Vector{T} - a2 = rand(rng..., T, 2, 3) ::Array{T, 2} - if T <: AbstractFloat - for a in [a0, a1..., a2...] - @test 0.0 <= a < 1.0 - end - end - for A in (Array{T}(5), Array{T}(2, 3)) - X = T == Bool ? T[0,1] : T[0,1,2] - rand!(rng..., A) ::typeof(A) - rand!(rng..., A, X) ::typeof(A) - rand!(rng..., sparse(A)) ::typeof(sparse(A)) - rand!(rng..., sparse(A), X) ::typeof(sparse(A)) - end - end end function hist(X,n) From bead2d4f2c92a7fc783241db00906bbbfb00656e Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Sun, 24 Jul 2016 17:50:35 +0000 Subject: [PATCH 0613/1117] Turn on LLVM assertions for arm and power. [ci skip] --- Make.inc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Make.inc b/Make.inc index 2264fea33433c..ec4649fe757a6 100644 --- a/Make.inc +++ b/Make.inc @@ -642,10 +642,12 @@ else ISX86:=0 endif -# If we are running on powerpc64 or ppc64, set certain options automatically +# If we are running on powerpc64 or ppc64le, set certain options automatically ifneq (,$(filter $(ARCH), powerpc64 ppc64le)) + JCFLAGS += -fsigned-char override LLVM_VER:=3.8.1 +override LLVM_ASSERTIONS:=1 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=POWER8 @@ -653,12 +655,13 @@ endif # If we are running on ARM, set certain options automatically ifneq (,$(findstring arm,$(ARCH))) -JCFLAGS += -fsigned-char +JCFLAGS += -fsigned-char override LLVM_VER:=3.8.1 -override USE_BLAS64:=0 +override LLVM_ASSERTIONS:=1 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=ARMV7 +override USE_BLAS64:=0 override USE_SYSTEM_LIBM:=1 endif From c962b48c7879205977557b59e94f180385c10ac3 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Mon, 25 Jul 2016 00:23:18 +0200 Subject: [PATCH 0614/1117] Remove non-standard unicode list char MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `•` is not part of the commonmark spec. Using this char for lists wasn't ever documented as far as I could tell. --- base/markdown/Common/block.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index bba06e7bc3647..298f75e1d934e 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -225,8 +225,8 @@ List(xs...) = List(vcat(xs...)) isordered(list::List) = list.ordered >= 0 -const BULLETS = r"^ {0,3}(\*|\+|•|-)( |$)" -const NUM_OR_BULLETS = r"^ {0,3}(\*|•|\+|-|\d+(\.|\)))( |$)" +const BULLETS = r"^ {0,3}(\*|\+|-)( |$)" +const NUM_OR_BULLETS = r"^ {0,3}(\*|\+|-|\d+(\.|\)))( |$)" function list(stream::IO, block::MD) withstream(stream) do From ab7b184b531720c52781f957e537ab56cb851d21 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Mon, 25 Jul 2016 00:26:39 +0200 Subject: [PATCH 0615/1117] Print operator `Binding`s correctly This prints `Binding`s for operators with a `:` char so that they are always valid Julia syntax, i.e. string(Binding(Base, :+)) == "Base.:+" rather than string(Binding(Base, :+)) == "Base.+" --- base/docs/bindings.jl | 8 +++++++- test/docs.jl | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/base/docs/bindings.jl b/base/docs/bindings.jl index 368075b0334c3..045b62b3bc4e4 100644 --- a/base/docs/bindings.jl +++ b/base/docs/bindings.jl @@ -31,7 +31,13 @@ macro var(x) esc(bindingexpr(x)) end -Base.show(io::IO, b::Binding) = b.mod === Main ? print(io, b.var) : print(io, b.mod, '.', b.var) +function Base.show(io::IO, b::Binding) + if b.mod === Main + print(io, b.var) + else + print(io, b.mod, '.', Base.isoperator(b.var) ? ":" : "", b.var) + end +end aliasof(b::Binding) = defined(b) ? (a = aliasof(resolve(b), b); defined(a) ? a : b) : b aliasof(d::DataType, b) = Binding(d.name.module, d.name.name) diff --git a/test/docs.jl b/test/docs.jl index 563bc9366f98f..4c08d8ba05198 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -867,6 +867,18 @@ let x = Binding(Main, :bindingdoesnotexist) @test @var(bindingdoesnotexist) == x end +let x = Binding(Main, :+) + @test parse(string(x)) == :(Base.:+) +end + +let x = Binding(Base, :parse) + @test parse(string(x)) == :(Base.parse) +end + +let x = Binding(Main, :⊕) + @test parse(string(x)) == :(⊕) +end + # Docs.helpmode tests: we test whether the correct expressions are being generated here, # rather than complete integration with Julia's REPL mode system. for (line, expr) in Pair[ From a9cc5eb7d6daad4cee7f856050d830e7f782913a Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Mon, 25 Jul 2016 00:31:22 +0200 Subject: [PATCH 0616/1117] Print markdown code with correct backticks This adds extra backticks and spaces around code fragments when printing if the fragment contains backticks. This allows `Markdown.parse` to properly re-parse text correctly, i.e. Markdown.plain(Markdown.parse("``` `x` ```")) == "``` `x` ```" rather than Markdown.plain(Markdown.parse("``` `x` ```")) == "``x``" --- base/markdown/render/plain.jl | 11 ++++++++++- test/markdown.jl | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/base/markdown/render/plain.jl b/base/markdown/render/plain.jl index cfd006393b902..096c5d66be8ad 100644 --- a/base/markdown/render/plain.jl +++ b/base/markdown/render/plain.jl @@ -105,7 +105,16 @@ plaininline(io::IO, md::Bold) = plaininline(io, "**", md.text, "**") plaininline(io::IO, md::Italic) = plaininline(io, "*", md.text, "*") -plaininline(io::IO, md::Code) = print(io, "`", md.code, "`") +function plaininline(io::IO, md::Code) + if contains(md.code, "`") + n = maximum(length(m) for m in matchall(r"(`+)", md.code)) + s = "`"^((iseven(n) ? 1 : 2) + n) + print(io, s, Base.startswith(md.code, "`") ? " " : "") + print(io, md.code, endswith(md.code, "`") ? " " : "", s) + else + print(io, "`", md.code, "`") + end +end plaininline(io::IO, br::LineBreak) = println(io) diff --git a/test/markdown.jl b/test/markdown.jl index 470b349b0e6e5..6a12e836e6294 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -55,6 +55,11 @@ code_in_code = md""" @test code_in_code == MD(Code("```")) @test plain(code_in_code) == "````\n```\n````\n" +let text = "Foo ```bar` ``baz`` ```\n", + md = Markdown.parse(text) + @test text == Markdown.plain(md) +end + @test md"A footnote [^foo]." == MD(Paragraph(["A footnote ", Footnote("foo", nothing), "."])) @test md"[^foo]: footnote" == MD(Paragraph([Footnote("foo", Any[" footnote"])])) From 81dccbd02d740107a75c134a939cdc789f0b78fb Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Mon, 25 Jul 2016 00:35:32 +0200 Subject: [PATCH 0617/1117] Allow escaped `|` chars in tables Markdown tables might require literal `|` to be written within a cell, such as when writing the function `|`. This allows `|` to be escaped using a `\`. --- base/markdown/GitHub/table.jl | 8 +++++--- test/markdown.jl | 9 +++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/base/markdown/GitHub/table.jl b/base/markdown/GitHub/table.jl index e4e74f152a0f4..9822aed14d1fd 100644 --- a/base/markdown/GitHub/table.jl +++ b/base/markdown/GitHub/table.jl @@ -8,10 +8,10 @@ end function parserow(stream::IO) withstream(stream) do line = readline(stream) |> chomp - row = split(line, "|") + row = split(line, r"(?<!\\)\|") length(row) == 1 && return row[1] == "" && shift!(row) - map!(strip, row) + map!(x -> strip(replace(x, "\\|", "|")), row) row[end] == "" && pop!(row) return row end @@ -103,7 +103,9 @@ _dash(width, align) = throw(ArgumentError("Invalid alignment $align")) function plain(io::IO, md::Table) - cells = mapmap(plaininline, md.rows) + cells = mapmap(md.rows) do each + replace(plaininline(each), "|", "\\|") + end padcells!(cells, md.align, len = length, min = 3) for i = indices(cells,1) print(io, "| ") diff --git a/test/markdown.jl b/test/markdown.jl index 6a12e836e6294..d854a857e40c4 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -362,6 +362,15 @@ let text = table = Markdown.parse(text) @test text == Markdown.plain(table) end +let text = + """ + | a | b | + |:-------- | ---:| + | `x \\| y` | 2 | + """, + table = Markdown.parse(text) + @test text == Markdown.plain(table) +end # LaTeX extension From 9b8aa7bf1fa20b30c4bb6ac5a30d78c9f9e12b2f Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sun, 24 Jul 2016 23:01:29 +0000 Subject: [PATCH 0618/1117] Remove sizeclass 12 on ARM and PPC32. We don't take alignment into account while doing allocation. This makes sure that allocation of size 8 uses the 16 size class instead of the 12 size class and is properly aligned on these architectures. --- src/julia.h | 2 +- src/julia_internal.h | 8 ++++++++ src/julia_threads.h | 6 ++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/julia.h b/src/julia.h index 24f5d4f5cf7f0..24bf039327a47 100644 --- a/src/julia.h +++ b/src/julia.h @@ -19,7 +19,7 @@ #include <setjmp.h> #ifndef _OS_WINDOWS_ # define jl_jmp_buf sigjmp_buf -# if defined(_CPU_ARM_) +# if defined(_CPU_ARM_) || defined(_CPU_PPC_) # define MAX_ALIGN 8 # elif defined(_CPU_AARCH64_) // int128 is 16 bytes aligned on aarch64 diff --git a/src/julia_internal.h b/src/julia_internal.h index fcc29f42b1791..db4a2f97383e8 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -54,6 +54,10 @@ void *jl_gc_perm_alloc(size_t sz); static const int jl_gc_sizeclasses[JL_GC_N_POOLS] = { #ifdef _P64 8, +#elif defined(_CPU_ARM_) || defined(_CPU_PPC_) + // ARM and PowerPC has max alignment of 8, + // make sure allocation of size 8 has that alignment. + 4, 8, #else 4, 8, 12, #endif @@ -87,6 +91,10 @@ STATIC_INLINE int JL_CONST_FUNC jl_gc_szclass(size_t sz) if (sz <= 8) return 0; const int N = 0; +#elif defined(_CPU_ARM_) || defined(_CPU_PPC_) + if (sz <= 8) + return (sz + 3) / 4 - 1; + const int N = 1; #else if (sz <= 12) return (sz + 3) / 4 - 1; diff --git a/src/julia_threads.h b/src/julia_threads.h index 50a9bc36c2820..e18eb7283e1d4 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -55,9 +55,11 @@ typedef struct { // variables for allocating objects from pools #ifdef _P64 -#define JL_GC_N_POOLS 41 +# define JL_GC_N_POOLS 41 +#elif defined(_CPU_ARM_) || defined(_CPU_PPC_) +# define JL_GC_N_POOLS 42 #else -#define JL_GC_N_POOLS 43 +# define JL_GC_N_POOLS 43 #endif jl_gc_pool_t norm_pools[JL_GC_N_POOLS]; } jl_thread_heap_t; From 3e2c6e7987087e169728d3fee5fd996b262ebfb9 Mon Sep 17 00:00:00 2001 From: Stefan O'Rear <stefanor@cox.net> Date: Sun, 24 Jul 2016 17:48:37 -0700 Subject: [PATCH 0619/1117] Use standard nomenclature for UNIX domain sockets UNIX modifies "domain", not "domain socket"; "UNIX domain" modifies "socket", and "domain socket" by itself is meaningless. This matches the phrasing and capitalization from POSIX.1: http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_10_17 --- base/docs/helpdb/Base.jl | 4 ++-- doc/manual/networking-and-streams.rst | 4 ++-- doc/stdlib/io-network.rst | 4 ++-- examples/clustermanager/simple/README | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 28f47e6e8496b..ddd020baebfa0 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1565,7 +1565,7 @@ connect(host=?, port) """ connect(path) -> PipeEndpoint -Connect to the Named Pipe / Domain Socket at `path`. +Connect to the named pipe / UNIX domain socket at `path`. """ connect(path) @@ -7286,7 +7286,7 @@ listen(addr,port) """ listen(path) -> PipeServer -Create and listen on a Named Pipe / Domain Socket. +Create and listen on a named pipe / UNIX domain socket. """ listen(path) diff --git a/doc/manual/networking-and-streams.rst b/doc/manual/networking-and-streams.rst index 3aac2fcd9b505..3ec703c26ee4f 100644 --- a/doc/manual/networking-and-streams.rst +++ b/doc/manual/networking-and-streams.rst @@ -197,12 +197,12 @@ create various other kinds of servers:: julia> listen(IPv6(0),2001) # Listens on port 2001 on all IPv6 interfaces TCPServer(active) - julia> listen("testsocket") # Listens on a domain socket/named pipe + julia> listen("testsocket") # Listens on a UNIX domain socket/named pipe PipeServer(active) Note that the return type of the last invocation is different. This is because this server does not listen on TCP, but rather on a named pipe (Windows) -or domain socket (UNIX). The difference +or UNIX domain socket (UNIX). The difference is subtle and has to do with the :func:`accept` and :func:`connect` methods. The :func:`accept` method retrieves a connection to the client that is connecting on the server we just created, while the :func:`connect` function connects to a server using the diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 2ddf77b578d4a..b4d56fae3f155 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -886,7 +886,7 @@ Network I/O .. Docstring generated from Julia source - Connect to the Named Pipe / Domain Socket at ``path``\ . + Connect to the named pipe / UNIX domain socket at ``path``\ . .. function:: listen([addr,]port) -> TCPServer @@ -898,7 +898,7 @@ Network I/O .. Docstring generated from Julia source - Create and listen on a Named Pipe / Domain Socket. + Create and listen on a named pipe / UNIX domain socket. .. function:: getaddrinfo(host) diff --git a/examples/clustermanager/simple/README b/examples/clustermanager/simple/README index 86906ada08b7b..b14d0699d4bc2 100644 --- a/examples/clustermanager/simple/README +++ b/examples/clustermanager/simple/README @@ -1,4 +1,4 @@ -This is a simple proof-of-concept that uses Unix Domain Sockets as transport. +This is a simple proof-of-concept that uses UNIX domain sockets as transport. All commands must be run from `examples/clustermanager/simple` directory From 50acb8f45289cf3834f197a9e3b2d23bc5b713bc Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sun, 24 Jul 2016 23:45:48 -0400 Subject: [PATCH 0620/1117] add file/line numbers to closest candidate printing intended to help with anonymous functions that don't have names and #16091 --- base/replutil.jl | 2 + test/replutil.jl | 95 +++++++++++++++++++++++++++++------------------- 2 files changed, 59 insertions(+), 38 deletions(-) diff --git a/base/replutil.jl b/base/replutil.jl index 54be916eb01e6..5b90ce02d757f 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -433,6 +433,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs::Vector=Any[]) # function itself doesn't match return else + # TODO: use the methodshow logic here use_constructor_syntax = isa(func, Type) print(buf, use_constructor_syntax ? func : typeof(func).name.mt.name) end @@ -520,6 +521,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs::Vector=Any[]) length(kwords) > 0 && print(buf, "; ", join(kwords, ", ")) end print(buf, ")") + print(buf, " at ", method.file, ":", method.line) if !isempty(kwargs) unexpected = Symbol[] if isempty(kwords) || !(any(endswith(string(kword), "...") for kword in kwords)) diff --git a/test/replutil.jl b/test/replutil.jl index 11288aa33e19d..1abd59d506efc 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -8,26 +8,28 @@ function test_have_color(buf, color, no_color) end end +cfile = " at $(@__FILE__):" +c1line = @__LINE__ + 1 method_c1(x::Float64, s::AbstractString...) = true buf = IOBuffer() Base.show_method_candidates(buf, Base.MethodError(method_c1,(1, 1, ""))) -no_color = "\nClosest candidates are:\n method_c1(!Matched::Float64, !Matched::AbstractString...)" +no_color = "\nClosest candidates are:\n method_c1(!Matched::Float64, !Matched::AbstractString...)$cfile$c1line" test_have_color(buf, - "\e[0m\nClosest candidates are:\n method_c1(\e[1m\e[31m::Float64\e[0m, \e[1m\e[31m::AbstractString...\e[0m)\e[0m", + "\e[0m\nClosest candidates are:\n method_c1(\e[1m\e[31m::Float64\e[0m, \e[1m\e[31m::AbstractString...\e[0m)$cfile$c1line\e[0m", no_color) -no_color = "\nClosest candidates are:\n method_c1(!Matched::Float64, ::AbstractString...)" +no_color = "\nClosest candidates are:\n method_c1(!Matched::Float64, ::AbstractString...)$cfile$c1line" Base.show_method_candidates(buf, Base.MethodError(method_c1,(1, "", ""))) test_have_color(buf, - "\e[0m\nClosest candidates are:\n method_c1(\e[1m\e[31m::Float64\e[0m, ::AbstractString...)\e[0m", + "\e[0m\nClosest candidates are:\n method_c1(\e[1m\e[31m::Float64\e[0m, ::AbstractString...)$cfile$c1line\e[0m", no_color) # should match -no_color = "\nClosest candidates are:\n method_c1(::Float64, ::AbstractString...)" +no_color = "\nClosest candidates are:\n method_c1(::Float64, ::AbstractString...)$cfile$c1line" Base.show_method_candidates(buf, Base.MethodError(method_c1,(1., "", ""))) test_have_color(buf, - "\e[0m\nClosest candidates are:\n method_c1(::Float64, ::AbstractString...)\e[0m", + "\e[0m\nClosest candidates are:\n method_c1(::Float64, ::AbstractString...)$cfile$c1line\e[0m", no_color) # Have no matches so should return empty @@ -36,8 +38,12 @@ test_have_color(buf, "", "") # matches the implicit constructor -> convert method Base.show_method_candidates(buf, Base.MethodError(Tuple{}, (1, 1, 1))) -@test contains(takebuf_string(buf), "\nClosest candidates are:\n Tuple{}{T}(") +let mc = takebuf_string(buf) + @test contains(mc, "\nClosest candidates are:\n Tuple{}{T}(") + @test !contains(mc, cfile) +end +c2line = @__LINE__ method_c2(x::Int32, args...) = true method_c2(x::Int32, y::Float64, args...) = true method_c2(x::Int32, y::Float64) = true @@ -45,30 +51,35 @@ method_c2(x::Int32, y::Int32, z::Int32) = true method_c2{T<:Real}(x::T, y::T, z::T) = true Base.show_method_candidates(buf, Base.MethodError(method_c2,(1., 1., 2))) -color = "\e[0m\nClosest candidates are:\n method_c2(\e[1m\e[31m::Int32\e[0m, ::Float64, ::Any...)\n method_c2(\e[1m\e[31m::Int32\e[0m, ::Any...)\n method_c2{T<:Real}(::T<:Real, ::T<:Real, \e[1m\e[31m::T<:Real\e[0m)\n ...\e[0m" -no_color = no_color = "\nClosest candidates are:\n method_c2(!Matched::Int32, ::Float64, ::Any...)\n method_c2(!Matched::Int32, ::Any...)\n method_c2{T<:Real}(::T<:Real, ::T<:Real, !Matched::T<:Real)\n ..." +color = "\e[0m\nClosest candidates are:\n method_c2(\e[1m\e[31m::Int32\e[0m, ::Float64, ::Any...)$cfile$(c2line+2)\n method_c2(\e[1m\e[31m::Int32\e[0m, ::Any...)$cfile$(c2line+1)\n method_c2{T<:Real}(::T<:Real, ::T<:Real, \e[1m\e[31m::T<:Real\e[0m)$cfile$(c2line+5)\n ...\e[0m" +no_color = no_color = "\nClosest candidates are:\n method_c2(!Matched::Int32, ::Float64, ::Any...)$cfile$(c2line+2)\n method_c2(!Matched::Int32, ::Any...)$cfile$(c2line+1)\n method_c2{T<:Real}(::T<:Real, ::T<:Real, !Matched::T<:Real)$cfile$(c2line+5)\n ..." test_have_color(buf, color, no_color) +c3line = @__LINE__ + 1 method_c3(x::Float64, y::Float64) = true Base.show_method_candidates(buf, Base.MethodError(method_c3,(1.,))) -color = "\e[0m\nClosest candidates are:\n method_c3(::Float64, \e[1m\e[31m::Float64\e[0m)\e[0m" -no_color = no_color = "\nClosest candidates are:\n method_c3(::Float64, !Matched::Float64)" +color = "\e[0m\nClosest candidates are:\n method_c3(::Float64, \e[1m\e[31m::Float64\e[0m)$cfile$c3line\e[0m" +no_color = no_color = "\nClosest candidates are:\n method_c3(::Float64, !Matched::Float64)$cfile$c3line" test_have_color(buf, color, no_color) # Test for the method error in issue #8651 +c4line = @__LINE__ method_c4() = true method_c4(x::AbstractString) = false Base.show_method_candidates(buf, MethodError(method_c4,("",))) -test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c4(::AbstractString)\n method_c4()\e[0m", "\nClosest candidates are:\n method_c4(::AbstractString)\n method_c4()") +test_have_color(buf, + "\e[0m\nClosest candidates are:\n method_c4(::AbstractString)$cfile$(c4line+2)\n method_c4()$cfile$(c4line+1)\e[0m", + "\nClosest candidates are:\n method_c4(::AbstractString)$cfile$(c4line+2)\n method_c4()$cfile$(c4line+1)") +c5line = @__LINE__ + 1 method_c5(::Type{Float64}) = true Base.show_method_candidates(buf, MethodError(method_c5,(Float64,))) -test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c5(::Type{Float64})\e[0m", - "\nClosest candidates are:\n method_c5(::Type{Float64})") +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c5(::Type{Float64})$cfile$c5line\e[0m", + "\nClosest candidates are:\n method_c5(::Type{Float64})$cfile$c5line") Base.show_method_candidates(buf, MethodError(method_c5,(Int32,))) -test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c5(\e[1m\e[31m::Type{Float64}\e[0m)\e[0m", - "\nClosest candidates are:\n method_c5(!Matched::Type{Float64})") +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c5(\e[1m\e[31m::Type{Float64}\e[0m)$cfile$c5line\e[0m", + "\nClosest candidates are:\n method_c5(!Matched::Type{Float64})$cfile$c5line") type Test_type end test_type = Test_type() @@ -77,19 +88,23 @@ for f in [getindex, setindex!] test_have_color(buf, "", "") end +PR16155line = @__LINE__ + 2 type PR16155 a::Int64 b end +PR16155line2 = @__LINE__ + 1 +(::Type{T}){T<:PR16155}(arg::Any) = "replace call-to-convert method from sysimg" Base.show_method_candidates(buf, MethodError(PR16155,(1.0, 2.0, Int64(3)))) -test_have_color(buf, "\e[0m\nClosest candidates are:\n PR16155(::Any, ::Any)\n PR16155(\e[1m\e[31m::Int64\e[0m, ::Any)\n PR16155{T}(::Any)\e[0m", - "\nClosest candidates are:\n PR16155(::Any, ::Any)\n PR16155(!Matched::Int64, ::Any)\n PR16155{T}(::Any)") +test_have_color(buf, "\e[0m\nClosest candidates are:\n PR16155(::Any, ::Any)$cfile$PR16155line\n PR16155(\e[1m\e[31m::Int64\e[0m, ::Any)$cfile$PR16155line\n PR16155{T<:PR16155}(::Any)$cfile$PR16155line2\n ...\e[0m", + "\nClosest candidates are:\n PR16155(::Any, ::Any)$cfile$PR16155line\n PR16155(!Matched::Int64, ::Any)$cfile$PR16155line\n PR16155{T<:PR16155}(::Any)$cfile$PR16155line2\n ...") Base.show_method_candidates(buf, MethodError(PR16155,(Int64(3), 2.0, Int64(3)))) -test_have_color(buf, "\e[0m\nClosest candidates are:\n PR16155(::Int64, ::Any)\n PR16155(::Any, ::Any)\n PR16155{T}(::Any)\e[0m", - "\nClosest candidates are:\n PR16155(::Int64, ::Any)\n PR16155(::Any, ::Any)\n PR16155{T}(::Any)") +test_have_color(buf, "\e[0m\nClosest candidates are:\n PR16155(::Int64, ::Any)$cfile$PR16155line\n PR16155(::Any, ::Any)$cfile$PR16155line\n PR16155{T<:PR16155}(::Any)$cfile$PR16155line2\n ...\e[0m", + "\nClosest candidates are:\n PR16155(::Int64, ::Any)$cfile$PR16155line\n PR16155(::Any, ::Any)$cfile$PR16155line\n PR16155{T<:PR16155}(::Any)$cfile$PR16155line2\n ...") +c6line = @__LINE__ method_c6(; x=1) = x method_c6(a; y=1) = y m_error = try method_c6(y=1) catch e; e; end @@ -99,6 +114,7 @@ m_error = try method_c6(1, x=1) catch e; e; end showerror(buf, m_error) error_out1 = takebuf_string(buf) +c6mline = @__LINE__ module TestKWError method_c6_in_module(; x=1) = x method_c6_in_module(a; y=1) = y @@ -111,36 +127,39 @@ showerror(buf, m_error) error_out3 = takebuf_string(buf) if Base.have_color - @test contains(error_out, "method_c6(; x)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") - @test contains(error_out, "method_c6(\e[1m\e[31m::Any\e[0m; y)") - @test contains(error_out1, "method_c6(::Any; y)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") - @test contains(error_out2, "method_c6_in_module(; x)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") - @test contains(error_out2, "method_c6_in_module(\e[1m\e[31m::Any\e[0m; y)") - @test contains(error_out3, "method_c6_in_module(::Any; y)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") + @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") + @test contains(error_out, "method_c6(\e[1m\e[31m::Any\e[0m; y)$cfile$(c6line + 2)") + @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") + @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") + @test contains(error_out2, "method_c6_in_module(\e[1m\e[31m::Any\e[0m; y)$cfile$(c6mline + 3)") + @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") else - @test contains(error_out, "method_c6(; x) got an unsupported keyword argument \"y\"") - @test contains(error_out, "method_c6(!Matched::Any; y)") - @test contains(error_out1, "method_c6(::Any; y) got an unsupported keyword argument \"x\"") - @test contains(error_out2, "method_c6_in_module(; x) got an unsupported keyword argument \"y\"") - @test contains(error_out2, "method_c6_in_module(!Matched::Any; y)") - @test contains(error_out3, "method_c6_in_module(::Any; y) got an unsupported keyword argument \"x\"") + @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1) got an unsupported keyword argument \"y\"") + @test contains(error_out, "method_c6(!Matched::Any; y)$cfile$(c6line + 2)") + @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2) got an unsupported keyword argument \"x\"") + @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2) got an unsupported keyword argument \"y\"") + @test contains(error_out2, "method_c6_in_module(!Matched::Any; y)$cfile$(c6mline + 3)") + @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3) got an unsupported keyword argument \"x\"") end +c7line = @__LINE__ + 1 method_c7(a, b; kargs...) = a Base.show_method_candidates(buf, MethodError(method_c7, (1, 1)), [(:x, 1), (:y, 2)]) -test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c7(::Any, ::Any; kargs...)\e[0m", - "\nClosest candidates are:\n method_c7(::Any, ::Any; kargs...)") +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c7(::Any, ::Any; kargs...)$cfile$c7line\e[0m", + "\nClosest candidates are:\n method_c7(::Any, ::Any; kargs...)$cfile$c7line") +c8line = @__LINE__ + 1 method_c8(a, b; y=1, w=1) = a Base.show_method_candidates(buf, MethodError(method_c8, (1, 1)), [(:x, 1), (:y, 2), (:z, 1), (:w, 1)]) -test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)\e[1m\e[31m got an unsupported keyword argument \"x\", \"z\"\e[0m\e[0m", - "\nClosest candidates are:\n method_c8(::Any, ::Any; y, w) got an unsupported keyword argument \"x\", \"z\"") +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line\e[1m\e[31m got an unsupported keyword argument \"x\", \"z\"\e[0m\e[0m", + "\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line got an unsupported keyword argument \"x\", \"z\"") +ac15639line = @__LINE__ addConstraint_15639(c::Int32) = c addConstraint_15639(c::Int64; uncset=nothing) = addConstraint_15639(Int32(c), uncset=uncset) Base.show_method_candidates(buf, MethodError(addConstraint_15639, (Int32(1),)), [(:uncset, nothing)]) -test_have_color(buf, "\e[0m\nClosest candidates are:\n addConstraint_15639(::Int32)\e[1m\e[31m got an unsupported keyword argument \"uncset\"\e[0m\n addConstraint_15639(\e[1m\e[31m::Int64\e[0m; uncset)\e[0m", - "\nClosest candidates are:\n addConstraint_15639(::Int32) got an unsupported keyword argument \"uncset\"\n addConstraint_15639(!Matched::Int64; uncset)") +test_have_color(buf, "\e[0m\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1)\e[1m\e[31m got an unsupported keyword argument \"uncset\"\e[0m\n addConstraint_15639(\e[1m\e[31m::Int64\e[0m; uncset)$cfile$(ac15639line + 2)\e[0m", + "\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1) got an unsupported keyword argument \"uncset\"\n addConstraint_15639(!Matched::Int64; uncset)$cfile$(ac15639line + 2)") macro except_str(expr, err_type) return quote From 9c14ec0031a6c6556cfe56fab8b98a078f725579 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Mon, 25 Jul 2016 05:10:47 +0000 Subject: [PATCH 0621/1117] Revert "Turn on LLVM assertions for arm and power." This reverts commit f4ce9848595a7f9e633489a41df954ea3cfbc83c. [ci skip] --- Make.inc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Make.inc b/Make.inc index ec4649fe757a6..2264fea33433c 100644 --- a/Make.inc +++ b/Make.inc @@ -642,12 +642,10 @@ else ISX86:=0 endif -# If we are running on powerpc64 or ppc64le, set certain options automatically +# If we are running on powerpc64 or ppc64, set certain options automatically ifneq (,$(filter $(ARCH), powerpc64 ppc64le)) - JCFLAGS += -fsigned-char override LLVM_VER:=3.8.1 -override LLVM_ASSERTIONS:=1 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=POWER8 @@ -655,13 +653,12 @@ endif # If we are running on ARM, set certain options automatically ifneq (,$(findstring arm,$(ARCH))) - JCFLAGS += -fsigned-char + override LLVM_VER:=3.8.1 -override LLVM_ASSERTIONS:=1 +override USE_BLAS64:=0 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=ARMV7 -override USE_BLAS64:=0 override USE_SYSTEM_LIBM:=1 endif From 584cc3a8d0a5f0fe9e996a580509dea566ac1588 Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Thu, 21 Jul 2016 10:43:20 +0200 Subject: [PATCH 0622/1117] Revise build number computation Use commits since last change to VERSION file instead of commits since last tag. Build number is appended if VERSION ends in -dev or -pre. To maintain consistency for 0.5.0-dev versions, the number of commits is incremented by one for these (last tag was one commit before change to VERSION). --- base/version.jl | 15 ++++++++++++--- base/version_git.sh | 14 +++++++++----- contrib/commit-name.sh | 33 ++++++++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/base/version.jl b/base/version.jl index 60410e390e563..219d1e08c6774 100644 --- a/base/version.jl +++ b/base/version.jl @@ -206,9 +206,18 @@ A `VersionNumber` object describing which version of Julia is in use. For detail [Version Number Literals](:ref:`man-version-number-literals`). """ const VERSION = try - # Include build number if we've got at least some distance from a tag (e.g. a release) - build_number = GIT_VERSION_INFO.build_number != 0 ? "+$(GIT_VERSION_INFO.build_number)" : "" - convert(VersionNumber, "$(VERSION_STRING)$(build_number)") + ver = convert(VersionNumber, VERSION_STRING) + if !isempty(ver.prerelease) + build_number = GIT_VERSION_INFO.build_number + if ver == v"0.5.0-pre" + # due to change of reference for counting commits from last tag to last change of VERSION file + build_number += 5578 + end + ver = VersionNumber(ver.major, ver.minor, ver.patch, ver.prerelease, (build_number,)) + elseif GIT_VERSION_INFO.build_number != 0 + println("WARNING: ignoring non-zero build number for VERSION") + end + ver catch e println("while creating Base.VERSION, ignoring error $e") VersionNumber(0) diff --git a/base/version_git.sh b/base/version_git.sh index 38699a59092ee..97d3cc05d78f5 100644 --- a/base/version_git.sh +++ b/base/version_git.sh @@ -30,7 +30,6 @@ origin=$(git config -l 2>/dev/null | grep 'remote\.\w*\.url.*JuliaLang/julia.git if [ -z "$origin" ]; then origin="origin/" fi -last_tag=$(git describe --tags --abbrev=0) git_time=$(git log -1 --pretty=format:%ct) #collect the contents @@ -41,10 +40,15 @@ if [ -n "$(git status --porcelain)" ]; then commit_short="$commit_short"* fi branch=$(git branch | sed -n '/\* /s///p') -# Some versions of wc (eg on OS X) add extra whitespace to their output. -# The sed(1) call stops this from breaking the generated Julia's indentation by -# stripping all non-digits. -build_number=$(git rev-list HEAD ^$last_tag | wc -l | sed -e 's/[^[:digit:]]//g') + +topdir=$(git rev-parse --show-toplevel) +verchanged=$(git blame -L ,1 -sl -- "$topdir/VERSION" | cut -f 1 -d " ") +if [ $verchanged = 0000000000000000000000000000000000000000 ]; then + # uncommited change to VERSION + build_number=0 +else + build_number=$(git rev-list --count HEAD "^$verchanged") +fi date_string=$git_time case $(uname) in diff --git a/contrib/commit-name.sh b/contrib/commit-name.sh index aa8b6a1f37dc8..09c8dcded1e75 100755 --- a/contrib/commit-name.sh +++ b/contrib/commit-name.sh @@ -6,7 +6,34 @@ gitref=${1:-HEAD} -last_tag=$(git describe --tags --abbrev=0 "$gitref") ver=$(git show "$gitref:VERSION") -nb=$(git rev-list --count "$gitref" "^$last_tag") -echo "$ver+$nb" +major=$(echo $ver | cut -f 1 -d .) +minor=$(echo $ver | cut -f 2 -d .) + +if [ $major = 0 -a $minor -lt 5 ]; then + # use tag based build number prior to 0.5.0- + last_tag=$(git describe --tags --abbrev=0 "$gitref") + nb=$(git rev-list --count "$gitref" "^$last_tag") + if [ $nb = 0 ]; then + echo $ver + else + echo "$ver+$nb" + fi +else + topdir=$(git rev-parse --show-toplevel) + verchanged=$(git blame -L ,1 -sl $gitref -- "$topdir/VERSION" | cut -f 1 -d " ") + nb=$(git rev-list --count "$gitref" "^$verchanged") + pre=$(echo $ver | cut -s -f 2 -d "-") + if [ $ver = "0.5.0-dev" ]; then + # bump to 0.5.0-dev was one commit after tag during 0.5.0-dev + nb=$(expr $nb + 1) + elif [ $ver = "0.5.0-pre" ]; then + # bump to 0.5.0-pre was 5578 commits after tag + nb=$(expr $nb + 5578) + fi + if [ -n "$pre" ]; then + echo "$ver+$nb" + else + echo $ver + fi +fi From 6bb779e9d098b8b620481c22c79edc0d3f68d517 Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Mon, 18 Jul 2016 08:30:37 +0200 Subject: [PATCH 0623/1117] Add version comparison test involving pre-releases --- test/version.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/version.jl b/test/version.jl index 8f0bb9ec73a09..68bdc958129b6 100644 --- a/test/version.jl +++ b/test/version.jl @@ -116,6 +116,7 @@ import Base.issupbuild # basic comparison VersionNumber(2, 3, 1) == VersionNumber(Int8(2), UInt32(3), Int32(1)) == v"2.3.1" @test v"2.3.0" < v"2.3.1" < v"2.4.8" < v"3.7.2" +@test v"0.6.0-" < v"0.6.0-dev" < v"0.6.0-dev.123" < v"0.6.0-dev.unknown" < v"0.6.0-pre" < v"0.6.0" #lowerbound and upperbound import Base: lowerbound, upperbound From dee3bc2cc1f844f7797764b466fbe50424910e8f Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Mon, 25 Jul 2016 09:53:11 -0400 Subject: [PATCH 0624/1117] Fix documentation for \ and move it to linalg/generic.jl --- base/docs/helpdb/Base.jl | 17 ----------------- base/linalg/generic.jl | 16 ++++++++++++++++ doc/stdlib/linalg.rst | 4 +++- doc/stdlib/parallel.rst | 4 ++-- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index ddd020baebfa0..6b18f4a2e111f 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -7055,23 +7055,6 @@ Matrix multiplication. """ Base.:(*)(::AbstractMatrix, ::AbstractMatrix) -""" - \\(A, B) - -Matrix division using a polyalgorithm. For input matrices `A` and `B`, the result `X` is -such that `A*X == B` when `A` is square. The solver that is used depends upon the structure -of `A`. A direct solver is used for upper or lower triangular `A`. For Hermitian `A` -(equivalent to symmetric `A` for non-complex `A`) the `BunchKaufman` factorization is used. -Otherwise an LU factorization is used. For rectangular `A` the result is the minimum-norm -least squares solution computed by a pivoted QR factorization of `A` and a rank estimate of -`A` based on the R factor. - -When `A` is sparse, a similar polyalgorithm is used. For indefinite matrices, the `LDLt` -factorization does not use pivoting during the numerical factorization and therefore the -procedure can fail even for invertible matrices. -""" -Base.:(\)(A,B) - """ .\\(x, y) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 2e91295d1a3fe..5a076989d7f6a 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -329,6 +329,22 @@ function inv{T}(A::AbstractMatrix{T}) A_ldiv_B!(factorize(convert(AbstractMatrix{S}, A)), eye(S, checksquare(A))) end +""" + \\(A, B) + +Matrix division using a polyalgorithm. For input matrices `A` and `B`, the result `X` is +such that `A*X == B` when `A` is square. The solver that is used depends upon the structure +of `A`. If `A` is upper or lower triangular (or diagonal), no factorization of `A` is +required and the system is solved with either forward or backward substitution. +For non-triangular square matrices, an LU factorization is used. + +For rectangular `A` the result is the minimum-norm least squares solution computed by a +pivoted QR factorization of `A` and a rank estimate of `A` based on the R factor. + +When `A` is sparse, a similar polyalgorithm is used. For indefinite matrices, the `LDLt` +factorization does not use pivoting during the numerical factorization and therefore the +procedure can fail even for invertible matrices. +""" function (\)(A::AbstractMatrix, B::AbstractVecOrMat) m, n = size(A) if m == n diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 713d6c55912b6..8352695c8b3c5 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -23,7 +23,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Matrix division using a polyalgorithm. For input matrices ``A`` and ``B``\ , the result ``X`` is such that ``A*X == B`` when ``A`` is square. The solver that is used depends upon the structure of ``A``\ . A direct solver is used for upper or lower triangular ``A``\ . For Hermitian ``A`` (equivalent to symmetric ``A`` for non-complex ``A``\ ) the ``BunchKaufman`` factorization is used. Otherwise an LU factorization is used. For rectangular ``A`` the result is the minimum-norm least squares solution computed by a pivoted QR factorization of ``A`` and a rank estimate of ``A`` based on the R factor. + Matrix division using a polyalgorithm. For input matrices ``A`` and ``B``\ , the result ``X`` is such that ``A*X == B`` when ``A`` is square. The solver that is used depends upon the structure of ``A``\ . If ``A`` is upper or lower triangular (or diagonal), no factorization of ``A`` is required and the system is solved with either forward or backward substitution. For non-triangular square matrices, an LU factorization is used. + + For rectangular ``A`` the result is the minimum-norm least squares solution computed by a pivoted QR factorization of ``A`` and a rank estimate of ``A`` based on the R factor. When ``A`` is sparse, a similar polyalgorithm is used. For indefinite matrices, the ``LDLt`` factorization does not use pivoting during the numerical factorization and therefore the procedure can fail even for invertible matrices. diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 472a8744868a7..93c5befca6d60 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -658,13 +658,13 @@ will) change in the future. .. Docstring generated from Julia source - Get the number of threads available to the Julia process. This is the inclusive upper bound on ``threadid()``\ . + Get the number of threads available to the Julia process. This is the inclusive upper bound on ``threadid()``\ . .. function:: Threads.@threads .. Docstring generated from Julia source - A macro to parallelize a for-loop to run with multiple threads. This spawns ``nthreads()`` number of threads, splits the iteration space amongst them, and iterates in parallel. A barrier is placed at the end of the loop which waits for all the threads to finish execution, and the loop returns. + A macro to parallelize a for-loop to run with multiple threads. This spawns ``nthreads()`` number of threads, splits the iteration space amongst them, and iterates in parallel. A barrier is placed at the end of the loop which waits for all the threads to finish execution, and the loop returns. .. function:: Threads.Atomic{T} From 717173e70ce6f52437e8656a5a34eb991cb6ab99 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Mon, 25 Jul 2016 11:13:48 -0400 Subject: [PATCH 0625/1117] NEWS: improvements and additions * miscellaneous copy improvements * note that open now respects umask on UNIX * note deprecation of integer-character comparisons [ci skip] --- NEWS.md | 91 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 30 deletions(-) diff --git a/NEWS.md b/NEWS.md index 746dc70713396..2b0fa6d93ef24 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,8 +4,9 @@ Julia v0.5.0 Release Notes New language features --------------------- - * Generator expressions, e.g. `f(i) for i in 1:n` ([#4470]). This returns an iterator - that computes the specified values on demand. + * Generator expressions: `f(i) for i in 1:n` ([#4470]). This returns an iterator + that computes the specified values on demand. This is useful for computing, e.g. + `sum(f(i) for i in 1:n)` without creating an intermediate array of values. * Generators and comprehensions support filtering using `if` ([#550]) and nested iteration using multiple `for` keywords ([#4867]). @@ -14,7 +15,7 @@ New language features and nested `f.(g.(args...))` calls are fused into a single `broadcast` loop ([#17300]). Similarly, the syntax `x .= ...` is equivalent to a `broadcast!(identity, x, ...)` call and fuses with nested "dot" calls; also, `x .+= y` and similar is now - equivalent to `x .= x .+ y`, rather than `=` ([#17510]). + equivalent to `x .= x .+ y`, rather than `x = x .+ y` ([#17510]). * Macro expander functions are now generic, so macros can have multiple definitions (e.g. for different numbers of arguments, or optional arguments) ([#8846], [#9627]). @@ -27,7 +28,7 @@ New language features * `x ∈ X` is now a synonym for `x in X` in `for` loops and comprehensions, as it already was in comparisons ([#13824]). - * `PROGRAM_FILE` global is now available for determining the name of the running script ([#14114]). + * The `PROGRAM_FILE` global is now available for determining the name of the running script ([#14114]). * The syntax `x.:sym` (e.g. `Base.:+`) is now supported, and `x.(:sym)` is deprecated ([#15032]). @@ -46,15 +47,6 @@ New language features operator names like `Base.≤` should now use `Base.:≤` (prefixed by `@compat` if you need 0.4 compatibility via the `Compat` package). -New architectures ------------------ - - This release greatly improves support for ARM, and introduces support for Power. - - * [ARM](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Aarm) ([#14194], [#14519], [#16645], [#16621]) - - * [Power](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Apower) ([#16455], [#16404]) - Language changes ---------------- @@ -68,7 +60,8 @@ Language changes * `using` and `import` are now case-sensitive even on case-insensitive filesystems (common on Mac and Windows) ([#13542]). - * Relational symbols are now allowed as infix operators ([#8036]). + * Relational algebra symbols are now allowed as infix operators ([#8036]): + `⨝`, `⟕`, `⟖`, `⟗` for joins and `▷` for anti-join. * A warning is always given when a method is overwritten (previously, this was done only when the new and old definitions were in separate modules) ([#14759]). @@ -77,7 +70,8 @@ Language changes This also applies to the `>:` operator. * Simple 2-argument comparisons like `A < B` are parsed as calls instead of using the - `:comparison` expression type ([#15524]). + `:comparison` expression type ([#15524]). The `:comparison` expression type is still + produced in ASTs when comparisons are chained (e.g. `A < B ≤ C`). * The `if` keyword cannot be followed immediately by a line break ([#15763]). @@ -105,19 +99,30 @@ Compiler/Runtime improvements * Machine SIMD types can be represented in Julia as a homogeneous tuple of `VecElement` ([#15244]). +New architectures +----------------- + + This release greatly improves support for ARM, and introduces support for Power. + + * [ARM](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Aarm): + [#14194], [#14519], [#16645], [#16621] + + * [Power](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Apower): + [#16455], [#16404] + Breaking changes ---------------- * The assignment operations `.+=`, `.*=` and so on now generate calls to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side - if the latter is a `a[...]` expression. This means that they will fail - if the left-hand side is immutable (or does not support `view`), and will - otherwise change the left-hand side in-place ([#17510], [#17546]). + if the latter is an indexing expression, (e.g. `a[...]`). This means that they will fail + if the left-hand side is immutable (or does not support `view`), and will otherwise + change the left-hand side in-place ([#17510], [#17546]). - * Method ambiguities no longer generate warnings when files are - loaded, nor do they dispatch to an arbitrarily-chosen method; - instead, a call that cannot be resolved to a single method results - in a `MethodError`. ([#6190]) + * Method ambiguities no longer generate warnings when files are loaded, + nor do they dispatch to an arbitrarily-chosen method; instead, a call that + cannot be resolved to a single method results in a `MethodError` at run time, + rather than the previous definition-time warning. ([#6190]) * `pmap` keyword arguments `err_retry=true` and `err_stop=false` are deprecated. Action to be taken on errors can be specified via the `on_error` keyword argument. @@ -127,9 +132,10 @@ Breaking changes If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of the desired shape ([#4211]). - * `mapslices` now re-uses temporary storage. Recipient functions - that expect input slices to be persistent should copy data to - other storage ([#17266]). + * `mapslices` now re-uses temporary storage. Recipient functions that expect + input slices to be persistent should copy data to other storage ([#17266]). + All usages of `mapslices` should be carefully audited since this change can cause + silent, incorrect behavior, rather than failing noisily. * Local variables and arguments are represented in lowered code as numbered `Slot` objects instead of as symbols ([#15609]). @@ -148,7 +154,19 @@ Library improvements * The `UTF8String` and `ASCIIString` types have been merged into a single `String` type ([#16058]). Use `isascii(s)` to check whether - a string contains only ASCII characters. + a string contains only ASCII characters. The `ascii(s)` function now + converts `s` to `String`, raising an `ArgumentError` exception if `s` is + not pure ASCII. + + * The `UTF16String` and `UTF32String` types and corresponding `utf16` and + `utf32` converter functions have been removed from the standard library. + If you need these types, they have been moved to the + [LegacyStrings](https://github.com/JuliaArchive/LegacyStrings.jl) + package. In the future, more robust Unicode string support will be provided + by the `StringEncodings` package. If you only need these types to call wide + string APIs (UTF-16 on Windows, UTF-32 on UNIX), consider using the new + `transcode` function (see below) or the `Cwstring` type as a `ccall` argument + type, which also ensures correct NUL termination of string data. * The basic string construction routines are now `string(args...)`, `String(s)`, `unsafe_string(ptr)` (formerly `bytestring(ptr)`), and @@ -157,6 +175,14 @@ Library improvements * A `transcode(T, src)` function is now exported for converting data between UTF-xx Unicode encodings ([#17323]). + * Comparisons between `Char`s and `Integer`s are now deprecated ([#16024]): + `'x' == 120` now produces a warning but still evaluates to `true`. In the + future it may evalaute to `false` or the comparison may be an error. To + compare characters with integers you should either convert the integer to + a character value or conver the character the corresponding code point + first: e.g. `'x' == Char(120)` or `Int('x') == 120`. The former is usually + preferrable. + * Support for Unicode 9 ([#17402]). * Most of the combinatorics functions have been moved from `Base` @@ -193,7 +219,9 @@ Library improvements * Arrays and linear algebra: * All dimensions indexed by scalars are now dropped, whereas previously only - trailing scalar dimensions would be omitted from the result ([#13612]). + trailing scalar dimensions would be omitted from the result ([#13612]). This + is a very major behavioral change, but should cause obvious failures. To retain + a dimension sliced with a scalar `i` slice with `i:i` instead. * Dimensions indexed by multidimensional arrays add dimensions. More generally, the dimensionality of the result is the sum of the dimensionalities of the indices ([#15431]). @@ -201,7 +229,7 @@ Library improvements * New `normalize` and `normalize!` convenience functions for normalizing vectors ([#13681]). - * QR + * QR matrix factorization: * New method for generic QR with column pivoting ([#13480]). @@ -223,7 +251,8 @@ Library improvements `Base.SparseArrays.dropstored!` ([#17404]). * New `foreach` function for calling a function on every element of a collection when - the results are not needed ([#13774]). + the results are not needed ([#13774]). As compared to `map(f, v)`, which allocates and + returns a result array, `foreach(f, v)` calls `f` on each element of `v`, returning nothing. * `Cmd(cmd; ...)` now accepts new Windows-specific options `windows_verbatim` (to alter Windows command-line generation) and `windows_hide` (to @@ -248,7 +277,7 @@ Library improvements `OS_NAME` has been replaced by `Sys.KERNEL` and always reports the name of the kernel (as reported by `uname`). The `@windows_only` and `@osx` family of macros have been replaced with functions such as `is_windows()` and `is_apple()`. - There's now also an `@static` macro that will evaluate the condition of an + There is now also a `@static` macro that will evaluate the condition of an if-statement at compile time, for when a static branch is required ([#16219]). * Prime number related functions have been moved from `Base` to the @@ -259,6 +288,8 @@ Library improvements * File handling: + * The `open` function now respects `umask` on UNIX when creating files ([#16466], [#16502]). + * A new function `walkdir()` returns an iterator that walks the directory tree of a directory. ([#1765]) ``` From 7251de8b5ad8fe82adf320edcfea671832db1c34 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 25 Jul 2016 11:35:54 -0400 Subject: [PATCH 0626/1117] more compact lowering of return type decls on short functions fixes part of #17600 --- src/julia-syntax.scm | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index a8e3e704d114b..c38b1af3e2734 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -174,11 +174,21 @@ (scope-block ,(if (eq? rett 'Any) body - (insert-after-meta - body - (let ((R (make-ssavalue))) - (list `(= ,R ,rett) - `(meta ret-type ,R))))))))) + (let ((meta (take-while (lambda (x) (and (pair? x) + (memq (car x) '(line meta)))) + (cdr body)))) + ;; wrap one-liners in `convert` instead of adding an ssavalue + (if (length= (cdr body) (+ 1 (length meta))) + (let ((val (last body))) + `(,(car body) ,@meta + ,(if (and (pair? val) (eq? (car val) 'return)) + `(return (call (top convert) ,rett ,(cadr val))) + `(call (top convert) ,rett ,val)))) + (let ((R (make-ssavalue))) + `(,(car body) ,@meta + (= ,R ,rett) + (meta ret-type ,R) + ,@(list-tail body (+ 1 (length meta)))))))))))) ;; convert list of names (sl) and list of upper bounds to expressions that ;; construct TypeVars From ebdaba39d1ee086dfa13e1dc8e1add135b5eda51 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Sat, 23 Jul 2016 11:52:29 -0400 Subject: [PATCH 0627/1117] Don't dynamically resize in transcoding UTF-16 to UTF-8 This is for security: it prevents extra copies of sensitive strings from being potentially leaked when the underlying array is grown. See #17560. --- base/c.jl | 65 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/base/c.jl b/base/c.jl index 28b9af1ca0a0a..561a6f79a5cc4 100644 --- a/base/c.jl +++ b/base/c.jl @@ -331,38 +331,69 @@ function transcode(::Type{UInt16}, src::Vector{UInt8}) end function transcode(::Type{UInt8}, src::Vector{UInt16}) - dst = UInt8[] - i, n = 1, length(src) - n > 0 || return dst - sizehint!(dst, n) + n = length(src) + n == 0 && return UInt8[] + + # Precompute m = sizeof(dst). This involves annoying duplication + # of the loop over the src array. However, this is not just an + # optimization: it is problematic for security reasons to grow + # dst dynamically, because Base.winprompt uses this function to + # convert passwords to UTF-8 and we don't want to make unintentional + # copies of the password data. + a = src[1] + i, m = 1, 0 + while true + if a < 0x80 + m += 1 + elseif a < 0x800 # 2-byte UTF-8 + m += 2 + elseif a & 0xfc00 == 0xd800 && i < length(src) + b = src[i += 1] + if (b & 0xfc00) == 0xdc00 # 2-unit UTF-16 sequence => 4-byte UTF-8 + m += 4 + else + m += 3 + a = b; continue + end + else + # 1-unit high UTF-16 or unpaired high surrogate + # either way, encode as 3-byte UTF-8 code point + m += 3 + end + i < n || break + a = src[i += 1] + end + + dst = Array{UInt8}(m) a = src[1] + i, j = 1, 0 while true if a < 0x80 # ASCII - push!(dst, a % UInt8) + dst[j += 1] = a % UInt8 elseif a < 0x800 # 2-byte UTF-8 - push!(dst, 0xc0 | ((a >> 6) % UInt8), - 0x80 | ((a % UInt8) & 0x3f)) + dst[j += 1] = 0xc0 | ((a >> 6) % UInt8) + dst[j += 1] = 0x80 | ((a % UInt8) & 0x3f) elseif a & 0xfc00 == 0xd800 && i < n b = src[i += 1] if (b & 0xfc00) == 0xdc00 # 2-unit UTF-16 sequence => 4-byte UTF-8 a += 0x2840 - push!(dst, 0xf0 | ((a >> 8) % UInt8), - 0x80 | ((a % UInt8) >> 2), - 0xf0 $ ((((a % UInt8) << 4) & 0x3f) $ (b >> 6) % UInt8), - 0x80 | ((b % UInt8) & 0x3f)) + dst[j += 1] = 0xf0 | ((a >> 8) % UInt8) + dst[j += 1] = 0x80 | ((a % UInt8) >> 2) + dst[j += 1] = 0xf0 $ ((((a % UInt8) << 4) & 0x3f) $ (b >> 6) % UInt8) + dst[j += 1] = 0x80 | ((b % UInt8) & 0x3f) else - push!(dst, 0xe0 | ((a >> 12) % UInt8), - 0x80 | (((a >> 6) % UInt8) & 0x3f), - 0x80 | ((a % UInt8) & 0x3f)) + dst[j += 1] = 0xe0 | ((a >> 12) % UInt8) + dst[j += 1] = 0x80 | (((a >> 6) % UInt8) & 0x3f) + dst[j += 1] = 0x80 | ((a % UInt8) & 0x3f) a = b; continue end else # 1-unit high UTF-16 or unpaired high surrogate # either way, encode as 3-byte UTF-8 code point - push!(dst, 0xe0 | ((a >> 12) % UInt8), - 0x80 | (((a >> 6) % UInt8) & 0x3f), - 0x80 | ((a % UInt8) & 0x3f)) + dst[j += 1] = 0xe0 | ((a >> 12) % UInt8) + dst[j += 1] = 0x80 | (((a >> 6) % UInt8) & 0x3f) + dst[j += 1] = 0x80 | ((a % UInt8) & 0x3f) end i < n || break a = src[i += 1] From a8b5821e06292c83020d09241c051fa4d7014e44 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Sat, 23 Jul 2016 17:30:25 -0400 Subject: [PATCH 0628/1117] Use String instead of AbstractString in LibGit2 type fields For short strings, the standard String type should always be used. --- base/libgit2/types.jl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 5a5de6d843fa3..17699a2235861 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -583,8 +583,8 @@ end StatusEntry() = StatusEntry(Cuint(0), C_NULL, C_NULL) immutable FetchHead - name::AbstractString - url::AbstractString + name::String + url::String oid::Oid ismerge::Bool end @@ -645,8 +645,8 @@ end # Structure has the same layout as SignatureStruct type Signature - name::AbstractString - email::AbstractString + name::String + email::String time::Int64 time_offset::Cint end @@ -707,9 +707,9 @@ end "Credentials that support only `user` and `password` parameters" type UserPasswordCredentials <: AbstractCredentials - user::AbstractString - pass::AbstractString - usesshagent::AbstractString # used for ssh-agent authentication + user::String + pass::String + usesshagent::String # used for ssh-agent authentication count::Int # authentication failure protection count UserPasswordCredentials(u::AbstractString,p::AbstractString) = new(u,p,"Y",3) end @@ -724,11 +724,11 @@ reset!(p::UserPasswordCredentials, cnt::Int=3) = (p.count = cnt) "SSH credentials type" type SSHCredentials <: AbstractCredentials - user::AbstractString - pass::AbstractString - pubkey::AbstractString - prvkey::AbstractString - usesshagent::AbstractString # used for ssh-agent authentication + user::String + pass::String + pubkey::String + prvkey::String + usesshagent::String # used for ssh-agent authentication SSHCredentials(u::AbstractString,p::AbstractString) = new(u,p,"","","Y") SSHCredentials() = SSHCredentials("","") @@ -736,9 +736,9 @@ end "Credentials that support caching" type CachedCredentials <: AbstractCredentials - cred::Dict{AbstractString,SSHCredentials} + cred::Dict{String,SSHCredentials} count::Int # authentication failure protection count - CachedCredentials() = new(Dict{AbstractString,SSHCredentials}(),3) + CachedCredentials() = new(Dict{String,SSHCredentials}(),3) end "Returns specific credential parameter value: first index is a credential parameter name, second index is a host name (with schema)" From 506dddf995bfefda47b8a745211cbff9af82b5ed Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Sat, 23 Jul 2016 08:28:25 -0400 Subject: [PATCH 0629/1117] Try harder not to leave passwords in memory (for #17560) --- base/libgit2/callbacks.jl | 223 ++++++++++++++++++++------------------ base/libgit2/types.jl | 33 +++++- base/pkg/entry.jl | 78 ++++++------- base/util.jl | 29 ++++- test/misc.jl | 17 +++ 5 files changed, 230 insertions(+), 150 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 36db11bf2973b..2de5af594d674 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -64,143 +64,152 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, # get credentials object from payload pointer creds = nothing + creds_are_temp = true if payload_ptr != C_NULL tmpobj = unsafe_pointer_to_objref(payload_ptr) if isa(tmpobj, AbstractCredentials) creds = tmpobj + creds_are_temp = false end end isusedcreds = checkused!(creds) - # use ssh key or ssh-agent - if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) - creds == nothing && (creds = SSHCredentials()) - credid = "ssh://$host" + try + # use ssh key or ssh-agent + if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) + creds == nothing && (creds = SSHCredentials()) + credid = "ssh://$host" + + # first try ssh-agent if credentials support its usage + if creds[:usesshagent, credid] === nothing || creds[:usesshagent, credid] == "Y" + err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring), cred, username_ptr) + creds[:usesshagent, credid] = "U" # used ssh-agent only one time + err == 0 && return Cint(0) + end - # first try ssh-agent if credentials support its usage - if creds[:usesshagent, credid] === nothing || creds[:usesshagent, credid] == "Y" - err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring), cred, username_ptr) - creds[:usesshagent, credid] = "U" # used ssh-agent only one time - err == 0 && return Cint(0) - end + errcls, errmsg = Error.last_error() + if errcls != Error.None + # Check if we used ssh-agent + if creds[:usesshagent, credid] == "U" + println("ERROR: $errmsg ssh-agent") + creds[:usesshagent, credid] = "E" # reported ssh-agent error + else + println("ERROR: $errmsg") + end + flush(STDOUT) + end - errcls, errmsg = Error.last_error() - if errcls != Error.None - # Check if we used ssh-agent - if creds[:usesshagent, credid] == "U" - println("ERROR: $errmsg ssh-agent") - creds[:usesshagent, credid] = "E" # reported ssh-agent error + # if username is not provided, then prompt for it + username = if username_ptr == Cstring(C_NULL) + uname = creds[:user, credid] # check if credentials were already used + uname !== nothing && !isusedcreds ? uname : prompt("Username for '$schema$host'") else - println("ERROR: $errmsg") + unsafe_string(username_ptr) end - flush(STDOUT) - end + creds[:user, credid] = username # save credentials - # if username is not provided, then prompt for it - username = if username_ptr == Cstring(C_NULL) - uname = creds[:user, credid] # check if credentials were already used - uname !== nothing && !isusedcreds ? uname : prompt("Username for '$schema$host'") - else - unsafe_string(username_ptr) - end - creds[:user, credid] = username # save credentials - - # For SSH we need a private key location - privatekey = if haskey(ENV,"SSH_KEY_PATH") - ENV["SSH_KEY_PATH"] - else - keydefpath = creds[:prvkey, credid] # check if credentials were already used - if keydefpath !== nothing && !isusedcreds - keydefpath # use cached value + # For SSH we need a private key location + privatekey = if haskey(ENV,"SSH_KEY_PATH") + ENV["SSH_KEY_PATH"] else - if keydefpath === nothing || isempty(keydefpath) - keydefpath = joinpath(homedir(),".ssh","id_rsa") + keydefpath = creds[:prvkey, credid] # check if credentials were already used + if keydefpath !== nothing && !isusedcreds + keydefpath # use cached value + else + if keydefpath === nothing || isempty(keydefpath) + keydefpath = joinpath(homedir(),".ssh","id_rsa") + end + prompt("Private key location for '$schema$username@$host'", default=keydefpath) end - prompt("Private key location for '$schema$username@$host'", default=keydefpath) end - end - creds[:prvkey, credid] = privatekey # save credentials - - # For SSH we need a public key location, look for environment vars SSH_* as well - publickey = if haskey(ENV,"SSH_PUB_KEY_PATH") - ENV["SSH_PUB_KEY_PATH"] - else - keydefpath = creds[:pubkey, credid] # check if credentials were already used - if keydefpath !== nothing && !isusedcreds - keydefpath # use cached value + creds[:prvkey, credid] = privatekey # save credentials + + # For SSH we need a public key location, look for environment vars SSH_* as well + publickey = if haskey(ENV,"SSH_PUB_KEY_PATH") + ENV["SSH_PUB_KEY_PATH"] else - if keydefpath === nothing || isempty(keydefpath) - keydefpath = privatekey*".pub" - end - if isfile(keydefpath) - keydefpath + keydefpath = creds[:pubkey, credid] # check if credentials were already used + if keydefpath !== nothing && !isusedcreds + keydefpath # use cached value else - prompt("Public key location for '$schema$username@$host'", default=keydefpath) + if keydefpath === nothing || isempty(keydefpath) + keydefpath = privatekey*".pub" + end + if isfile(keydefpath) + keydefpath + else + prompt("Public key location for '$schema$username@$host'", default=keydefpath) + end end end - end - creds[:pubkey, credid] = publickey # save credentials - - passphrase = if haskey(ENV,"SSH_KEY_PASS") - ENV["SSH_KEY_PASS"] - else - passdef = creds[:pass, credid] # check if credentials were already used - if passdef === nothing || isusedcreds - if is_windows() - passdef = Base.winprompt( - "Your SSH Key requires a password, please enter it now:", - "Passphrase required", privatekey; prompt_username = false) - isnull(passdef) && return Cint(Error.EAUTH) - passdef = Base.get(passdef)[2] - else - passdef = prompt("Passphrase for $privatekey", password=true) + creds[:pubkey, credid] = publickey # save credentials + + passphrase = if haskey(ENV,"SSH_KEY_PASS") + ENV["SSH_KEY_PASS"] + else + passdef = creds[:pass, credid] # check if credentials were already used + if passdef === nothing || isusedcreds + if is_windows() + passdef = Base.winprompt( + "Your SSH Key requires a password, please enter it now:", + "Passphrase required", privatekey; prompt_username = false) + isnull(passdef) && return Cint(Error.EAUTH) + passdef = Base.get(passdef)[2] + else + passdef = prompt("Passphrase for $privatekey", password=true) + end end end - end - creds[:pass, credid] = passphrase # save credentials + creds[:pass, credid] = passphrase # save credentials - isempty(username) && return Cint(Error.EAUTH) + isempty(username) && return Cint(Error.EAUTH) - err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), - cred, username, publickey, privatekey, passphrase) - err == 0 && return Cint(0) - end + err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), + cred, username, publickey, privatekey, passphrase) + err == 0 && return Cint(0) + end - if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) - creds == nothing && (creds = UserPasswordCredentials()) - credid = "$schema$host" - - username = creds[:user, credid] - userpass = creds[:pass, credid] - if is_windows() - if username === nothing || userpass === nothing || isusedcreds - res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", - username === nothing ? "" : username; prompt_username = true) - isnull(res) && return Cint(Error.EAUTH) - username, userpass = Base.get(res) - end - else - if username === nothing || isusedcreds - username = prompt("Username for '$schema$host'") - end + if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) + creds == nothing && (creds = UserPasswordCredentials()) + credid = "$schema$host" + + username = creds[:user, credid] + userpass = creds[:pass, credid] + if is_windows() + if username === nothing || userpass === nothing || isusedcreds + res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", + username === nothing ? "" : username; prompt_username = true) + isnull(res) && return Cint(Error.EAUTH) + username, userpass = Base.get(res) + end + else + if username === nothing || isusedcreds + username = prompt("Username for '$schema$host'") + end - if userpass === nothing || isusedcreds - userpass = prompt("Password for '$schema$username@$host'", password=true) + if userpass === nothing || isusedcreds + userpass = prompt("Password for '$schema$username@$host'", password=true) + end end - end - creds[:user, credid] = username # save credentials - creds[:pass, credid] = userpass # save credentials + creds[:user, credid] = username # save credentials + creds[:pass, credid] = userpass # save credentials - isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) + isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) - err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring, Cstring), - cred, username, userpass) - err == 0 && return Cint(0) + err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cstring), + cred, username, userpass) + err == 0 && return Cint(0) + end + finally + # if credentials are not passed back to caller via payload, + # then zero any passwords immediately. + if creds_are_temp && creds !== nothing + securezero!(creds) + end end - return Cint(err) end diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 17699a2235861..a3e9ecb3019ea 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -705,13 +705,19 @@ function getobjecttype(obj_type::Cint) end end +import Base.securezero! + "Credentials that support only `user` and `password` parameters" type UserPasswordCredentials <: AbstractCredentials user::String pass::String usesshagent::String # used for ssh-agent authentication count::Int # authentication failure protection count - UserPasswordCredentials(u::AbstractString,p::AbstractString) = new(u,p,"Y",3) + function UserPasswordCredentials(u::AbstractString,p::AbstractString) + c = new(u,p,"Y",3) + finalizer(c, securezero!) + return c + end end "Checks if credentials were used or failed authentication, see `LibGit2.credentials_callback`" function checkused!(p::UserPasswordCredentials) @@ -721,6 +727,13 @@ function checkused!(p::UserPasswordCredentials) end "Resets authentication failure protection count" reset!(p::UserPasswordCredentials, cnt::Int=3) = (p.count = cnt) +function securezero!(cred::UserPasswordCredentials) + securezero!(cred.user) + securezero!(cred.pass) + securezero!(cred.usesshagent) + cred.count = 0 + return cred +end "SSH credentials type" type SSHCredentials <: AbstractCredentials @@ -730,9 +743,21 @@ type SSHCredentials <: AbstractCredentials prvkey::String usesshagent::String # used for ssh-agent authentication - SSHCredentials(u::AbstractString,p::AbstractString) = new(u,p,"","","Y") + function SSHCredentials(u::AbstractString,p::AbstractString) + c = new(u,p,"","","Y") + finalizer(c, securezero!) + return c + end SSHCredentials() = SSHCredentials("","") end +function securezero!(cred::SSHCredentials) + securezero!(cred.user) + securezero!(cred.pass) + securezero!(cred.pubkey) + securezero!(cred.prvkey) + securezero!(cred.usesshagent) + return cred +end "Credentials that support caching" type CachedCredentials <: AbstractCredentials @@ -775,3 +800,7 @@ function checkused!(p::CachedCredentials) end "Resets authentication failure protection count" reset!(p::CachedCredentials, cnt::Int=3) = (p.count = cnt) +function securezero!(p::CachedCredentials) + foreach(securezero!, values(p.cred)) + return p +end diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 120f6f114b0e0..6a0cf4dda1f6b 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -406,49 +406,53 @@ function update(branch::AbstractString, upkgs::Set{String}) push!(deferred_errors, PkgError("Package $pkg: unable to update cache.", cex)) end end - creds = LibGit2.CachedCredentials() fixed = Read.fixed(avail,instd,dont_update) - stopupdate = false - for (pkg,ver) in fixed - ispath(pkg,".git") || continue - pkg in dont_update && continue - with(GitRepo, pkg) do repo - if LibGit2.isattached(repo) - if LibGit2.isdirty(repo) - warn("Package $pkg: skipping update (dirty)...") - elseif Read.ispinned(repo) - info("Package $pkg: skipping update (pinned)...") - else - prev_sha = string(LibGit2.head_oid(repo)) - success = true - try - LibGit2.fetch(repo, payload = Nullable(creds)) - LibGit2.reset!(creds) - LibGit2.merge!(repo, fastforward=true) - catch err - cex = CapturedException(err, catch_backtrace()) - push!(deferred_errors, PkgError("Package $pkg cannot be updated.", cex)) - success = false - stopupdate = isa(err, InterruptException) - end - if success - post_sha = string(LibGit2.head_oid(repo)) - branch = LibGit2.branch(repo) - info("Updating $pkg $branch...", - prev_sha != post_sha ? " $(prev_sha[1:8]) → $(post_sha[1:8])" : "") + creds = LibGit2.CachedCredentials() + try + stopupdate = false + for (pkg,ver) in fixed + ispath(pkg,".git") || continue + pkg in dont_update && continue + with(GitRepo, pkg) do repo + if LibGit2.isattached(repo) + if LibGit2.isdirty(repo) + warn("Package $pkg: skipping update (dirty)...") + elseif Read.ispinned(repo) + info("Package $pkg: skipping update (pinned)...") + else + prev_sha = string(LibGit2.head_oid(repo)) + success = true + try + LibGit2.fetch(repo, payload = Nullable(creds)) + LibGit2.reset!(creds) + LibGit2.merge!(repo, fastforward=true) + catch err + cex = CapturedException(err, catch_backtrace()) + push!(deferred_errors, PkgError("Package $pkg cannot be updated.", cex)) + success = false + stopupdate = isa(err, InterruptException) + end + if success + post_sha = string(LibGit2.head_oid(repo)) + branch = LibGit2.branch(repo) + info("Updating $pkg $branch...", + prev_sha != post_sha ? " $(prev_sha[1:8]) → $(post_sha[1:8])" : "") + end end end end - end - stopupdate && break - if haskey(avail,pkg) - try - Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) - catch err - cex = CapturedException(err, catch_backtrace()) - push!(deferred_errors, PkgError("Package $pkg: unable to update cache.", cex)) + stopupdate && break + if haskey(avail,pkg) + try + Cache.prefetch(pkg, Read.url(pkg), [a.sha1 for (v,a)=avail[pkg]]) + catch err + cex = CapturedException(err, catch_backtrace()) + push!(deferred_errors, PkgError("Package $pkg: unable to update cache.", cex)) + end end end + finally + Base.securezero!(creds) end info("Computing changes...") resolve(reqs, avail, instd, fixed, free, upkgs) diff --git a/base/util.jl b/base/util.jl index c456472ee694a..6a75c35f660ed 100644 --- a/base/util.jl +++ b/base/util.jl @@ -313,6 +313,21 @@ end julia_exename() = ccall(:jl_is_debugbuild,Cint,())==0 ? "julia" : "julia-debug" +""" + securezero!(o) + +`securezero!` fills the memory associated with an object `o` with zeros. +Unlike `fill!(o,0)` and similar code, which might be optimized away by +the compiler for objects about to be discarded, the `securezero!` function +will always be called. +""" +function securezero! end +@noinline securezero!{T<:Number}(a::AbstractArray{T}) = fill!(a, 0) +securezero!(s::String) = securezero!(s.data) +@noinline unsafe_securezero!{T}(p::Ptr{T}, len::Integer=1) = + ccall(:memset, Ptr{T}, (Ptr{T}, Cint, Csize_t), p, 0, len*sizeof(T)) +unsafe_securezero!(p::Ptr{Void}, len::Integer=1) = Ptr{Void}(unsafe_securezero!(Ptr{UInt8}(p), len)) + if is_windows() function getpass(prompt::AbstractString) print(prompt) @@ -336,7 +351,7 @@ function getpass(prompt::AbstractString) return unsafe_string(pointer(p), plen) # use unsafe_string rather than String(p[1:plen]) # to be absolutely certain we never make an extra copy finally - fill!(p, 0) # don't leave password in memory + securezero!(p) end return "" @@ -418,11 +433,17 @@ if is_windows() # Step 4: Free the encrypted buffer # ccall(:SecureZeroMemory, Ptr{Void}, (Ptr{Void}, Csize_t), outbuf_data[], outbuf_size[]) - not an actual function + unsafe_securezero!(outbuf_data[], outbuf_size[]) ccall((:CoTaskMemFree, "ole32.dll"), Void, (Ptr{Void},), outbuf_data[]) - # Done - Nullable((String(transcode(UInt8, usernamebuf[1:usernamelen[]-1])), - String(transcode(UInt8, passbuf[1:passlen[]-1])))) + # Done. + passbuf_ = passbuf[1:passlen[]-1] + result = Nullable((String(transcode(UInt8, usernamebuf[1:usernamelen[]-1])), + String(transcode(UInt8, passbuf_)))) + securezero!(passbuf_) + securezero!(passbuf) + + return result end end diff --git a/test/misc.jl b/test/misc.jl index 16066830498e9..7a2e5738089f8 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -393,3 +393,20 @@ end optstring = sprint(show, Base.JLOptions()) @test startswith(optstring, "JLOptions(") @test endswith(optstring, ")") + +# Base.securezero! functions (#17579) +import Base: securezero!, unsafe_securezero! +let a = [1,2,3] + @test securezero!(a) === a == [0,0,0] + a[:] = 1:3 + @test unsafe_securezero!(pointer(a), length(a)) == pointer(a) + @test a == [0,0,0] + a[:] = 1:3 + @test unsafe_securezero!(Ptr{Void}(pointer(a)), sizeof(a)) == Ptr{Void}(pointer(a)) + @test a == [0,0,0] +end +let creds = Base.LibGit2.CachedCredentials() + creds[:pass, "foo"] = "bar" + securezero!(creds) + @test creds[:pass, "foo"] == "\0\0\0" +end From f8c21b880bf1e40790c9a869ea46bd3141f888c2 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 25 Jul 2016 11:11:36 -0500 Subject: [PATCH 0630/1117] Make StepRange safe for more number types --- base/range.jl | 25 ++++++++++++++++--------- test/dates/ranges.jl | 22 +++++++++++----------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/base/range.jl b/base/range.jl index 62997616404a4..200181c72b9cb 100644 --- a/base/range.jl +++ b/base/range.jl @@ -31,15 +31,7 @@ function steprange_last{T}(start::T, step, stop) last = stop else if (step > z) != (stop > start) - # empty range has a special representation where stop = start-1 - # this is needed to avoid the wrap-around that can happen computing - # start - step, which leads to a range that looks very large instead - # of empty. - if step > z - last = start - one(stop-start) - else - last = start + one(stop-start) - end + last = steprange_last_empty(start, step, stop) else diff = stop - start if T<:Signed && (diff > zero(diff)) != (stop > start) @@ -58,6 +50,21 @@ function steprange_last{T}(start::T, step, stop) last end +function steprange_last_empty{T<:Integer}(start::T, step, stop) + # empty range has a special representation where stop = start-1 + # this is needed to avoid the wrap-around that can happen computing + # start - step, which leads to a range that looks very large instead + # of empty. + if step > zero(step) + last = start - one(stop-start) + else + last = start + one(stop-start) + end + last +end +# For types where x+one(x) may not be well-defined +steprange_last_empty(start, step, stop) = start - step + steprem(start,stop,step) = (stop-start) % step StepRange{T,S}(start::T, step::S, stop::T) = StepRange{T,S}(start, step, stop) diff --git a/test/dates/ranges.jl b/test/dates/ranges.jl index b21d6111b3102..0f8b8919f355f 100644 --- a/test/dates/ranges.jl +++ b/test/dates/ranges.jl @@ -14,7 +14,7 @@ function test_all_combos() @test length(dr) == 0 @test isempty(dr) @test first(dr) == f1 - @test last(dr) == f1-one(l1 - f1) + @test last(dr) < f1 @test length([i for i in dr]) == 0 @test_throws ArgumentError minimum(dr) @test_throws ArgumentError maximum(dr) @@ -23,8 +23,8 @@ function test_all_combos() @test [dr;] == T[] @test isempty(reverse(dr)) @test length(reverse(dr)) == 0 - @test first(reverse(dr)) == f1-one(l1 - f1) - @test last(reverse(dr)) == f1 + @test first(reverse(dr)) < f1 + @test last(reverse(dr)) >= f1 @test issorted(dr) @test sortperm(dr) == 1:1:0 @test !(f1 in dr) @@ -66,7 +66,7 @@ function test_all_combos() @test length(dr) == 0 @test isempty(dr) @test first(dr) == l1 - @test last(dr) == l1+one(l1 - f1) + @test last(dr) > l1 @test length([i for i in dr]) == 0 @test_throws ArgumentError minimum(dr) @test_throws ArgumentError maximum(dr) @@ -75,8 +75,8 @@ function test_all_combos() @test [dr;] == T[] @test isempty(reverse(dr)) @test length(reverse(dr)) == 0 - @test first(reverse(dr)) == l1+one(l1 - f1) - @test last(reverse(dr)) == l1 + @test first(reverse(dr)) > l1 + @test last(reverse(dr)) <= l1 @test !issorted(dr) @test sortperm(dr) == 0:-1:1 @test !(l1 in dr) @@ -119,7 +119,7 @@ function test_all_combos() @test length(dr) == 0 @test isempty(dr) @test first(dr) == f1 - @test last(dr) == f1-one(l1 - f1) + @test last(dr) < f1 @test length([i for i in dr]) == 0 @test_throws ArgumentError minimum(dr) @test_throws ArgumentError maximum(dr) @@ -128,8 +128,8 @@ function test_all_combos() @test [dr;] == T[] @test isempty(reverse(dr)) @test length(reverse(dr)) == 0 - @test first(reverse(dr)) == f1-one(l1 - f1) - @test last(reverse(dr)) == f1 + @test first(reverse(dr)) < f1 + @test last(reverse(dr)) >= f1 @test issorted(dr) @test sortperm(dr) == 1:1:0 @test !(f1 in dr) @@ -171,7 +171,7 @@ function test_all_combos() @test length(dr) == 0 @test isempty(dr) @test first(dr) == l1 - @test last(dr) == l1+one(l1 - f1) + @test last(dr) > l1 @test length([i for i in dr]) == 0 @test_throws ArgumentError minimum(dr) @test_throws ArgumentError maximum(dr) @@ -180,7 +180,7 @@ function test_all_combos() @test [dr;] == T[] @test isempty(reverse(dr)) @test length(reverse(dr)) == 0 - @test first(reverse(dr)) == l1+one(l1 - f1) + @test first(reverse(dr)) > l1 @test last(reverse(dr)) <= l1 @test !issorted(dr) @test sortperm(dr) == 0:-1:1 From 4fb1f31d932f58b1bf6f4c6197c53a8c081861c7 Mon Sep 17 00:00:00 2001 From: Erik Schnetter <schnetter@gmail.com> Date: Mon, 25 Jul 2016 12:36:31 -0400 Subject: [PATCH 0631/1117] Don't require a `.git` suffix for the master repository name (#17606) Closes #13881. --- base/version_git.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/version_git.sh b/base/version_git.sh index 97d3cc05d78f5..ebcd0965a7587 100644 --- a/base/version_git.sh +++ b/base/version_git.sh @@ -26,7 +26,7 @@ if [ "$#" = "2" -a "$2" = "NO_GIT" ]; then exit 0 fi # Collect temporary variables -origin=$(git config -l 2>/dev/null | grep 'remote\.\w*\.url.*JuliaLang/julia.git' | sed -n 's/remote\.\([a-zA-Z]*\)\..*/\1\//p') +origin=$(git config -l 2>/dev/null | grep 'remote\.\w*\.url.*JuliaLang/julia' | sed -n 's/remote\.\([a-zA-Z]*\)\..*/\1\//p') if [ -z "$origin" ]; then origin="origin/" fi From 279f11b8d4e736584066951042f716804991d430 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Mon, 25 Jul 2016 12:50:53 -0400 Subject: [PATCH 0632/1117] NEWS: fix a couple of types --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 2b0fa6d93ef24..52af7857f76e8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -177,9 +177,9 @@ Library improvements * Comparisons between `Char`s and `Integer`s are now deprecated ([#16024]): `'x' == 120` now produces a warning but still evaluates to `true`. In the - future it may evalaute to `false` or the comparison may be an error. To + future it may evaluate to `false` or the comparison may be an error. To compare characters with integers you should either convert the integer to - a character value or conver the character the corresponding code point + a character value or convert the character the corresponding code point first: e.g. `'x' == Char(120)` or `Int('x') == 120`. The former is usually preferrable. From 0203992ef84e8434bcf52309db5b6a0eb76fe3dc Mon Sep 17 00:00:00 2001 From: Maxim Grechkin <grechkin@cs.washington.edu> Date: Mon, 25 Jul 2016 12:08:43 -0700 Subject: [PATCH 0633/1117] a bit more fixes for correlation + tests --- base/statistics.jl | 8 ++++---- test/statistics.jl | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/base/statistics.jl b/base/statistics.jl index 9629d105a93d7..d85569ec52526 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -324,7 +324,7 @@ function cov2cor!{T}(C::AbstractMatrix{T}, xsd::AbstractArray) end C[j,j] = one(T) for i = j+1:nx - C[i,j] /= (xsd[i] * xsd[j]) + C[i,j] = clamp(C[i,j] / (xsd[i] * xsd[j]), -1, 1) end end return C @@ -334,7 +334,7 @@ function cov2cor!(C::AbstractMatrix, xsd::Number, ysd::AbstractArray) length(ysd) == ny || throw(DimensionMismatch("inconsistent dimensions")) for (j, y) in enumerate(ysd) # fixme (iter): here and in all `cov2cor!` we assume that `C` is efficiently indexed by integers for i in 1:nx - C[i,j] /= (xsd * y) + C[i,j] = clamp(C[i, j] / (xsd * y), -1, 1) end end return C @@ -344,7 +344,7 @@ function cov2cor!(C::AbstractMatrix, xsd::AbstractArray, ysd::Number) length(xsd) == nx || throw(DimensionMismatch("inconsistent dimensions")) for j in 1:ny for (i, x) in enumerate(xsd) - C[i,j] /= (x * ysd) + C[i,j] = clamp(C[i,j] / (x * ysd), -1, 1) end end return C @@ -355,7 +355,7 @@ function cov2cor!(C::AbstractMatrix, xsd::AbstractArray, ysd::AbstractArray) throw(DimensionMismatch("inconsistent dimensions")) for (i, x) in enumerate(xsd) for (j, y) in enumerate(ysd) - C[i,j] /= x*y + C[i,j] = clamp(C[i,j] / (x * y), -1, 1) end end return C diff --git a/test/statistics.jl b/test/statistics.jl index 412d6da5824e3..c4f53aa38d7de 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -308,6 +308,16 @@ for vd in [1, 2], zm in [true, false] @inferred cor(X, Y, vd) end +@test cor(repmat(1:17, 1, 17))[2] <= 1.0 +@test cor(1:17, 1:17) <= 1.0 +@test cor(1:17, 18:34) <= 1.0 +let tmp = linspace(1, 85, 100) + tmp2 = collect(tmp) + @test cor(tmp, tmp) <= 1.0 + @test cor(tmp, tmp2) <= 1.0 +end + + @test midpoints(1.0:1.0:10.0) == 1.5:1.0:9.5 @test midpoints(1:10) == 1.5:9.5 @test midpoints(Float64[1.0:1.0:10.0;]) == Float64[1.5:1.0:9.5;] From f45a8337362c0b52106422390d2ed79648308097 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 25 Jul 2016 14:25:50 -0500 Subject: [PATCH 0634/1117] Add docstring for Base.OneTo [ci skip] --- base/range.jl | 7 +++++++ doc/stdlib/math.rst | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/base/range.jl b/base/range.jl index 200181c72b9cb..358ada339af9e 100644 --- a/base/range.jl +++ b/base/range.jl @@ -83,6 +83,13 @@ unitrange_last{T}(start::T, stop::T) = ifelse(stop >= start, convert(T,start+floor(stop-start)), convert(T,start-one(stop-start))) +""" + Base.OneTo(n) + +Define an `AbstractUnitRange` that behaves like `1:n`, with the added +distinction that the lower limit is guaranteed (by the type system) to +be 1. +""" immutable OneTo{T<:Integer} <: AbstractUnitRange{T} stop::T OneTo(stop) = new(max(zero(T), stop)) diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index b411d4e0da5ed..75844d7c06f9c 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -311,6 +311,12 @@ Mathematical Operators Construct a range by length, given a starting value and optional step (defaults to 1). +.. function:: Base.OneTo(n) + + .. Docstring generated from Julia source + + Define an ``AbstractUnitRange`` that behaves like ``1:n``\ , with the added distinction that the lower limit is guaranteed (by the type system) to be 1. + .. _==: .. function:: ==(x, y) From c84694d2ec0721d20ab6f5af1cc2dda0003356f0 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 25 Jul 2016 16:34:07 -0400 Subject: [PATCH 0635/1117] fix type inference issues with Float16 due to bootstrap order --- base/float16.jl | 11 +---------- base/math.jl | 18 ++++++++++++++++++ base/sysimg.jl | 2 +- test/numbers.jl | 24 +++--------------------- 4 files changed, 23 insertions(+), 32 deletions(-) diff --git a/base/float16.jl b/base/float16.jl index 69fbbc20ba421..bfd72a8f66f67 100644 --- a/base/float16.jl +++ b/base/float16.jl @@ -144,22 +144,13 @@ end for op in (:<,:<=,:isless) @eval ($op)(a::Float16, b::Float16) = ($op)(Float32(a), Float32(b)) end -for func in (:sin,:cos,:tan,:asin,:acos,:atan,:sinh,:cosh,:tanh,:asinh,:acosh, - :atanh,:exp,:log,:log2,:log10,:sqrt,:lgamma,:log1p,:erf,:erfc) - @eval begin - $func(a::Float16) = Float16($func(Float32(a))) - $func(a::Complex32) = Complex32($func(Complex64(a))) - end -end -for func in (:div,:fld,:cld,:rem,:mod,:atan2,:hypot) +for func in (:div,:fld,:cld,:rem,:mod) @eval begin $func(a::Float16,b::Float16) = Float16($func(Float32(a),Float32(b))) end end -ldexp(a::Float16, b::Integer) = Float16(ldexp(Float32(a), b)) - ^(x::Float16, y::Integer) = Float16(Float32(x)^y) rationalize{T<:Integer}(::Type{T}, x::Float16; tol::Real=eps(x)) = rationalize(T, Float32(x); tol=tol) diff --git a/base/math.jl b/base/math.jl index 41ad90358a211..32267c994de7f 100644 --- a/base/math.jl +++ b/base/math.jl @@ -430,6 +430,24 @@ end # generic fallback; for number types, promotion.jl does promotion muladd(x,y,z) = x*y+z +# Float16 definitions + +for func in (:sin,:cos,:tan,:asin,:acos,:atan,:sinh,:cosh,:tanh,:asinh,:acosh, + :atanh,:exp,:log,:log2,:log10,:sqrt,:lgamma,:log1p,:erf,:erfc) + @eval begin + $func(a::Float16) = Float16($func(Float32(a))) + $func(a::Complex32) = Complex32($func(Complex64(a))) + end +end + +for func in (:atan2,:hypot) + @eval begin + $func(a::Float16,b::Float16) = Float16($func(Float32(a),Float32(b))) + end +end + +ldexp(a::Float16, b::Integer) = Float16(ldexp(Float32(a), b)) + # More special functions include("special/trig.jl") include("special/bessel.jl") diff --git a/base/sysimg.jl b/base/sysimg.jl index 89217dbf71b8a..f2cf7fb13d96d 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -101,6 +101,7 @@ include("multinverses.jl") using .MultiplicativeInverses include("abstractarraymath.jl") include("arraymath.jl") +include("float16.jl") # SIMD loops include("simdloop.jl") @@ -186,7 +187,6 @@ include("math.jl") importall .Math const (√)=sqrt const (∛)=cbrt -include("float16.jl") # multidimensional arrays include("cartesian.jl") diff --git a/test/numbers.jl b/test/numbers.jl index cab8bc626f208..a85a5b543490f 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2790,13 +2790,7 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Complex{Int}, Complex{UInt}, Complex32, Complex64, Complex128) for S in types for op in (+, -) - if S === Float16 - # broken, fixme then remove this branch - @test_throws ErrorException @inferred(Base.promote_op(op, S)) - T = Base.promote_op(op, S) - else - T = @inferred Base.promote_op(op, S) - end + T = @inferred Base.promote_op(op, S) t = @inferred op(one(S)) @test T === typeof(t) end @@ -2806,13 +2800,7 @@ let types = (Base.BitInteger_types..., BigInt, Bool, for R in types, S in types for op in (+, -, *, /, ^) - if R === Float16 || S === Float16 - # broken, fixme then remove this branch - @test_throws ErrorException @inferred(Base.promote_op(op, R, S)) - T = Base.promote_op(op, R, S) - else - T = @inferred Base.promote_op(op, R, S) - end + T = @inferred Base.promote_op(op, R, S) t = @inferred op(one(R), one(S)) @test T === typeof(t) end @@ -2824,13 +2812,7 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Float16, Float32, Float64, BigFloat) for S in types, T in types for op in (<, >, <=, >=, (==)) - if S === Float16 || T === Float16 - # broken, fixme then remove this branch - @test_throws ErrorException @inferred(Base.promote_op(op, S, T)) - @test Base.promote_op(op, S, T) === Bool - else - @test @inferred(Base.promote_op(op, S, T)) === Bool - end + @test @inferred(Base.promote_op(op, S, T)) === Bool end end end From b63413b344ab4098998f2870a4d95acfaafee6fe Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 25 Jul 2016 15:40:44 -0500 Subject: [PATCH 0636/1117] Be consistent about dropping a dimension in slicedim. Fixes #17589. --- base/abstractarraymath.jl | 9 ++++++--- base/arraymath.jl | 35 ----------------------------------- base/bitarray.jl | 2 +- test/arrayops.jl | 12 +++++++++--- test/bitarray.jl | 2 +- 5 files changed, 17 insertions(+), 43 deletions(-) diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 43efa5063b48c..349b5533927a8 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -59,9 +59,12 @@ imag{T<:Real}(x::AbstractArray{T}) = zero(x) \(A::Number, B::AbstractArray) = B ./ A # index A[:,:,...,i,:,:,...] where "i" is in dimension "d" -# TODO: more optimized special cases -slicedim(A::AbstractArray, d::Integer, i) = - A[[ n==d ? i : (indices(A,n)) for n in 1:ndims(A) ]...] +function slicedim(A::AbstractArray, d::Integer, i) + d >= 1 || throw(ArgumentError("dimension must be ≥ 1")) + nd = ndims(A) + d > nd && (i == 1 || throw_boundserror(A, (ntuple(k->Colon(),nd)..., ntuple(k->1,d-1-nd)..., i))) + A[( n==d ? i : indices(A,n) for n in 1:nd )...] +end function flipdim(A::AbstractVector, d::Integer) d > 0 || throw(ArgumentError("dimension to flip must be positive")) diff --git a/base/arraymath.jl b/base/arraymath.jl index 7ec3eacf6e5b2..696ab37b871ab 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -96,41 +96,6 @@ end ## data movement ## -# TODO?: replace with slice? -function slicedim(A::Array, d::Integer, i::Integer) - if d < 1 - throw(ArgumentError("dimension must be ≥ 1")) - end - d_in = size(A) - leading = d_in[1:(d-1)] - d_out = tuple(leading..., 1, d_in[(d+1):end]...) - - M = prod(leading) - N = length(A) - stride = M * d_in[d] - - B = similar(A, d_out) - index_offset = 1 + (i-1)*M - - l = 1 - - if M==1 - for j=0:stride:(N-stride) - B[l] = A[j + index_offset] - l += 1 - end - else - for j=0:stride:(N-stride) - offs = j + index_offset - for k=0:(M-1) - B[l] = A[offs + k] - l += 1 - end - end - end - return B -end - function flipdim{T}(A::Array{T}, d::Integer) if d < 1 throw(ArgumentError("dimension d must be ≥ 1")) diff --git a/base/bitarray.jl b/base/bitarray.jl index ecb6efd9ae16a..0fc33f46a1e5b 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1243,7 +1243,7 @@ end function slicedim(A::BitArray, d::Integer, i::Integer) d_in = size(A) leading = d_in[1:(d-1)] - d_out = tuple(leading..., 1, d_in[(d+1):end]...) + d_out = tuple(leading..., d_in[(d+1):end]...) M = prod(leading) N = length(A) diff --git a/test/arrayops.jl b/test/arrayops.jl index 00e4ca331153c..60cd54e437f50 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1457,9 +1457,15 @@ A = 1:5 B = 1.5:5.5 @test A + B == 2.5:2.0:10.5 -#slice dim error -A = zeros(5,5) -@test_throws ArgumentError slicedim(A,0,1) +# slicedim +for A in (reshape(collect(1:20), 4, 5), + reshape(1:20, 4, 5)) + @test slicedim(A, 1, 2) == collect(2:4:20) + @test slicedim(A, 2, 2) == collect(5:8) + @test_throws ArgumentError slicedim(A,0,1) + @test slicedim(A, 3, 1) == A + @test_throws BoundsError slicedim(A, 3, 2) +end ### ### LinearSlow workout diff --git a/test/bitarray.jl b/test/bitarray.jl index 7aef9ec100a96..d401baa93ccda 100644 --- a/test/bitarray.jl +++ b/test/bitarray.jl @@ -857,7 +857,7 @@ b1 = bitrand(s1, s2, s3, s4) for d = 1 : 4 j = rand(1:size(b1, d)) #for j = 1 : size(b1, d) - @check_bit_operation slicedim(b1, d, j) BitArray{4} + @check_bit_operation slicedim(b1, d, j) BitArray{3} #end @check_bit_operation flipdim(b1, d) BitArray{4} end From 23bc0d3a22822934ea7552e856733e5de4efaccf Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 25 Jul 2016 17:05:16 -0400 Subject: [PATCH 0637/1117] NEWS: explain `breaking changes` section and use it a bit better [ci skip] --- NEWS.md | 49 +++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/NEWS.md b/NEWS.md index 52af7857f76e8..cbbc09a17facd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -66,25 +66,14 @@ Language changes * A warning is always given when a method is overwritten (previously, this was done only when the new and old definitions were in separate modules) ([#14759]). - * `A <: B` is parsed as `Expr(:(<:), :A, :B)` in all cases ([#9503]). - This also applies to the `>:` operator. - - * Simple 2-argument comparisons like `A < B` are parsed as calls instead of using the - `:comparison` expression type ([#15524]). The `:comparison` expression type is still - produced in ASTs when comparisons are chained (e.g. `A < B ≤ C`). - * The `if` keyword cannot be followed immediately by a line break ([#15763]). + * Juxtaposition of numeric literals ending in `.` (e.g. `1.x`) is no longer + allowed ([#15731]). + * The built-in `NTuple` type has been removed; `NTuple{N,T}` is now implemented internally as `Tuple{Vararg{T,N}}` ([#11242]). - * Array comprehensions preserve the dimensions of the input ranges. For example, - `[2x for x in A]` will have the same dimensions as `A` ([#16622]). - - * The result type of an array comprehension depends only on the types of elements - computed, instead of using type inference ([#7258]). If the result is empty, then - type inference is still used to determine the element type. - * Use of the syntax `x::T` to declare the type of a local variable is deprecated. In the future this will always mean type assertion, and declarations should use `local x::T` instead ([#16071]). @@ -113,6 +102,8 @@ New architectures Breaking changes ---------------- +This section lists changes that do not have deprecation warnings. + * The assignment operations `.+=`, `.*=` and so on now generate calls to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side if the latter is an indexing expression, (e.g. `a[...]`). This means that they will fail @@ -124,9 +115,12 @@ Breaking changes cannot be resolved to a single method results in a `MethodError` at run time, rather than the previous definition-time warning. ([#6190]) - * `pmap` keyword arguments `err_retry=true` and `err_stop=false` are deprecated. - Action to be taken on errors can be specified via the `on_error` keyword argument. - Retry is specified via `retry_n`, `retry_on` and `retry_max_delay` ([#15409], [#15975], [#16663]). + * Array comprehensions preserve the dimensions of the input ranges. For example, + `[2x for x in A]` will have the same dimensions as `A` ([#16622]). + + * The result type of an array comprehension depends only on the types of elements + computed, instead of using type inference ([#7258]). If the result is empty, then + type inference is still used to determine the element type. * `reshape` is now defined to always share data with the original array. If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of @@ -144,8 +138,12 @@ Breaking changes is now divided among the fields `code`, `slotnames`, `slottypes`, `slotflags`, `gensymtypes`, `rettype`, `nargs`, and `isva` in the `LambdaInfo` type ([#15609]). - * Juxtaposition of numeric literals ending in `.` (e.g. `1.x`) is no longer - allowed ([#15731]). + * `A <: B` is parsed as `Expr(:(<:), :A, :B)` in all cases ([#9503]). + This also applies to the `>:` operator. + + * Simple 2-argument comparisons like `A < B` are parsed as calls instead of using the + `:comparison` expression type ([#15524]). The `:comparison` expression type is still + produced in ASTs when comparisons are chained (e.g. `A < B ≤ C`). Library improvements -------------------- @@ -179,15 +177,12 @@ Library improvements `'x' == 120` now produces a warning but still evaluates to `true`. In the future it may evaluate to `false` or the comparison may be an error. To compare characters with integers you should either convert the integer to - a character value or convert the character the corresponding code point + a character value or convert the character to the corresponding code point first: e.g. `'x' == Char(120)` or `Int('x') == 120`. The former is usually preferrable. * Support for Unicode 9 ([#17402]). - * Most of the combinatorics functions have been moved from `Base` - to the [Combinatorics.jl package](https://github.com/JuliaLang/Combinatorics.jl) ([#13897]). - * Packages: * The package system (`Pkg`) is now based on the `libgit2` library, rather @@ -211,6 +206,13 @@ Library improvements package provides the old-style `handler` functionality, for compatibility with code that needs to support both Julia v0.4 and v0.5. + * Most of the combinatorics functions have been moved from `Base` + to the [Combinatorics.jl package](https://github.com/JuliaLang/Combinatorics.jl) ([#13897]). + + * `pmap` keyword arguments `err_retry=true` and `err_stop=false` are deprecated. + Action to be taken on errors can be specified via the `on_error` keyword argument. + Retry is specified via `retry_n`, `retry_on` and `retry_max_delay` ([#15409], [#15975], [#16663]). + * The functions `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the function argument as the first argument to allow for do-block syntax ([#13338]). @@ -303,7 +305,6 @@ Library improvements * A new function `chown()` changes the ownership of files. ([#15007]) - Deprecated or removed --------------------- From e5671db4a733eee834f3d69477c599f432dab7ca Mon Sep 17 00:00:00 2001 From: Morten Piibeleht <morten.piibeleht@gmail.com> Date: Tue, 26 Jul 2016 01:01:05 +0300 Subject: [PATCH 0638/1117] doc: Fix a footnote Without the space the footnote is not rendered properly. --- doc/manual/arrays.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 20a5e0342cf4f..e04ae53de2ae0 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -90,7 +90,7 @@ Function Description defaulting to the element type and dimensions of ``A`` if omitted. :func:`reinterpret(type, A) <reinterpret>` an array with the same binary data as the given array, but with the specified element type -:func:`rand(dims) <rand>` :obj:`Array` of ``Float64``\ s with random, iid[#]_ and uniformly +:func:`rand(dims) <rand>` :obj:`Array` of ``Float64``\ s with random, iid [#iid]_ and uniformly distributed values in the half-open interval :math:`[0, 1)` :func:`randn(dims) <randn>` :obj:`Array` of ``Float64``\ s with random, iid and standard normally distributed random values @@ -101,7 +101,7 @@ Function Description :func:`fill(x, dims) <fill>` create an array filled with the value ``x`` =================================================== ===================================================================== -.. [#] *iid*, independently and identically distributed. +.. [#iid] *iid*, independently and identically distributed. The syntax ``[A, B, C, ...]`` constructs a 1-d array (vector) of its arguments. From cbc657f676b8d14535dec97902234d7e589177be Mon Sep 17 00:00:00 2001 From: Morten Piibeleht <morten.piibeleht@gmail.com> Date: Tue, 26 Jul 2016 01:13:56 +0300 Subject: [PATCH 0639/1117] doc: Add a missing paren --- doc/devdocs/reflection.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/devdocs/reflection.rst b/doc/devdocs/reflection.rst index 185016495c732..161ec02b61566 100644 --- a/doc/devdocs/reflection.rst +++ b/doc/devdocs/reflection.rst @@ -124,7 +124,7 @@ highlighting to the output of :func:`code_typed` (see :ref:`man-code-warntype`). Closer to the machine, the LLVM intermediate representation of a function may be printed using by :func:`code_llvm(f::Function, (Argtypes...)) <code_llvm>`, and finally the compiled machine code is -available using :func:`code_native(f::Function, (Argtypes...) <code_native>` (this will trigger JIT +available using :func:`code_native(f::Function, (Argtypes...)) <code_native>` (this will trigger JIT compilation/code generation for any function which has not previously been called). For convenience, there are macro versions of the above functions which take standard function calls From 0311c8a3ba879c807c7c40898a0ae6901e1d4099 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 25 Jul 2016 22:25:15 -0700 Subject: [PATCH 0640/1117] fix llvm patch repeat application issue (#17592) dependency on configure should be order-only (after the `|`) since a patch modifies the file. also use CMakeLists.txt since llvm's configure will be going away --- deps/llvm.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 2798acf817191..99cc7829e71cb 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -407,7 +407,7 @@ endif # LLVM_VER LLVM_PATCH_PREV:= LLVM_PATCH_LIST:= define LLVM_PATCH -$$(LLVM_SRC_DIR)/$1.patch-applied: $$(LLVM_SRC_DIR)/configure | $$(SRCDIR)/patches/$1.patch $$(LLVM_PATCH_PREV) +$$(LLVM_SRC_DIR)/$1.patch-applied: | $$(LLVM_SRC_DIR)/CMakeLists.txt $$(SRCDIR)/patches/$1.patch $$(LLVM_PATCH_PREV) cd $$(LLVM_SRC_DIR) && patch -p1 < $$(SRCDIR)/patches/$1.patch echo 1 > $$@ LLVM_PATCH_PREV := $$(LLVM_SRC_DIR)/$1.patch-applied From 43098c8e2bc27049ea1501a2ab604006ffd4afe4 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 00:59:15 -0700 Subject: [PATCH 0641/1117] Use withenv in temp_pkg_dir, and rename the copy in replcompletions to avoid overwriting the same function name, and restore the env var to its original value when done --- test/pkg.jl | 22 ++++++++++++---------- test/replcompletions.jl | 26 +++++++++++++++----------- test/replutil.jl | 1 - 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index e97861b60d481..281222f997464 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -3,16 +3,18 @@ import Base.Pkg.PkgError function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) - # Used in tests below to setup and teardown a sandboxed package directory - const tmpdir = ENV["JULIA_PKGDIR"] = joinpath(tempdir(),randstring()) - @test !isdir(Pkg.dir()) - try - Pkg.init() - @test isdir(Pkg.dir()) - Pkg.resolve() - fn() - finally - remove_tmp_dir && rm(tmpdir, recursive=true) + # Used in tests below to set up and tear down a sandboxed package directory + const tmpdir = joinpath(tempdir(),randstring()) + withenv("JULIA_PKGDIR" => tmpdir) do + @test !isdir(Pkg.dir()) + try + Pkg.init() + @test isdir(Pkg.dir()) + Pkg.resolve() + fn() + finally + remove_tmp_dir && rm(tmpdir, recursive=true) + end end end diff --git a/test/replcompletions.jl b/test/replcompletions.jl index 89b59ed7855db..1b9ee804bd6e5 100644 --- a/test/replcompletions.jl +++ b/test/replcompletions.jl @@ -57,16 +57,20 @@ module CompletionFoo end test_repl_comp_dict = CompletionFoo.test_dict -function temp_pkg_dir(fn::Function) - # Used in tests below to setup and teardown a sandboxed package directory - const tmpdir = ENV["JULIA_PKGDIR"] = joinpath(tempdir(),randstring()) - @test !isdir(Pkg.dir()) - try - mkpath(Pkg.dir()) - @test isdir(Pkg.dir()) - fn() - finally - rm(tmpdir, recursive=true) +function temp_pkg_dir_noinit(fn::Function) + # Used in tests below to set up and tear down a sandboxed package directory + # Unlike the version in test/pkg.jl, this does not run Pkg.init so does not + # clone METADATA (only pkg and libgit2-online tests should need internet access) + const tmpdir = joinpath(tempdir(),randstring()) + withenv("JULIA_PKGDIR" => tmpdir) do + @test !isdir(Pkg.dir()) + try + mkpath(Pkg.dir()) + @test isdir(Pkg.dir()) + fn() + finally + rm(tmpdir, recursive=true) + end end end @@ -382,7 +386,7 @@ c, r, res = test_complete(s) # Test completion of packages mkp(p) = ((@assert !isdir(p)); mkpath(p)) -temp_pkg_dir() do +temp_pkg_dir_noinit() do # Complete <Mod>/src/<Mod>.jl and <Mod>.jl/src/<Mod>.jl # but not <Mod>/ if no corresponding .jl file is found pkg_dir = Pkg.dir("CompletionFooPackage", "src") diff --git a/test/replutil.jl b/test/replutil.jl index 1abd59d506efc..4ad026f3a8f80 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -395,7 +395,6 @@ end # Issue #13032 withenv("JULIA_EDITOR" => nothing, "VISUAL" => nothing, "EDITOR" => nothing) do - # Make sure editor doesn't error when no ENV editor is set. @test isa(Base.editor(), Array) From 7b7ad296d5bcc86ef71020a46f0e3895b4e9d702 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 01:15:33 -0700 Subject: [PATCH 0642/1117] Add a test for type and line number info in backtraces so that #10595 does not regress, and #17251 is tracked --- test/cmdlineargs.jl | 31 ++++++++++++++++++++++++------- test/file.jl | 4 ++-- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index d1c6b7df4ab38..23ed71e1a2cb8 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1,5 +1,13 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +catcmd = `cat` +if is_windows() + try # use busybox-w32 on windows + success(`busybox`) + catcmd = `busybox cat` + end +end + let exename = `$(Base.julia_cmd()) --precompiled=yes` # --version let v = split(readstring(`$exename -v`), "julia version ")[end] @@ -246,13 +254,6 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` @test readchomp(`$exename -e 'println(ARGS);' ''`) == "String[\"\"]" # issue #12679 - catcmd = `cat` - if is_windows() - try # use busybox-w32 on windows - success(`busybox`) - catcmd = `busybox cat` - end - end @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --compile=yes -ioo`),stderr=catcmd)) == "ERROR: unknown option `-o`" @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no -p`),stderr=catcmd)) == "ERROR: option `-p/--procs` is missing an argument" @test readchomp(pipeline(ignorestatus(`$exename --startup-file=no --inline`),stderr=catcmd)) == "ERROR: option `--inline` is missing an argument" @@ -287,3 +288,19 @@ let exename = `$(Base.julia_cmd())` @test readchomp(`$exename --precompiled=yes -E "Bool(Base.JLOptions().use_precompiled)"`) == "true" @test readchomp(`$exename --precompiled=no -E "Bool(Base.JLOptions().use_precompiled)"`) == "false" end + +# backtrace contains type and line number info (esp. on windows #17179) +for precomp in ("yes", "no") + bt = readstring(pipeline(ignorestatus(`$(Base.julia_cmd()) --precompiled=$precomp + -E 'include("____nonexistent_file")'`), stderr=catcmd)) + @test contains(bt, "in include_from_node1") + if is_windows() && Sys.WORD_SIZE == 32 && precomp == "yes" + # fixme, issue #17251 + @test_broken contains(bt, "in include_from_node1(::String) at $(joinpath(".","loading.jl"))") + else + @test contains(bt, "in include_from_node1(::String) at $(joinpath(".","loading.jl"))") + end + lno = match(r"at \.[/\\]loading.jl:(\d+)", bt) + @test length(lno.captures) == 1 + @test parse(Int, lno.captures[1]) > 0 +end diff --git a/test/file.jl b/test/file.jl index 956b54d2952b4..5a76143a8f59b 100644 --- a/test/file.jl +++ b/test/file.jl @@ -263,8 +263,8 @@ test_monitor_wait_poll() test_watch_file_timeout(0.1) test_watch_file_change(6) -@test_throws Base.UVError watch_file("nonexistantfile", 10) -@test_throws Base.UVError poll_file("nonexistantfile", 2, 10) +@test_throws Base.UVError watch_file("____nonexistent_file", 10) +@test_throws Base.UVError poll_file("____nonexistent_file", 2, 10) ############## # mark/reset # From 47d306b2b63df8a69e7428465e4b5cb148b3d758 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 25 May 2016 05:04:40 -0700 Subject: [PATCH 0643/1117] Remove many unnecessary end-of-line semicolons --- base/dates/accessors.jl | 24 +- base/linalg/matmul.jl | 36 +-- test/core.jl | 2 +- test/grisu.jl | 574 ++++++++++++++++----------------- test/linalg/dense.jl | 2 +- test/linalg/pinv.jl | 118 +++---- test/meta.jl | 2 +- test/mmap.jl | 4 +- test/parallel_exec.jl | 2 +- test/perf/kernel/bench_eu.jl | 16 +- test/perf/kernel/getdivgrad.jl | 2 +- test/perf/kernel/gk.jl | 6 +- test/perf/kernel/perf.jl | 6 +- test/perf/kernel/raytracer.jl | 2 +- test/random.jl | 2 +- test/read.jl | 2 +- test/repl.jl | 4 +- test/sparsedir/sparse.jl | 20 +- test/staged.jl | 4 +- test/strings/basic.jl | 4 +- 20 files changed, 416 insertions(+), 416 deletions(-) diff --git a/base/dates/accessors.jl b/base/dates/accessors.jl index 4592f30370d7e..4ef616a09a86d 100644 --- a/base/dates/accessors.jl +++ b/base/dates/accessors.jl @@ -3,33 +3,33 @@ # Convert # of Rata Die days to proleptic Gregorian calendar y,m,d,w # Reference: http://mysite.verizon.net/aesir_research/date/date0.htm function yearmonthday(days) - z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4); - y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153); + z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4) + y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153) d = c - div(153m-457,5); return m > 12 ? (y+1,m-12,d) : (y,m,d) end function year(days) - z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4); - y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153); + z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4) + y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153) return m > 12 ? y+1 : y end function yearmonth(days) - z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4); - y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153); + z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4) + y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153) return m > 12 ? (y+1,m-12) : (y,m) end function month(days) - z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4); - y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153); + z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4) + y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153) return m > 12 ? m-12 : m end function monthday(days) - z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4); - y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153); + z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4) + y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153) d = c - div(153m-457,5); return m > 12 ? (m-12,d) : (m,d) end function day(days) - z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4); - y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153); + z = days + 306; h = 100z - 25; a = fld(h,3652425); b = a - fld(a,4) + y = fld(100b+h,36525); c = b + z - 365y - fld(y,4); m = div(5c+456,153) return c - div(153m-457,5) end # https://en.wikipedia.org/wiki/Talk:ISO_week_date#Algorithms diff --git a/base/linalg/matmul.jl b/base/linalg/matmul.jl index 0f998330395b6..6c9117a3cd06e 100644 --- a/base/linalg/matmul.jl +++ b/base/linalg/matmul.jl @@ -652,31 +652,31 @@ end function matmul3x3!{T,S,R}(C::AbstractMatrix{R}, tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) @inbounds begin if tA == 'T' - A11 = transpose(A[1,1]); A12 = transpose(A[2,1]); A13 = transpose(A[3,1]); - A21 = transpose(A[1,2]); A22 = transpose(A[2,2]); A23 = transpose(A[3,2]); - A31 = transpose(A[1,3]); A32 = transpose(A[2,3]); A33 = transpose(A[3,3]); + A11 = transpose(A[1,1]); A12 = transpose(A[2,1]); A13 = transpose(A[3,1]) + A21 = transpose(A[1,2]); A22 = transpose(A[2,2]); A23 = transpose(A[3,2]) + A31 = transpose(A[1,3]); A32 = transpose(A[2,3]); A33 = transpose(A[3,3]) elseif tA == 'C' - A11 = ctranspose(A[1,1]); A12 = ctranspose(A[2,1]); A13 = ctranspose(A[3,1]); - A21 = ctranspose(A[1,2]); A22 = ctranspose(A[2,2]); A23 = ctranspose(A[3,2]); - A31 = ctranspose(A[1,3]); A32 = ctranspose(A[2,3]); A33 = ctranspose(A[3,3]); + A11 = ctranspose(A[1,1]); A12 = ctranspose(A[2,1]); A13 = ctranspose(A[3,1]) + A21 = ctranspose(A[1,2]); A22 = ctranspose(A[2,2]); A23 = ctranspose(A[3,2]) + A31 = ctranspose(A[1,3]); A32 = ctranspose(A[2,3]); A33 = ctranspose(A[3,3]) else - A11 = A[1,1]; A12 = A[1,2]; A13 = A[1,3]; - A21 = A[2,1]; A22 = A[2,2]; A23 = A[2,3]; - A31 = A[3,1]; A32 = A[3,2]; A33 = A[3,3]; + A11 = A[1,1]; A12 = A[1,2]; A13 = A[1,3] + A21 = A[2,1]; A22 = A[2,2]; A23 = A[2,3] + A31 = A[3,1]; A32 = A[3,2]; A33 = A[3,3] end if tB == 'T' - B11 = transpose(B[1,1]); B12 = transpose(B[2,1]); B13 = transpose(B[3,1]); - B21 = transpose(B[1,2]); B22 = transpose(B[2,2]); B23 = transpose(B[3,2]); - B31 = transpose(B[1,3]); B32 = transpose(B[2,3]); B33 = transpose(B[3,3]); + B11 = transpose(B[1,1]); B12 = transpose(B[2,1]); B13 = transpose(B[3,1]) + B21 = transpose(B[1,2]); B22 = transpose(B[2,2]); B23 = transpose(B[3,2]) + B31 = transpose(B[1,3]); B32 = transpose(B[2,3]); B33 = transpose(B[3,3]) elseif tB == 'C' - B11 = ctranspose(B[1,1]); B12 = ctranspose(B[2,1]); B13 = ctranspose(B[3,1]); - B21 = ctranspose(B[1,2]); B22 = ctranspose(B[2,2]); B23 = ctranspose(B[3,2]); - B31 = ctranspose(B[1,3]); B32 = ctranspose(B[2,3]); B33 = ctranspose(B[3,3]); + B11 = ctranspose(B[1,1]); B12 = ctranspose(B[2,1]); B13 = ctranspose(B[3,1]) + B21 = ctranspose(B[1,2]); B22 = ctranspose(B[2,2]); B23 = ctranspose(B[3,2]) + B31 = ctranspose(B[1,3]); B32 = ctranspose(B[2,3]); B33 = ctranspose(B[3,3]) else - B11 = B[1,1]; B12 = B[1,2]; B13 = B[1,3]; - B21 = B[2,1]; B22 = B[2,2]; B23 = B[2,3]; - B31 = B[3,1]; B32 = B[3,2]; B33 = B[3,3]; + B11 = B[1,1]; B12 = B[1,2]; B13 = B[1,3] + B21 = B[2,1]; B22 = B[2,2]; B23 = B[2,3] + B31 = B[3,1]; B32 = B[3,2]; B33 = B[3,3] end C[1,1] = A11*B11 + A12*B21 + A13*B31 diff --git a/test/core.jl b/test/core.jl index c2d85714956f4..56bbe9d3e7ff7 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2955,7 +2955,7 @@ f10978(T::TupleType10978) = isa(T, TupleType10978) @test f10978(Tuple{Int}) # issue #10995 -#typealias TupleType{T<:Tuple} Type{T}; +#typealias TupleType{T<:Tuple} Type{T} f10995(::Any) = (while false; end; nothing) f10995(T::TupleType10978) = (while false; end; @assert isa(T, TupleType10978)) g10995(x) = f10995(typeof(x)) diff --git a/test/grisu.jl b/test/grisu.jl index 6fb7568717f09..3cf09b0d893c0 100644 --- a/test/grisu.jl +++ b/test/grisu.jl @@ -3,19 +3,19 @@ using Base.Grisu function trimrep(buffer) - len = length(unsafe_string(pointer(buffer))) - ind = len - for i = len:-1:1 - buffer[i] != 0x30 && break - ind -= 1 - end - buffer[ind+1] = 0 - return unsafe_string(pointer(buffer)) + len = length(unsafe_string(pointer(buffer))) + ind = len + for i = len:-1:1 + buffer[i] != 0x30 && break + ind -= 1 + end + buffer[ind+1] = 0 + return unsafe_string(pointer(buffer)) end const bufsize = 500 -buffer = Array{UInt8}(bufsize); -fill!(buffer,0); +buffer = Array{UInt8}(bufsize) +fill!(buffer,0) bignums = [Grisu.Bignums.Bignum(),Grisu.Bignums.Bignum(),Grisu.Bignums.Bignum(),Grisu.Bignums.Bignum()] # Start by checking the byte-order. @@ -227,63 +227,63 @@ status,len,point = Grisu.fastshortest(min_double, buffer) @test status @test "5" == trimrep(buffer) @test -323 == point -fill!(buffer,0); +fill!(buffer,0) max_double = 1.7976931348623157e308 status,len,point = Grisu.fastshortest(max_double, buffer) @test status @test "17976931348623157" == trimrep(buffer) @test 309 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(4294967272.0, buffer) @test status @test "4294967272" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(4.1855804968213567e298, buffer) @test status @test "4185580496821357" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(5.5626846462680035e-309, buffer) @test status @test "5562684646268003" == trimrep(buffer) @test -308 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(2147483648.0, buffer) @test status @test "2147483648" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(3.5844466002796428e+298, buffer) @test !status # Not all Grisu.fastshortest variants manage to compute this number. if status - @test "35844466002796428" == trimrep(buffer) - @test 299 == point - fill!(buffer,0); + @test "35844466002796428" == trimrep(buffer) + @test 299 == point + fill!(buffer,0) end smallest_normal64 = 0x0010000000000000 v = reinterpret(Float64,smallest_normal64) status,len,point = Grisu.fastshortest(v, buffer) if status - @test "22250738585072014" == trimrep(buffer) - @test -307 == point - fill!(buffer,0); + @test "22250738585072014" == trimrep(buffer) + @test -307 == point + fill!(buffer,0) end largest_denormal64 = 0x000FFFFFFFFFFFFF v = reinterpret(Float64,largest_denormal64) status,len,point = Grisu.fastshortest(v, buffer) if status - @test "2225073858507201" == trimrep(buffer) - @test -307 == point - fill!(buffer,0); + @test "2225073858507201" == trimrep(buffer) + @test -307 == point + fill!(buffer,0) end @@ -292,59 +292,59 @@ status,len,point = Grisu.fastshortest(min_float, buffer) @test status @test "1" == trimrep(buffer) @test -44 == point -fill!(buffer,0); +fill!(buffer,0) max_float = 3.4028234f38 #Float32(3.4028234e38) status,len,point = Grisu.fastshortest(max_float, buffer) @test status @test "34028235" == trimrep(buffer) @test 39 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(Float32(4294967272.0), buffer) @test status @test "42949673" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(Float32(3.32306998946228968226e+35), buffer) @test status @test "332307" == trimrep(buffer) @test 36 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(Float32(1.2341e-41), buffer) @test status @test "12341" == trimrep(buffer) @test -40 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(Float32(3.3554432e7), buffer) @test status @test "33554432" == trimrep(buffer) @test 8 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(Float32(3.26494756798464e14), buffer) @test status @test "32649476" == trimrep(buffer) @test 15 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastshortest(Float32(3.91132223637771935344e37), buffer) if status # Not all Grisu.fastshortest variants manage to compute this number. - @test "39113222" == trimrep(buffer) - @test 38 == point - fill!(buffer,0); + @test "39113222" == trimrep(buffer) + @test 38 == point + fill!(buffer,0) end smallest_normal32 = 0x00800000 v = reinterpret(Float32,smallest_normal32) status,len,point = Grisu.fastshortest(v, buffer) if status - @test "11754944" == trimrep(buffer) - @test -37 == point - fill!(buffer,0); + @test "11754944" == trimrep(buffer) + @test -37 == point + fill!(buffer,0) end largest_denormal32 = 0x007FFFFF @@ -353,21 +353,21 @@ status,len,point = Grisu.fastshortest(v, buffer) @test status @test "11754942" == trimrep(buffer) @test -37 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastprecision(1.0, 3, buffer) @test status @test 3 >= len-1 @test "1" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastprecision(1.5, 10, buffer) if status - @test 10 >= len-1 - @test "15" == trimrep(buffer) - @test 1 == point - fill!(buffer,0); + @test 10 >= len-1 + @test "15" == trimrep(buffer) + @test 1 == point + fill!(buffer,0) end min_double = 5e-324 @@ -375,47 +375,47 @@ status,len,point = Grisu.fastprecision(min_double, 5,buffer) @test status @test "49407" == trimrep(buffer) @test -323 == point -fill!(buffer,0); +fill!(buffer,0) max_double = 1.7976931348623157e308 status,len,point = Grisu.fastprecision(max_double, 7,buffer) @test status @test "1797693" == trimrep(buffer) @test 309 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastprecision(4294967272.0, 14,buffer) if status - @test 14 >= len-1 - @test "4294967272" == trimrep(buffer) - @test 10 == point - fill!(buffer,0); + @test 14 >= len-1 + @test "4294967272" == trimrep(buffer) + @test 10 == point + fill!(buffer,0) end status,len,point = Grisu.fastprecision(4.1855804968213567e298, 17,buffer) @test status @test "41855804968213567" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastprecision(5.5626846462680035e-309, 1,buffer) @test status @test "6" == trimrep(buffer) @test -308 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastprecision(2147483648.0, 5,buffer) @test status @test "21475" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastprecision(3.5844466002796428e+298, 10,buffer) @test status @test 10 >= len-1 @test "35844466" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) smallest_normal64 = 0x0010000000000000 v = reinterpret(Float64,smallest_normal64) @@ -423,7 +423,7 @@ status,len,point = Grisu.fastprecision(v, 17, buffer) @test status @test "22250738585072014" == trimrep(buffer) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) largest_denormal64 = 0x000FFFFFFFFFFFFF v = reinterpret(Float64,largest_denormal64) @@ -432,614 +432,614 @@ status,len,point = Grisu.fastprecision(v, 17, buffer) @test 20 >= len-1 @test "22250738585072009" == trimrep(buffer) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) v = 3.3161339052167390562200598e-237 status,len,point = Grisu.fastprecision(v, 18, buffer) @test status @test "331613390521673906" == trimrep(buffer) @test -236 == point -fill!(buffer,0); +fill!(buffer,0) v = 7.9885183916008099497815232e+191 status,len,point = Grisu.fastprecision(v, 4, buffer) @test status @test "7989" == trimrep(buffer) @test 192 == point -fill!(buffer,0); +fill!(buffer,0) #fastfixedtoa status,len,point = Grisu.fastfixedtoa(1.0, 0,1, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1.0, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1.0, 0,0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0xFFFFFFFF, 0,5, buffer) @test "4294967295" == unsafe_string(pointer(buffer)) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(4294967296.0, 0,5, buffer) @test "4294967296" == unsafe_string(pointer(buffer)) #todo @test 10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1e21, 0,5, buffer) @test "1" == unsafe_string(pointer(buffer)) #todo extra '0's @test 22 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(999999999999999868928.00, 0,2, buffer) @test "999999999999999868928" == unsafe_string(pointer(buffer)) #todo extra '0' @test 21 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(6.9999999999999989514240000e+21, 0,5, buffer) @test "6999999999999998951424" == unsafe_string(pointer(buffer)) #todo short several '9's @test 22 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1.5, 0,5, buffer) @test "15" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1.55, 0,5, buffer) @test "155" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1.55, 0,1, buffer) @test "16" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1.00000001, 0,15, buffer) @test "100000001" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.1, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 0 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.01, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.001, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -2 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0001, 0,10, buffer) #todo @test "1" == unsafe_string(pointer(buffer)) @test -3 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00001, 0,10, buffer) #todo @test "1" == unsafe_string(pointer(buffer)) @test -4 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000001, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -5 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000001, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -6 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000001, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -7 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000001, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -8 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000001, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -9 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000000001, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000001, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -11 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000001, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -12 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000000000001, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -13 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000001, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -14 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000000001, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -15 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000000000000001, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -16 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000001, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -17 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000000000001, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -18 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000000000000000001, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -19 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.10000000004, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 0 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.01000000004, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00100000004, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -2 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00010000004, 0,10, buffer) #todo @test "1" == unsafe_string(pointer(buffer)) @test -3 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00001000004, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -4 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000100004, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -5 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000010004, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -6 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000001004, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -7 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000000104, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -8 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000001000004, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -9 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000100004, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000010004, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -11 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000001004, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -12 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000000104, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -13 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000001000004, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -14 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000100004, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -15 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000010004, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -16 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000001004, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -17 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000000104, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -18 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000000014, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -19 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.10000000006, 0,10, buffer) @test "1000000001" == unsafe_string(pointer(buffer)) @test 0 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.01000000006, 0,10, buffer) @test "100000001" == unsafe_string(pointer(buffer)) @test -1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00100000006, 0,10, buffer) @test "10000001" == unsafe_string(pointer(buffer)) @test -2 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00010000006, 0,10, buffer) @test "1000001" == unsafe_string(pointer(buffer)) @test -3 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00001000006, 0,10, buffer) @test "100001" == unsafe_string(pointer(buffer)) @test -4 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000100006, 0,10, buffer) @test "10001" == unsafe_string(pointer(buffer)) @test -5 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000010006, 0,10, buffer) @test "1001" == unsafe_string(pointer(buffer)) @test -6 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000001006, 0,10, buffer) @test "101" == unsafe_string(pointer(buffer)) @test -7 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000000106, 0,10, buffer) @test "11" == unsafe_string(pointer(buffer)) @test -8 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000001000006, 0,15, buffer) @test "100001" == unsafe_string(pointer(buffer)) @test -9 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000100006, 0,15, buffer) @test "10001" == unsafe_string(pointer(buffer)) @test -10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000010006, 0,15, buffer) @test "1001" == unsafe_string(pointer(buffer)) @test -11 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000001006, 0,15, buffer) @test "101" == unsafe_string(pointer(buffer)) @test -12 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000000000000106, 0,15, buffer) @test "11" == unsafe_string(pointer(buffer)) @test -13 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000001000006, 0,20, buffer) @test "100001" == unsafe_string(pointer(buffer)) @test -14 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000100006, 0,20, buffer) @test "10001" == unsafe_string(pointer(buffer)) @test -15 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000010006, 0,20, buffer) @test "1001" == unsafe_string(pointer(buffer)) @test -16 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000001006, 0,20, buffer) @test "101" == unsafe_string(pointer(buffer)) @test -17 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000000106, 0,20, buffer) @test "11" == unsafe_string(pointer(buffer)) @test -18 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000000000000000016, 0,20, buffer) @test "2" == unsafe_string(pointer(buffer)) @test -19 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.6, 0,0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.96, 0,1, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.996, 0,2, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.9996, 0,3, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.99996, 0,4, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.999996, 0,5, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.9999996, 0,6, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.99999996, 0,7, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.999999996, 0,8, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.9999999996, 0,9, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.99999999996, 0,10, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.999999999996, 0,11, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.9999999999996, 0,12, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.99999999999996, 0,13, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.999999999999996, 0,14, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.9999999999999996, 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00999999999999996, 0,16, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000999999999999996, 0,17, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -2 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.0000999999999999996, 0,18, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -3 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.00000999999999999996, 0,19, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -4 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.000000999999999999996, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -5 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(323423.234234, 0,10, buffer) @test "323423234234" == unsafe_string(pointer(buffer)) @test 6 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(12345678.901234, 0,4, buffer) @test "123456789012" == unsafe_string(pointer(buffer)) @test 8 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(98765.432109, 0,5, buffer) @test "9876543211" == unsafe_string(pointer(buffer)) @test 5 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(42, 0,20, buffer) @test "42" == unsafe_string(pointer(buffer)) @test 2 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(0.5, 0,0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1e-23, 0,10, buffer) @test "" == unsafe_string(pointer(buffer)) @test -10 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1e-123, 0,2, buffer) @test "" == unsafe_string(pointer(buffer)) @test -2 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1e-123, 0,0, buffer) @test "" == unsafe_string(pointer(buffer)) @test 0 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1e-23, 0,20, buffer) @test "" == unsafe_string(pointer(buffer)) @test -20 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1e-21, 0,20, buffer) @test "" == unsafe_string(pointer(buffer)) @test -20 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1e-22, 0,20, buffer) @test "" == unsafe_string(pointer(buffer)) @test -20 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(6e-21, 0,20, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -19 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(9.1193616301674545152000000e+19, 0,0,buffer) @test "91193616301674545152" == unsafe_string(pointer(buffer)) @test 20 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(4.8184662102767651659096515e-04, 0,19,buffer) @test "4818466210276765" == unsafe_string(pointer(buffer)) @test -3 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1.9023164229540652612705182e-23, 0,8,buffer) @test "" == unsafe_string(pointer(buffer)) @test -8 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(1000000000000000128.0, 0,0,buffer) @test "1000000000000000128" == unsafe_string(pointer(buffer)) @test 19 == point -fill!(buffer,0); +fill!(buffer,0) #bignumdtoa status,len,point = Grisu.bignumdtoa(1.0, Grisu.SHORTEST, 0, buffer,bignums) @test "1" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(1.0, Grisu.FIXED, 3, buffer,bignums) @test 3 >= len - 1 - point @test "1" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(1.0, Grisu.PRECISION, 3, buffer,bignums) @test 3 >= len - 1 @test "1" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(1.5, Grisu.SHORTEST, 0, buffer,bignums) @test "15" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(1.5, Grisu.FIXED, 10, buffer,bignums) @test 10 >= len - 1 - point @test "15" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(1.5, Grisu.PRECISION, 10, buffer,bignums) @test 10 >= len - 1 @test "15" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) min_double = 5e-324 status,len,point = Grisu.bignumdtoa(min_double, Grisu.SHORTEST, 0, buffer,bignums) @test "5" == trimrep(buffer) @test -323 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(min_double, Grisu.FIXED, 5, buffer,bignums) @@ -1050,166 +1050,166 @@ status,len,point = Grisu.bignumdtoa(min_double, Grisu.PRECISION, 5, buffer,bignu @test 5 >= len - 1 @test "49407" == trimrep(buffer) @test -323 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) max_double = 1.7976931348623157e308 status,len,point = Grisu.bignumdtoa(max_double, Grisu.SHORTEST, 0, buffer,bignums) @test "17976931348623157" == trimrep(buffer) @test 309 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(max_double, Grisu.PRECISION, 7, buffer,bignums) @test 7 >= len - 1 @test "1797693" == trimrep(buffer) @test 309 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(4294967272.0, Grisu.SHORTEST, 0, buffer,bignums) @test "4294967272" == unsafe_string(pointer(buffer)) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(4294967272.0, Grisu.FIXED, 5, buffer,bignums) @test "429496727200000" == unsafe_string(pointer(buffer)) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(4294967272.0, Grisu.PRECISION, 14, buffer,bignums) @test 14 >= len - 1 @test "4294967272" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(4.1855804968213567e298, Grisu.SHORTEST, 0,buffer,bignums) @test "4185580496821357" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(4.1855804968213567e298, Grisu.PRECISION, 20,buffer,bignums) @test 20 >= len - 1 @test "41855804968213567225" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(5.5626846462680035e-309, Grisu.SHORTEST, 0, buffer,bignums) @test "5562684646268003" == trimrep(buffer) @test -308 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(5.5626846462680035e-309, Grisu.PRECISION, 1, buffer,bignums) @test 1 >= len - 1 @test "6" == trimrep(buffer) @test -308 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(2147483648.0, Grisu.SHORTEST, 0, buffer,bignums) @test "2147483648" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(2147483648.0, Grisu.FIXED, 2, buffer,bignums) @test 2 >= len - 1 - point @test "2147483648" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(2147483648.0, Grisu.PRECISION, 5, buffer,bignums) @test 5 >= len - 1 @test "21475" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(3.5844466002796428e+298, Grisu.SHORTEST, 0, buffer,bignums) @test "35844466002796428" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(3.5844466002796428e+298, Grisu.PRECISION, 10, buffer,bignums) @test 10 >= len - 1 @test "35844466" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) v = reinterpret(Float64,0x0010000000000000) status,len,point = Grisu.bignumdtoa(v, Grisu.SHORTEST, 0, buffer,bignums) @test "22250738585072014" == trimrep(buffer) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(v, Grisu.PRECISION, 20, buffer,bignums) @test 20 >= len - 1 @test "22250738585072013831" == trimrep(buffer) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) v = reinterpret(Float64,0x000FFFFFFFFFFFFF) status,len,point = Grisu.bignumdtoa(v, Grisu.SHORTEST, 0, buffer,bignums) @test "2225073858507201" == trimrep(buffer) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(v, Grisu.PRECISION, 20, buffer,bignums) @test 20 >= len - 1 @test "2225073858507200889" == trimrep(buffer) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(4128420500802942e-24, Grisu.SHORTEST, 0, buffer,bignums) @test "4128420500802942" == trimrep(buffer) @test -8 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) -v = 3.9292015898194142585311918e-10; +v = 3.9292015898194142585311918e-10 status,len,point = Grisu.bignumdtoa(v, Grisu.SHORTEST, 0, buffer,bignums) @test "39292015898194143" == trimrep(buffer) -v = 4194304.0; +v = 4194304.0 status,len,point = Grisu.bignumdtoa(v, Grisu.FIXED, 5, buffer,bignums) @test 5 >= len - 1 - point @test "4194304" == trimrep(buffer) -v = 3.3161339052167390562200598e-237; +v = 3.3161339052167390562200598e-237 status,len,point = Grisu.bignumdtoa(v, Grisu.PRECISION, 19, buffer,bignums) @test 19 >= len - 1 @test "3316133905216739056" == trimrep(buffer) @test -236 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) -v = 7.9885183916008099497815232e+191; +v = 7.9885183916008099497815232e+191 status,len,point = Grisu.bignumdtoa(v, Grisu.PRECISION, 4, buffer,bignums) @test 4 >= len - 1 @test "7989" == trimrep(buffer) @test 192 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) -v = 1.0000000000000012800000000e+17; +v = 1.0000000000000012800000000e+17 status,len,point = Grisu.bignumdtoa(v, Grisu.FIXED, 1, buffer,bignums) @test 1 >= len - 1 - point @test "100000000000000128" == trimrep(buffer) @test 18 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) @@ -1217,64 +1217,64 @@ min_float = Float32(1e-45) status,len,point = Grisu.bignumdtoa(min_float, Grisu.SHORTEST, 0, buffer,bignums) @test "1" == trimrep(buffer) @test -44 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) max_float = Float32(3.4028234e38) status,len,point = Grisu.bignumdtoa(max_float, Grisu.SHORTEST, 0, buffer,bignums) @test "34028235" == trimrep(buffer) @test 39 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(Float32(4294967272.0), Grisu.SHORTEST, 0, buffer,bignums) @test "42949673" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(Float32(3.32306998946228968226e+35), Grisu.SHORTEST, 0, buffer,bignums) @test "332307" == trimrep(buffer) @test 36 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(Float32(1.2341e-41), Grisu.SHORTEST, 0, buffer,bignums) @test "12341" == trimrep(buffer) @test -40 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(Float32(3.3554432e7), Grisu.SHORTEST, 0, buffer,bignums) @test "33554432" == trimrep(buffer) @test 8 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(Float32(3.26494756798464e14), Grisu.SHORTEST, 0, buffer,bignums) @test "32649476" == trimrep(buffer) @test 15 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) status,len,point = Grisu.bignumdtoa(Float32(3.91132223637771935344e37), Grisu.SHORTEST, 0, buffer,bignums) @test "39113222" == trimrep(buffer) @test 38 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) v = reinterpret(Float32,0x00800000) status,len,point = Grisu.bignumdtoa(v, Grisu.SHORTEST, 0, buffer,bignums) @test "11754944" == trimrep(buffer) @test -37 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) v = reinterpret(Float32,0x007FFFFF) status,len,point = Grisu.bignumdtoa(v, Grisu.SHORTEST, 0, buffer,bignums) @test "11754942" == trimrep(buffer) @test -37 == point -fill!(buffer,0); +fill!(buffer,0) map(x->Grisu.Bignums.zero!(x),bignums) #Float16 @@ -1283,405 +1283,405 @@ status,len,point = Grisu.fastshortest(min_double,buffer) @test status @test "6104" == trimrep(buffer) @test -4 == point -fill!(buffer,0); +fill!(buffer,0) max_double = realmax(Float16) status,len,point = Grisu.fastshortest(max_double,buffer) @test status @test "655" == trimrep(buffer) @test 5 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastprecision(Float16(1.0), 3, buffer) @test status @test 3 >= len-1 @test "1" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastprecision(Float16(1.5), 10, buffer) if status - @test 10 >= len-1 - @test "15" == trimrep(buffer) - @test 1 == point - fill!(buffer,0); + @test 10 >= len-1 + @test "15" == trimrep(buffer) + @test 1 == point + fill!(buffer,0) end status,len,point = Grisu.fastfixedtoa(Float16(1.0), 0,1, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(1.0), 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(1.0), 0,0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(1.5), 0,5, buffer) @test "15" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(1.55), 0,5, buffer) @test "15498" == unsafe_string(pointer(buffer)) #todo @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(1.55), 0,1, buffer) @test "15" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(1.00000001), 0,15, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.1), 0,10, buffer) @test "999755859" == unsafe_string(pointer(buffer)) @test -1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.01), 0,10, buffer) @test "100021362" == unsafe_string(pointer(buffer)) @test -1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.001), 0,10, buffer) @test "10004044" == unsafe_string(pointer(buffer)) @test -2 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.0001), 0,10, buffer) #todo @test "1000166" == unsafe_string(pointer(buffer)) @test -3 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.00001), 0,10, buffer) #todo @test "100136" == unsafe_string(pointer(buffer)) @test -4 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.000001), 0,10, buffer) @test "10133" == unsafe_string(pointer(buffer)) @test -5 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.0000001), 0,10, buffer) @test "1192" == unsafe_string(pointer(buffer)) @test -6 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.6), 0,0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.96), 0,1, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.996), 0,2, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.9996), 0,3, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.99996), 0,4, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.999996), 0,5, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.9999996), 0,6, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.99999996), 0,7, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(42), 0,20, buffer) @test "42" == unsafe_string(pointer(buffer)) @test 2 == point -fill!(buffer,0); +fill!(buffer,0) status,len,point = Grisu.fastfixedtoa(Float16(0.5), 0,0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) #dtoa len,point,neg = Grisu.grisu(0.0, Grisu.SHORTEST, 0, buffer) @test "0" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(Float32(0.0), Grisu.SHORTEST, 0, buffer) @test "0" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(0.0, Grisu.FIXED, 2, buffer) @test 1 >= len-1 @test "0" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(0.0, Grisu.PRECISION, 3, buffer) @test 1 >= len-1 @test "0" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(1.0, Grisu.SHORTEST, 0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(Float32(1.0), Grisu.SHORTEST, 0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(1.0, Grisu.FIXED, 3, buffer) @test 3 >= len-1-point @test "1" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(1.0, Grisu.PRECISION, 3, buffer) @test 3 >= len-1 @test "1" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(1.5, Grisu.SHORTEST, 0, buffer) @test "15" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(Float32(1.5), Grisu.SHORTEST, 0, buffer) @test "15" == unsafe_string(pointer(buffer)) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(1.5, Grisu.FIXED, 10, buffer) @test 10 >= len-1-point @test "15" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(1.5, Grisu.PRECISION, 10, buffer) @test 10 >= len-1 @test "15" == trimrep(buffer) @test 1 == point -fill!(buffer,0); +fill!(buffer,0) min_double = 5e-324 len,point,neg = Grisu.grisu(min_double, Grisu.SHORTEST, 0, buffer) @test "5" == unsafe_string(pointer(buffer)) @test -323 == point -fill!(buffer,0); +fill!(buffer,0) min_float = 1e-45 len,point,neg = Grisu.grisu(Float32(min_float), Grisu.SHORTEST, 0, buffer) @test "1" == unsafe_string(pointer(buffer)) @test -44 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(min_double, Grisu.FIXED, 5, buffer) @test 5 >= len-1-point @test "" == trimrep(buffer) @test -5 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(min_double, Grisu.PRECISION, 5, buffer) @test 5 >= len-1 @test "49407" == trimrep(buffer) @test -323 == point -fill!(buffer,0); +fill!(buffer,0) max_double = 1.7976931348623157e308 len,point,neg = Grisu.grisu(max_double, Grisu.SHORTEST, 0, buffer) @test "17976931348623157" == unsafe_string(pointer(buffer)) @test 309 == point -fill!(buffer,0); +fill!(buffer,0) max_float = 3.4028234e38 len,point,neg = Grisu.grisu(Float32(max_float), Grisu.SHORTEST, 0, buffer) @test "34028235" == unsafe_string(pointer(buffer)) @test 39 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(max_double, Grisu.PRECISION, 7, buffer) @test 7 >= len-1 @test "1797693" == trimrep(buffer) @test 309 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(4294967272.0, Grisu.SHORTEST, 0, buffer) @test "4294967272" == unsafe_string(pointer(buffer)) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(Float32(4294967272.0), Grisu.SHORTEST, 0, buffer) @test "42949673" == unsafe_string(pointer(buffer)) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(4294967272.0, Grisu.FIXED, 5, buffer) @test 5 >= len-1-point @test "4294967272" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(4294967272.0, Grisu.PRECISION, 14, buffer) @test 14 >= len-1 @test "4294967272" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(4.1855804968213567e298, Grisu.SHORTEST, 0, buffer) @test "4185580496821357" == unsafe_string(pointer(buffer)) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(4.1855804968213567e298, Grisu.PRECISION, 20, buffer) @test 20 >= len-1 @test "41855804968213567225" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(5.5626846462680035e-309, Grisu.SHORTEST, 0, buffer) @test "5562684646268003" == unsafe_string(pointer(buffer)) @test -308 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(5.5626846462680035e-309, Grisu.PRECISION, 1, buffer) @test 1 >= len-1 @test "6" == trimrep(buffer) @test -308 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(-2147483648.0, Grisu.SHORTEST, 0, buffer) @test 1 == neg @test "2147483648" == unsafe_string(pointer(buffer)) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(Float32(-2147483648.), Grisu.SHORTEST, 0, buffer) @test 1 == neg @test "21474836" == unsafe_string(pointer(buffer)) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(-2147483648.0, Grisu.FIXED, 2, buffer) @test 2 >= len-1-point @test "2147483648" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(-2147483648.0, Grisu.PRECISION, 5, buffer) @test 5 >= len-1 @test "21475" == trimrep(buffer) @test 10 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(-3.5844466002796428e+298, Grisu.SHORTEST, 0, buffer) @test 1 == neg @test "35844466002796428" == unsafe_string(pointer(buffer)) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(-3.5844466002796428e+298, Grisu.PRECISION, 10, buffer) @test 1 == neg @test 10 >= len-1 @test "35844466" == trimrep(buffer) @test 299 == point -fill!(buffer,0); +fill!(buffer,0) v = reinterpret(Float64,0x0010000000000000) len,point,neg = Grisu.grisu(v, Grisu.SHORTEST, 0, buffer) @test "22250738585072014" == unsafe_string(pointer(buffer)) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) f = reinterpret(Float32,0x00800000) len,point,neg = Grisu.grisu(f, Grisu.SHORTEST, 0, buffer) @test "11754944" == unsafe_string(pointer(buffer)) @test -37 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(v, Grisu.PRECISION, 20, buffer) @test 20 >= len-1 @test "22250738585072013831" == trimrep(buffer) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) v = reinterpret(Float64,0x000FFFFFFFFFFFFF) len,point,neg = Grisu.grisu(v, Grisu.SHORTEST, 0, buffer) @test "2225073858507201" == unsafe_string(pointer(buffer)) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) f = reinterpret(Float32,0x007FFFFF) len,point,neg = Grisu.grisu(f, Grisu.SHORTEST, 0, buffer) @test "11754942" == unsafe_string(pointer(buffer)) @test -37 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(v, Grisu.PRECISION, 20, buffer) @test 20 >= len-1 @test "2225073858507200889" == trimrep(buffer) @test -307 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(4128420500802942e-24, Grisu.SHORTEST, 0, buffer) @test 0 == neg @test "4128420500802942" == unsafe_string(pointer(buffer)) @test -8 == point -fill!(buffer,0); +fill!(buffer,0) v = -3.9292015898194142585311918e-10 len,point,neg = Grisu.grisu(v, Grisu.SHORTEST, 0, buffer) @test "39292015898194143" == unsafe_string(pointer(buffer)) -fill!(buffer,0); +fill!(buffer,0) f = Float32(-3.9292015898194142585311918e-10) len,point,neg = Grisu.grisu(f, Grisu.SHORTEST, 0, buffer) @test "39292017" == unsafe_string(pointer(buffer)) -fill!(buffer,0); +fill!(buffer,0) v = 4194304.0 len,point,neg = Grisu.grisu(v, Grisu.FIXED, 5, buffer) @test 5 >= len-1-point @test "4194304" == trimrep(buffer) -fill!(buffer,0); +fill!(buffer,0) v = 3.3161339052167390562200598e-237 len,point,neg = Grisu.grisu(v, Grisu.PRECISION, 19, buffer) @test 19 >= len-1 @test "3316133905216739056" == trimrep(buffer) @test -236 == point -fill!(buffer,0); +fill!(buffer,0) len,point,neg = Grisu.grisu(0.0, Grisu.SHORTEST, 0, buffer) @test !neg diff --git a/test/linalg/dense.jl b/test/linalg/dense.jl index 3f1c7a3c45995..1f7ae605fd773 100644 --- a/test/linalg/dense.jl +++ b/test/linalg/dense.jl @@ -422,7 +422,7 @@ for elty in (Float64, Complex{Float64}) end -A8 = 100 * [-1+1im 0 0 1e-8; 0 1 0 0; 0 0 1 0; 0 0 0 1]; +A8 = 100 * [-1+1im 0 0 1e-8; 0 1 0 0; 0 0 1 0; 0 0 0 1] @test expm(logm(A8)) ≈ A8 # issue 5116 diff --git a/test/linalg/pinv.jl b/test/linalg/pinv.jl index 221a17339e0f2..1f5c8049dc20e 100644 --- a/test/linalg/pinv.jl +++ b/test/linalg/pinv.jl @@ -70,7 +70,7 @@ function randn_float64(m::Integer, n::Integer) b=Array{Float64}(m,n) for i=1:n for j=1:m - b[j,i]=convert(Float64,a[j,i]); + b[j,i]=convert(Float64,a[j,i]) end end return b @@ -81,7 +81,7 @@ function randn_float32(m::Integer, n::Integer) b=Array{Float32}(m,n) for i=1:n for j=1:m - b[j,i]=convert(Float32,a[j,i]); + b[j,i]=convert(Float32,a[j,i]) end end return b @@ -93,17 +93,17 @@ function test_pinv(a,m,n,tol1,tol2,tol3) apinv = @inferred pinv(a) @test_approx_eq_eps vecnorm(a*apinv*a - a)/vecnorm(a) 0 tol1 - x0 = randn(n); b = a*x0; x = apinv*b; + x0 = randn(n); b = a*x0; x = apinv*b @test_approx_eq_eps vecnorm(a*x-b)/vecnorm(b) 0 tol1 debug && println(vecnorm(a*apinv*a - a)/vecnorm(a)) debug && println(vecnorm(a*x-b)/vecnorm(b)) debug && println("=== julia pinv, tol=sqrt(eps(1.0)) ===") - apinv = pinv(a,sqrt(eps(real(one(eltype(a)))))); + apinv = pinv(a,sqrt(eps(real(one(eltype(a)))))) @test_approx_eq_eps vecnorm(a*apinv*a - a)/vecnorm(a) 0 tol2 - x0 = randn(n); b = a*x0; x = apinv*b; + x0 = randn(n); b = a*x0; x = apinv*b @test_approx_eq_eps vecnorm(a*x-b)/vecnorm(b) 0 tol2 debug && println(vecnorm(a*apinv*a - a)/vecnorm(a)) debug && println(vecnorm(a*x-b)/vecnorm(b)) @@ -123,21 +123,21 @@ let default_tol = (real(one(eltya))) * max(m,n) * 10 - debug && println("\n--- dense/ill-conditioned matrix ---\n"); - ### a = randn_float64(m,n) * hilb(eltya,n); - a = hilb(eltya,m,n); + debug && println("\n--- dense/ill-conditioned matrix ---\n") + ### a = randn_float64(m,n) * hilb(eltya,n) + a = hilb(eltya,m,n) test_pinv(a,m,n,1e-2,1e-5,1e-5) - debug && println("\n--- dense/diagonal matrix ---\n"); - a = onediag(eltya,m,n); + debug && println("\n--- dense/diagonal matrix ---\n") + a = onediag(eltya,m,n) test_pinv(a,m,n,default_tol,default_tol,default_tol) - debug && println("\n--- dense/tri-diagonal matrix ---\n"); - a = tridiag(eltya,m,n); + debug && println("\n--- dense/tri-diagonal matrix ---\n") + a = tridiag(eltya,m,n) test_pinv(a,m,n,default_tol,1e-5,default_tol) - debug && println("\n--- Diagonal matrix ---\n"); - a = onediag_sparse(eltya,m); + debug && println("\n--- Diagonal matrix ---\n") + a = onediag_sparse(eltya,m) test_pinv(a,m,m,default_tol,default_tol,default_tol) m = 100 @@ -146,21 +146,21 @@ let default_tol = (real(one(eltya))) * max(m,n) * 10 - debug && println("\n--- dense/ill-conditioned matrix ---\n"); - ### a = randn_float64(m,n) * hilb(eltya,n); - a = hilb(eltya,m,n); + debug && println("\n--- dense/ill-conditioned matrix ---\n") + ### a = randn_float64(m,n) * hilb(eltya,n) + a = hilb(eltya,m,n) test_pinv(a,m,n,1e-2,1e-5,1e-5) - debug && println("\n--- dense/diagonal matrix ---\n"); - a = onediag(eltya,m,n); + debug && println("\n--- dense/diagonal matrix ---\n") + a = onediag(eltya,m,n) test_pinv(a,m,n,default_tol,default_tol,default_tol) - debug && println("\n--- dense/tri-diagonal matrix ---\n"); - a = tridiag(eltya,m,n); + debug && println("\n--- dense/tri-diagonal matrix ---\n") + a = tridiag(eltya,m,n) test_pinv(a,m,n,default_tol,1e-5,default_tol) - debug && println("\n--- Diagonal matrix ---\n"); - a = onediag_sparse(eltya,m); + debug && println("\n--- Diagonal matrix ---\n") + a = onediag_sparse(eltya,m) test_pinv(a,m,m,default_tol,default_tol,default_tol) m = 100 @@ -169,21 +169,21 @@ let default_tol = (real(one(eltya))) * max(m,n) * 10 - debug && println("\n--- dense/ill-conditioned matrix ---\n"); - ### a = randn_float64(m,n) * hilb(eltya,n); - a = hilb(eltya,m,n); + debug && println("\n--- dense/ill-conditioned matrix ---\n") + ### a = randn_float64(m,n) * hilb(eltya,n) + a = hilb(eltya,m,n) test_pinv(a,m,n,1e-2,1e-5,1e-5) - debug && println("\n--- dense/diagonal matrix ---\n"); - a = onediag(eltya,m,n); + debug && println("\n--- dense/diagonal matrix ---\n") + a = onediag(eltya,m,n) test_pinv(a,m,n,default_tol,default_tol,default_tol) - debug && println("\n--- dense/tri-diagonal matrix ---\n"); - a = tridiag(eltya,m,n); + debug && println("\n--- dense/tri-diagonal matrix ---\n") + a = tridiag(eltya,m,n) test_pinv(a,m,n,default_tol,1e-5,default_tol) - debug && println("\n--- Diagonal matrix ---\n"); - a = onediag_sparse(eltya,m); + debug && println("\n--- Diagonal matrix ---\n") + a = onediag_sparse(eltya,m) test_pinv(a,m,m,default_tol,default_tol,default_tol) end end @@ -199,21 +199,21 @@ for eltya in (Float32, Complex64) default_tol = (real(one(eltya))) * max(m,n) * 10 - debug && println("\n--- dense/ill-conditioned matrix ---\n"); -### a = randn_float32(m,n) * hilb(eltya,n); - a = hilb(eltya,m,n); + debug && println("\n--- dense/ill-conditioned matrix ---\n") +### a = randn_float32(m,n) * hilb(eltya,n) + a = hilb(eltya,m,n) test_pinv(a,m,n,1e0,1e-2,1e-2) - debug && println("\n--- dense/diagonal matrix ---\n"); - a = onediag(eltya,m,n); + debug && println("\n--- dense/diagonal matrix ---\n") + a = onediag(eltya,m,n) test_pinv(a,m,n,default_tol,default_tol,default_tol) - debug && println("\n--- dense/tri-diagonal matrix ---\n"); - a = tridiag(eltya,m,n); + debug && println("\n--- dense/tri-diagonal matrix ---\n") + a = tridiag(eltya,m,n) test_pinv(a,m,n,default_tol,1e-2,default_tol) - debug && println("\n--- Diagonal matrix ---\n"); - a = onediag_sparse(eltya,m); + debug && println("\n--- Diagonal matrix ---\n") + a = onediag_sparse(eltya,m) test_pinv(a,m,m,default_tol,default_tol,default_tol) m = 100 @@ -222,21 +222,21 @@ for eltya in (Float32, Complex64) default_tol = (real(one(eltya))) * max(m,n) * 10 - debug && println("\n--- dense/ill-conditioned matrix ---\n"); -### a = randn_float32(m,n) * hilb(eltya,n); - a = hilb(eltya,m,n); + debug && println("\n--- dense/ill-conditioned matrix ---\n") +### a = randn_float32(m,n) * hilb(eltya,n) + a = hilb(eltya,m,n) test_pinv(a,m,n,1e0,1e-2,1e-2) - debug && println("\n--- dense/diagonal matrix ---\n"); - a = onediag(eltya,m,n); + debug && println("\n--- dense/diagonal matrix ---\n") + a = onediag(eltya,m,n) test_pinv(a,m,n,default_tol,default_tol,default_tol) - debug && println("\n--- dense/tri-diagonal matrix ---\n"); - a = tridiag(eltya,m,n); + debug && println("\n--- dense/tri-diagonal matrix ---\n") + a = tridiag(eltya,m,n) test_pinv(a,m,n,default_tol,1e-2,default_tol) - debug && println("\n--- Diagonal matrix ---\n"); - a = onediag_sparse(eltya,m); + debug && println("\n--- Diagonal matrix ---\n") + a = onediag_sparse(eltya,m) test_pinv(a,m,m,default_tol,default_tol,default_tol) m = 100 @@ -245,21 +245,21 @@ for eltya in (Float32, Complex64) default_tol = (real(one(eltya))) * max(m,n) * 10 - debug && println("\n--- dense/ill-conditioned matrix ---\n"); -### a = randn_float32(m,n) * hilb(eltya,n); - a = hilb(eltya,m,n); + debug && println("\n--- dense/ill-conditioned matrix ---\n") +### a = randn_float32(m,n) * hilb(eltya,n) + a = hilb(eltya,m,n) test_pinv(a,m,n,1e0,1e-2,1e-2) - debug && println("\n--- dense/diagonal matrix ---\n"); - a = onediag(eltya,m,n); + debug && println("\n--- dense/diagonal matrix ---\n") + a = onediag(eltya,m,n) test_pinv(a,m,n,default_tol,default_tol,default_tol) - debug && println("\n--- dense/tri-diagonal matrix ---\n"); - a = tridiag(eltya,m,n); + debug && println("\n--- dense/tri-diagonal matrix ---\n") + a = tridiag(eltya,m,n) test_pinv(a,m,n,default_tol,1e-2,default_tol) - debug && println("\n--- Diagonal matrix ---\n"); - a = onediag_sparse(eltya,m); + debug && println("\n--- Diagonal matrix ---\n") + a = onediag_sparse(eltya,m) test_pinv(a,m,m,default_tol,default_tol,default_tol) end diff --git a/test/meta.jl b/test/meta.jl index c133013ea16cb..dfa7c98c980fa 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -130,7 +130,7 @@ using Base.Meta @test isexpr(:(1+1),Vector([:call])) @test isexpr(1,:call)==false @test isexpr(:(1+1),:call,3) -ioB = IOBuffer(); +ioB = IOBuffer() show_sexpr(ioB,:(1+1)) show_sexpr(ioB,QuoteNode(1),1) diff --git a/test/mmap.jl b/test/mmap.jl index 432f6cf210306..d6566ec17b9a3 100644 --- a/test/mmap.jl +++ b/test/mmap.jl @@ -283,10 +283,10 @@ finalize(m); m = nothing; gc() file = tempname() touch(file) open(file, "r+") do s - A = Mmap.mmap(s, Vector{UInt8}, (10,), 0); + A = Mmap.mmap(s, Vector{UInt8}, (10,), 0) Mmap.sync!(A) finalize(A); A = nothing; gc() - A = Mmap.mmap(s, Vector{UInt8}, (10,), 1); + A = Mmap.mmap(s, Vector{UInt8}, (10,), 1) Mmap.sync!(A) finalize(A); A = nothing; gc() end diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index cf03ca8a2e486..81d3d47a20507 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -582,7 +582,7 @@ num_small_requests = 10000 # test parallel sends of large arrays from multiple tasks to the same remote worker ntasks = 10 rr_list = [Channel() for x in 1:ntasks] -a=ones(2*10^5); +a = ones(2*10^5) for rr in rr_list @async let rr=rr try diff --git a/test/perf/kernel/bench_eu.jl b/test/perf/kernel/bench_eu.jl index d09f795e8d0b9..287440de2f487 100644 --- a/test/perf/kernel/bench_eu.jl +++ b/test/perf/kernel/bench_eu.jl @@ -6,12 +6,12 @@ function bench_eu_devec(numPaths) steps = 250 r = 0.05 - sigma = .4; - T = 1; + sigma = .4 + T = 1 dt = T/(steps) - K = 100; + K = 100 - S = 100 * ones(numPaths,1); + S = 100 * ones(numPaths,1) t1 = (r-0.5*sigma.^2)*dt t2 = sigma*sqrt(dt) @@ -27,12 +27,12 @@ end function bench_eu_vec(numPaths) steps = 250 r = 0.05 - sigma = .4; - T = 1; + sigma = .4 + T = 1 dt = T/(steps) - K = 100; + K = 100 - S = 100 * ones(numPaths,1); + S = 100 * ones(numPaths,1) t1 = (r-0.5*sigma.^2)*dt t2 = sigma*sqrt(dt) diff --git a/test/perf/kernel/getdivgrad.jl b/test/perf/kernel/getdivgrad.jl index e998bf49651d3..aed68e649bed2 100644 --- a/test/perf/kernel/getdivgrad.jl +++ b/test/perf/kernel/getdivgrad.jl @@ -11,7 +11,7 @@ function getDivGrad(n1,n2,n3) # DIV from faces to cell-centers Div = [D1 D2 D3] - return Div*Div'; + return Div*Div' end #----------------- 1D finite difference on staggered grid diff --git a/test/perf/kernel/gk.jl b/test/perf/kernel/gk.jl index ca0c0491b7aa7..84a3803fd4f65 100644 --- a/test/perf/kernel/gk.jl +++ b/test/perf/kernel/gk.jl @@ -10,7 +10,7 @@ function myunifskew(n) A = zeros(n, n) - #print("A[i,j] initialized with zeros \n"); + #print("A[i,j] initialized with zeros \n") for i=1:n for j=1:i-1 @@ -20,12 +20,12 @@ function myunifskew(n) temp = rand() A[i,j]= temp A[j,i]= -A[i,j] - #print("welcome"); + #print("welcome") else temp = rand() A[j,i]= temp A[i,j]= -A[j,i] - #print("welcome"); + #print("welcome") end end diff --git a/test/perf/kernel/perf.jl b/test/perf/kernel/perf.jl index 928ca1ef6f0b8..b5e50514288cc 100644 --- a/test/perf/kernel/perf.jl +++ b/test/perf/kernel/perf.jl @@ -33,13 +33,13 @@ include("gk.jl") @timeit gk(350,[0.1]) "gk" "Grigoriadis Khachiyan matrix games" # issue #942 -s = sparse(ones(280,280)); +s = sparse(ones(280,280)) @timeit s*s "sparsemul" "Sparse matrix - sparse matrix multiplication" -s2 = sparse(rand(1:2000,10^5), kron([1:10^4;],ones(Int,10)), ones(Int,10^5), 2000, 10^4); +s2 = sparse(rand(1:2000,10^5), kron([1:10^4;],ones(Int,10)), ones(Int,10^5), 2000, 10^4) @timeit s2*s2' "sparsemul2" "Sparse matrix - matrix multiplication with fill-in" # issue #938 -x = 1:600000; +x = 1:600000 @timeit sparse(x,x,x) "sparserange" "Construction of a sparse array from ranges" # issue 4707 diff --git a/test/perf/kernel/raytracer.jl b/test/perf/kernel/raytracer.jl index 06721486d8483..e1ac918e9fb39 100644 --- a/test/perf/kernel/raytracer.jl +++ b/test/perf/kernel/raytracer.jl @@ -131,7 +131,7 @@ function Raytracer(levels, n, ss) for dy in 0:1:(ss-1) d = Vec(x+dx*1./ss-n/2., y+dy*1./ss-n/2., n*1.0) ray = Ray(Vec(0., 0., -4.0), unitize(d)) - g += ray_trace(light, ray, scene); + g += ray_trace(light, ray, scene) end end # write(f, trunc(UInt8, 0.5 + 255. * g / (ss*ss))) diff --git a/test/random.jl b/test/random.jl index 4115a612e4b5f..591725f1fded0 100644 --- a/test/random.jl +++ b/test/random.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # Issue #6573 -srand(0); rand(); x = rand(384); +srand(0); rand(); x = rand(384) @test find(x .== rand()) == [] @test rand() != rand() diff --git a/test/read.jl b/test/read.jl index 38547dedbee93..ac73232b8fcd0 100644 --- a/test/read.jl +++ b/test/read.jl @@ -202,7 +202,7 @@ for (name, f) in l verbose && println("$name readbytes!...") l = length(text) for n = [1, 2, l-2, l-1, l, l+1, l+2] - a1 = Vector{UInt8}(n); + a1 = Vector{UInt8}(n) a2 = Vector{UInt8}(n) s1 = io() s2 = IOBuffer(text) diff --git a/test/repl.jl b/test/repl.jl index e6b3369c75d39..fd4d657ecc6af 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -130,7 +130,7 @@ if !is_windows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER isopen(t) || return error("Stuck waiting for history test") end - s1 = "12345678"; s2 = "23456789"; + s1 = "12345678"; s2 = "23456789" write(stdin_write, s1, '\n') readuntil(stdout_read, s1) write(stdin_write, s2, '\n') @@ -190,7 +190,7 @@ function AddCustomMode(repl) b = Dict{Any,Any}[skeymap, mk, LineEdit.history_keymap, LineEdit.default_keymap, LineEdit.escape_defaults] foobar_mode.keymap_dict = LineEdit.keymap(b) - main_mode.keymap_dict = LineEdit.keymap_merge(main_mode.keymap_dict, foobar_keymap); + main_mode.keymap_dict = LineEdit.keymap_merge(main_mode.keymap_dict, foobar_keymap) foobar_mode, search_prompt end diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 3666e96e9bc89..fbe6e18f9a3b4 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -918,19 +918,19 @@ function test_getindex_algs{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, I::AbstractVector, end let M=2^14, N=2^4 - Irand = randperm(M); - Jrand = randperm(N); - SA = [sprand(M, N, d) for d in [1., 0.1, 0.01, 0.001, 0.0001, 0.]]; - IA = [sort(Irand[1:round(Int,n)]) for n in [M, M*0.1, M*0.01, M*0.001, M*0.0001, 0.]]; + Irand = randperm(M) + Jrand = randperm(N) + SA = [sprand(M, N, d) for d in [1., 0.1, 0.01, 0.001, 0.0001, 0.]] + IA = [sort(Irand[1:round(Int,n)]) for n in [M, M*0.1, M*0.01, M*0.001, M*0.0001, 0.]] debug = false if debug - println("row sizes: $([round(Int,nnz(S)/S.n) for S in SA])"); - println("I sizes: $([length(I) for I in IA])"); + println("row sizes: $([round(Int,nnz(S)/S.n) for S in SA])") + println("I sizes: $([length(I) for I in IA])") @printf(" S | I | binary S | binary I | linear | best\n") end - J = Jrand; + J = Jrand for I in IA for S in SA res = Any[1,2,3] @@ -967,7 +967,7 @@ let M = 2^8, N=2^3 I = sort([Irand; Irand; Irand]) J = [Jrand; Jrand] - SA = [sprand(M, N, d) for d in [1., 0.1, 0.01, 0.001, 0.0001, 0.]]; + SA = [sprand(M, N, d) for d in [1., 0.1, 0.01, 0.001, 0.0001, 0.]] for S in SA res = Any[1,2,3] for searchtype in [0, 1, 2] @@ -983,8 +983,8 @@ let M = 2^14, N=2^4 J = randperm(N) Jsorted = sort(J) - SA = [sprand(M, N, d) for d in [1., 0.1, 0.01, 0.001, 0.0001, 0.]]; - IA = [I[1:round(Int,n)] for n in [M, M*0.1, M*0.01, M*0.001, M*0.0001, 0.]]; + SA = [sprand(M, N, d) for d in [1., 0.1, 0.01, 0.001, 0.0001, 0.]] + IA = [I[1:round(Int,n)] for n in [M, M*0.1, M*0.01, M*0.001, M*0.0001, 0.]] debug = false if debug @printf(" | | | times | memory |\n") diff --git a/test/staged.jl b/test/staged.jl index 8f064b56b5e3a..f7e38b4c11fb1 100644 --- a/test/staged.jl +++ b/test/staged.jl @@ -60,13 +60,13 @@ splat2(3, 3:5) print(stagediobuf, indx) :(nothing) end -A = rand(5,5,3); +A = rand(5,5,3) splat3(A, 1:2, 1:2, 1) @test takebuf_string(stagediobuf) == "(UnitRange{$intstr},UnitRange{$intstr},$intstr)" splat3(A, 1:2, 1, 1:2) @test takebuf_string(stagediobuf) == "(UnitRange{$intstr},$intstr,UnitRange{$intstr})" -B = view(A, 1:3, 2, 1:3); +B = view(A, 1:3, 2, 1:3) @generated function mygetindex(S::SubArray, indexes::Real...) T, N, A, I = S.parameters if N != length(indexes) diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 45f5f91605880..a286be8b4a91f 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -158,11 +158,11 @@ end immutable tstStringType <: AbstractString data::Array{UInt8,1} end -tstr = tstStringType("12"); +tstr = tstStringType("12") @test_throws ErrorException endof(tstr) @test_throws ErrorException next(tstr, Bool(1)) -gstr = GenericString("12"); +gstr = GenericString("12") @test typeof(string(gstr))==GenericString @test convert(Array{UInt8}, gstr) ==[49;50] From a4b88a3066955c49444ff62c73ef14a1e68424b8 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 12 Jul 2016 01:08:16 -0700 Subject: [PATCH 0644/1117] Use === for comparison to nothing --- base/libgit2/callbacks.jl | 4 ++-- base/replutil.jl | 4 ++-- base/require.jl | 8 ++++---- base/subarray.jl | 2 +- contrib/build_sysimg.jl | 12 ++++++------ doc/manual/metaprogramming.rst | 4 ++-- doc/manual/strings.rst | 2 +- examples/clustermanager/0mq/ZMQCM.jl | 2 +- test/ambiguous.jl | 2 +- test/core.jl | 2 +- test/file.jl | 2 +- test/goto.jl | 4 ++-- test/linalg/tridiag.jl | 2 +- test/pkg.jl | 6 +++--- test/serialize.jl | 2 +- test/show.jl | 2 +- test/staged.jl | 2 +- test/version.jl | 18 +++++++++--------- 18 files changed, 40 insertions(+), 40 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 2de5af594d674..620c2e840ee73 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -77,7 +77,7 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, try # use ssh key or ssh-agent if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) - creds == nothing && (creds = SSHCredentials()) + creds === nothing && (creds = SSHCredentials()) credid = "ssh://$host" # first try ssh-agent if credentials support its usage @@ -172,7 +172,7 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, end if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) - creds == nothing && (creds = UserPasswordCredentials()) + creds === nothing && (creds = UserPasswordCredentials()) credid = "$schema$host" username = creds[:user, credid] diff --git a/base/replutil.jl b/base/replutil.jl index 5b90ce02d757f..3c26308f06b64 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -241,7 +241,7 @@ function showerror(io::IO, ex::DomainError, bt; backtrace=true) end function showerror(io::IO, ex::SystemError) - if ex.extrainfo == nothing + if ex.extrainfo === nothing print(io, "SystemError: $(ex.prefix): $(Libc.strerror(ex.errnum))") else print(io, "SystemError (with $(ex.extrainfo)): $(ex.prefix): $(Libc.strerror(ex.errnum))") @@ -297,7 +297,7 @@ function showerror(io::IO, ex::MethodError) f_is_function = true # See #13033 T = striptype(ex.args[1]) - if T == nothing + if T === nothing print(io, "First argument to `convert` must be a Type, got ", ex.args[1]) else print(io, "Cannot `convert` an object of type ", arg_types_param[2], " to an object of type ", T) diff --git a/base/require.jl b/base/require.jl index 1b2f648894488..ea2e64d267c84 100644 --- a/base/require.jl +++ b/base/require.jl @@ -40,7 +40,7 @@ require(f::AbstractString, fs::AbstractString...) = (require(f); for x in fs req function require(name::String) path = find_in_node1_path(name) - path == nothing && error("$name not found") + path === nothing && error("$name not found") if myid() == 1 && toplevel_load refs = Any[ @spawnat p _require(path) for p in filter(x->x!=1, procs()) ] @@ -85,7 +85,7 @@ end function include_from_node1(path::AbstractString) prev = source_path(nothing) - path = (prev == nothing) ? abspath(path) : joinpath(dirname(prev),path) + path = (prev === nothing) ? abspath(path) : joinpath(dirname(prev),path) tls = task_local_storage() tls[:SOURCE_PATH] = path local result @@ -99,7 +99,7 @@ function include_from_node1(path::AbstractString) result = include_string(remotecall_fetch(readstring, 1, path), path) end finally - if prev == nothing + if prev === nothing delete!(tls, :SOURCE_PATH) else tls[:SOURCE_PATH] = prev @@ -122,7 +122,7 @@ function reload_path(path::AbstractString) had || delete!(package_list, path) rethrow(e) finally - if prev != nothing + if prev !== nothing tls[:SOURCE_PATH] = prev end end diff --git a/base/subarray.jl b/base/subarray.jl index 390b6fc5aa07e..4d4fa370410b6 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -313,7 +313,7 @@ should transform to """ function replace_ref_end!(ex,withex=nothing) if isa(ex,Symbol) && ex == :end - withex == nothing && error("Invalid use of end") + withex === nothing && error("Invalid use of end") return withex elseif isa(ex,Expr) if ex.head == :ref diff --git a/contrib/build_sysimg.jl b/contrib/build_sysimg.jl index 1d7b40d105cb2..f84dd8f5d57d6 100644 --- a/contrib/build_sysimg.jl +++ b/contrib/build_sysimg.jl @@ -25,7 +25,7 @@ directives such as `using MyPackage` to include that package in the new system i system image will not replace an older image unless `force` is set to true. """ function build_sysimg(sysimg_path=nothing, cpu_target="native", userimg_path=nothing; force=false, debug=false) - if sysimg_path == nothing + if sysimg_path === nothing sysimg_path = default_sysimg_path(debug) end @@ -39,7 +39,7 @@ function build_sysimg(sysimg_path=nothing, cpu_target="native", userimg_path=not end # Canonicalize userimg_path before we enter the base_dir - if userimg_path != nothing + if userimg_path !== nothing userimg_path = abspath(userimg_path) end @@ -59,7 +59,7 @@ function build_sysimg(sysimg_path=nothing, cpu_target="native", userimg_path=not end # Copy in userimg.jl if it exists - if userimg_path != nothing + if userimg_path !== nothing if !isfile(userimg_path) error("$userimg_path is not found, ensure it is an absolute path!") end @@ -86,7 +86,7 @@ function build_sysimg(sysimg_path=nothing, cpu_target="native", userimg_path=not println("$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $inference_path.ji --startup-file=no sysimg.jl") run(`$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $inference_path.ji --startup-file=no sysimg.jl`) - if cc != nothing + if cc !== nothing link_sysimg(sysimg_path, cc, debug) else info("System image successfully built at $sysimg_path.ji") @@ -103,7 +103,7 @@ function build_sysimg(sysimg_path=nothing, cpu_target="native", userimg_path=not end finally # Cleanup userimg.jl - if userimg_path != nothing && isfile("userimg.jl") + if userimg_path !== nothing && isfile("userimg.jl") rm("userimg.jl") end end @@ -148,7 +148,7 @@ end # Link sys.o into sys.$(dlext) function link_sysimg(sysimg_path=nothing, cc=find_system_compiler(), debug=false) - if sysimg_path == nothing + if sysimg_path === nothing sysimg_path = default_sysimg_path(debug) end julia_libdir = dirname(Libdl.dlpath(debug ? "libjulia-debug" : "libjulia")) diff --git a/doc/manual/metaprogramming.rst b/doc/manual/metaprogramming.rst index a2fca9211a7e2..8ab35ac404e63 100644 --- a/doc/manual/metaprogramming.rst +++ b/doc/manual/metaprogramming.rst @@ -831,7 +831,7 @@ executed. Consider if the regular expression occurs in a loop:: for line = lines m = match(r"^\s*(?:#|$)", line) - if m == nothing + if m === nothing # non-comment else # comment @@ -847,7 +847,7 @@ this:: re = Regex("^\\s*(?:#|\$)") for line = lines m = match(re, line) - if m == nothing + if m === nothing # non-comment else # comment diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 4da392ec1edec..674ee174973a8 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -659,7 +659,7 @@ the interactive prompt. Other than not printing, it is a completely normal value and you can test for it programmatically:: m = match(r"^\s*(?:#|$)", line) - if m == nothing + if m === nothing println("not a comment") else println("blank or comment") diff --git a/examples/clustermanager/0mq/ZMQCM.jl b/examples/clustermanager/0mq/ZMQCM.jl index 8c8b56d0d7d0f..1f305939b2ca5 100644 --- a/examples/clustermanager/0mq/ZMQCM.jl +++ b/examples/clustermanager/0mq/ZMQCM.jl @@ -239,7 +239,7 @@ function start_worker(zid, cookie) #println("worker recv data from $from_zid") streams = get(manager.map_zmq_julia, from_zid, nothing) - if streams == nothing + if streams === nothing # First time.. (r_s, w_s) = setup_connection(from_zid, REMOTE_INITIATED) Base.process_messages(r_s, w_s) diff --git a/test/ambiguous.jl b/test/ambiguous.jl index 4a34f57957781..3b2296ce0a630 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -19,7 +19,7 @@ for m in mt ln = getline(m) atarget = ambigs[ln] if isempty(atarget) - @test m.ambig == nothing + @test m.ambig === nothing else aln = Int[getline(a) for a in m.ambig] @test sort(aln) == atarget diff --git a/test/core.jl b/test/core.jl index 56bbe9d3e7ff7..d2a6d50d8fd77 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2310,7 +2310,7 @@ function test_wr() test_wr(ref, wref) pop!(ref) gc() - @test wref[1].value == nothing + @test wref[1].value === nothing end test_wr() diff --git a/test/file.jl b/test/file.jl index 5a76143a8f59b..5da9ac7e89816 100644 --- a/test/file.jl +++ b/test/file.jl @@ -179,7 +179,7 @@ if !is_windows() end else # test that chown doesn't cause any errors for Windows - @test chown(file, -2, -2) == nothing + @test chown(file, -2, -2) === nothing end ####################################################################### diff --git a/test/goto.jl b/test/goto.jl index a352744862c32..b2395853a5f74 100644 --- a/test/goto.jl +++ b/test/goto.jl @@ -76,7 +76,7 @@ function goto_test6() @label a end -@test goto_test6() == nothing +@test goto_test6() === nothing function goto_test7(x) @@ -86,7 +86,7 @@ function goto_test7(x) end end -@test goto_test7(false) == nothing +@test goto_test7(false) === nothing module GotoMacroTest macro goto_test8_macro() diff --git a/test/linalg/tridiag.jl b/test/linalg/tridiag.jl index 53fb78a416fb4..a5e7b8599df0c 100644 --- a/test/linalg/tridiag.jl +++ b/test/linalg/tridiag.jl @@ -211,7 +211,7 @@ end function test_approx_eq_vecs{S<:Real,T<:Real}(a::StridedVecOrMat{S}, b::StridedVecOrMat{T}, error=nothing) n = size(a, 1) @test n==size(b,1) && size(a,2)==size(b,2) - error==nothing && (error=n^3*(eps(S)+eps(T))) + error===nothing && (error=n^3*(eps(S)+eps(T))) for i=1:n ev1, ev2 = a[:,i], b[:,i] deviation = min(abs(norm(ev1-ev2)),abs(norm(ev1+ev2))) diff --git a/test/pkg.jl b/test/pkg.jl index 281222f997464..946612d22604f 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -67,7 +67,7 @@ temp_pkg_dir() do @test !isempty(Pkg.available()) @test_throws PkgError Pkg.installed("MyFakePackage") - @test Pkg.installed("Example") == nothing + @test Pkg.installed("Example") === nothing # check that versioninfo(io, true) doesn't error and produces some output # (done here since it calls Pkg.status which might error or clone metadata) @@ -97,7 +97,7 @@ temp_pkg_dir() do end Pkg.setprotocol!("") - @test Pkg.Cache.rewrite_url_to == nothing + @test Pkg.Cache.rewrite_url_to === nothing Pkg.setprotocol!("https") Pkg.add("Example") @test [keys(Pkg.installed())...] == ["Example"] @@ -395,7 +395,7 @@ temp_pkg_dir() do # issue #15948 let package = "Example" Pkg.rm(package) # Remove package if installed - @test Pkg.installed(package) == nothing # Registered with METADATA but not installed + @test Pkg.installed(package) === nothing # Registered with METADATA but not installed msg = readstring(ignorestatus(`$(Base.julia_cmd()) -f -e "redirect_stderr(STDOUT); Pkg.build(\"$package\")"`)) @test contains(msg, "$package is not an installed package") @test !contains(msg, "signal (15)") diff --git a/test/serialize.jl b/test/serialize.jl index 67a7249c2b91c..d36622738d950 100644 --- a/test/serialize.jl +++ b/test/serialize.jl @@ -288,7 +288,7 @@ create_serialization_stream() do s # user-defined type array r = deserialize(s) @test r.storage[:v] == 2 @test r.state == :done - @test r.exception == nothing + @test r.exception === nothing end immutable MyErrorTypeTest <: Exception end diff --git a/test/show.jl b/test/show.jl index 7e2545c7683f1..5c5327b381147 100644 --- a/test/show.jl +++ b/test/show.jl @@ -287,7 +287,7 @@ let oldout = STDOUT, olderr = STDERR rderr, wrerr = redirect_stderr() @test wrerr === STDERR err = @async readstring(rderr) - @test dump(Int64) == nothing + @test dump(Int64) === nothing if !is_windows() close(wrout) close(wrerr) diff --git a/test/staged.jl b/test/staged.jl index f7e38b4c11fb1..f74d3b2244673 100644 --- a/test/staged.jl +++ b/test/staged.jl @@ -187,7 +187,7 @@ let gf_err2 @test_throws ErrorException gf_err2(code_llvm) @test_throws ErrorException gf_err2(code_native) @test gf_err_ref[] == 6 - @test gf_err2(code_lowered) == nothing + @test gf_err2(code_lowered) === nothing end # issue #15043 diff --git a/test/version.jl b/test/version.jl index 68bdc958129b6..a89f4efa82a01 100644 --- a/test/version.jl +++ b/test/version.jl @@ -204,27 +204,27 @@ end # check_new_version import Base.check_new_version -@test check_new_version([v"1", v"2"], v"3") == nothing +@test check_new_version([v"1", v"2"], v"3") === nothing @test_throws AssertionError check_new_version([v"2", v"1"], v"3") @test_throws ErrorException check_new_version([v"1", v"2"], v"2") -@test check_new_version(VersionNumber[], v"0") == nothing -@test check_new_version(VersionNumber[], v"0.0.1") == nothing +@test check_new_version(VersionNumber[], v"0") === nothing +@test check_new_version(VersionNumber[], v"0.0.1") === nothing @test_throws ErrorException check_new_version(VersionNumber[], v"0.0.2") -@test check_new_version(VersionNumber[], v"0.1") == nothing +@test check_new_version(VersionNumber[], v"0.1") === nothing @test_throws ErrorException check_new_version(VersionNumber[], v"0.2") -@test check_new_version(VersionNumber[], v"1") == nothing +@test check_new_version(VersionNumber[], v"1") === nothing @test_throws ErrorException check_new_version(VersionNumber[], v"2") @test_throws ErrorException check_new_version(VersionNumber[v"1", v"2", v"3"], v"2") @test_throws ErrorException check_new_version([v"1", v"2"], v"4") @test_throws ErrorException check_new_version([v"1", v"2"], v"2-rc") -@test check_new_version([v"1", v"2"], v"2.0.1") == nothing -@test check_new_version([v"1", v"2"], v"2.1") == nothing -@test check_new_version([v"1", v"2"], v"3") == nothing +@test check_new_version([v"1", v"2"], v"2.0.1") === nothing +@test check_new_version([v"1", v"2"], v"2.1") === nothing +@test check_new_version([v"1", v"2"], v"3") === nothing # banner import Base.banner io = IOBuffer() -@test banner(io) == nothing +@test banner(io) === nothing @test length(takebuf_string(io)) > 50 # julia_version.h version test From f1be468999b447fb8f5b20d088d21fce3cd2469f Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 22 Jul 2016 18:54:57 -0700 Subject: [PATCH 0645/1117] use getfield instead of eval in doc/DocCheck.jl --- doc/DocCheck.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/DocCheck.jl b/doc/DocCheck.jl index 525d047069f35..9abd4470e0e48 100644 --- a/doc/DocCheck.jl +++ b/doc/DocCheck.jl @@ -36,7 +36,7 @@ undefined_exports() = undefined(Base) function undocumented(m::Module) undoc = Dict{Symbol, Array}() for v in sort(names(m)) - if isdefined(m,v) && !isdocumented(eval(m,v)) && !isdeprecated(m,v) + if isdefined(m,v) && !isdocumented(getfield(m,v)) && !isdeprecated(m,v) ms = modfuncjoin(m,v) haskey(undoc, v) ? push!(undoc[v], ms) : (undoc[v] = [ms]) end From 457d19a9e445e72b554f3a7f7fe640daeaf5b743 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 11 Jul 2016 20:08:55 -0700 Subject: [PATCH 0646/1117] Remove use of 'abstract type' when discussing parametric composite types this is a bit confusing and we don't have a great word for it, but since Point wasn't declared with abstract here we probably shouldn't use that term --- doc/manual/types.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 9cca9fd17e58a..8c1fc728c812c 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -657,7 +657,7 @@ However, ``Point`` itself is also a valid type object: Point{T} Here the ``T`` is the dummy type symbol used in the original declaration -of ``Point``. What does ``Point`` by itself mean? It is an abstract type +of ``Point``. What does ``Point`` by itself mean? It is a type that contains all the specific instances ``Point{Float64}``, ``Point{AbstractString}``, etc.: From dc29497d697d255d933c4f89c93b4cf8c0927018 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 18 Jul 2016 12:49:59 -0700 Subject: [PATCH 0647/1117] Minor doc and comment nitpicks Grammar changes, 'is rewrite' -> 'is rewritten', 'ARM and PowerPC has' -> 'ARM and PowerPC have' Indentation changes in test/core.jl and doc/manual/types.rst Code highlighting in some docstrings Fix a Sphinx warning about an undefined label Match parentheticals and fix spelling in NEWS.md, run NEWS-update --- NEWS.md | 9 +++++-- base/array.jl | 6 ++--- base/inference.jl | 2 +- base/multidimensional.jl | 2 +- base/show.jl | 6 ++--- doc/manual/performance-tips.rst | 2 +- doc/manual/types.rst | 44 ++++++++++++++++----------------- doc/stdlib/collections.rst | 4 +-- doc/stdlib/io-network.rst | 6 ++--- src/julia-syntax.scm | 2 +- src/julia_internal.h | 2 +- test/core.jl | 20 +++++++-------- 12 files changed, 55 insertions(+), 50 deletions(-) diff --git a/NEWS.md b/NEWS.md index cbbc09a17facd..a9d2f8125c84c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -106,7 +106,7 @@ This section lists changes that do not have deprecation warnings. * The assignment operations `.+=`, `.*=` and so on now generate calls to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side - if the latter is an indexing expression, (e.g. `a[...]`). This means that they will fail + if the latter is an indexing expression, e.g. `a[...]`). This means that they will fail if the left-hand side is immutable (or does not support `view`), and will otherwise change the left-hand side in-place ([#17510], [#17546]). @@ -179,7 +179,7 @@ Library improvements compare characters with integers you should either convert the integer to a character value or convert the character to the corresponding code point first: e.g. `'x' == Char(120)` or `Int('x') == 120`. The former is usually - preferrable. + preferable. * Support for Unicode 9 ([#17402]). @@ -342,6 +342,7 @@ Deprecated or removed [#550]: https://github.com/JuliaLang/julia/issues/550 [#964]: https://github.com/JuliaLang/julia/issues/964 [#1090]: https://github.com/JuliaLang/julia/issues/1090 +[#1765]: https://github.com/JuliaLang/julia/issues/1765 [#4163]: https://github.com/JuliaLang/julia/issues/4163 [#4211]: https://github.com/JuliaLang/julia/issues/4211 [#4470]: https://github.com/JuliaLang/julia/issues/4470 @@ -381,6 +382,7 @@ Deprecated or removed [#14519]: https://github.com/JuliaLang/julia/issues/14519 [#14759]: https://github.com/JuliaLang/julia/issues/14759 [#14798]: https://github.com/JuliaLang/julia/issues/14798 +[#15007]: https://github.com/JuliaLang/julia/issues/15007 [#15032]: https://github.com/JuliaLang/julia/issues/15032 [#15172]: https://github.com/JuliaLang/julia/issues/15172 [#15192]: https://github.com/JuliaLang/julia/issues/15192 @@ -395,6 +397,7 @@ Deprecated or removed [#15731]: https://github.com/JuliaLang/julia/issues/15731 [#15763]: https://github.com/JuliaLang/julia/issues/15763 [#15975]: https://github.com/JuliaLang/julia/issues/15975 +[#16024]: https://github.com/JuliaLang/julia/issues/16024 [#16058]: https://github.com/JuliaLang/julia/issues/16058 [#16071]: https://github.com/JuliaLang/julia/issues/16071 [#16107]: https://github.com/JuliaLang/julia/issues/16107 @@ -405,7 +408,9 @@ Deprecated or removed [#16403]: https://github.com/JuliaLang/julia/issues/16403 [#16404]: https://github.com/JuliaLang/julia/issues/16404 [#16455]: https://github.com/JuliaLang/julia/issues/16455 +[#16466]: https://github.com/JuliaLang/julia/issues/16466 [#16481]: https://github.com/JuliaLang/julia/issues/16481 +[#16502]: https://github.com/JuliaLang/julia/issues/16502 [#16621]: https://github.com/JuliaLang/julia/issues/16621 [#16622]: https://github.com/JuliaLang/julia/issues/16622 [#16645]: https://github.com/JuliaLang/julia/issues/16645 diff --git a/base/array.jl b/base/array.jl index bf637fc688a69..23c8044854ee5 100644 --- a/base/array.jl +++ b/base/array.jl @@ -202,7 +202,7 @@ promote_rule{T,n,S}(::Type{Array{T,n}}, ::Type{Array{S,n}}) = Array{promote_type """ collect(element_type, collection) -Return an Array with the given element type of all items in a collection or iterable. +Return an `Array` with the given element type of all items in a collection or iterable. The result has the same shape and number of dimensions as `collection`. """ collect{T}(::Type{T}, itr) = _collect(T, itr, iteratorsize(itr)) @@ -226,8 +226,8 @@ _similar_for(c, T, itr, isz) = similar(c, T) """ collect(collection) -Return an Array of all items in a collection or iterator. For associative collections, returns -Pair{KeyType, ValType}. If the argument is array-like or is an iterator with the `HasShape()` +Return an `Array` of all items in a collection or iterator. For associative collections, returns +`Pair{KeyType, ValType}`. If the argument is array-like or is an iterator with the `HasShape()` trait, the result will have the same shape and number of dimensions as the argument. """ collect(itr) = _collect(1:1 #= Array =#, itr, iteratoreltype(itr), iteratorsize(itr)) diff --git a/base/inference.jl b/base/inference.jl index c5c497419b3fb..008813275b54b 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2967,7 +2967,7 @@ function inlining_pass(e::Expr, sv, linfo) # by the interpreter and inlining might put in something it can't handle, # like another ccall (or try to move the variables out into the function) if is_known_call(e, Core.Intrinsics.ccall, sv) - # 4 is rewrite to 2 below to handle the callee. + # 4 is rewritten to 2 below to handle the callee. i0 = 4 isccall = true elseif is_known_call(e, Core.Intrinsics.llvmcall, sv) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 8cf6e59fdb8f8..f4f4a676d2112 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -157,7 +157,7 @@ end # IteratorsMD using .IteratorsMD ## Bounds-checking with CartesianIndex -@inline checkbounds_indices(::Type{Bool}, ::Tuple{}, I::Tuple{CartesianIndex,Vararg{Any}}) = +@inline checkbounds_indices(::Type{Bool}, ::Tuple{}, I::Tuple{CartesianIndex,Vararg{Any}}) = checkbounds_indices(Bool, (), (I[1].I..., tail(I)...)) @inline checkbounds_indices(::Type{Bool}, IA::Tuple{Any}, I::Tuple{CartesianIndex,Vararg{Any}}) = checkbounds_indices(Bool, IA, (I[1].I..., tail(I)...)) diff --git a/base/show.jl b/base/show.jl index 99ae006e4063e..2915b8dc903ef 100644 --- a/base/show.jl +++ b/base/show.jl @@ -7,7 +7,7 @@ print(io::IO, s::Symbol) = (write(io,s); nothing) IOContext provides a mechanism for passing output configuration settings among `show` methods. -In short, it is an immutable dictionary that is a subclass of IO. It supports standard +In short, it is an immutable dictionary that is a subclass of `IO`. It supports standard dictionary operations such as `getindex`, and can also be used as an I/O stream. """ immutable IOContext{IO_t <: IO} <: AbstractPipe @@ -43,14 +43,14 @@ IOContext(io::IO, context::IO) = IOContext(io) """ IOContext(io::IO, context::IOContext) -Create a IOContext that wraps an alternate IO but inherits the properties of `context`. +Create an `IOContext` that wraps an alternate `IO` but inherits the properties of `context`. """ IOContext(io::IO, context::IOContext) = IOContext(io, context.dict) """ IOContext(io::IO, KV::Pair) -Create an `IOContext` that wraps a given stream, adding the specified key=>value pair to +Create an `IOContext` that wraps a given stream, adding the specified `key=>value` pair to the properties of that stream (note that `io` can itself be an `IOContext`). - use `(key => value) in dict` to see if this particular combination is in the properties set diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index f98db0d5b8130..e5aabd82f53c0 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -947,7 +947,7 @@ Taken to its extreme, pre-allocation can make your code uglier, so performance measurements and some judgment may be required. However, for "vectorized" (element-wise) functions, the convenient syntax ``x .= f.(y)`` can be used for in-place operations with fused loops -and no temporary arrays (:ref:`dot-vectorizing`). +and no temporary arrays (:ref:`man-dot-vectorizing`). Avoid string interpolation for I/O diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 8c1fc728c812c..22504cc76fc88 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -120,8 +120,8 @@ using :func:`convert`: .. doctest:: foo-func julia> function foo() - x::Int8 = 100 - x + x::Int8 = 100 + x end foo (generic function with 1 method) @@ -247,7 +247,7 @@ An important use of abstract types is to provide default implementations for concrete types. To give a simple example, consider:: function myplus(x,y) - x+y + x+y end The first thing to note is that the above argument declarations are equivalent @@ -262,7 +262,7 @@ arguments based on the generic function given above, i.e., it implicitly defines and compiles:: function myplus(x::Int,y::Int) - x+y + x+y end and finally, it invokes this specific method. @@ -374,9 +374,9 @@ operator: .. doctest:: julia> type Foo - bar - baz::Int - qux::Float64 + bar + baz::Int + qux::Float64 end Fields with no type annotation default to :obj:`Any`, and can accordingly @@ -473,8 +473,8 @@ It is also possible to define *immutable* composite types by using the keyword ``immutable`` instead of ``type``:: immutable Complex - real::Float64 - imag::Float64 + real::Float64 + imag::Float64 end Such types behave much like other composite types, except that instances @@ -616,16 +616,16 @@ Parametric Composite Types abstract Pointy{T} type Point{T} <: Pointy{T} - x::T - y::T + x::T + y::T end Type parameters are introduced immediately after the type name, surrounded by curly braces:: type Point{T} - x::T - y::T + x::T + y::T end This declaration defines a new parametric type, ``Point{T}``, holding @@ -849,8 +849,8 @@ example, have declared ``Point{T}`` to be a subtype of ``Pointy{T}`` as follows:: type Point{T} <: Pointy{T} - x::T - y::T + x::T + y::T end Given such a declaration, for each choice of ``T``, we have ``Point{T}`` @@ -879,7 +879,7 @@ Consider if we create a point-like implementation that only requires a single coordinate because the point is on the diagonal line *x = y*:: type DiagPoint{T} <: Pointy{T} - x::T + x::T end Now both ``Point{Float64}`` and ``DiagPoint{Float64}`` are @@ -924,8 +924,8 @@ Type parameters for parametric composite types can be restricted in the same manner:: type Point{T<:Real} <: Pointy{T} - x::T - y::T + x::T + y::T end To give a real-world example of how all this parametric type @@ -934,8 +934,8 @@ machinery can be useful, here is the actual definition of Julia's for simplicity), representing an exact ratio of integers:: immutable Rational{T<:Integer} <: Real - num::T - den::T + num::T + den::T end It only makes sense to take ratios of integer values, so the parameter @@ -954,8 +954,8 @@ of one field. For example, a 2-element tuple type resembles the following immutable type:: immutable Tuple2{A,B} - a::A - b::B + a::A + b::B end However, there are three key differences: diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 169edd46d6e40..a033f8716f556 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -676,13 +676,13 @@ Iterable Collections .. Docstring generated from Julia source - Return an Array of all items in a collection or iterator. For associative collections, returns Pair{KeyType, ValType}. If the argument is array-like or is an iterator with the ``HasShape()`` trait, the result will have the same shape and number of dimensions as the argument. + Return an ``Array`` of all items in a collection or iterator. For associative collections, returns ``Pair{KeyType, ValType}``\ . If the argument is array-like or is an iterator with the ``HasShape()`` trait, the result will have the same shape and number of dimensions as the argument. .. function:: collect(element_type, collection) .. Docstring generated from Julia source - Return an Array with the given element type of all items in a collection or iterable. The result has the same shape and number of dimensions as ``collection``\ . + Return an ``Array`` with the given element type of all items in a collection or iterable. The result has the same shape and number of dimensions as ``collection``\ . .. function:: issubset(a, b) ⊆(a,b) -> Bool diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index b4d56fae3f155..cb1ecfdb361c8 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -404,13 +404,13 @@ General I/O IOContext provides a mechanism for passing output configuration settings among ``show`` methods. - In short, it is an immutable dictionary that is a subclass of IO. It supports standard dictionary operations such as ``getindex``\ , and can also be used as an I/O stream. + In short, it is an immutable dictionary that is a subclass of ``IO``\ . It supports standard dictionary operations such as ``getindex``\ , and can also be used as an I/O stream. .. function:: IOContext(io::IO, KV::Pair) .. Docstring generated from Julia source - Create an ``IOContext`` that wraps a given stream, adding the specified key=>value pair to the properties of that stream (note that ``io`` can itself be an ``IOContext``\ ). + Create an ``IOContext`` that wraps a given stream, adding the specified ``key=>value`` pair to the properties of that stream (note that ``io`` can itself be an ``IOContext``\ ). * use ``(key => value) in dict`` to see if this particular combination is in the properties set * use ``get(dict, key, default)`` to retrieve the most recent value for a particular key @@ -425,7 +425,7 @@ General I/O .. Docstring generated from Julia source - Create a IOContext that wraps an alternate IO but inherits the properties of ``context``\ . + Create an ``IOContext`` that wraps an alternate ``IO`` but inherits the properties of ``context``\ . .. function:: IOContext(io::IO; properties...) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index c38b1af3e2734..ba0123fb8da3d 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2533,7 +2533,7 @@ f(x) = yt(x) (return (null))))))) -;; better versions of above, but they gets handled wrong in many places +;; better versions of above, but they get handled wrong in many places ;; need to fix that in order to handle #265 fully (and use the definitions) ;; template for generating a closure type with parameters diff --git a/src/julia_internal.h b/src/julia_internal.h index db4a2f97383e8..d967de1ff00cb 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -55,7 +55,7 @@ static const int jl_gc_sizeclasses[JL_GC_N_POOLS] = { #ifdef _P64 8, #elif defined(_CPU_ARM_) || defined(_CPU_PPC_) - // ARM and PowerPC has max alignment of 8, + // ARM and PowerPC have max alignment of 8, // make sure allocation of size 8 has that alignment. 4, 8, #else diff --git a/test/core.jl b/test/core.jl index d2a6d50d8fd77..2ed5c170e4181 100644 --- a/test/core.jl +++ b/test/core.jl @@ -3130,13 +3130,13 @@ end # issue 11874 immutable Foo11874 - x::Int + x::Int end function bar11874(x) - local y::Foo11874 - y = x - nothing + local y::Foo11874 + y = x + nothing end Base.convert(::Type{Foo11874},x::Int) = float(x) @@ -3376,9 +3376,9 @@ gg13183{X}(x::X...) = 1==0 ? gg13183(x, x) : 0 # issue 8932 (llvm return type legalizer error) immutable Vec3_8932 - x::Float32 - y::Float32 - z::Float32 + x::Float32 + y::Float32 + z::Float32 end f8932(a::Vec3_8932, b::Vec3_8932) = Vec3_8932(a.x % b.x, a.y % b.y, a.z % b.z) a8932 = Vec3_8932(1,1,1) @@ -3416,9 +3416,9 @@ immutable X13647 b::Bool end function f13647(x, y) - z = false - z = y - x === z + z = false + z = y + x === z end @test f13647(X13647(1, false), X13647(1, false)) @test !f13647(X13647(1, false), X13647(1, true)) From 7412d0302389e2579d26d98d69b004309ddd1789 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 03:07:56 -0700 Subject: [PATCH 0648/1117] Don't mention UNIX twice in the span of 4 words --- doc/manual/networking-and-streams.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/networking-and-streams.rst b/doc/manual/networking-and-streams.rst index 3ec703c26ee4f..cbaa39eba5985 100644 --- a/doc/manual/networking-and-streams.rst +++ b/doc/manual/networking-and-streams.rst @@ -202,8 +202,8 @@ create various other kinds of servers:: Note that the return type of the last invocation is different. This is because this server does not listen on TCP, but rather on a named pipe (Windows) -or UNIX domain socket (UNIX). The difference -is subtle and has to do with the :func:`accept` and :func:`connect` methods. The :func:`accept` +or UNIX domain socket. The difference is subtle and has to do with the +:func:`accept` and :func:`connect` methods. The :func:`accept` method retrieves a connection to the client that is connecting on the server we just created, while the :func:`connect` function connects to a server using the specified method. The :func:`connect` function takes the same arguments as From 55dbbcc11d388bac92ee3a0605885f11f6ccc408 Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Tue, 12 Jul 2016 11:44:56 +0200 Subject: [PATCH 0649/1117] Include "build number" in pre-release version Fixes #17330 --- base/version.jl | 12 ++++++------ base/version_git.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/version.jl b/base/version.jl index 219d1e08c6774..6bdb1822e35c5 100644 --- a/base/version.jl +++ b/base/version.jl @@ -208,13 +208,13 @@ A `VersionNumber` object describing which version of Julia is in use. For detail const VERSION = try ver = convert(VersionNumber, VERSION_STRING) if !isempty(ver.prerelease) - build_number = GIT_VERSION_INFO.build_number - if ver == v"0.5.0-pre" - # due to change of reference for counting commits from last tag to last change of VERSION file - build_number += 5578 + if GIT_VERSION_INFO.build_number >= 0 + ver = VersionNumber(ver.major, ver.minor, ver.patch, (ver.prerelease..., GIT_VERSION_INFO.build_number), ver.build) + else + println("WARNING: no build number found for pre-release version") + ver = VersionNumber(ver.major, ver.minor, ver.patch, (ver.prerelease..., "unknown"), ver.build) end - ver = VersionNumber(ver.major, ver.minor, ver.patch, ver.prerelease, (build_number,)) - elseif GIT_VERSION_INFO.build_number != 0 + elseif GIT_VERSION_INFO.build_number > 0 println("WARNING: ignoring non-zero build number for VERSION") end ver diff --git a/base/version_git.sh b/base/version_git.sh index ebcd0965a7587..110ef86ee0d9b 100644 --- a/base/version_git.sh +++ b/base/version_git.sh @@ -73,7 +73,7 @@ fork_master_timestamp=$(git show -s $(git merge-base HEAD $(echo $origin)master) # Check for errrors and emit default value for missing numbers. if [ -z "$build_number" ]; then - build_number="0" + build_number="-1" fi if [ -z "$fork_master_distance" ]; then fork_master_distance="-1" From 0c89e9862aaa78ae0b655f143d31fda391fe96bf Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Mon, 18 Jul 2016 08:44:09 +0200 Subject: [PATCH 0650/1117] Adjust contrib/commit-name.sh to new version number format Assumes the change to take place when switching VERSION to 0.6.0-dev. --- contrib/commit-name.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/commit-name.sh b/contrib/commit-name.sh index 09c8dcded1e75..f808694884b68 100755 --- a/contrib/commit-name.sh +++ b/contrib/commit-name.sh @@ -32,7 +32,11 @@ else nb=$(expr $nb + 5578) fi if [ -n "$pre" ]; then - echo "$ver+$nb" + if [ $major = 0 -a $minor -le 5 ]; then + echo "$ver+$nb" + else + echo "$ver.$nb" + fi else echo $ver fi From 9e948c1f27a0e9ed7ad8e546f031606a420fbae4 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 04:51:57 -0700 Subject: [PATCH 0651/1117] Use mktempdir() do ... block to clean up even if a test fails --- test/repl.jl | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/repl.jl b/test/repl.jl index fd4d657ecc6af..f75411b165b58 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -67,30 +67,30 @@ if !is_windows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER # calling readuntil() to suppress the warning it (currently) gives for # long strings. origpwd = pwd() - tmpdir = mktempdir() - write(stdin_write, ";") - readuntil(stdout_read, "shell> ") - write(stdin_write, "cd $(escape_string(tmpdir))\n") - readuntil(stdout_read, "cd $(escape_string(tmpdir))"[max(1,end-39):end]) - readuntil(stdout_read, realpath(tmpdir)[max(1,end-39):end]) - readuntil(stdout_read, "\n") - readuntil(stdout_read, "\n") - @test pwd() == realpath(tmpdir) - write(stdin_write, ";") - readuntil(stdout_read, "shell> ") - write(stdin_write, "cd -\n") - readuntil(stdout_read, origpwd[max(1,end-39):end]) - readuntil(stdout_read, "\n") - readuntil(stdout_read, "\n") - @test pwd() == origpwd - write(stdin_write, ";") - readuntil(stdout_read, "shell> ") - write(stdin_write, "cd\n") - readuntil(stdout_read, homedir()[max(1,end-39):end]) - readuntil(stdout_read, "\n") - readuntil(stdout_read, "\n") - @test pwd() == homedir() - rm(tmpdir) + mktempdir() do tmpdir + write(stdin_write, ";") + readuntil(stdout_read, "shell> ") + write(stdin_write, "cd $(escape_string(tmpdir))\n") + readuntil(stdout_read, "cd $(escape_string(tmpdir))"[max(1,end-39):end]) + readuntil(stdout_read, realpath(tmpdir)[max(1,end-39):end]) + readuntil(stdout_read, "\n") + readuntil(stdout_read, "\n") + @test pwd() == realpath(tmpdir) + write(stdin_write, ";") + readuntil(stdout_read, "shell> ") + write(stdin_write, "cd -\n") + readuntil(stdout_read, origpwd[max(1,end-39):end]) + readuntil(stdout_read, "\n") + readuntil(stdout_read, "\n") + @test pwd() == origpwd + write(stdin_write, ";") + readuntil(stdout_read, "shell> ") + write(stdin_write, "cd\n") + readuntil(stdout_read, homedir()[max(1,end-39):end]) + readuntil(stdout_read, "\n") + readuntil(stdout_read, "\n") + @test pwd() == homedir() + end cd(origpwd) # Test that accepting a REPL result immediately shows up, not From cf4b668d53b00fb1d4dbf47ccff0c5695363b683 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 04:55:28 -0700 Subject: [PATCH 0652/1117] Escape more of the regex --- test/cmdlineargs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 23ed71e1a2cb8..07dcd3c5e06a5 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -300,7 +300,7 @@ for precomp in ("yes", "no") else @test contains(bt, "in include_from_node1(::String) at $(joinpath(".","loading.jl"))") end - lno = match(r"at \.[/\\]loading.jl:(\d+)", bt) + lno = match(r"at \.[\/\\]loading\.jl:(\d+)", bt) @test length(lno.captures) == 1 @test parse(Int, lno.captures[1]) > 0 end From fc5f77126b0c28b36f9c6e20190e9248f31ea62c Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Tue, 26 Jul 2016 16:32:56 +0200 Subject: [PATCH 0653/1117] Fix markdown footnote parsing Footnotes were not able to parse anything other than single paragraphs. Multiple nested blocks were ignored, which doesn't match the behaviour of any markdown parsers that do support footnote syntax -- most notably Pandoc. Also add missing rendering methods for `Footnote` for all output formats and some additional tests. --- base/markdown/Common/Common.jl | 4 +- base/markdown/Common/block.jl | 35 ++++++++++ base/markdown/Common/inline.jl | 19 +++--- base/markdown/GitHub/GitHub.jl | 4 +- base/markdown/Julia/Julia.jl | 4 +- base/markdown/render/html.jl | 16 +++++ base/markdown/render/latex.jl | 9 +++ base/markdown/render/plain.jl | 24 +++++-- base/markdown/render/rst.jl | 26 +++++--- base/markdown/render/terminal/render.jl | 17 +++-- test/markdown.jl | 88 ++++++++++++++++++++++++- 11 files changed, 211 insertions(+), 35 deletions(-) diff --git a/base/markdown/Common/Common.jl b/base/markdown/Common/Common.jl index edb630bcf79b8..0152fd04b6cf3 100644 --- a/base/markdown/Common/Common.jl +++ b/base/markdown/Common/Common.jl @@ -3,8 +3,8 @@ include("block.jl") include("inline.jl") -@flavor common [list, indentcode, blockquote, admonition, hashheader, horizontalrule, +@flavor common [list, indentcode, blockquote, admonition, footnote, hashheader, horizontalrule, paragraph, linebreak, escapes, inline_code, - asterisk_bold, asterisk_italic, image, footnote, link] + asterisk_bold, asterisk_italic, image, footnote_link, link] diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index 298f75e1d934e..598096f1e4181 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -121,6 +121,41 @@ function indentcode(stream::IO, block::MD) end end +# -------- +# Footnote +# -------- + +type Footnote + id::String + text +end + +function footnote(stream::IO, block::MD) + withstream(stream) do + regex = r"^\[\^(\w+)\]:" + str = startswith(stream, regex) + if isempty(str) + return false + else + ref = match(regex, str).captures[1] + buffer = IOBuffer() + write(buffer, readline(stream)) + while !eof(stream) + if startswith(stream, " ") + write(buffer, readline(stream)) + elseif blankline(stream) + write(buffer, '\n') + else + break + end + end + content = parse(seekstart(buffer)).content + push!(block, Footnote(ref, content)) + return true + end + end +end + # –––––– # Quotes # –––––– diff --git a/base/markdown/Common/inline.jl b/base/markdown/Common/inline.jl index 960c9ad9cc61b..8e48dc9256796 100644 --- a/base/markdown/Common/inline.jl +++ b/base/markdown/Common/inline.jl @@ -92,18 +92,17 @@ function link(stream::IO, md::MD) end end -type Footnote - id::String - text -end - @trigger '[' -> -function footnote(stream::IO, md::MD) +function footnote_link(stream::IO, md::MD) withstream(stream) do - startswith(stream, "[^") || return - id = readuntil(stream, ']', match = '[') - id ≡ nothing && return - Footnote(id, startswith(stream, ':') ? parseinline(stream, md) : nothing) + regex = r"^\[\^(\w+)\]" + str = startswith(stream, regex) + if isempty(str) + return + else + ref = match(regex, str).captures[1] + return Footnote(ref, nothing) + end end end diff --git a/base/markdown/GitHub/GitHub.jl b/base/markdown/GitHub/GitHub.jl index ffd6e0b5b103a..319d5938aa49c 100644 --- a/base/markdown/GitHub/GitHub.jl +++ b/base/markdown/GitHub/GitHub.jl @@ -58,8 +58,8 @@ function github_paragraph(stream::IO, md::MD) return true end -@flavor github [list, indentcode, blockquote, admonition, fencedcode, hashheader, +@flavor github [list, indentcode, blockquote, admonition, footnote, fencedcode, hashheader, github_table, github_paragraph, linebreak, escapes, en_dash, inline_code, asterisk_bold, - asterisk_italic, image, footnote, link] + asterisk_italic, image, footnote_link, link] diff --git a/base/markdown/Julia/Julia.jl b/base/markdown/Julia/Julia.jl index f588d47456801..2297b8339899c 100644 --- a/base/markdown/Julia/Julia.jl +++ b/base/markdown/Julia/Julia.jl @@ -8,7 +8,7 @@ include("interp.jl") @flavor julia [blocktex, blockinterp, hashheader, list, indentcode, fencedcode, - blockquote, admonition, github_table, horizontalrule, setextheader, paragraph, + blockquote, admonition, footnote, github_table, horizontalrule, setextheader, paragraph, linebreak, escapes, tex, interp, en_dash, inline_code, - asterisk_bold, asterisk_italic, image, footnote, link] + asterisk_bold, asterisk_italic, image, footnote_link, link] diff --git a/base/markdown/render/html.jl b/base/markdown/render/html.jl index 96a3c8b08f972..a0404bf8f845e 100644 --- a/base/markdown/render/html.jl +++ b/base/markdown/render/html.jl @@ -88,6 +88,15 @@ function html(io::IO, md::BlockQuote) end end +function html(io::IO, f::Footnote) + withtag(io, :div, :class => "footnote", :id => "footnote-$(f.id)") do + withtag(io, :p, :class => "footnote-title") do + print(io, f.id) + end + html(io, f.text) + end +end + function html(io::IO, md::Admonition) withtag(io, :div, :class => "admonition $(md.category)") do withtag(io, :p, :class => "admonition-title") do @@ -150,6 +159,13 @@ function htmlinline(io::IO, md::Image) tag(io, :img, :src=>md.url, :alt=>md.alt) end + +function htmlinline(io::IO, f::Footnote) + withtag(io, :a, :href => "#footnote-$(f.id)", :class => "footnote") do + print(io, "[", f.id, "]") + end +end + function htmlinline(io::IO, link::Link) withtag(io, :a, :href=>link.url) do htmlinline(io, link.text) diff --git a/base/markdown/render/latex.jl b/base/markdown/render/latex.jl index ec17ce1daae7e..b86f072a7f724 100644 --- a/base/markdown/render/latex.jl +++ b/base/markdown/render/latex.jl @@ -58,6 +58,13 @@ function latex(io::IO, md::BlockQuote) end end + +function latex(io::IO, f::Footnote) + print(io, "\\footnotetext[", f.id, "]{") + latex(io, f.text) + println(io, "}") +end + function latex(io::IO, md::Admonition) wrapblock(io, "quote") do wrapinline(io, "textbf") do @@ -134,6 +141,8 @@ function latexinline(io::IO, md::Image) end end +latexinline(io::IO, f::Footnote) = print(io, "\\footnotemark[", f.id, "]") + function latexinline(io::IO, md::Link) wrapinline(io, "href") do print(io, md.url) diff --git a/base/markdown/render/plain.jl b/base/markdown/render/plain.jl index 096c5d66be8ad..e43ada6dcb991 100644 --- a/base/markdown/render/plain.jl +++ b/base/markdown/render/plain.jl @@ -53,6 +53,23 @@ function plain(io::IO, q::BlockQuote) println(io) end +function plain(io::IO, f::Footnote) + print(io, "[^", f.id, "]:") + s = sprint(io -> plain(io, f.text)) + lines = split(rstrip(s), "\n") + # Single line footnotes are printed on the same line as their label + # rather than taking up an additional line. + if length(lines) == 1 + println(io, " ", lines[1]) + else + println(io) + for line in lines + println(io, isempty(line) ? "" : " ", line) + end + println(io) + end +end + function plain(io::IO, md::Admonition) s = sprint(buf -> plain(buf, md.content)) title = md.title == ucfirst(md.category) ? "" : " \"$(md.title)\"" @@ -90,12 +107,9 @@ end plaininline(io::IO, md::Vector) = !isempty(md) && plaininline(io, md...) -plaininline(io::IO, link::Link) = plaininline(io, "[", link.text, "](", link.url, ")") +plaininline(io::IO, f::Footnote) = print(io, "[^", f.id, "]") -function plaininline(io::IO, md::Footnote) - print(io, "[^", md.id, "]") - md.text ≡ nothing || (print(io, ":"); plaininline(io, md.text)) -end +plaininline(io::IO, link::Link) = plaininline(io, "[", link.text, "](", link.url, ")") plaininline(io::IO, md::Image) = plaininline(io, "![", md.alt, "](", md.url, ")") diff --git a/base/markdown/render/rst.jl b/base/markdown/render/rst.jl index f50f9394b4635..6f88bd1b44ddb 100644 --- a/base/markdown/render/rst.jl +++ b/base/markdown/render/rst.jl @@ -57,6 +57,23 @@ function rst(io::IO, q::BlockQuote) println(io) end +function rst(io::IO, f::Footnote) + print(io, ".. [", f.id, "]") + s = sprint(io -> rst(io, f.text)) + lines = split(rstrip(s), "\n") + # Single line footnotes are printed on the same line as their label + # rather than taking up an additional line. + if length(lines) == 1 + println(io, " ", lines[1]) + else + println(io) + for line in lines + println(io, isempty(line) ? "" : " ", rstrip(line)) + end + println(io) + end +end + function rst(io::IO, md::Admonition) s = sprint(buf -> rst(buf, md.content)) title = md.title == ucfirst(md.category) ? "" : md.title @@ -105,14 +122,7 @@ function rstinline(io::IO, md::Link) end end -function rstinline(io::IO, md::Footnote) - if md.text ≡ nothing - print(io, "[", md.id, "]_") - else - print(io, ".. [", md.id, "]") - rstinline(io, md.text) - end -end +rstinline(io::IO, f::Footnote) = print(io, "[", f.id, "]_") rstescape(s) = replace(s, "\\", "\\\\") diff --git a/base/markdown/render/terminal/render.jl b/base/markdown/render/terminal/render.jl index 5d5ab014a751a..db7b80cfc3b40 100644 --- a/base/markdown/render/terminal/render.jl +++ b/base/markdown/render/terminal/render.jl @@ -40,6 +40,16 @@ function term(io::IO, md::Admonition, columns) end end +function term(io::IO, f::Footnote, columns) + print(io, " "^margin, "| ") + with_output_format(:bold, print, io, "[^$(f.id)]") + println(io, "\n", " "^margin, "|") + s = sprint(io -> term(io, f.text, columns - 10)) + for line in split(rstrip(s), "\n") + println(io, " "^margin, "|", line) + end +end + function term(io::IO, md::List, columns) for (i, point) in enumerate(md.items) print(io, " "^2margin, isordered(md) ? "$(i + md.ordered - 1). " : "• ") @@ -121,15 +131,12 @@ function terminline(io::IO, md::Image) terminline(io, "(Image: $(md.alt))") end +terminline(io::IO, f::Footnote) = with_output_format(:bold, terminline, io, "[^$(f.id)]") + function terminline(io::IO, md::Link) terminline(io, md.text) end -function terminline(io::IO, md::Footnote) - print(io, "[^", md.id, "]") - md.text ≡ nothing || (print(io, ":"); terminline(io, md.text)) -end - function terminline(io::IO, code::Code) print_with_format(:cyan, io, code.code) end diff --git a/test/markdown.jl b/test/markdown.jl index d854a857e40c4..ba517a8360c55 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -62,7 +62,93 @@ end @test md"A footnote [^foo]." == MD(Paragraph(["A footnote ", Footnote("foo", nothing), "."])) -@test md"[^foo]: footnote" == MD(Paragraph([Footnote("foo", Any[" footnote"])])) +@test md"[^foo]: footnote" == MD([Footnote("foo", Any[Paragraph(Any["footnote"])])]) + +let text = + """ + A paragraph with some footnotes,[^1] and another.[^note] + + [^1]: Footnote text for the first. + + [^note]: A longer footnote: + + Indented paragraphs are part of the footnote. + + some.code + + And *another* paragraph. + + This isn't part of the footnote. + """, + md = Markdown.parse(text) + @test length(md.content) == 4 + @test isa(md.content[1], Markdown.Paragraph) + @test isa(md.content[2], Markdown.Footnote) + @test isa(md.content[3], Markdown.Footnote) + @test isa(md.content[4], Markdown.Paragraph) + + @test md.content[2].id == "1" + @test md.content[3].id == "note" + + @test length(md.content[3].text) == 4 + + let expected = + """ + A paragraph with some footnotes,[^1] and another.[^note] + + [^1]: Footnote text for the first. + + [^note]: + A longer footnote: + + Indented paragraphs are part of the footnote. + + ``` + some.code + ``` + + And *another* paragraph. + + + This isn't part of the footnote. + """ + @test Markdown.plain(md) == expected + end + let expected = + """ + A paragraph with some footnotes,[1]_ and another.[note]_ + + .. [1] Footnote text for the first. + + .. [note] + A longer footnote: + + Indented paragraphs are part of the footnote. + + .. code-block:: julia + + some.code + + And *another* paragraph. + + + This isn't part of the footnote. + """ + @test Markdown.rst(md) == expected + end + let html = Markdown.html(md) + @test contains(html, ",<a href=\"#footnote-1\" class=\"footnote\">[1]</a>") + @test contains(html, ".<a href=\"#footnote-note\" class=\"footnote\">[note]</a>") + @test contains(html, "<div class=\"footnote\" id=\"footnote-1\"><p class=\"footnote-title\">1</p>") + @test contains(html, "<div class=\"footnote\" id=\"footnote-note\"><p class=\"footnote-title\">note</p>") + end + let latex = Markdown.latex(md) + @test contains(latex, ",\\footnotemark[1]") + @test contains(latex, ".\\footnotemark[note]") + @test contains(latex, "\n\\footnotetext[1]{Footnote text for") + @test contains(latex, "\n\\footnotetext[note]{A longer footnote:\n") + end +end let doc = md""" * one From a3cb0f3cedeeea682836e11992a7979aecce2748 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 25 Jul 2016 16:56:10 -0400 Subject: [PATCH 0654/1117] fix #17572, error during inference due to `<:` on non-Type --- base/inference.jl | 5 +++++ test/inference.jl | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/base/inference.jl b/base/inference.jl index 008813275b54b..943f3ee2bbcab 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1231,6 +1231,8 @@ function ⊑(a::ANY, b::ANY) return isa(a.val, b) elseif isa(b,Const) return a === Bottom + elseif !(isa(a,Type) || isa(a,TypeVar)) || !(isa(b,Type) || isa(b,TypeVar)) + return a === b else return a <: b end @@ -1246,6 +1248,9 @@ function tmerge(typea::ANY, typeb::ANY) typeb ⊑ typea && return typea typea, typeb = widenconst(typea), widenconst(typeb) typea === typeb && return typea + if !(isa(typea,Type) || isa(typea,TypeVar)) || !(isa(typeb,Type) || isa(typeb,TypeVar)) + return Any + end if (typea <: Tuple) && (typeb <: Tuple) if isa(typea, DataType) && isa(typeb, DataType) && length(typea.parameters) == length(typeb.parameters) && !isvatuple(typea) && !isvatuple(typeb) return typejoin(typea, typeb) diff --git a/test/inference.jl b/test/inference.jl index 05216b4786c83..5776354b91e6e 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -262,3 +262,10 @@ typealias NInt{N} Tuple{Vararg{Int, N}} fNInt(x::NInt) = (x...) gNInt() = fNInt(x) @test Base.return_types(gNInt, ()) == Any[NInt] + +# issue #17572 +function f17572{A}(::Type{Val{A}}) + return Tuple{Int}(Tuple{A}((1,))) +end +# test that inference doesn't error +@test isa(code_typed(f17572, (Type{Val{0}},)), Array) From 54c68b8015fdc4ff83c8dd5bbb36677e9868006b Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 12:24:54 -0400 Subject: [PATCH 0655/1117] Bump openspecfun to 0.5.3 should hopefully fix #17602 rpath issues --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/openspecfun.version | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 deps/checksums/openspecfun-39699a1c1824bf88410cabb8a7438af91ea98f4c.tar.gz/md5 create mode 100644 deps/checksums/openspecfun-39699a1c1824bf88410cabb8a7438af91ea98f4c.tar.gz/sha512 delete mode 100644 deps/checksums/openspecfun-e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad.tar.gz/md5 delete mode 100644 deps/checksums/openspecfun-e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad.tar.gz/sha512 diff --git a/deps/checksums/openspecfun-39699a1c1824bf88410cabb8a7438af91ea98f4c.tar.gz/md5 b/deps/checksums/openspecfun-39699a1c1824bf88410cabb8a7438af91ea98f4c.tar.gz/md5 new file mode 100644 index 0000000000000..c667bb83eeef0 --- /dev/null +++ b/deps/checksums/openspecfun-39699a1c1824bf88410cabb8a7438af91ea98f4c.tar.gz/md5 @@ -0,0 +1 @@ +00130a0879f691240df5f3ea25e6f0ba diff --git a/deps/checksums/openspecfun-39699a1c1824bf88410cabb8a7438af91ea98f4c.tar.gz/sha512 b/deps/checksums/openspecfun-39699a1c1824bf88410cabb8a7438af91ea98f4c.tar.gz/sha512 new file mode 100644 index 0000000000000..6848380958cdb --- /dev/null +++ b/deps/checksums/openspecfun-39699a1c1824bf88410cabb8a7438af91ea98f4c.tar.gz/sha512 @@ -0,0 +1 @@ +8681fc2f31696de2a6850cb4318c14ad39d3514742b066cf16058da87966d08e0d197fbb5104229fa19b61819a12ae5c76ea0279749f4338923cd4741ba31132 diff --git a/deps/checksums/openspecfun-e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad.tar.gz/md5 b/deps/checksums/openspecfun-e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad.tar.gz/md5 deleted file mode 100644 index 3ce5174da8623..0000000000000 --- a/deps/checksums/openspecfun-e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -d31760863615a0b7cb85c6a8a92312e9 diff --git a/deps/checksums/openspecfun-e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad.tar.gz/sha512 b/deps/checksums/openspecfun-e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad.tar.gz/sha512 deleted file mode 100644 index 2232eadc68cea..0000000000000 --- a/deps/checksums/openspecfun-e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -530ee030df1caee894965697927d8e8f96c5801c7b1720aae9322984fb3182f54043d2f77ce9a723436dd59e59c482abc3ce5f6447ffd0a138d214d06b076dbe diff --git a/deps/openspecfun.version b/deps/openspecfun.version index 270c822587f8b..f0df8e91d01d8 100644 --- a/deps/openspecfun.version +++ b/deps/openspecfun.version @@ -1,2 +1,2 @@ -OPENSPECFUN_BRANCH=v0.5.2 -OPENSPECFUN_SHA1=e4df5ae16af1bb4d28729a2fb5a8c2f3a49cf7ad +OPENSPECFUN_BRANCH=v0.5.3 +OPENSPECFUN_SHA1=39699a1c1824bf88410cabb8a7438af91ea98f4c From b5a362f2dac60e6d5d64390eb510974937158154 Mon Sep 17 00:00:00 2001 From: Jake Bolewski <jakebolewski@gmail.com> Date: Tue, 26 Jul 2016 12:30:42 -0400 Subject: [PATCH 0656/1117] Add missing colon getindex method for AbstractString closes #17624 --- base/strings/basic.jl | 1 + test/strings/basic.jl | 3 +++ 2 files changed, 4 insertions(+) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 399d383f8be1e..8e2eedca19399 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -69,6 +69,7 @@ start(s::AbstractString) = 1 done(s::AbstractString,i) = (i > endof(s)) getindex(s::AbstractString, i::Int) = next(s,i)[1] getindex(s::AbstractString, i::Integer) = s[Int(i)] +getindex(s::AbstractString, i::Colon) = s getindex{T<:Integer}(s::AbstractString, r::UnitRange{T}) = s[Int(first(r)):Int(last(r))] # TODO: handle other ranges with stride ±1 specially? getindex(s::AbstractString, v::AbstractVector) = diff --git a/test/strings/basic.jl b/test/strings/basic.jl index a286be8b4a91f..1008ce9566a17 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -419,3 +419,6 @@ foobaz(ch) = reinterpret(Char, typemax(UInt32)) # issue #17271: endof() doesn't throw an error even with invalid strings @test endof(String(b"\x90")) == 0 @test endof(String(b"\xce")) == 1 + +# issue #17624, missing getindex method for String +@test "abc"[:] == "abc" From dafaadbf6ef15e7837f19db855ba5d71989d6fe2 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Tue, 26 Jul 2016 09:45:46 -0700 Subject: [PATCH 0657/1117] Add rpath makefile patch from https://github.com/JuliaLang/julia/issues/17602#issuecomment-235066570 but with RPATH_ESCAPED_ORIGIN not cut off --- Make.inc | 2 ++ deps/openspecfun.mk | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index 2264fea33433c..91a6e310338e1 100644 --- a/Make.inc +++ b/Make.inc @@ -845,10 +845,12 @@ ifeq ($(OS), WINNT) else ifeq ($(OS), Darwin) RPATH := -Wl,-rpath,'@executable_path/$(build_libdir_rel)' RPATH_ORIGIN := -Wl,-rpath,'@loader_path/' + RPATH_ESCAPED_ORIGIN := $(RPATH_ORIGIN) RPATH_LIB := -Wl,-rpath,'@loader_path/' -Wl,-rpath,'@loader_path/julia/' else RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin + RPATH_ESCAPED_ORIGIN := -Wl,-rpath,'\$$\$$ORIGIN' -Wl,-z,origin RPATH_LIB := -Wl,-rpath,'$$ORIGIN' -Wl,-rpath,'$$ORIGIN/julia' -Wl,-z,origin endif diff --git a/deps/openspecfun.mk b/deps/openspecfun.mk index 0df44fd0b4357..781e47e208f76 100644 --- a/deps/openspecfun.mk +++ b/deps/openspecfun.mk @@ -11,7 +11,7 @@ endif OPENSPECFUN_OBJ_TARGET := $(build_shlibdir)/libopenspecfun.$(SHLIB_EXT) OPENSPECFUN_OBJ_SOURCE := $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/libopenspecfun.$(SHLIB_EXT) -OPENSPECFUN_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" +OPENSPECFUN_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" LDFLAGS="$(RPATH_ESCAPED_ORIGIN)" ifeq ($(USE_SYSTEM_LIBM),0) OPENSPECFUN_FLAGS += USE_OPENLIBM=1 From 2201882d1b36c9ef1f48f14c186f76e02e7a7ebc Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 09:47:08 -0700 Subject: [PATCH 0658/1117] Define empty new RPATH variables on windows and line wrap in openspecfun.mk --- Make.inc | 2 ++ deps/openspecfun.mk | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Make.inc b/Make.inc index 91a6e310338e1..8bd586dd92cb8 100644 --- a/Make.inc +++ b/Make.inc @@ -842,6 +842,8 @@ endif ifeq ($(OS), WINNT) RPATH := RPATH_ORIGIN := + RPATH_ESCAPED_ORIGIN := + RPATH_LIB := else ifeq ($(OS), Darwin) RPATH := -Wl,-rpath,'@executable_path/$(build_libdir_rel)' RPATH_ORIGIN := -Wl,-rpath,'@loader_path/' diff --git a/deps/openspecfun.mk b/deps/openspecfun.mk index 781e47e208f76..6e3ccce12d02a 100644 --- a/deps/openspecfun.mk +++ b/deps/openspecfun.mk @@ -11,10 +11,12 @@ endif OPENSPECFUN_OBJ_TARGET := $(build_shlibdir)/libopenspecfun.$(SHLIB_EXT) OPENSPECFUN_OBJ_SOURCE := $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/libopenspecfun.$(SHLIB_EXT) -OPENSPECFUN_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" LDFLAGS="$(RPATH_ESCAPED_ORIGIN)" +OPENSPECFUN_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" \ + USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" \ + CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" LDFLAGS="$(RPATH_ESCAPED_ORIGIN)" ifeq ($(USE_SYSTEM_LIBM),0) - OPENSPECFUN_FLAGS += USE_OPENLIBM=1 + OPENSPECFUN_FLAGS += USE_OPENLIBM=1 $(OPENSPECFUN_OBJ_SOURCE): $(OPENLIBM_OBJ_TARGET) endif From 7309ee0aeafda10a06c108f642772b2ae36d9793 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 26 Jul 2016 12:47:44 -0400 Subject: [PATCH 0659/1117] fix a few doctest failures [ci skip] --- doc/devdocs/types.rst | 8 ++++---- doc/manual/metaprogramming.rst | 9 +++------ doc/manual/stacktraces.rst | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index 7f9c089eafa25..b97d9ffb6e7b5 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -245,11 +245,11 @@ bound :obj:`TypeVar` objects with a hash (``#T`` instead of ``T``): .. doctest:: - julia> jl_(first(methods(candid))) - Method(sig=Tuple{Main.#candid, Array{#T<:Any, N<:Any}, #T<:Any}, va=false, isstaged=false, tvars=#T<:Any, func=Main.candid(?), invokes=nothing, next=nothing) + julia> jl_(first(methods(candid)).sig) + Tuple{Main.#candid, Array{#T<:Any, N<:Any}, #T<:Any} - julia> jl_(first(methods(sneaky))) - Method(sig=Tuple{Main.#sneaky, Array{#T<:Any, N<:Any}, T<:Any}, va=false, isstaged=false, tvars=#T<:Any, func=Main.sneaky(?), invokes=nothing, next=nothing) + julia> jl_(first(methods(sneaky)).sig) + Tuple{Main.#sneaky, Array{#T<:Any, N<:Any}, T<:Any} Even though both print as ``T``, in ``sneaky`` the second ``T`` is not bound, and hence it isn't constrained to be the same type as the diff --git a/doc/manual/metaprogramming.rst b/doc/manual/metaprogramming.rst index 8ab35ac404e63..7c541fb1853b7 100644 --- a/doc/manual/metaprogramming.rst +++ b/doc/manual/metaprogramming.rst @@ -629,14 +629,11 @@ Compare: Expr head: Symbol string args: Array{Any}((5,)) - 1: String - data: Array{UInt8}((3,)) UInt8[0x61,0x20,0x28] + 1: String "a (" 2: Symbol a - 3: String - data: Array{UInt8}((18,)) UInt8[0x29,0x20,0x73,0x68,0x6f,0x75,0x6c,0x64,0x20,0x65,0x71,0x75,0x61,0x6c,0x20,0x62,0x20,0x28] + 3: String ") should equal b (" 4: Symbol b - 5: String - data: Array{UInt8}((2,)) UInt8[0x29,0x21] + 5: String ")!" typ: Any So now instead of getting a plain string in ``msg_body``, the macro is diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index b86f0362ff1c4..e6c55ba6f4c61 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -89,7 +89,7 @@ Each :obj:`StackFrame` contains the function name, file name, line number, lambd 234 julia> top_frame.linfo - Nullable{LambdaInfo}(LambdaInfo for eval(::Module, ::Any) + Nullable{LambdaInfo}(LambdaInfo for eval(::Module, ::Any)) ... julia> top_frame.inlined From 8426c4e94eb82b7bb3a904a671eb5550b1f81dc5 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 26 Jul 2016 13:04:19 -0400 Subject: [PATCH 0660/1117] return LambdaInfo instead of array from macro versions of code_{lowered,typed} this makes it easy to see the code interactively again --- base/interactiveutil.jl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 61f6a111090f8..c4301ce159264 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -338,8 +338,8 @@ function gen_call_with_extracted_types(fcn, ex0) exret end -for fname in [:which, :less, :edit, :functionloc, :code_typed, :code_warntype, - :code_lowered, :code_llvm, :code_llvm_raw, :code_native] +for fname in [:which, :less, :edit, :functionloc, :code_warntype, + :code_llvm, :code_llvm_raw, :code_native] @eval begin macro ($fname)(ex0) gen_call_with_extracted_types($(Expr(:quote,fname)), ex0) @@ -347,6 +347,18 @@ for fname in [:which, :less, :edit, :functionloc, :code_typed, :code_warntype, end end +for fname in [:code_typed, :code_lowered] + @eval begin + macro ($fname)(ex0) + thecall = gen_call_with_extracted_types($(Expr(:quote,fname)), ex0) + quote + results = $thecall + length(results) == 1 ? results[1] : results + end + end + end +end + """ @which From b9dc8924597e94bf791776a47837eba51833b8b2 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 26 Jul 2016 13:34:22 -0400 Subject: [PATCH 0661/1117] remove dead code in inlining pass --- base/inference.jl | 173 ++++------------------------------------------ 1 file changed, 15 insertions(+), 158 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 943f3ee2bbcab..e4e5cfb08b269 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2488,41 +2488,9 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end methsig = method.sig - incompletematch = false if !(atype <: metharg) - incompletematch = true - if !inline_incompletematch_allowed || !isdefined(Main,:Base) - # provide global disable if this optimization is not desirable - # need Main.Base defined for MethodError - return invoke_NF() - end - end - - ## This code tries to limit the argument list length only when it is - ## growing due to recursion. - ## It might be helpful for some things, but turns out not to be - ## necessary to get max performance from recursive varargs functions. - # if length(atypes) > MAX_TUPLETYPE_LEN - # # check call stack to see if this argument list is growing - # st = inference_stack - # while !isa(st, EmptyCallStack) - # if st.code === linfo.def.code && length(atypes) > length(st.types) - # atypes = limit_tuple_type(atypes) - # meth = _methods(f, atypes, 1) - # if meth === false || length(meth) != 1 - # return NF - # end - # meth = meth[1]::Tuple - # linfo2 = meth[3].func.code - # if linfo2 !== linfo - # return NF - # end - # linfo = linfo2 - # break - # end - # st = st.prev - # end - # end + return invoke_NF() + end (linfo, ty, inferred) = typeinf(method, metharg, methsp, false) if linfo === nothing || !inferred @@ -2532,31 +2500,6 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference # in this case function can be inlined to a constant return inline_as_constant(linfo.constval, argexprs, sv) elseif linfo !== nothing && !linfo.inlineable - # TODO - #= - if incompletematch - # inline a typeassert-based call-site, rather than a - # full generic lookup, using the inliner to handle - # all the fiddly details - numarg = length(argexprs) - newnames = unique_names(ast,numarg) - spnames = [] - spvals = [] - locals = [] - newcall = Expr(:call, e.args[1]) - newcall.typ = ty - for i = 1:numarg - name = newnames[i] - argtype = exprtype(argexprs[i],sv) - push!(locals, Any[name,argtype,0]) - push!(newcall.args, argtype===Any ? name : SymbolNode(name, argtype)) - end - body.args = Any[Expr(:return, newcall)] - ast = Expr(:lambda, newnames, Any[[], locals, [], 0], body) - else - return invoke_NF() - end - =# return invoke_NF() elseif linfo === nothing || linfo.code === nothing (linfo, ty, inferred) = typeinf(method, metharg, methsp, true) @@ -2618,101 +2561,27 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference prelude_stmts = Any[] stmts_free = true # true = all entries of stmts are effect_free - # when 1 method matches the inferred types, there is still a chance - # of a no-method error at run time, unless the inferred types are a - # subset of the method signature. - if incompletematch - t = Expr(:call) # tuple(args...) - t.typ = Tuple - argexprs2 = t.args - icall = LabelNode(label_counter(body.args)+1) - partmatch = Expr(:gotoifnot, false, icall.label) - thrw = Expr(:call, :throw, Expr(:call, GlobalRef(Main.Base,:MethodError), Expr(:call, top_tuple, e.args[1], QuoteNode(:inline)), t)) - thrw.typ = Bottom - end - for i=na:-1:1 # stmts_free needs to be calculated in reverse-argument order #args_i = args[i] aei = argexprs[i] aeitype = argtype = widenconst(exprtype(aei,sv)) - needtypeassert = false - if incompletematch - if isva - if nm == 0 - methitype = Tuple{} - elseif i > nm - methitype = methargs[end] - if isvarargtype(methitype) - methitype = Tuple{methitype} - else - methitype = Tuple{} - end - else - methitype = tupletype_tail(metharg,i) - end - isva = false - else - if i < nm - methitype = methargs[i] - else - methitype = methargs[end] - if isvarargtype(methitype) - methitype = methitype.parameters[1] - else - @assert i==nm - end - end - end - if isa(methitype, TypeVar) - methitype = methitype.ub - end - if !(aeitype <: methitype) - #TODO: make Undef a faster special-case? - needtypeassert = true - aeitype = methitype - end - end # ok for argument to occur more than once if the actual argument # is a symbol or constant, or is not affected by previous statements # that will exist after the inlining pass finishes - if needtypeassert - vnew1 = unique_name(enclosing_ast, ast) - add_variable(enclosing_ast, vnew1, aeitype, true) - v1 = (aeitype===Any ? vnew1 : SymbolNode(vnew1,aeitype)) - push!(spvals, v1) - vnew2 = unique_name(enclosing_ast, ast) - v2 = (argtype===Any ? vnew2 : SymbolNode(vnew2,argtype)) - unshift!(body.args, Expr(:(=), args_i, v2)) - args[i] = args_i = vnew2 - aeitype = argtype - affect_free = stmts_free - occ = 3 - # it's really late in codegen, so we expand the typeassert manually: cond = !isa(vnew2, methitype) | cond - cond = Expr(:call, Intrinsics.isa, v2, methitype) - cond.typ = Bool - cond = Expr(:call, Intrinsics.not_int, cond) - cond.typ = Bool - cond = Expr(:call, Intrinsics.or_int, cond, partmatch.args[1]) - cond.typ = Bool - cond = Expr(:call, Intrinsics.box, Bool, cond) - cond.typ = Bool - partmatch.args[1] = cond - else - affect_free = stmts_free # false = previous statements might affect the result of evaluating argument - occ = 0 - for j = length(body.args):-1:1 - b = body.args[j] - if occ < 6 - occ += occurs_more(b, x->(isa(x,Slot)&&x.id==i), 6) - end - # TODO: passing `sv` here is wrong since it refers to the enclosing function - if occ > 0 && affect_free && !effect_free(b, sv, true) #TODO: we could short-circuit this test better by memoizing effect_free(b) in the for loop over i - affect_free = false - end - if occ > 5 && !affect_free - break - end + affect_free = stmts_free # false = previous statements might affect the result of evaluating argument + occ = 0 + for j = length(body.args):-1:1 + b = body.args[j] + if occ < 6 + occ += occurs_more(b, x->(isa(x,Slot)&&x.id==i), 6) + end + # TODO: passing `sv` here is wrong since it refers to the enclosing function + if occ > 0 && affect_free && !effect_free(b, sv, true) #TODO: we could short-circuit this test better by memoizing effect_free(b) in the for loop over i + affect_free = false + end + if occ > 5 && !affect_free + break end end free = effect_free(aei,sv,true) @@ -2728,15 +2597,6 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference stmts_free = false end end - if incompletematch - unshift!(argexprs2, (argtype===Any ? args_i : SymbolNode(a,argtype))) - end - end - if incompletematch && partmatch.args[1] != false - unshift!(body.args, icall) - unshift!(body.args, thrw) - unshift!(body.args, partmatch) - unshift!(argexprs2, top_tuple) end # re-number the SSAValues and copy their type-info to the new ast @@ -2873,9 +2733,6 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end return (expr, stmts) end -# The inlining incomplete matches optimization currently -# doesn't work on Tuples of TypeVars -const inline_incompletematch_allowed = false inline_worthy(body::ANY, cost::Integer) = true From 41ef7f31a3077e5eccc62b72977b62523577b2bb Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 26 Jul 2016 14:30:08 -0400 Subject: [PATCH 0662/1117] formatting fixes and remove dead code also fix 2 compiler warnings --- src/alloc.c | 22 ++++----- src/builtins.c | 2 +- src/ccall.cpp | 12 ++--- src/cgmemmgr.cpp | 2 +- src/cgutils.cpp | 15 +++--- src/codegen.cpp | 24 ++++------ src/debuginfo.cpp | 11 +++-- src/dlload.c | 20 ++++---- src/dump.c | 6 +-- src/gc.c | 1 - src/gf.c | 112 ++------------------------------------------ src/init.c | 14 +++--- src/jloptions.c | 3 +- src/jltypes.c | 1 - src/julia.h | 4 +- src/llvm-gcroot.cpp | 21 +++++---- src/signals-win.c | 2 +- src/stackwalk.c | 28 +---------- src/threadgroup.c | 6 --- src/threadgroup.h | 44 +++++++++-------- src/threading.c | 1 - src/threading.h | 20 ++++---- src/typemap.c | 19 ++++---- 23 files changed, 123 insertions(+), 267 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 898e2c2f6f419..42c6931c20cbe 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -72,9 +72,8 @@ jl_value_t *jl_readonlymemory_exception; union jl_typemap_t jl_cfunction_list; jl_sym_t *call_sym; jl_sym_t *invoke_sym; -jl_sym_t *dots_sym; +jl_sym_t *dots_sym; jl_sym_t *empty_sym; jl_sym_t *module_sym; jl_sym_t *slot_sym; -jl_sym_t *empty_sym; jl_sym_t *export_sym; jl_sym_t *import_sym; jl_sym_t *importall_sym; jl_sym_t *toplevel_sym; jl_sym_t *quote_sym; jl_sym_t *amp_sym; @@ -84,11 +83,10 @@ jl_sym_t *line_sym; jl_sym_t *jl_incomplete_sym; jl_sym_t *goto_sym; jl_sym_t *goto_ifnot_sym; jl_sym_t *label_sym; jl_sym_t *return_sym; jl_sym_t *lambda_sym; jl_sym_t *assign_sym; -jl_sym_t *body_sym; +jl_sym_t *body_sym; jl_sym_t *globalref_sym; jl_sym_t *method_sym; jl_sym_t *core_sym; jl_sym_t *enter_sym; jl_sym_t *leave_sym; jl_sym_t *exc_sym; jl_sym_t *error_sym; -jl_sym_t *globalref_sym; jl_sym_t *new_sym; jl_sym_t *using_sym; jl_sym_t *const_sym; jl_sym_t *thunk_sym; jl_sym_t *anonymous_sym; jl_sym_t *underscore_sym; @@ -851,11 +849,10 @@ jl_datatype_t *jl_new_uninitialized_datatype(void) return t; } -static jl_datatype_layout_t *jl_get_layout( - uint32_t nfields, - uint32_t alignment, - int haspadding, - jl_fielddesc32_t desc[]) +static jl_datatype_layout_t *jl_get_layout(uint32_t nfields, + uint32_t alignment, + int haspadding, + jl_fielddesc32_t desc[]) { // compute the smallest fielddesc type that can hold the layout description int fielddesc_type = 0; @@ -882,8 +879,8 @@ static jl_datatype_layout_t *jl_get_layout( // allocate a new descriptor uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); - jl_datatype_layout_t *flddesc = (jl_datatype_layout_t*)jl_gc_perm_alloc( - sizeof(jl_datatype_layout_t) + nfields * fielddesc_size); + jl_datatype_layout_t *flddesc = + (jl_datatype_layout_t*)jl_gc_perm_alloc(sizeof(jl_datatype_layout_t) + nfields * fielddesc_size); flddesc->nfields = nfields; flddesc->alignment = alignment; flddesc->haspadding = haspadding; @@ -923,7 +920,8 @@ static jl_datatype_layout_t *jl_get_layout( // A non-zero result *must* match the LLVM rules for a vector type <nfields x t>. // For sake of Ahead-Of-Time (AOT) compilation, this routine has to work // without LLVM being available. -unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) { +unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) +{ if (!jl_is_vecelement_type(t)) return 0; // LLVM 3.7 and 3.8 either crash or generate wrong code for many diff --git a/src/builtins.c b/src/builtins.c index 2026934123e1a..c16890238bc51 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -768,7 +768,7 @@ JL_DLLEXPORT void *(jl_symbol_name)(jl_sym_t *s) return jl_symbol_name(s); } -//WARNING: THIS FUNCTION IS NEVER CALLED BUT INLINE BY CCALL +// WARNING: THIS FUNCTION IS NEVER CALLED BUT INLINE BY CCALL JL_DLLEXPORT void *jl_array_ptr(jl_array_t *a) { return a->data; diff --git a/src/ccall.cpp b/src/ccall.cpp index f078cd36d2ca3..9303786cc0b14 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1270,21 +1270,21 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) if (jl_is_expr(args[2])) { jl_expr_t *rtexpr = (jl_expr_t*)args[2]; if (rtexpr->head == call_sym && jl_expr_nargs(rtexpr) == 4 && - static_eval(jl_exprarg(rtexpr, 0), ctx, true, false) == jl_builtin_apply_type && - static_eval(jl_exprarg(rtexpr, 1), ctx, true, false) == (jl_value_t*)jl_array_type) { + static_eval(jl_exprarg(rtexpr, 0), ctx, true, false) == jl_builtin_apply_type && + static_eval(jl_exprarg(rtexpr, 1), ctx, true, false) == (jl_value_t*)jl_array_type) { // `Array` used as return type just returns a julia object reference rt = (jl_value_t*)jl_any_type; static_rt = true; } else if (rtexpr->head == call_sym && jl_expr_nargs(rtexpr) == 3 && - static_eval(jl_exprarg(rtexpr, 0), ctx, true, false) == jl_builtin_apply_type && - static_eval(jl_exprarg(rtexpr, 1), ctx, true, false) == (jl_value_t*)jl_pointer_type) { + static_eval(jl_exprarg(rtexpr, 0), ctx, true, false) == jl_builtin_apply_type && + static_eval(jl_exprarg(rtexpr, 1), ctx, true, false) == (jl_value_t*)jl_pointer_type) { // substitute Ptr{Void} for statically-unknown pointer type rt = (jl_value_t*)jl_voidpointer_type; } else if (rtexpr->head == call_sym && jl_expr_nargs(rtexpr) == 3 && - static_eval(jl_exprarg(rtexpr, 0), ctx, true, false) == jl_builtin_apply_type && - static_eval(jl_exprarg(rtexpr, 1), ctx, true, false) == (jl_value_t*)jl_ref_type) { + static_eval(jl_exprarg(rtexpr, 0), ctx, true, false) == jl_builtin_apply_type && + static_eval(jl_exprarg(rtexpr, 1), ctx, true, false) == (jl_value_t*)jl_ref_type) { // `Ref{T}` used as return type just returns T (from a jl_value_t*) rt = (jl_value_t*)jl_any_type; static_rt = true; diff --git a/src/cgmemmgr.cpp b/src/cgmemmgr.cpp index b0e0c3d79eea6..5c3b187ecab25 100644 --- a/src/cgmemmgr.cpp +++ b/src/cgmemmgr.cpp @@ -290,7 +290,7 @@ static void write_self_mem(void *dest, void *ptr, size_t size) return; if (ret == -1 && (errno == EAGAIN || errno == EINTR)) continue; - assert(ret < size); + assert((size_t)ret < size); size -= ret; ptr = (char*)ptr + ret; dest = (char*)dest + ret; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index c58f60e271990..9ad239345595d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -86,9 +86,9 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed return jl_pvalue_dillvmt; // always return the boxed representation for types with hidden content if (jl_is_abstracttype(jt) || !jl_is_datatype(jt) || jl_is_array_type(jt) || - jt == (jl_value_t*)jl_sym_type || jt == (jl_value_t*)jl_module_type || - jt == (jl_value_t*)jl_simplevector_type || jt == (jl_value_t*)jl_datatype_type || - jt == (jl_value_t*)jl_lambda_info_type) + jt == (jl_value_t*)jl_sym_type || jt == (jl_value_t*)jl_module_type || + jt == (jl_value_t*)jl_simplevector_type || jt == (jl_value_t*)jl_datatype_type || + jt == (jl_value_t*)jl_lambda_info_type) return jl_pvalue_dillvmt; if (jl_is_typector(jt) || jl_is_typevar(jt)) return jl_pvalue_dillvmt; @@ -266,7 +266,8 @@ static Value *emit_bitcast(Value *v, Type *jl_value) PointerType::get(cast<PointerType>(jl_value)->getElementType(), v->getType()->getPointerAddressSpace()); return builder.CreateBitCast(v, jl_value_addr); - } else { + } + else { return builder.CreateBitCast(v, jl_value); } } @@ -804,7 +805,8 @@ static Value *emit_bounds_check(const jl_cgval_t &ainfo, jl_value_t *ty, Value * // // Parameter ptr should be the pointer argument for the LoadInst or StoreInst. // It is currently unused, but might be used in the future for a more precise answer. -static unsigned julia_alignment(Value* /*ptr*/, jl_value_t *jltype, unsigned alignment) { +static unsigned julia_alignment(Value* /*ptr*/, jl_value_t *jltype, unsigned alignment) +{ if (!alignment && ((jl_datatype_t*)jltype)->layout->alignment > MAX_ALIGN) { // Type's natural alignment exceeds strictest alignment promised in heap, so return the heap alignment. return MAX_ALIGN; @@ -1089,7 +1091,8 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, else { if (strct.V->getType()->isVectorTy()) { fldv = builder.CreateExtractElement(strct.V, ConstantInt::get(T_int32, idx)); - } else { + } + else { // VecElement types are unwrapped in LLVM. assert( strct.V->getType()->isSingleValueType() ); fldv = strct.V; diff --git a/src/codegen.cpp b/src/codegen.cpp index 74303cefb818a..7ac5ceea6d81f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -399,7 +399,6 @@ struct jl_cgval_t { jl_value_t *constant; // constant value (rooted in linfo.def.roots) Value *gcroot; // the gcroot associated with V (if it has one) jl_value_t *typ; // the original type of V, never NULL - //Type *T; // cached result of julia_type_to_llvm(typ) bool isboxed; // whether this value is a jl_value_t* allocated on the heap with the right type tag bool isghost; // whether this value is "ghost" bool isimmutable; // V points to something that is definitely immutable (e.g. single-assignment, but including memory) @@ -413,7 +412,6 @@ struct jl_cgval_t { constant(NULL), gcroot(gcroot), typ(typ), - //T(julia_type_to_llvm(typ)), isboxed(isboxed), isghost(false), isimmutable(isboxed && jl_is_immutable_datatype(typ)), @@ -427,7 +425,6 @@ struct jl_cgval_t { constant(((jl_datatype_t*)typ)->instance), gcroot(NULL), typ(typ), - //T(T_void), isboxed(false), isghost(true), isimmutable(true), @@ -441,7 +438,6 @@ struct jl_cgval_t { constant(v.constant), gcroot(v.gcroot), typ(typ), - //T(V.T), isboxed(v.isboxed), isghost(v.isghost), isimmutable(v.isimmutable), @@ -620,6 +616,7 @@ static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_value_t *typ } return jl_cgval_t(v, froot, isboxed, typ); } + static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_datatype_t *typ, jl_codectx_t *ctx, bool needsroot = true) { return mark_julia_type(v, isboxed, (jl_value_t*)typ, ctx, needsroot); @@ -633,6 +630,7 @@ static inline jl_cgval_t remark_julia_type(const jl_cgval_t &v, jl_value_t *typ) } return jl_cgval_t(v, typ); } + static inline jl_cgval_t mark_julia_const(jl_value_t *jv) { jl_value_t *typ; @@ -648,7 +646,6 @@ static inline jl_cgval_t mark_julia_const(jl_value_t *jv) return constant; } - // --- utilities --- static void emit_write_barrier(jl_codectx_t*, Value*, Value*); @@ -1365,7 +1362,8 @@ const int logdata_blocksize = 32; // target getting nearby lines in the same gen typedef uint64_t logdata_block[logdata_blocksize]; typedef StringMap< std::vector<logdata_block*> > logdata_t; -static void visitLine( std::vector<logdata_block*> &vec, int line, Value *addend, const char* name ) { +static void visitLine(std::vector<logdata_block*> &vec, int line, Value *addend, const char* name) +{ unsigned block = line / logdata_blocksize; line = line % logdata_blocksize; if (vec.size() <= block) @@ -1428,7 +1426,7 @@ extern "C" JL_DLLEXPORT void jl_clear_malloc_data(void) for (itb = bytes.begin(); itb != bytes.end(); itb++) { if (*itb) { logdata_block &data = **itb; - for (size_t i = 0; i < logdata_blocksize; i++) { + for (int i = 0; i < logdata_blocksize; i++) { if (data[i] > 0) data[i] = 1; } @@ -1503,7 +1501,6 @@ extern "C" void jl_write_malloc_log(void) write_log_data(mallocData, ".mem"); } - // --- code gen for intrinsic functions --- #include "intrinsics.cpp" @@ -3395,7 +3392,7 @@ static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx) } static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_tupletype_t *argt, - jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt) + jl_typemap_entry_t *sf, jl_value_t *declrt, jl_tupletype_t *sigt) { // Generate a c-callable wrapper bool toboxed; @@ -5564,10 +5561,10 @@ static void init_julia_llvm_env(Module *m) // Helper to figure out what features to set for the LLVM target // If the user specifies native (or does not specify) we default // using the API provided by LLVM -static inline SmallVector<std::string,10> getTargetFeatures() { +static inline SmallVector<std::string,10> getTargetFeatures() +{ StringMap<bool> HostFeatures; - if (!strcmp(jl_options.cpu_target,"native")) - { + if (!strcmp(jl_options.cpu_target,"native")) { // On earlier versions of LLVM this is empty llvm::sys::getHostCPUFeatures(HostFeatures); } @@ -5605,8 +5602,7 @@ static inline SmallVector<std::string,10> getTargetFeatures() { } SmallVector<std::string,10> attr; - for (StringMap<bool>::const_iterator it = HostFeatures.begin(); it != HostFeatures.end(); it++) - { + for (StringMap<bool>::const_iterator it = HostFeatures.begin(); it != HostFeatures.end(); it++) { std::string att = it->getValue() ? it->getKey().str() : std::string("-") + it->getKey().str(); attr.append(1, att); diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 454c53e738747..b61abe00e817c 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -110,7 +110,8 @@ struct ObjectInfo { // so that when we see it get emitted, we can add a link back to the linfo // that it came from (providing name, type signature, file info, etc.) static StringMap<jl_lambda_info_t*> linfo_in_flight; -static std::string mangle(const std::string &Name, const DataLayout &DL) { +static std::string mangle(const std::string &Name, const DataLayout &DL) +{ #if defined(USE_MCJIT) || defined(USE_ORCJIT) std::string MangledName; { @@ -129,7 +130,7 @@ void jl_add_linfo_in_flight(StringRef name, jl_lambda_info_t *linfo, const DataL #if defined(_OS_WINDOWS_) static void create_PRUNTIME_FUNCTION(uint8_t *Code, size_t Size, StringRef fnname, - uint8_t *Section, size_t Allocated, uint8_t *UnwindData) + uint8_t *Section, size_t Allocated, uint8_t *UnwindData) { // GC safe DWORD mod_size = 0; @@ -1009,7 +1010,8 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, #endif *slide = -(int64_t)fbase; #ifdef _OS_DARWIN_ - } else { + } + else { // If we're here the, the dsym does not match the dylib. Use the original // object instead. For consistency (and to make sure we get a sensible size // for the memory buffer), we also use a fresh copy mapped from @@ -1214,7 +1216,8 @@ JL_DLLEXPORT uint64_t jl_get_section_start(uint64_t fptr) uint64_t ret = 0; if (fit != objmap.end() && fptr < fit->first + fit->second.SectionSize) { ret = fit->first; - } else { + } + else { obfiletype::iterator objit = objfilemap.lower_bound(fptr); // Ideally we'd have a containment check here, but we can't really // get the shared library size easily. diff --git a/src/dlload.c b/src/dlload.c index 73ca19aeab5da..da290a1f32a3c 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -129,9 +129,9 @@ static void *jl_load_dynamic_library_(const char *modname, unsigned flags, int t // file not found error due to the extra or missing extension name. int hasext = endswith_extension(modname); -/* - this branch returns handle of libjulia -*/ + /* + this branch returns handle of libjulia + */ if (modname == NULL) { #ifdef _OS_WINDOWS_ if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, @@ -144,9 +144,9 @@ static void *jl_load_dynamic_library_(const char *modname, unsigned flags, int t #endif goto done; } -/* - this branch shortcuts absolute paths -*/ + /* + this branch shortcuts absolute paths + */ #ifdef _OS_WINDOWS_ else if (modname[1] == ':') { #else @@ -159,10 +159,10 @@ static void *jl_load_dynamic_library_(const char *modname, unsigned flags, int t if (jl_stat(modname, (char*)&stbuf) == 0) goto notfound; } -/* - this branch permutes all base paths in DL_LOAD_PATH with all extensions - note: skip when !jl_base_module to avoid UndefVarError(:DL_LOAD_PATH) -*/ + /* + this branch permutes all base paths in DL_LOAD_PATH with all extensions + note: skip when !jl_base_module to avoid UndefVarError(:DL_LOAD_PATH) + */ else if (jl_base_module != NULL) { jl_array_t *DL_LOAD_PATH = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("DL_LOAD_PATH")); if (DL_LOAD_PATH != NULL) { diff --git a/src/dump.c b/src/dump.c index b10909d27c4d8..f422c9a1bf189 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1783,7 +1783,8 @@ JL_DLLEXPORT int jl_read_verify_header(ios_t *s) readstr_verify(s, jl_git_commit()) && !read_uint8(s)); } -static void jl_finalize_serializer(jl_serializer_state *s) { +static void jl_finalize_serializer(jl_serializer_state *s) +{ size_t i, l; // save module initialization order if (jl_module_init_order != NULL) { @@ -2392,8 +2393,7 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) return restored; } -JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, - size_t sz) +JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz) { ios_t f; jl_array_t *modules; diff --git a/src/gc.c b/src/gc.c index f62f056b011d4..d8b8f15963c92 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1103,7 +1103,6 @@ static jl_value_t **mark_stack_base = NULL; static size_t mark_stack_size = 0; static size_t mark_sp = 0; - static void grow_mark_stack(void) { size_t newsz = mark_stack_size>0 ? mark_stack_size*2 : 32000; diff --git a/src/gf.c b/src/gf.c index 99d26eb5d2588..3f0f1e3a43edb 100644 --- a/src/gf.c +++ b/src/gf.c @@ -966,7 +966,8 @@ static jl_array_t *check_ambiguous_matches(union jl_typemap_t defs, return env.shadowed; } -static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue) { +static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue) +{ // method overwritten jl_method_t *method = (jl_method_t*)newentry->func.method; jl_module_t *newmod = method->module; @@ -1783,7 +1784,7 @@ STATIC_INLINE uint32_t int32hash_fast(uint32_t a) // a = (a+0xd3a2646c) ^ (a<<9); // a = (a+0xfd7046c5) + (a<<3); // a = (a^0xb55a4f09) ^ (a>>16); - return a; + return a; // identity hashing seems to work well enough here } STATIC_INLINE int sig_match_fast(jl_value_t **args, jl_value_t **sig, size_t i, size_t n) @@ -1803,35 +1804,6 @@ STATIC_INLINE int sig_match_fast(jl_value_t **args, jl_value_t **sig, size_t i, return 1; } -//STATIC_INLINE jl_typemap_entry_t *sig2_match_fast(jl_value_t **args, jl_typemap_entry_t *entry1, jl_typemap_entry_t *entry2, size_t n) -//{ -// // NOTE: This function is a huge performance hot spot!! -// jl_value_t **sig1 = jl_svec_data(entry1->sig->parameters); -// jl_value_t **sig2 = jl_svec_data(entry2->sig->parameters); -// size_t i; -// for (i = 0; i < n; i++) { -// jl_value_t *decl1 = sig1[i]; -// jl_value_t *decl2 = sig2[i]; -// jl_value_t *a = args[i]; -// jl_value_t *ta = jl_typeof(a); -// if (ta != decl1) { -// /* -// we are only matching concrete types here, and those types are -// hash-consed, so pointer comparison should work. -// */ -// return sig_match_fast(args, sig2, i, n) ? entry2 : NULL; -// } -// if (ta != decl2) { -// /* -// we are only matching concrete types here, and those types are -// hash-consed, so pointer comparison should work. -// */ -// return sig_match_fast(args, sig1, i, n) ? entry1 : NULL; -// } -// } -// return entry1; -//} - jl_typemap_entry_t *call_cache[N_CALL_CACHE]; static uint8_t pick_which[N_CALL_CACHE]; #ifdef JL_GF_PROFILE @@ -1887,84 +1859,6 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) otherwise instantiate the generic method and use it */ uint32_t callsite = int32hash_fast((uintptr_t)__builtin_return_address(0)); - // implementation 1 -// uint32_t cache_idx1 = (callsite) & (N_CALL_CACHE - 1); -// uint32_t cache_idx2 = (callsite >> 16) & (N_CALL_CACHE - 1); -// jl_typemap_entry_t *entry1 = call_cache[cache_idx1]; -// jl_typemap_entry_t *entry2 = call_cache[cache_idx2]; -// jl_methtable_t *mt = NULL; -// jl_typemap_entry_t *entry = NULL; -// if (entry1 && nargs == jl_svec_len(entry1->sig->parameters)) { -// if (entry2 && nargs == jl_svec_len(entry2->sig->parameters)) -// entry = sig2_match_fast(args, entry1, entry2, nargs); -// else if (sig_match_fast(args, jl_svec_data(entry1->sig->parameters), 0, nargs)) -// entry = entry1; -// } -// else if (entry2 && nargs == jl_svec_len(entry2->sig->parameters)) { -// if (sig_match_fast(args, jl_svec_data(entry2->sig->parameters), 0, nargs)) -// entry = entry2; -// } -// -// if (entry == NULL) { -// jl_value_t *F = args[0]; -// mt = jl_gf_mtable(F); -// entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt)); -// if (entry && entry->isleafsig && entry->simplesig == (void*)jl_nothing && entry->guardsigs == jl_emptysvec) { -// switch (++pick_which[cache_idx1] & 1) { -// case 0: -// call_cache[cache_idx1] = entry; -// break; -// case 1: -// call_cache[cache_idx2] = entry; -// break; -// } -// } -// } - - // implementation 2 -// uint32_t cache_idx1 = (callsite) & (N_CALL_CACHE - 1); -// uint32_t cache_idx2 = (callsite >> 8) & (N_CALL_CACHE - 1); -// uint32_t cache_idx3 = (callsite >> 16) & (N_CALL_CACHE - 1); -// uint32_t cache_idx4 = (callsite >> 24) & (N_CALL_CACHE - 1); -// jl_typemap_entry_t *entry = call_cache[cache_idx1]; -// jl_methtable_t *mt = NULL; -// if (!entry || nargs != jl_svec_len(entry->sig->parameters) || -// !sig_match_fast(args, jl_svec_data(entry->sig->parameters), nargs)) { -// entry = call_cache[cache_idx2]; -// if (!entry || nargs != jl_svec_len(entry->sig->parameters) || -// !sig_match_fast(args, jl_svec_data(entry->sig->parameters), nargs)) { -// entry = call_cache[cache_idx3]; -// if (!entry || nargs != jl_svec_len(entry->sig->parameters) || -// !sig_match_fast(args, jl_svec_data(entry->sig->parameters), nargs)) { -// entry = call_cache[cache_idx4]; -// if (!entry || nargs != jl_svec_len(entry->sig->parameters) || -// !sig_match_fast(args, jl_svec_data(entry->sig->parameters), nargs)) { -// -// jl_value_t *F = args[0]; -// mt = jl_gf_mtable(F); -// entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt)); -// if (entry && entry->isleafsig && entry->simplesig == (void*)jl_nothing && entry->guardsigs == jl_emptysvec) { -// switch (++pick_which[cache_idx1] & 3) { -// case 0: -// call_cache[cache_idx1] = entry; -// break; -// case 1: -// call_cache[cache_idx2] = entry; -// break; -// case 2: -// call_cache[cache_idx3] = entry; -// break; -// case 3: -// call_cache[cache_idx4] = entry; -// break; -// } -// } -// } -// } -// } -// } - - // implementation 3 // compute the entry hashes // use different parts of the value // so that a collision across all of diff --git a/src/init.c b/src/init.c index 2ddaac18c79c3..53e2dd62b578f 100644 --- a/src/init.c +++ b/src/init.c @@ -476,13 +476,13 @@ static char *abspath(const char *in) } static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) -{ // this function resolves the paths in jl_options to absolute file locations as needed - // and it replaces the pointers to `julia_home`, `julia_bin`, `image_file`, and output file paths - // it may fail, print an error, and exit(1) if any of these paths are longer than PATH_MAX - // - // note: if you care about lost memory, you should call the appropriate `free()` function - // on the original pointer for each `char*` you've inserted into `jl_options`, after - // calling `julia_init()` +{ // this function resolves the paths in jl_options to absolute file locations as needed + // and it replaces the pointers to `julia_home`, `julia_bin`, `image_file`, and output file paths + // it may fail, print an error, and exit(1) if any of these paths are longer than PATH_MAX + // + // note: if you care about lost memory, you should call the appropriate `free()` function + // on the original pointer for each `char*` you've inserted into `jl_options`, after + // calling `julia_init()` char *free_path = (char*)malloc(PATH_MAX); size_t path_size = PATH_MAX; if (uv_exepath(free_path, &path_size)) { diff --git a/src/jloptions.c b/src/jloptions.c index ab91b109a3cb5..3913ee7d33c45 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -182,7 +182,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) } } jl_errorf("unknown option `-%c`", optopt); - } else { + } + else { jl_errorf("unknown option `%s`", argv[lastind]); } break; diff --git a/src/jltypes.c b/src/jltypes.c index cc2d1acfa33b2..c35060cba9b38 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2563,7 +2563,6 @@ void jl_reset_instantiate_inner_types(jl_datatype_t *t) partial_inst.len = 0; } - // subtype comparison static int jl_subtype_le(jl_value_t *a, jl_value_t *b, int ta, int invariant); diff --git a/src/julia.h b/src/julia.h index 24bf039327a47..734361d28f2ee 100644 --- a/src/julia.h +++ b/src/julia.h @@ -556,7 +556,7 @@ extern JL_DLLEXPORT jl_value_t *jl_nothing; // some important symbols extern jl_sym_t *call_sym; extern jl_sym_t *invoke_sym; -extern jl_sym_t *empty_sym; +extern jl_sym_t *empty_sym; extern jl_sym_t *body_sym; extern jl_sym_t *dots_sym; extern jl_sym_t *vararg_sym; extern jl_sym_t *quote_sym; extern jl_sym_t *newvar_sym; extern jl_sym_t *top_sym; extern jl_sym_t *dot_sym; @@ -570,7 +570,6 @@ extern jl_sym_t *importall_sym; extern jl_sym_t *using_sym; extern jl_sym_t *goto_sym; extern jl_sym_t *goto_ifnot_sym; extern jl_sym_t *label_sym; extern jl_sym_t *return_sym; extern jl_sym_t *lambda_sym; extern jl_sym_t *assign_sym; -extern jl_sym_t *body_sym; extern jl_sym_t *method_sym; extern jl_sym_t *slot_sym; extern jl_sym_t *enter_sym; extern jl_sym_t *leave_sym; extern jl_sym_t *exc_sym; extern jl_sym_t *new_sym; @@ -1143,7 +1142,6 @@ JL_DLLEXPORT jl_value_t *jl_get_field(jl_value_t *o, const char *fld); JL_DLLEXPORT jl_value_t *jl_value_ptr(jl_value_t *a); // arrays - JL_DLLEXPORT jl_array_t *jl_new_array(jl_value_t *atype, jl_value_t *dims); JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_value_t *dims); diff --git a/src/llvm-gcroot.cpp b/src/llvm-gcroot.cpp index 74facedd86866..d0129583b8b9b 100644 --- a/src/llvm-gcroot.cpp +++ b/src/llvm-gcroot.cpp @@ -88,17 +88,21 @@ static void tbaa_decorate_gcframe(Instruction *inst, Instruction *user = dyn_cast<Instruction>(*I); if (!user) { continue; - } else if (isa<GetElementPtrInst>(user)) { + } + else if (isa<GetElementPtrInst>(user)) { if (__likely(user->getOperand(0) == inst)) { tbaa_decorate_gcframe(user, visited, tbaa_gcframe); } - } else if (isa<StoreInst>(user)) { + } + else if (isa<StoreInst>(user)) { if (user->getOperand(1) == inst) { user->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_gcframe); } - } else if (isa<LoadInst>(user)) { + } + else if (isa<LoadInst>(user)) { user->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_gcframe); - } else if (isa<BitCastInst>(user)) { + } + else if (isa<BitCastInst>(user)) { tbaa_decorate_gcframe(user, visited, tbaa_gcframe); } } @@ -174,11 +178,10 @@ class JuliaGCAllocator { Instruction *JuliaGCAllocator::get_pgcstack(Instruction *ptlsStates) { Constant *offset = ConstantInt::getSigned(T_int32, offsetof(jl_tls_states_t, pgcstack) / sizeof(void*)); - return GetElementPtrInst::Create( - LLVM37_param(NULL) - ptlsStates, - ArrayRef<Value*>(offset), - "jl_pgcstack"); + return GetElementPtrInst::Create(LLVM37_param(NULL) + ptlsStates, + ArrayRef<Value*>(offset), + "jl_pgcstack"); } frame_register JuliaGCAllocator::get_gcroot(Value *ptr) diff --git a/src/signals-win.c b/src/signals-win.c index a9b77c11a1ad2..1f74c77d7f2c2 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -318,7 +318,6 @@ JL_DLLEXPORT void jl_install_sigint_handler(void) SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigint_handler,1); } - volatile HANDLE hBtThread = 0; static DWORD WINAPI profile_bt( LPVOID lparam ) { @@ -365,6 +364,7 @@ static DWORD WINAPI profile_bt( LPVOID lparam ) hBtThread = 0; return 0; } + JL_DLLEXPORT int jl_profile_start_timer(void) { running = 1; diff --git a/src/stackwalk.c b/src/stackwalk.c index 18f991efa3a4a..e67308ea01b25 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -137,7 +137,6 @@ JL_DLLEXPORT jl_value_t *jl_get_backtrace(void) } - #if defined(_OS_WINDOWS_) #ifdef _CPU_X86_64_ static UNWIND_HISTORY_TABLE HistoryTable; @@ -229,7 +228,8 @@ static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) #endif } -static int readable_pointer(LPCVOID pointer) { +static int readable_pointer(LPCVOID pointer) +{ // Check whether the pointer is valid and executable before dereferencing // to avoid segfault while recording. See #10638. MEMORY_BASIC_INFORMATION mInfo; @@ -345,30 +345,6 @@ size_t rec_backtrace_ctx_dwarf(uintptr_t *data, size_t maxsize, #endif #endif -// Always Set *func_name and *file_name to malloc'd pointers (non-NULL) -/*static int frame_info_from_ip(char **func_name, - char **file_name, size_t *line_num, - char **inlinedat_file, size_t *inlinedat_line, - jl_lambda_info_t **outer_linfo, - size_t ip, int skipC, int skipInline) -{ - // This function is not allowed to reference any TLS variables since - // it can be called from an unmanaged thread on OSX. - static const char *name_unknown = "???"; - int fromC = 0; - - jl_getFunctionInfo(func_name, file_name, line_num, inlinedat_file, inlinedat_line, outer_linfo, - ip, &fromC, skipC, skipInline); - if (!*func_name) { - *func_name = strdup(name_unknown); - *line_num = ip; - } - if (!*file_name) { - *file_name = strdup(name_unknown); - } - return fromC; - }*/ - JL_DLLEXPORT jl_value_t *jl_lookup_code_address(void *ip, int skipC) { jl_ptls_t ptls = jl_get_ptls_states(); diff --git a/src/threadgroup.c b/src/threadgroup.c index c728204c17d7b..7b4db25d1853d 100644 --- a/src/threadgroup.c +++ b/src/threadgroup.c @@ -59,7 +59,6 @@ int ti_threadgroup_create(uint8_t num_sockets, uint8_t num_cores, return 0; } - int ti_threadgroup_addthread(ti_threadgroup_t *tg, int16_t ext_tid, int16_t *tgtid) { @@ -76,7 +75,6 @@ int ti_threadgroup_addthread(ti_threadgroup_t *tg, int16_t ext_tid, return 0; } - int ti_threadgroup_initthread(ti_threadgroup_t *tg, int16_t ext_tid) { ti_thread_sense_t *ts; @@ -95,7 +93,6 @@ int ti_threadgroup_initthread(ti_threadgroup_t *tg, int16_t ext_tid) return 0; } - int ti_threadgroup_member(ti_threadgroup_t *tg, int16_t ext_tid, int16_t *tgtid) { if (ext_tid < 0 || ext_tid >= tg->num_threads) @@ -113,14 +110,12 @@ int ti_threadgroup_member(ti_threadgroup_t *tg, int16_t ext_tid, int16_t *tgtid) return 0; } - int ti_threadgroup_size(ti_threadgroup_t *tg, int16_t *tgsize) { *tgsize = tg->num_threads; return 0; } - int ti_threadgroup_fork(ti_threadgroup_t *tg, int16_t ext_tid, void **bcast_val) { if (tg->tid_map[ext_tid] == 0) { @@ -170,7 +165,6 @@ int ti_threadgroup_fork(ti_threadgroup_t *tg, int16_t ext_tid, void **bcast_val) return 0; } - int ti_threadgroup_join(ti_threadgroup_t *tg, int16_t ext_tid) { int i; diff --git a/src/threadgroup.h b/src/threadgroup.h index 522350a38b5a5..02edf30d15571 100644 --- a/src/threadgroup.h +++ b/src/threadgroup.h @@ -11,36 +11,34 @@ typedef struct { volatile int sense; } ti_thread_sense_t; - // thread group typedef struct { - int16_t *tid_map, num_threads, added_threads; - uint8_t num_sockets, num_cores, num_threads_per_core; + int16_t *tid_map, num_threads, added_threads; + uint8_t num_sockets, num_cores, num_threads_per_core; // fork/join/barrier - volatile uint8_t group_sense; - ti_thread_sense_t **thread_sense; - void *envelope; + volatile uint8_t group_sense; + ti_thread_sense_t **thread_sense; + void *envelope; // to let threads sleep - uv_mutex_t alarm_lock; - uv_cond_t alarm; - uint64_t sleep_threshold; + uv_mutex_t alarm_lock; + uv_cond_t alarm; + uint64_t sleep_threshold; } ti_threadgroup_t; - -int ti_threadgroup_create(uint8_t num_sockets, uint8_t num_cores, - uint8_t num_threads_per_core, - ti_threadgroup_t **newtg); -int ti_threadgroup_addthread(ti_threadgroup_t *tg, int16_t ext_tid, - int16_t *tgtid); -int ti_threadgroup_initthread(ti_threadgroup_t *tg, int16_t ext_tid); -int ti_threadgroup_member(ti_threadgroup_t *tg, int16_t ext_tid, - int16_t *tgtid); -int ti_threadgroup_size(ti_threadgroup_t *tg, int16_t *tgsize); -int ti_threadgroup_fork(ti_threadgroup_t *tg, int16_t ext_tid, - void **bcast_val); -int ti_threadgroup_join(ti_threadgroup_t *tg, int16_t ext_tid); -int ti_threadgroup_destroy(ti_threadgroup_t *tg); +int ti_threadgroup_create(uint8_t num_sockets, uint8_t num_cores, + uint8_t num_threads_per_core, + ti_threadgroup_t **newtg); +int ti_threadgroup_addthread(ti_threadgroup_t *tg, int16_t ext_tid, + int16_t *tgtid); +int ti_threadgroup_initthread(ti_threadgroup_t *tg, int16_t ext_tid); +int ti_threadgroup_member(ti_threadgroup_t *tg, int16_t ext_tid, + int16_t *tgtid); +int ti_threadgroup_size(ti_threadgroup_t *tg, int16_t *tgsize); +int ti_threadgroup_fork(ti_threadgroup_t *tg, int16_t ext_tid, + void **bcast_val); +int ti_threadgroup_join(ti_threadgroup_t *tg, int16_t ext_tid); +int ti_threadgroup_destroy(ti_threadgroup_t *tg); #endif /* THREADGROUP_H */ diff --git a/src/threading.c b/src/threading.c index 1c422009584ad..b4de746abe526 100644 --- a/src/threading.c +++ b/src/threading.c @@ -13,7 +13,6 @@ . make code generation thread-safe and remove the lock */ - #include <stdint.h> #include <stdio.h> #include <stdlib.h> diff --git a/src/threading.h b/src/threading.h index 3d70d86b511b8..7ad732b07bcb1 100644 --- a/src/threading.h +++ b/src/threading.h @@ -23,32 +23,28 @@ enum { TI_THREAD_WORK }; - // passed to thread function typedef struct { - int16_t volatile state; - int16_t tid; - ti_threadgroup_t *tg; + int16_t volatile state; + int16_t tid; + ti_threadgroup_t *tg; } ti_threadarg_t; - // commands to thread function enum { TI_THREADWORK_DONE, TI_THREADWORK_RUN }; - // work command to thread function typedef struct { - uint8_t command; - jl_function_t *fun; - jl_svec_t *args; - jl_value_t *ret; - jl_module_t *current_module; + uint8_t command; + jl_function_t *fun; + jl_svec_t *args; + jl_value_t *ret; + jl_module_t *current_module; } ti_threadwork_t; - // thread function void ti_threadfun(void *arg); diff --git a/src/typemap.c b/src/typemap.c index 06a4b2627caf3..9d308db03d336 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -301,7 +301,8 @@ static void mtcache_rehash(struct jl_ordereddict_t *pa, size_t newlen, jl_value_ // Recursively rehash a TypeMap (for example, after deserialization) void jl_typemap_rehash(union jl_typemap_t ml, int8_t offs); -void jl_typemap_rehash_array(struct jl_ordereddict_t *pa, jl_value_t *parent, int8_t tparam, int8_t offs) { +void jl_typemap_rehash_array(struct jl_ordereddict_t *pa, jl_value_t *parent, int8_t tparam, int8_t offs) +{ size_t i, len = jl_array_len(pa->values); for (i = 0; i < len; i++) { union jl_typemap_t ml; @@ -368,11 +369,9 @@ static union jl_typemap_t *mtcache_hash_bp(struct jl_ordereddict_t *pa, jl_value return NULL; } - // ----- Sorted Type Signature Lookup Matching ----- // -jl_value_t *jl_lookup_match(jl_value_t *a, jl_value_t *b, jl_svec_t **penv, - jl_svec_t *tvars) +jl_value_t *jl_lookup_match(jl_value_t *a, jl_value_t *b, jl_svec_t **penv, jl_svec_t *tvars) { jl_value_t *ti = jl_type_intersection_matching(a, b, penv, tvars); if (ti == (jl_value_t*)jl_bottom_type) @@ -449,7 +448,7 @@ int is_cache_leaf(jl_value_t *ty) } static int jl_typemap_intersection_array_visitor(struct jl_ordereddict_t *a, jl_value_t *ty, int tparam, - int offs, struct typemap_intersection_env *closure) + int offs, struct typemap_intersection_env *closure) { size_t i, l = jl_array_len(a->values); union jl_typemap_t *data = (union jl_typemap_t*)jl_array_data(a->values); @@ -518,7 +517,7 @@ static int jl_typemap_intersection_node_visitor(jl_typemap_entry_t *ml, struct t } int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs, - struct typemap_intersection_env *closure) + struct typemap_intersection_env *closure) { if (jl_typeof(map.unknown) == (jl_value_t*)jl_typemap_level_type) { jl_typemap_level_t *cache = map.node; @@ -698,7 +697,7 @@ static jl_typemap_entry_t *jl_typemap_lookup_by_type_(jl_typemap_entry_t *ml, jl // this is the general entry point for looking up a type in the cache // (as a subtype, or with typeseq) jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_tupletype_t *types, jl_svec_t **penv, - int8_t subtype_inexact__sigseq_useenv, int8_t subtype, int8_t offs) + int8_t subtype_inexact__sigseq_useenv, int8_t subtype, int8_t offs) { if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) { jl_typemap_level_t *cache = ml_or_cache.node; @@ -931,8 +930,8 @@ static void jl_typemap_list_insert_(jl_typemap_entry_t **pml, jl_value_t *parent } static void jl_typemap_insert_generic(union jl_typemap_t *pml, jl_value_t *parent, - jl_typemap_entry_t *newrec, jl_value_t *key, int8_t offs, - const struct jl_typemap_info *tparams) + jl_typemap_entry_t *newrec, jl_value_t *key, int8_t offs, + const struct jl_typemap_info *tparams) { if (jl_typeof(pml->unknown) == (jl_value_t*)jl_typemap_level_type) { jl_typemap_level_insert_(pml->node, newrec, offs, tparams); @@ -961,7 +960,7 @@ static int jl_typemap_array_insert_(struct jl_ordereddict_t *cache, jl_value_t * } static void jl_typemap_level_insert_(jl_typemap_level_t *cache, jl_typemap_entry_t *newrec, int8_t offs, - const struct jl_typemap_info *tparams) + const struct jl_typemap_info *tparams) { size_t l = jl_field_count(newrec->sig); // compute the type at offset `offs` into `sig`, which may be a Vararg From 503270f1eb7c347eb9f752f082dfbba542dc68cb Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Tue, 26 Jul 2016 13:48:57 -0500 Subject: [PATCH 0663/1117] Use generic OrdinalRange interface instead of fields for arithmetic --- base/operators.jl | 2 +- test/ranges.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/operators.jl b/base/operators.jl index 3f3a8fc95977c..22febbc58a633 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -490,7 +490,7 @@ for f in (:+, :-) r1l = length(r1) (r1l == length(r2) || throw(DimensionMismatch("argument dimensions must match"))) - range($f(r1.start,r2.start), $f(step(r1),step(r2)), r1l) + range($f(first(r1),first(r2)), $f(step(r1),step(r2)), r1l) end function $f{T<:AbstractFloat}(r1::FloatRange{T}, r2::FloatRange{T}) diff --git a/test/ranges.jl b/test/ranges.jl index d6c5a0787a95b..0c0607c06ce9e 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -756,6 +756,7 @@ r = Base.OneTo(3) @test_throws BoundsError r[0] @test r+1 === 2:4 @test 2*r === 2:2:6 +@test r+r === 2:2:6 k = 0 for i in r @test i == (k+=1) From 2f62492791af094fbb4168e58632dfc5c7faeb19 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 12:03:59 -0700 Subject: [PATCH 0664/1117] Hotfix for a bug in #17592 No rule to make target CMakeLists.txt in a clean build, my bad --- deps/llvm.mk | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 99cc7829e71cb..efea7d7fcd9bd 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -258,7 +258,7 @@ $(LLVM_LLDB_TAR): | $(SRCDIR)/srccache endif ifeq ($(BUILD_LLDB),1) $(LLVM_SRC_DIR)/tools/lldb: -$(LLVM_SRC_DIR)/configure: $(LLVM_SRC_DIR)/tools/lldb +$(LLVM_SRC_DIR)/CMakeLists.txt: $(LLVM_SRC_DIR)/tools/lldb endif # LLDB still relies on plenty of python 2.x infrastructure, without checking @@ -278,13 +278,13 @@ ifeq ($(USEICC),1) LIBCXX_EXTRA_FLAGS := -Bstatic -lirc -Bdynamic endif -$(LLVM_SRC_DIR)/projects/libcxx: $(LLVM_LIBCXX_TAR) | $(LLVM_SRC_DIR)/configure +$(LLVM_SRC_DIR)/projects/libcxx: $(LLVM_LIBCXX_TAR) | $(LLVM_SRC_DIR)/CMakeLists.txt ([ ! -d $@ ] && \ git clone $(LLVM_GIT_URL_LIBCXX) $@ ) || \ (cd $@ && \ git pull --ff-only) $(LLVM_SRC_DIR)/projects/libcxx/.git/HEAD: | $(LLVM_SRC_DIR)/projects/libcxx/.git/HEAD -$(LLVM_SRC_DIR)/projects/libcxxabi: $(LLVM_LIBCXXABI_TAR) | $(LLVM_SRC_DIR)/configure +$(LLVM_SRC_DIR)/projects/libcxxabi: $(LLVM_LIBCXXABI_TAR) | $(LLVM_SRC_DIR)/CMakeLists.txt ([ ! -d $@ ] && \ git clone $(LLVM_GIT_URL_LIBCXXABI) $@ ) || \ (cd $@ && \ @@ -320,7 +320,7 @@ LIBCXX_DEPENDENCY := $(build_libdir)/libc++abi.so.1.0 $(build_libdir)/libc++.so. get-llvm: get-libcxx get-libcxxabi endif -$(LLVM_SRC_DIR)/configure: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) +$(LLVM_SRC_DIR)/CMakeLists.txt: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) ifneq ($(LLVM_CLANG_TAR),) $(JLCHECKSUM) $(LLVM_CLANG_TAR) endif @@ -468,7 +468,7 @@ endif ifeq ($(LLVM_USE_CMAKE),1) -$(LLVM_BUILDDIR_withtype)/CMakeCache.txt: $(LLVM_SRC_DIR)/configure $(LLVM_PATCH_LIST) | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) +$(LLVM_BUILDDIR_withtype)/CMakeCache.txt: $(LLVM_SRC_DIR)/CMakeLists.txt $(LLVM_PATCH_LIST) | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) mkdir -p $(dir $@) cd $(dir $@) && \ export PATH=$(llvm_python_workaround):$$PATH && \ @@ -485,11 +485,11 @@ $(LLVM_OBJ_SOURCE): $(LLVM_BUILDDIR_withtype)/CMakeCache.txt | $(llvm_python_wor else -$(LLVM_BUILDDIR_withtype)/config.status: $(LLVM_SRC_DIR)/configure $(LLVM_PATCH_LIST) | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) +$(LLVM_BUILDDIR_withtype)/config.status: $(LLVM_SRC_DIR)/CMakeLists.txt $(LLVM_PATCH_LIST) | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) mkdir -p $(dir $@) cd $(dir $@) && \ export PATH=$(llvm_python_workaround):$$PATH && \ - $< $(CONFIGURE_COMMON) $(LLVM_CC) $(LLVM_FLAGS) + $(LLVM_SRC_DIR)/configure $(CONFIGURE_COMMON) $(LLVM_CC) $(LLVM_FLAGS) touch -c $@ $(LLVM_OBJ_SOURCE): $(LLVM_BUILDDIR_withtype)/config.status | $(llvm_python_workaround) @@ -535,7 +535,7 @@ distclean-llvm: ifneq ($(LLVM_VER),svn) get-llvm: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) else -get-llvm: $(LLVM_SRC_DIR)/configure +get-llvm: $(LLVM_SRC_DIR)/CMakeLists.txt endif ifeq ($(LLVM_USE_CMAKE),1) configure-llvm: $(LLVM_BUILDDIR_withtype)/CMakeCache.txt From 3fbfa7bb2f3e281aad0911578140bc52c629ed42 Mon Sep 17 00:00:00 2001 From: Erik Schnetter <schnetter@gmail.com> Date: Mon, 25 Jul 2016 13:57:07 -0400 Subject: [PATCH 0665/1117] Correct `invmod` for unsigned integers and for negative moduli - add type signature to restrict arguments to Integer - use type promotion - correct existing test case for negative modulos - clarify documentation - add test cases - also update description of `gcdx`, add a few comments to the source code. Closes #17595. --- base/docs/helpdb/Base.jl | 4 +++- base/gmp.jl | 19 +++++++++++++------ base/intfuncs.jl | 30 +++++++++++++++++++----------- doc/stdlib/math.rst | 6 +++--- test/examples.jl | 2 +- test/intfuncs.jl | 11 +++++++---- test/numbers.jl | 9 +++++---- 7 files changed, 51 insertions(+), 30 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 6b18f4a2e111f..f7c6d85fd6560 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -7127,7 +7127,9 @@ qr """ invmod(x,m) -Take the inverse of `x` modulo `m`: `y` such that ``xy = 1 \\pmod m``. +Take the inverse of `x` modulo `m`: `y` such that ``x y = 1 \\pmod m``, +with ``div(x,y) = 0``. This is undefined for ``m = 0``, or if +``gcd(x,m) \\neq 1``. """ invmod diff --git a/base/gmp.jl b/base/gmp.jl index 975a94814fc87..70c63e273b2d7 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -260,14 +260,21 @@ for (fJ, fC) in ((:+, :add), (:-,:sub), (:*, :mul), end function invmod(x::BigInt, y::BigInt) - z = BigInt() - y = abs(y) - if y == 1 - return big(0) + z = zero(BigInt) + ya = abs(y) + if ya == 1 + return z + end + if (y==0 || ccall((:__gmpz_invert, :libgmp), Cint, (Ptr{BigInt}, Ptr{BigInt}, Ptr{BigInt}), &z, &x, &ya) == 0) + throw(DomainError()) end - if (y==0 || ccall((:__gmpz_invert, :libgmp), Cint, (Ptr{BigInt}, Ptr{BigInt}, Ptr{BigInt}), &z, &x, &y) == 0) - error("no inverse exists") + # GMP always returns a positive inverse; we instead want to + # normalize such that div(z, y) == 0, i.e. we want a negative z + # when y is negative. + if y < 0 + z = z + y end + # The postcondition is: mod(z * x, y) == mod(big(1), m) && div(z, y) == 0 return z end diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 64fdabb4d01fd..a46a3046be5a8 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -53,7 +53,7 @@ lcm{T<:Integer}(abc::AbstractArray{T}) = reduce(lcm,abc) Computes the greatest common (positive) divisor of `x` and `y` and their Bézout coefficients, i.e. the integer coefficients `u` and `v` that satisfy -``ux+vy = d = gcd(x,y)``. +``ux+vy = d = gcd(x,y)``. ``gcdx(x,y)`` returns ``(d,u,v)``. ```jldoctest julia> gcdx(12, 42) @@ -67,15 +67,20 @@ julia> gcdx(240, 46) !!! note Bézout coefficients are *not* uniquely defined. `gcdx` returns the minimal - Bézout coefficients that are computed by the extended Euclid algorithm. - (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients `u` - and `v` are minimal in the sense that ``|u| < |\\frac y d|`` and ``|v| < - |\\frac x d|``. Furthermore, the signs of `u` and `v` are chosen so that `d` - is positive. + Bézout coefficients that are computed by the extended Euclidean algorithm. + (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) + For signed integers, these coefficients `u` and `v` are minimal in + the sense that ``|u| < |y/d|`` and ``|v| < |x/d|``. Furthermore, + the signs of `u` and `v` are chosen so that `d` is positive. + For unsigned integers, the coefficients `u` and `v` might be near + their `typemax`, and the identity then holds only via the unsigned + integers' modulo arithmetic. """ function gcdx{T<:Integer}(a::T, b::T) + # a0, b0 = a, b s0, s1 = one(T), zero(T) t0, t1 = s1, s0 + # The loop invariant is: s0*a0 + t0*b0 == a while b != 0 q = div(a, b) a, b = b, rem(a, b) @@ -87,13 +92,16 @@ end gcdx(a::Integer, b::Integer) = gcdx(promote(a,b)...) # multiplicative inverse of n mod m, error if none -function invmod(n, m) +function invmod{T<:Integer}(n::T, m::T) g, x, y = gcdx(n, m) - if g != 1 || m == 0 - error("no inverse exists") - end - x < 0 ? abs(m) + x : x + (g != 1 || m == 0) && throw(DomainError()) + # Note that m might be negative here. + # For unsigned T, x might be close to typemax; add m to force a wrap-around. + r = mod(x + m, m) + # The postcondition is: mod(r * n, m) == mod(T(1), m) && div(r, m) == 0 + r end +invmod(n::Integer, m::Integer) = invmod(promote(n,m)...) # ^ for any x supporting * to_power_type(x::Number) = oftype(x*x, x) diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 75844d7c06f9c..f6e0fe5496fca 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1266,7 +1266,7 @@ Mathematical Functions .. Docstring generated from Julia source - Computes the greatest common (positive) divisor of ``x`` and ``y`` and their Bézout coefficients, i.e. the integer coefficients ``u`` and ``v`` that satisfy :math:`ux+vy = d = gcd(x,y)`\ . + Computes the greatest common (positive) divisor of ``x`` and ``y`` and their Bézout coefficients, i.e. the integer coefficients ``u`` and ``v`` that satisfy :math:`ux+vy = d = gcd(x,y)`\ . :math:`gcdx(x,y)` returns :math:`(d,u,v)`\ . .. doctest:: @@ -1279,7 +1279,7 @@ Mathematical Functions (2,-9,47) .. note:: - Bézout coefficients are *not* uniquely defined. ``gcdx`` returns the minimal Bézout coefficients that are computed by the extended Euclid algorithm. (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) These coefficients ``u`` and ``v`` are minimal in the sense that :math:`|u| < |\frac y d|` and :math:`|v| < |\frac x d|`\ . Furthermore, the signs of ``u`` and ``v`` are chosen so that ``d`` is positive. + Bézout coefficients are *not* uniquely defined. ``gcdx`` returns the minimal Bézout coefficients that are computed by the extended Euclidean algorithm. (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) For signed integers, these coefficients ``u`` and ``v`` are minimal in the sense that :math:`|u| < |y/d|` and :math:`|v| < |x/d|`\ . Furthermore, the signs of ``u`` and ``v`` are chosen so that ``d`` is positive. For unsigned integers, the coefficients ``u`` and ``v`` might be near their ``typemax``\ , and the identity then holds only via the unsigned integers' modulo arithmetic. .. function:: ispow2(n) -> Bool @@ -1322,7 +1322,7 @@ Mathematical Functions .. Docstring generated from Julia source - Take the inverse of ``x`` modulo ``m``\ : ``y`` such that :math:`xy = 1 \pmod m`\ . + Take the inverse of ``x`` modulo ``m``\ : ``y`` such that :math:`x y = 1 \pmod m`\ , with :math:`div(x,y) = 0`\ . This is undefined for :math:`m = 0`\ , or if :math:`gcd(x,m) \neq 1`\ . .. function:: powermod(x, p, m) diff --git a/test/examples.jl b/test/examples.jl index 494a93deebd82..08c6d13c3863e 100644 --- a/test/examples.jl +++ b/test/examples.jl @@ -18,7 +18,7 @@ x = ModInts.ModInt{256}(13) y = inv(x) @test y == ModInts.ModInt{256}(197) @test x*y == ModInts.ModInt{256}(1) -@test_throws ErrorException inv(ModInts.ModInt{8}(4)) +@test_throws DomainError inv(ModInts.ModInt{8}(4)) include(joinpath(dir, "ndgrid.jl")) r = repmat(1:10,1,10) diff --git a/test/intfuncs.jl b/test/intfuncs.jl index 50a1344ba7daa..7a6c84c54670e 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -35,10 +35,13 @@ end @test gcdx(5, -12) == (1, 5, 2) @test gcdx(-25, -4) == (1, -1, 6) -@test invmod(6, 31) == 26 -@test invmod(-1, 3) == 2 -@test invmod(-1, -3) == 2 -@test_throws ErrorException invmod(0, 3) +@test invmod(6, 31) === 26 +@test invmod(-1, 3) === 2 +@test invmod(1, -3) === -2 +@test invmod(-1, -3) === -1 +@test invmod(0x2, 0x3) === 0x2 +@test invmod(2, 0x3) === 2 +@test_throws DomainError invmod(0, 3) @test powermod(2, 3, 5) == 3 @test powermod(2, 3, -5) == -2 diff --git a/test/numbers.jl b/test/numbers.jl index a85a5b543490f..8ef819c0685ae 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2167,11 +2167,12 @@ for T in (Int32,Int64), ii = -20:20, jj = -20:20 @test d == gcd(ib,jb) @test lcm(i,j) == lcm(ib,jb) @test gcdx(i,j) == gcdx(ib,jb) - if j == 0 - @test_throws ErrorException invmod(i,j) - @test_throws ErrorException invmod(ib,jb) - elseif d == 1 + if j == 0 || d != 1 + @test_throws DomainError invmod(i,j) + @test_throws DomainError invmod(ib,jb) + else n = invmod(i,j) + @test div(n, j) == 0 @test n == invmod(ib,jb) @test mod(n*i,j) == mod(1,j) end From ac666d039c92a5321f4e9137db75aa93d3450dc7 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 16:14:28 -0400 Subject: [PATCH 0666/1117] Run contrib/add_license_to_files.jl --- src/jitlayers.h | 2 ++ src/timing.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/jitlayers.h b/src/jitlayers.h index cc27c6a84cdba..6ed1c43782fcf 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -1,3 +1,5 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + #include <llvm/IR/LLVMContext.h> #include <llvm/IR/Constants.h> #include <llvm/IR/Module.h> diff --git a/src/timing.cpp b/src/timing.cpp index 8a4db8d7d3170..772e75eb041fa 100644 --- a/src/timing.cpp +++ b/src/timing.cpp @@ -1,3 +1,5 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + #include <inttypes.h> #include "julia.h" #include "options.h" From 28c795a05f1ad50621a51aa854c637a42d6fd0d1 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 16:10:23 -0400 Subject: [PATCH 0667/1117] Bump VERSION to 0.5.0-rc0 See #17418 - this is not entirely ready for release since there are a few known bugs we need to fix, but it is feature-complete and ready for package authors to test against. Master is still in bug/doc/test-fix only mode, we will branch release-0.5 when closer to RC1 which will be more complete. Note that VERSION will stay at 0.5.0-rc0 until rc1 (or branching, whichever comes first), builds between rc0 and rc1 will be v"0.5.0-rc0+n". --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4d64de0d547d2..ce3afe3b9964a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0-pre +0.5.0-rc0 From 3cdf5efed8ffe98a5da516988e30a91306032aac Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Tue, 26 Jul 2016 15:35:34 -0500 Subject: [PATCH 0668/1117] Be symmetric in type-promotion in intersect --- base/array.jl | 2 +- test/ranges.jl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/base/array.jl b/base/array.jl index 23c8044854ee5..19e548054e338 100644 --- a/base/array.jl +++ b/base/array.jl @@ -938,7 +938,7 @@ end # These are moderately efficient, preserve order, and remove dupes. function intersect(v1, vs...) - ret = Array{eltype(v1)}(0) + ret = Array{promote_eltype(v1, vs...)}(0) for v_elem in v1 inall = true for vsi in vs diff --git a/test/ranges.jl b/test/ranges.jl index 0c0607c06ce9e..610531f32ba58 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -116,6 +116,9 @@ end @test intersect(UnitRange(1,2),3) == UnitRange(3,2) @test intersect(UnitRange(1,2), UnitRange(1,5), UnitRange(3,7), UnitRange(4,6)) == UnitRange(4,3) +@test intersect(1:3, 2) === intersect(2, 1:3) === 2:2 +@test intersect(1.0:3.0, 2) == intersect(2, 1.0:3.0) == [2.0] + @test sort(UnitRange(1,2)) == UnitRange(1,2) @test sort!(UnitRange(1,2)) == UnitRange(1,2) @test sort(1:10, rev=true) == collect(10:-1:1) From d4a5d80200724947fabf1190993ddf59441d1101 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Tue, 26 Jul 2016 15:36:05 -0500 Subject: [PATCH 0669/1117] Generalizations for AbstractUnitRange in intersect and findin --- base/range.jl | 6 +++--- test/ranges.jl | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/base/range.jl b/base/range.jl index 358ada339af9e..5c60b3078edc1 100644 --- a/base/range.jl +++ b/base/range.jl @@ -557,7 +557,7 @@ intersect{T<:Integer}(i::Integer, r::AbstractUnitRange{T}) = i < first(r) ? (first(r):i) : i > last(r) ? (i:last(r)) : (i:i) -intersect{T<:Integer}(r::UnitRange{T}, i::Integer) = intersect(i, r) +intersect{T<:Integer}(r::AbstractUnitRange{T}, i::Integer) = intersect(i, r) function intersect{T1<:Integer, T2<:Integer}(r::AbstractUnitRange{T1}, s::StepRange{T2}) if isempty(s) @@ -661,12 +661,12 @@ function _findin{T1<:Integer, T2<:Integer}(r::Range{T1}, span::AbstractUnitRange ifirst, ilast end -function findin{T1<:Integer, T2<:Integer}(r::UnitRange{T1}, span::UnitRange{T2}) +function findin{T1<:Integer, T2<:Integer}(r::AbstractUnitRange{T1}, span::AbstractUnitRange{T2}) ifirst, ilast = _findin(r, span) ifirst:ilast end -function findin{T1<:Integer, T2<:Integer}(r::Range{T1}, span::UnitRange{T2}) +function findin{T1<:Integer, T2<:Integer}(r::Range{T1}, span::AbstractUnitRange{T2}) ifirst, ilast = _findin(r, span) ifirst:1:ilast end diff --git a/test/ranges.jl b/test/ranges.jl index 610531f32ba58..c92ebeb385bbf 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -766,6 +766,11 @@ for i in r end @test intersect(r, Base.OneTo(2)) == Base.OneTo(2) @test intersect(r, 0:5) == 1:3 +@test intersect(r, 2) === intersect(2, r) === 2:2 +@test findin(r, r) === findin(r, 1:length(r)) === findin(1:length(r), r) === 1:length(r) +r2 = Base.OneTo(7) +@test findin(r2, 2:length(r2)-1) === 2:length(r2)-1 +@test findin(2:length(r2)-1, r2) === 1:length(r2)-2 io = IOBuffer() show(io, r) str = takebuf_string(io) From 3759feb1c5610a15795e16484f78f113585782a5 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Tue, 26 Jul 2016 20:59:00 -0400 Subject: [PATCH 0670/1117] default dotview to getindex, using view only for AbstractArray --- base/broadcast.jl | 19 +++++++------------ doc/manual/functions.rst | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 4c2a881489dd1..1dbb93d24493c 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -440,25 +440,20 @@ end ############################################################ # x[...] .= f.(y...) ---> broadcast!(f, dotview(x, ...), y...). -# The dotview function defaults to view, but we override it in +# The dotview function defaults to getindex, but we override it in # a few cases to get the expected in-place behavior without affecting # explicit calls to view. (All of this can go away if slices # are changed to generate views by default.) -dotview(args...) = view(args...) +dotview(args...) = getindex(args...) +dotview(A::AbstractArray, args...) = view(A, args...) # avoid splatting penalty in common cases: for nargs = 0:5 args = Symbol[Symbol("x",i) for i = 1:nargs] - eval(Expr(:(=), Expr(:call, :dotview, args...), Expr(:call, :view, args...))) + eval(Expr(:(=), Expr(:call, :dotview, args...), + Expr(:call, :getindex, args...))) + eval(Expr(:(=), Expr(:call, :dotview, :(A::AbstractArray), args...), + Expr(:call, :view, :A, args...))) end -# for a[i...] .= ... where a is an array-of-arrays, just pass a[i...] directly -# to broadcast! -dotview{T<:AbstractArray,N,I<:Integer}(a::AbstractArray{T,N}, i::Vararg{I,N}) = - a[i...] - -# dict[k] .= ... should work if dict[k] is an array -dotview(a::Associative, k) = a[k] -dotview(a::Associative, k1, k2, ks...) = a[tuple(k1,k2,ks...)] - end # module diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 139a63323dd7d..6fbe105171586 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -660,7 +660,7 @@ calls do not allocate new arrays over and over again for the results except that, as above, the ``broadcast!`` loop is fused with any nested "dot" calls. For example, ``X .= sin.(Y)`` is equivalent to ``broadcast!(sin, X, Y)``, overwriting ``X`` with ``sin.(Y)`` in-place. -If the left-hand side is a ``getindex`` expression, e.g. +If the left-hand side is an array-indexing expression, e.g. ``X[2:end] .= sin.(Y)``, then it translates to ``broadcast!`` on a ``view``, e.g. ``broadcast!(sin, view(X, 2:endof(X)), Y)``, so that the left-hand side is updated in-place. From b7deca1068c27508532001cdd41c58f8a30a70a9 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Tue, 26 Jul 2016 21:12:31 -0400 Subject: [PATCH 0671/1117] Make sure the LLVM metadata is initialized before our custom LLVM passes Fix #17600 --- src/codegen.cpp | 10 +++++++--- src/llvm-gcroot.cpp | 1 + src/llvm-ptls.cpp | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 7ac5ceea6d81f..74a39a26c1fd3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4886,8 +4886,9 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs #define V128_BUG #endif -static void init_julia_llvm_env(Module *m) +static void init_julia_llvm_meta(void) { + mbuilder = new MDBuilder(jl_LLVMContext); MDNode *tbaa_root = mbuilder->createTBAARoot("jtbaa"); tbaa_gcframe = tbaa_make_child("jtbaa_gcframe", tbaa_root); tbaa_stack = tbaa_make_child("jtbaa_stack", tbaa_root); @@ -4904,7 +4905,10 @@ static void init_julia_llvm_env(Module *m) tbaa_arraylen = tbaa_make_child("jtbaa_arraylen", tbaa_array); tbaa_arrayflags = tbaa_make_child("jtbaa_arrayflags", tbaa_array); tbaa_const = tbaa_make_child("jtbaa_const", tbaa_root, true); +} +static void init_julia_llvm_env(Module *m) +{ // every variable or function mapped in this function must be // exported from libjulia, to support static compilation T_int1 = Type::getInt1Ty(jl_LLVMContext); @@ -5731,6 +5735,8 @@ extern "C" void jl_init_codegen(void) jl_TargetMachine->setFastISel(true); #endif + init_julia_llvm_meta(); + #ifdef USE_ORCJIT jl_ExecutionEngine = new JuliaOJIT(*jl_TargetMachine); #else @@ -5747,8 +5753,6 @@ extern "C" void jl_init_codegen(void) jl_ExecutionEngine->DisableLazyCompilation(); #endif - mbuilder = new MDBuilder(jl_LLVMContext); - // Now that the execution engine exists, initialize all modules jl_setup_module(engine_module); jl_setup_module(m); diff --git a/src/llvm-gcroot.cpp b/src/llvm-gcroot.cpp index d0129583b8b9b..5c9da2b175eb2 100644 --- a/src/llvm-gcroot.cpp +++ b/src/llvm-gcroot.cpp @@ -983,5 +983,6 @@ void jl_dump_bb_uses(std::map<BasicBlock*, std::map<frame_register, liveness::id Pass *createLowerGCFramePass(MDNode *tbaa_gcframe) { + assert(tbaa_gcframe); return new LowerGCFrame(tbaa_gcframe); } diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 5ebb3ce5d3ed5..178faf6c3b28e 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -189,5 +189,6 @@ static RegisterPass<LowerPTLS> X("LowerPTLS", "LowerPTLS Pass", Pass *createLowerPTLSPass(bool imaging_mode, MDNode *tbaa_const) { + assert(tbaa_const); return new LowerPTLS(imaging_mode, tbaa_const); } From c793eaa053c03970e9389870e2fbe693d9c44bd0 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Tue, 26 Jul 2016 21:53:20 -0400 Subject: [PATCH 0672/1117] re-fix array-of-arrays case --- base/broadcast.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/broadcast.jl b/base/broadcast.jl index 1dbb93d24493c..937d66375bd9a 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -447,6 +447,7 @@ end dotview(args...) = getindex(args...) dotview(A::AbstractArray, args...) = view(A, args...) +dotview{T<:AbstractArray}(A::AbstractArray{T}, args...) = getindex(A, args...) # avoid splatting penalty in common cases: for nargs = 0:5 args = Symbol[Symbol("x",i) for i = 1:nargs] From b2462ecd93c7557cd099d04996efb49428f62a1f Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Wed, 27 Jul 2016 08:09:17 +0200 Subject: [PATCH 0673/1117] Fix typos in commit-name.sh comments [ci skip] --- contrib/commit-name.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/commit-name.sh b/contrib/commit-name.sh index f808694884b68..1b81c846b8a5e 100755 --- a/contrib/commit-name.sh +++ b/contrib/commit-name.sh @@ -1,8 +1,8 @@ #!/bin/sh # This file is a part of Julia. License is MIT: http://julialang.org/license -# Need to be run from a julia repo clone -# First argument (Optional) is a ref to the commit +# Needs to be run from a julia repo clone +# First argument (optional) is a ref to the commit gitref=${1:-HEAD} From 5c12a19b0e16ac94354f9a8592fe85bdff6a4c68 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Wed, 27 Jul 2016 09:17:50 +0200 Subject: [PATCH 0674/1117] Add summary of supported markdown syntax Outline all the syntax that Julia's markdown parser currently supports and provide examples of each. This should not be considered a complete markdown tutorial, but just a summary and style guide for Julia's flavour of markdown. --- doc/manual/documentation.rst | 340 ++++++++++++++++++++++++++++++++++- 1 file changed, 333 insertions(+), 7 deletions(-) diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index 7b6d8945e097c..4d38658912d20 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -494,15 +494,340 @@ Examining it's definition should serve as an example of how to use ``@__doc__`` ``@__doc__`` has no effect when a macro that uses it is not documented. -Markdown Syntax Notes ---------------------- +Markdown syntax +--------------- -Julia's Markdown parser supports most of the basic Markdown elements, -including paragraphs, code blocks, bulleted lists and basic links. It's -also a work in progress, however, and support for more advanced things -like tables is in the works. +The following markdown syntax is supported in Julia. -Markdown.jl supports interpolation in a very similar way to basic string +Inline elements +~~~~~~~~~~~~~~~ + +Here "inline" refers to elements that can be found within blocks of text, i.e. paragraphs. These include the following elements. + +Bold +^^^^ + +Surround words with two asterisks, ``**``, to display the enclosed text in boldface. + +.. code-block:: text + + A paragraph containing a **bold** word. + +Italics +^^^^^^^ + +Surround words with one asterisk, ``*``, to display the enclosed text in italics. + +.. code-block:: text + + A paragraph containing an *emphasised* word. + +Literals +^^^^^^^^ + +Surround text that should be displayed exactly as written with single backticks, ````` . + +.. code-block:: text + + A paragraph containing a `literal` word. + +Literals should be used when writing text that refers to names of variables, functions, or other parts of a Julia program. + +.. tip:: + + To include a backtick character within literal text use three backticks rather than one to enclose the text. + + .. code-block:: text + + A paragraph containing a ``` `backtick` character ```. + + By extension any odd number of backticks may be used to enclose a lesser number of backticks. + +:math:`\LaTeX` +^^^^^^^^^^^^^^ + +Surround text that should be displayed as mathematics using :math:`\LaTeX` syntax with double backticks, `````` . + +.. code-block:: text + + A paragraph containing some ``\LaTeX`` markup. + +.. tip:: + + As with literals in the previous section, if literal backticks need to be written within double backticks use an even number greater than two. Note that if a single literal backtick needs to be included within :math:`\LaTeX` markup then two enclosing backticks is sufficient. + +Links +^^^^^ + +Links to either external or internal addresses can be written using the following syntax, where the text enclosed in square brackets, ``[ ]``, is the name of the link and the text enclosed in parentheses, ``( )``, is the URL. + +.. code-block:: text + + A paragraph containing a link to [Julia](http://www.julialang.org). + +Footnote references +^^^^^^^^^^^^^^^^^^^ + +Named and numbered footnote references can be written using the following syntax. A footnote name must be a single alphanumeric word containing no punctuation. + +.. code-block:: text + + A paragraph containing a numbered footnote [^1] and a named one [^named]. + +.. note:: + + The text associated with a footnote can be written anywhere within the same page as the footnote reference. The syntax used to define the footnote text is discussed in the `Footnotes`_ section below. + +Toplevel elements +~~~~~~~~~~~~~~~~~ + +The following elements can be written either at the "toplevel" of a document or within another "toplevel" element. + +Paragraphs +^^^^^^^^^^ + +A paragraph is a block of plain text, possibly containing any number of inline elements defined in the `Inline elements`_ section above, with one or more blank lines above and below it. + +.. code-block:: text + + This is a paragraph. + + And this is *another* one containing some emphasised text. + A new line, but still part of the same paragraph. + +Headers +^^^^^^^ + +A document can be split up into different sections using headers. Headers use the following syntax: + +.. code-block:: text + + # Level One + ## Level Two + ### Level Three + #### Level Four + ##### Level Five + ###### Level Six + +A header line can contain any inline syntax in the same way as a paragraph can. + +.. tip:: + + Try to avoid using too many levels of header within a single document. A heavily nested document may be indicative of a need to restructure it or split it into several pages covering separate topics. + +Code blocks +^^^^^^^^^^^ + +Source code can be displayed as a literal block using an indent of four spaces as shown in the following example. + +.. code-block:: text + + This is a paragraph. + + function func(x) + # ... + end + + Another paragraph. + +Additionally, code blocks can be enclosed using triple backticks with an optional "language" to specify how a block of code should be highlighted. + +.. code-block:: text + + A code block without a "language": + + ``` + function func(x) + # ... + end + ``` + + and another one with the "language" specified as `julia`: + + ```julia + function func(x) + # ... + end + ``` + +.. note:: + + "Fenced" code blocks, as shown in the last example, should be prefered over indented code blocks since there is no way to specify what language an indented code block is written in. + +Block quotes +^^^^^^^^^^^^ + +Text from external sources, such as quotations from books or websites, can be quoted using ``>`` characters prepended to each line of the quote as follows. + +.. code-block:: text + + Here's a quote: + + > Julia is a high-level, high-performance dynamic programming language for + > technical computing, with syntax that is familiar to users of other + > technical computing environments. + +Note that a single space must appear after the ``>`` character on each line. Quoted blocks may themselves contain other toplevel or inline elements. + +Images +^^^^^^ + +The syntax for images is similar to the link syntax mentioned above. Prepending a ``!`` character to a link will display an image from the specified URL rather than a link to it. + +.. code-block:: text + + ![alternative text](link/to/image.png) + +Lists +^^^^^ + +Unordered lists can be written by prepending each item in a list with either ``*``, ``+``, or ``-``. + +.. code-block:: text + + A list of items: + + * item one + * item two + * item three + +Note the two spaces before each ``*`` and the single space after each one. + +Lists can contain other nested toplevel elements such as lists, code blocks, or quoteblocks. A blank line should be left between each list item when including any toplevel elements within a list. + +.. code-block:: text + + Another list: + + * item one + + * item two + + ``` + f(x) = x + ``` + + * And a sublist: + + + sub-item one + + sub-item two + +.. note:: + + The contents of each item in the list must line up with the first line of the item. In the above example the fenced code block must be indented by four spaces to align with the ``i`` in ``item two``. + +Ordered lists are written by replacing the "bullet" character, either ``*``, ``+``, or ``-``, with a positive integer followed by either ``.`` or ``)``. + +.. code-block:: text + + Two ordered lists: + + 1. item one + 2. item two + 3. item three + + + 5) item five + 6) item six + 7) item seven + +An ordered list may start from a number other than one, as in the second list of the above example, where it is numbered from five. As with unordered lists, ordered lists can contain nested toplevel elements. + +Display equations +^^^^^^^^^^^^^^^^^ + +Large :math:`\LaTeX` equations that do not fit inline within a paragraph may be written as display equations using a fenced code block with the "language" ``math`` as in the example below. + +.. code-block:: text + + ```math + f(a) = \frac{1}{2\pi}\int_{0}^{2\pi} (\alpha+R\cos(\theta))d\theta + ``` + +Footnotes +^^^^^^^^^ + +This syntax is paired with the inline syntax for `Footnote references`_. Make sure to read that section as well. + +Footnote text is defined using the following syntax, which is similar to footnote reference syntax, aside from the ``:`` character that is appended to the footnote label. + +.. code-block:: text + + [^1]: Numbered footnote text. + + [^note]: + + Named footnote text containing several toplevel elements. + + * item one + * item two + * item three + + ```julia + function func(x) + # ... + end + ``` + +.. note:: + + No checks are done during parsing to make sure that all footnote references have matching footnotes. + +Horizontal rules +^^^^^^^^^^^^^^^^ + +The equivalent of an ``<hr>`` HTML tag can be written using the following syntax: + +.. code-block:: text + + Text above the line. + + --- + + And text below the line. + +Tables +^^^^^^ + +Basic tables can be written using the syntax described below. Note that markdown tables have limited features and cannot contain nested toplevel elements unlike other elements discussed above -- only inline elements are allowed. Tables must always contain a header row with column names. Cells cannot span multiple rows or columns of the table. + +.. code-block:: text + + | Column One | Column Two | Column Three | + |:---------- | ---------- |:------------:| + | Row `1` | Column `2` | | + | *Row* 2 | **Row** 2 | Column ``3`` | + +.. note:: + + As illustrated in the above example each column of ``|`` characters must be aligned vertically. + + A ``:`` character on either end of a column's header separator (the row containing ``-`` characters) specifies whether the row is left-aligned, right-aligned, or (when ``:`` appears on both ends) center-aligned. Providing no ``:`` characters will default to right-aligning the column. + +Admonitions +^^^^^^^^^^^ + +Specially formatted blocks with titles such as "Notes", "Warning", or "Tips" are known as admonitions and are used when some part of a document needs special attention. They can be defined using the following ``!!!`` syntax: + +.. code-block:: text + + !!! note + + This is the content of the note. + + !!! warning "Beware!" + + And this is another one. + + This warning admonition has a custom title: `"Beware!"`. + +Admonitions, like most other toplevel elements, can contain other toplevel elements. When no title text, specified after the admonition type in double quotes, is included then the title used will be the type of the block, i.e. ``"Note"`` in the case of the ``note`` admonition. + + +Markdown Syntax Extensions +-------------------------- + +Julia's markdown supports interpolation in a very similar way to basic string literals, with the difference that it will store the object itself in the Markdown tree (as opposed to converting it to a string). When the Markdown content is rendered the usual ``show`` methods will be @@ -513,3 +838,4 @@ references) without cluttering the basic syntax. In principle, the Markdown parser itself can also be arbitrarily extended by packages, or an entirely custom flavour of Markdown can be used, but this should generally be unnecessary. + From 02d534a0beb506ddc41ba5967516a3dcc86c81a7 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Sat, 23 Jul 2016 14:24:30 -0400 Subject: [PATCH 0675/1117] Update NEWS with tooling improvements. Remove DEBUGGER.md. [ci skip] --- DEBUGGER.md | 6 ------ NEWS.md | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) delete mode 100644 DEBUGGER.md diff --git a/DEBUGGER.md b/DEBUGGER.md deleted file mode 100644 index 6caf8e7f69a77..0000000000000 --- a/DEBUGGER.md +++ /dev/null @@ -1,6 +0,0 @@ -The Julia Debugger ------------------- - -To install the Julia debugger, run `Pkg.add("Gallium")` and see documentation at - -https://github.com/Keno/Gallium.jl#gallium diff --git a/NEWS.md b/NEWS.md index a9d2f8125c84c..91cb0498bbdac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,22 @@ Julia v0.5.0 Release Notes ========================== +Language tooling improvements +----------------------------- + + * The [Julia debugger](https://github.com/Keno/Gallium.jl) makes its debut + with this release. Install it with `Pkg.add("Gallium")`, and the + [documentation](https://github.com/Keno/Gallium.jl#gallium) should + get you going. The [JuliaCon + talk](https://www.youtube.com/watch?v=e6-hcOHO0tc&list=PLP8iPy9hna6SQPwZUDtAM59-wPzCPyD_S&index=5) + on Gallium shows off various features of the debugger. + + * The [Juno IDE](http://junolab.org) has matured significantly, and now + also includes support for plotting and debugging. + + * [Cxx.jl](https://github.com/Keno/Cxx.jl) provides a convenient FFI for + calling C++ code from Julia. + New language features --------------------- @@ -35,11 +51,6 @@ New language features * Function return type syntax `function f()::T` has been added ([#1090]). Values returned from a function with such a declaration will be converted to the specified type `T`. - * Experimental support for arrays with indexing starting at values - different from 1. The array types are expected to be defined in - packages, but now Julia provides an API for writing generic - algorithms for arbitrary indexing schemes ([#16260]). - * Many more operators now support `.` prefixes (e.g. `.≤`) ([#17393]). However, users are discouraged from overloading these, since they are mainly parsed in order to implement backwards compatibility with planned automatic @@ -47,6 +58,19 @@ New language features operator names like `Base.≤` should now use `Base.:≤` (prefixed by `@compat` if you need 0.4 compatibility via the `Compat` package). +Experimental language features +------------------------------ + + * Support for + [multi-threading](http://docs.julialang.org/en/latest/manual/parallel-computing/#multi-threading-experimental). Loops + with independent iterations can be easily parallelized with the + `Threads.@threads` macro. + + * Support for arrays with indexing starting at values different from + 1. The array types are expected to be defined in packages, but now + Julia provides an API for writing generic algorithms for arbitrary + indexing schemes ([#16260]). + Language changes ---------------- @@ -88,6 +112,9 @@ Compiler/Runtime improvements * Machine SIMD types can be represented in Julia as a homogeneous tuple of `VecElement` ([#15244]). + * The performance of higher-order and anonymous functions has been greatly improved. + For example, `map(x->2x, A)` performs as well as `2.*A`([#13412]). + New architectures ----------------- From 46081f0f146f5a01851a6585cbec2621d150708a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 27 Jul 2016 10:58:08 -0400 Subject: [PATCH 0676/1117] more NEWS additions fixes #16982, mention `writemime` deprecation [ci skip] --- NEWS.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/NEWS.md b/NEWS.md index 91cb0498bbdac..f2982ce5516db 100644 --- a/NEWS.md +++ b/NEWS.md @@ -115,6 +115,12 @@ Compiler/Runtime improvements * The performance of higher-order and anonymous functions has been greatly improved. For example, `map(x->2x, A)` performs as well as `2.*A`([#13412]). + * On windows, a DLL of standard library code is now precompiled and used by default, + improving startup time ([#16953]). + + * LLVM has been upgraded to version 3.7.1, improving the quality of generated + code and debug info. However compile times may be slightly longer ([#14623]). + New architectures ----------------- @@ -332,6 +338,9 @@ Library improvements * A new function `chown()` changes the ownership of files. ([#15007]) + * Display properties can now be passed among output functions (e.g. `show`) + using an `IOContext` object ([#13825]). + Deprecated or removed --------------------- @@ -364,6 +373,9 @@ Deprecated or removed * The no-op `transpose` fallback has been deprecated. Consider introducing suitable `transpose` methods or calling `permutedims(x, [2,1])` ([#13171], [#17075], [#17374]). + * `writemime` is deprecated, and output methods specifying a MIME type are now + methods of `show` ([#14052]). + [PkgDev]: https://github.com/JuliaLang/PkgDev.jl <!--- generated by NEWS-update.jl: --> [#550]: https://github.com/JuliaLang/julia/issues/550 @@ -399,7 +411,9 @@ Deprecated or removed [#13774]: https://github.com/JuliaLang/julia/issues/13774 [#13780]: https://github.com/JuliaLang/julia/issues/13780 [#13824]: https://github.com/JuliaLang/julia/issues/13824 +[#13825]: https://github.com/JuliaLang/julia/issues/13825 [#13897]: https://github.com/JuliaLang/julia/issues/13897 +[#14052]: https://github.com/JuliaLang/julia/issues/14052 [#14114]: https://github.com/JuliaLang/julia/issues/14114 [#14194]: https://github.com/JuliaLang/julia/issues/14194 [#14243]: https://github.com/JuliaLang/julia/issues/14243 @@ -407,6 +421,7 @@ Deprecated or removed [#14424]: https://github.com/JuliaLang/julia/issues/14424 [#14469]: https://github.com/JuliaLang/julia/issues/14469 [#14519]: https://github.com/JuliaLang/julia/issues/14519 +[#14623]: https://github.com/JuliaLang/julia/issues/14623 [#14759]: https://github.com/JuliaLang/julia/issues/14759 [#14798]: https://github.com/JuliaLang/julia/issues/14798 [#15007]: https://github.com/JuliaLang/julia/issues/15007 @@ -443,6 +458,7 @@ Deprecated or removed [#16645]: https://github.com/JuliaLang/julia/issues/16645 [#16663]: https://github.com/JuliaLang/julia/issues/16663 [#16731]: https://github.com/JuliaLang/julia/issues/16731 +[#16953]: https://github.com/JuliaLang/julia/issues/16953 [#16972]: https://github.com/JuliaLang/julia/issues/16972 [#17037]: https://github.com/JuliaLang/julia/issues/17037 [#17075]: https://github.com/JuliaLang/julia/issues/17075 From 211f6132167242537744967ff6a5f0726e04d2d8 Mon Sep 17 00:00:00 2001 From: Lyndon White <oxinabox@ucc.asn.au> Date: Wed, 27 Jul 2016 23:27:05 +0800 Subject: [PATCH 0677/1117] Made for better branch prediction --- base/math.jl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/base/math.jl b/base/math.jl index 32267c994de7f..c4648cf1e3010 100644 --- a/base/math.jl +++ b/base/math.jl @@ -133,15 +133,17 @@ end @inline function exp2(x::Base.BitInteger) if x > 1023 Inf64 - elseif x < -1074 - Float64(0.0) elseif x <= -1023 - # Result will be a subnormal number - reinterpret(Float64, Int64(1) << (x + 1074)) + if x < -1074 + # Result will be a subnormal number + reinterpret(Float64, (1 % Int64) << (x + 1074)) + else #-1073 < x <= -1023 + 0.0 + end else # If x is a Int128, and is outside the range of Int64, then it is not -1023<x<=1023 # We will cast everything to Int64 to avoid errors in case of Int128 - reinterpret(Float64, (exponent_bias(Float64) + Int64(x)) << significand_bits(Float64)) + reinterpret(Float64, (exponent_bias(Float64) + (x % Int64)) << significand_bits(Float64)) end end From d714553d379504891abe9a6b99eec89224c05f42 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 27 Jul 2016 10:37:39 -0700 Subject: [PATCH 0678/1117] prepare_release.sh: strip -rc# from julianightlies download url add note about code signing, and add arm [ci skip] --- contrib/prepare_release.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/contrib/prepare_release.sh b/contrib/prepare_release.sh index 0d564ab163f07..24727a461cc61 100755 --- a/contrib/prepare_release.sh +++ b/contrib/prepare_release.sh @@ -14,6 +14,8 @@ if [ -z "$tag" ]; then fi version=$(cat VERSION) majmin=$(cut -d. -f1-2 VERSION) +# remove -rc# if present +majminpatch=$(cut -d- -f1 VERSION) if [ "$tag" != "v$version" ]; then echo "error: tagged commit does not match content of VERSION file" >&2 exit 1 @@ -33,21 +35,27 @@ rm -rf julia-$version # download and rename binaries, with -latest copies julianightlies="https://s3.amazonaws.com/julianightlies/bin" curl -L -o julia-$version-linux-x86_64.tar.gz \ - $julianightlies/linux/x64/$majmin/julia-$version-$shashort-linux64.tar.gz + $julianightlies/linux/x64/$majmin/julia-$majminpatch-$shashort-linux64.tar.gz cp julia-$version-linux-x86_64.tar.gz julia-$majmin-latest-linux-x86_64.tar.gz curl -L -o julia-$version-linux-i686.tar.gz \ - $julianightlies/linux/x86/$majmin/julia-$version-$shashort-linux32.tar.gz + $julianightlies/linux/x86/$majmin/julia-$majminpatch-$shashort-linux32.tar.gz cp julia-$version-linux-i686.tar.gz julia-$majmin-latest-linux-i686.tar.gz +curl -L -o julia-$version-linux-arm.tar.gz \ + $julianightlies/linux/arm/$majmin/julia-$majminpatch-$shashort-linuxarm.tar.gz +cp julia-$version-linux-arm.tar.gz julia-$majmin-latest-linux-arm.tar.gz curl -L -o "julia-$version-osx10.7 .dmg" \ - $julianightlies/osx/x64/$majmin/julia-$version-$shashort-osx.dmg + $julianightlies/osx/x64/$majmin/julia-$majminpatch-$shashort-osx.dmg cp "julia-$version-osx10.7 .dmg" "julia-$majmin-latest-osx10.7 .dmg" curl -L -o julia-$version-win64.exe \ - $julianightlies/winnt/x64/$majmin/julia-$version-$shashort-win64.exe + $julianightlies/winnt/x64/$majmin/julia-$majminpatch-$shashort-win64.exe cp julia-$version-win64.exe julia-$majmin-latest-win64.exe curl -L -o julia-$version-win32.exe \ - $julianightlies/winnt/x86/$majmin/julia-$version-$shashort-win32.exe + $julianightlies/winnt/x86/$majmin/julia-$majminpatch-$shashort-win32.exe cp julia-$version-win32.exe julia-$majmin-latest-win32.exe +echo "Note: if windows code signing is not working on the buildbots, then the" +echo "checksums need to be re-calculated after the binaries are manually signed!" + shasum -a 256 julia-$version* | grep -v sha256 | grep -v md5 > julia-$version.sha256 md5sum julia-$version* | grep -v sha256 | grep -v md5 > julia-$version.md5 From 1fcc686219bf374d1b57e70fe066132dbc6bc774 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 27 Jul 2016 14:36:08 -0400 Subject: [PATCH 0679/1117] fix return type lowering bug in #17613 --- src/julia-syntax.scm | 17 ++++++++++------- test/core.jl | 9 +++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index ba0123fb8da3d..a3c39d62b15a7 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -176,14 +176,17 @@ body (let ((meta (take-while (lambda (x) (and (pair? x) (memq (car x) '(line meta)))) - (cdr body)))) + (cdr body))) + (val (last body))) ;; wrap one-liners in `convert` instead of adding an ssavalue - (if (length= (cdr body) (+ 1 (length meta))) - (let ((val (last body))) - `(,(car body) ,@meta - ,(if (and (pair? val) (eq? (car val) 'return)) - `(return (call (top convert) ,rett ,(cadr val))) - `(call (top convert) ,rett ,val)))) + (if (and (length= (cdr body) (+ 1 (length meta))) + (not (expr-contains-p return? (if (return? val) + (cadr val) + val)))) + `(,(car body) ,@meta + ,(if (return? val) + `(return (call (top convert) ,rett ,(cadr val))) + `(call (top convert) ,rett ,val))) (let ((R (make-ssavalue))) `(,(car body) ,@meta (= ,R ,rett) diff --git a/test/core.jl b/test/core.jl index 2ed5c170e4181..394f7882dd0b3 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4364,6 +4364,15 @@ g1090{T}(x::T)::T = x+1.0 @test g1090(1) === 2 @test g1090(Float32(3)) === Float32(4) +function f17613_2(x)::Float64 + try + return x + catch + return x+1 + end +end +@test isa(f17613_2(1), Float64) + # issue #16783 function f16783() T = UInt32 From 6962e3f16c3344c642817b2e610d458af78d7724 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Fri, 22 Jul 2016 16:08:39 -0400 Subject: [PATCH 0680/1117] Don't prompt for passwords for non-encrypted keys Encrypted keys have `Proc-Type: 4,ENCRYPTED`, so check for that before requiring a passkey. Part of #17541. --- base/libgit2/callbacks.jl | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 620c2e840ee73..50f4421329ee7 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -145,11 +145,22 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, end creds[:pubkey, credid] = publickey # save credentials - passphrase = if haskey(ENV,"SSH_KEY_PASS") - ENV["SSH_KEY_PASS"] + passphrase_required = true + if !isfile(privatekey) + warn("Private key not found") else + # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" + open(privatekey) do f + passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") + end + end + + passphrase = if haskey(ENV,"SSH_KEY_PASS") + ENV["SSH_KEY_PASS"] + else passdef = creds[:pass, credid] # check if credentials were already used - if passdef === nothing || isusedcreds + passdef === nothing && (passdef = "") + if passphrase_required && (isempty(passdef) || isusedcreds) if is_windows() passdef = Base.winprompt( "Your SSH Key requires a password, please enter it now:", @@ -160,8 +171,9 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, passdef = prompt("Passphrase for $privatekey", password=true) end end + passdef end - creds[:pass, credid] = passphrase # save credentials + creds[:pass, credid] = passphrase isempty(username) && return Cint(Error.EAUTH) From c2fa62475e7e2d5b99b2849276db94c94c15e194 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Fri, 22 Jul 2016 16:40:11 -0400 Subject: [PATCH 0681/1117] Try ~/.ssh/id_rsa first by default Only prompt for a different private key. If authentication with the first key fails. This can be a little annoying if one wants to use a different private key, but the default one is encrypted, so it queries for a password, without asking for the key first. However, in that case, one probably wants to use the environment variables or the ssh agent anyway, so I'm not too concerned. --- base/libgit2/callbacks.jl | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 50f4421329ee7..1e7bb0edc1663 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -114,15 +114,23 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, ENV["SSH_KEY_PATH"] else keydefpath = creds[:prvkey, credid] # check if credentials were already used - if keydefpath !== nothing && !isusedcreds + keydefpath === nothing && (keydefpath = "") + if !isempty(keydefpath) && !isusedcreds keydefpath # use cached value else - if keydefpath === nothing || isempty(keydefpath) - keydefpath = joinpath(homedir(),".ssh","id_rsa") + defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") + if isempty(keydefpath) && isfile(defaultkeydefpath) + keydefpath = defaultkeydefpath + else + keydefpath = + prompt("Private key location for '$schema$username@$host'", default=keydefpath) end - prompt("Private key location for '$schema$username@$host'", default=keydefpath) end end + + # If the private key changed, invalidate the cached public key + (privatekey != creds[:prvkey, credid]) && + (creds[:pubkey, credid] = "") creds[:prvkey, credid] = privatekey # save credentials # For SSH we need a public key location, look for environment vars SSH_* as well From c9e78553dae1d141f6c45ed6926f334e9ce379fe Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Fri, 22 Jul 2016 17:17:13 -0400 Subject: [PATCH 0682/1117] Pre-populate username prompt if present in the URL --- base/libgit2/callbacks.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 1e7bb0edc1663..40b24542d250a 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -59,6 +59,8 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, # parse url for schema and host urlparts = match(urlmatcher, url) schema = urlparts.captures[1] + urlusername = urlparts.captures[4] + urlusername = urlusername === nothing ? "" : String(urlusername) host = urlparts.captures[5] schema = schema === nothing ? "" : schema*"://" @@ -200,13 +202,14 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, if is_windows() if username === nothing || userpass === nothing || isusedcreds res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", - username === nothing ? "" : username; prompt_username = true) + username === nothing || isempty(username) ? + urlusername : username; prompt_username = true) isnull(res) && return Cint(Error.EAUTH) username, userpass = Base.get(res) end else if username === nothing || isusedcreds - username = prompt("Username for '$schema$host'") + username = prompt("Username for '$schema$host'", default = urlusername) end if userpass === nothing || isusedcreds From 90620d6ccd7b47b516ec27455fc83859d2fb54e3 Mon Sep 17 00:00:00 2001 From: Lyndon White <oxinabox@ucc.asn.au> Date: Thu, 28 Jul 2016 11:51:55 +0800 Subject: [PATCH 0683/1117] correct denormal vs zero + microoptimistations (previous commit made denormals zero.) --- base/math.jl | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/base/math.jl b/base/math.jl index c4648cf1e3010..3d062532243bb 100644 --- a/base/math.jl +++ b/base/math.jl @@ -134,16 +134,13 @@ end if x > 1023 Inf64 elseif x <= -1023 - if x < -1074 - # Result will be a subnormal number - reinterpret(Float64, (1 % Int64) << (x + 1074)) - else #-1073 < x <= -1023 - 0.0 - end + # if -1073 < x <= -1023 then Result will be a subnormal number + # Hex literal with padding must be use to work on 32bit machine + reinterpret(Float64, 0x0000_0000_0000_0001 << ((x + 1074)) % UInt) else - # If x is a Int128, and is outside the range of Int64, then it is not -1023<x<=1023 # We will cast everything to Int64 to avoid errors in case of Int128 - reinterpret(Float64, (exponent_bias(Float64) + (x % Int64)) << significand_bits(Float64)) + # If x is a Int128, and is outside the range of Int64, then it is not -1023<x<=1023 + reinterpret(Float64, (exponent_bias(Float64) + (x % Int64)) << (significand_bits(Float64)) % UInt) end end From ccd98755aec6a4d9b6a468da8d380be8d1b22f0a Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 27 Jul 2016 11:27:14 -0700 Subject: [PATCH 0684/1117] also gpg sign arm binaries --- contrib/prepare_release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/prepare_release.sh b/contrib/prepare_release.sh index 24727a461cc61..f8d50353b7344 100755 --- a/contrib/prepare_release.sh +++ b/contrib/prepare_release.sh @@ -63,6 +63,7 @@ gpg -u julia --armor --detach-sig julia-$version-full.tar.gz gpg -u julia --armor --detach-sig julia-$version.tar.gz gpg -u julia --armor --detach-sig julia-$version-linux-x86_64.tar.gz gpg -u julia --armor --detach-sig julia-$version-linux-i686.tar.gz +gpg -u julia --armor --detach-sig julia-$version-linux-arm.tar.gz echo "All files prepared. Attach julia-$version.tar.gz and julia-$version-full.tar.gz" echo "to github releases, upload all binaries and checksums to julialang S3. Be sure" From e729ba86290e01bf15ab1d78894f6781daa924c0 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 13:38:36 -0400 Subject: [PATCH 0685/1117] Upgrade utf8proc to 2.0.2 so its tests compile --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + deps/utf8proc.mk | 1 - deps/utf8proc.version | 4 ++-- 6 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 delete mode 100644 deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/sha512 create mode 100644 deps/checksums/utf8proc-e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b.tar.gz/md5 create mode 100644 deps/checksums/utf8proc-e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b.tar.gz/sha512 diff --git a/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 b/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 deleted file mode 100644 index 46106d558178e..0000000000000 --- a/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -9b562e952121df2438496d878ae086af diff --git a/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/sha512 b/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/sha512 deleted file mode 100644 index a3ca40fbb0bad..0000000000000 --- a/deps/checksums/utf8proc-a1fe9955bbc75ffb923c1219bf58befd2688e34c.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -790a103fd00501822dcb8744fc52e87ed1065b829d1f14aed74204b32186ffec5784c4f4901c4f3bd8b5529dc508461cf9374fd7bc3eadff0a183bb443a7abb2 diff --git a/deps/checksums/utf8proc-e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b.tar.gz/md5 b/deps/checksums/utf8proc-e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b.tar.gz/md5 new file mode 100644 index 0000000000000..2b394ed2436f8 --- /dev/null +++ b/deps/checksums/utf8proc-e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b.tar.gz/md5 @@ -0,0 +1 @@ +c85e1275fb558d9bad5c3fa0b496e457 diff --git a/deps/checksums/utf8proc-e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b.tar.gz/sha512 b/deps/checksums/utf8proc-e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b.tar.gz/sha512 new file mode 100644 index 0000000000000..08976e711026b --- /dev/null +++ b/deps/checksums/utf8proc-e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b.tar.gz/sha512 @@ -0,0 +1 @@ +e7dd0a575f94621c7b56eca28fced5934c3dd6675877842287d04ad74bacaf446d8e8c8cdce14963c819b09786037c0c46679ff2dc42818497bb1acbbb9751e6 diff --git a/deps/utf8proc.mk b/deps/utf8proc.mk index 95c156cb8fd5e..259560e16a85d 100644 --- a/deps/utf8proc.mk +++ b/deps/utf8proc.mk @@ -13,7 +13,6 @@ UTF8PROC_MFLAGS := CC="$(CC) $(DEPS_CFLAGS)" CFLAGS="$(CFLAGS) $(UTF8PROC_CF $(UTF8PROC_SRC_TARGET): $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/Makefile $(MAKE) -C $(dir $<) $(UTF8PROC_MFLAGS) libutf8proc.a touch -c $@ -# make check doesn't actually compile (missing prototype / implementation for getline) $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/checked: $(UTF8PROC_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) $(MAKE) -C $(dir $@) $(UTF8PROC_MFLAGS) check diff --git a/deps/utf8proc.version b/deps/utf8proc.version index c194a01b80cf6..ab5f84a39687f 100644 --- a/deps/utf8proc.version +++ b/deps/utf8proc.version @@ -1,2 +1,2 @@ -UTF8PROC_BRANCH=v2.0.1 -UTF8PROC_SHA1=a1fe9955bbc75ffb923c1219bf58befd2688e34c +UTF8PROC_BRANCH=v2.0.2 +UTF8PROC_SHA1=e3a5ed7b8bb5d0c6bb313d3e1f4d072c04113c4b From cdfe244f2ac35efc764ab723f536f735f40f3f3e Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 26 Jul 2016 13:21:42 -0400 Subject: [PATCH 0686/1117] Append a bit to dSFMT.h.patch in order to enable the dsfmt tests --- deps/dsfmt.mk | 7 ++++++- deps/patches/dSFMT.h.patch | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/deps/dsfmt.mk b/deps/dsfmt.mk index d8011de1a34a3..76ac86a5692a3 100644 --- a/deps/dsfmt.mk +++ b/deps/dsfmt.mk @@ -27,6 +27,11 @@ $(BUILDDIR)/dsfmt-$(DSFMT_VER)/config.status: $(SRCDIR)/srccache/dsfmt-$(DSFMT_V $(DSFMT_OBJ_SOURCE): $(BUILDDIR)/dsfmt-$(DSFMT_VER)/config.status cd $(dir $<) && \ $(CC) $(CPPFLAGS) $(DSFMT_CFLAGS) $(LDFLAGS) dSFMT.c -o libdSFMT.$(SHLIB_EXT) +$(BUILDDIR)/dsfmt-$(DSFMT_VER)/checked: $(DSFMT_OBJ_SOURCE) +ifeq ($(OS),$(BUILD_OS)) + $(MAKE) -C $(dir $@) std-check sse2-check +endif + echo 1 > $@ $(build_shlibdir)/libdSFMT%$(SHLIB_EXT) $(build_includedir)/dSFMT%h: $(DSFMT_OBJ_SOURCE) | $(build_includedir) $(build_shlibdir) cp $(dir $<)/dSFMT.h $(build_includedir) cp $< $(build_shlibdir)/libdSFMT.$(SHLIB_EXT) && \ @@ -40,5 +45,5 @@ distclean-dsfmt: get-dsfmt: $(SRCDIR)/srccache/dsfmt-$(DSFMT_VER).tar.gz configure-dsfmt: $(BUILDDIR)/dsfmt-$(DSFMT_VER)/config.status compile-dsfmt: $(DSFMT_OBJ_SOURCE) -check-dsfmt: compile-dsfmt +check-dsfmt: $(BUILDDIR)/dsfmt-$(DSFMT_VER)/checked install-dsfmt: $(DSFMT_OBJ_TARGET) diff --git a/deps/patches/dSFMT.h.patch b/deps/patches/dSFMT.h.patch index 1e87273d518be..b35a7fe3364d5 100644 --- a/deps/patches/dSFMT.h.patch +++ b/deps/patches/dSFMT.h.patch @@ -347,3 +347,16 @@ dsfmt_gv_fill_array_close1_open2(array, size); } #endif /* DSFMT_DO_NOT_USE_OLD_NAMES */ +diff --git a/test.c b/test.c +index 82d55db..d65db9a 100644 +--- a/test.c ++++ b/test.c +@@ -4,7 +4,7 @@ + #include <time.h> + #include <string.h> + #define DSFMT_DO_NOT_USE_OLD_NAMES +-#include "dSFMT.h" ++#include "dSFMT.h.orig" + + #define NUM_RANDS 50000 + #define TIC_MAG 1 From 481a20c713adca4c234a6c9cab9c988ee8cc80c8 Mon Sep 17 00:00:00 2001 From: Matthias Reisinger <matthias.j.reisinger@gmail.com> Date: Thu, 28 Jul 2016 11:26:57 +0200 Subject: [PATCH 0687/1117] Fix build for LLVM 4.0 after commit r276654 In LLVM commit r276654 the signature of Target::createMCAsmBackend() changed and now requires a fourth parameter of type MCTargetOptions. For our purposes it is sufficient to pass a default constructed MCTargetOptions instance since this seems to preserve the original behaviour. --- src/disasm.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/disasm.cpp b/src/disasm.cpp index 433d5020f94a7..df3f15604e11b 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -475,7 +475,10 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #else CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, *STI, Ctx); #endif -#ifdef LLVM34 +#ifdef LLVM40 + MCTargetOptions Options; + MAB = TheTarget->createMCAsmBackend(*MRI, TripleName, MCPU, Options); +#elif defined(LLVM34) MAB = TheTarget->createMCAsmBackend(*MRI, TripleName, MCPU); #else MAB = TheTarget->createMCAsmBackend(TripleName, MCPU); From 8a7895ae9d8d3d497d68fe1ff85b3a5b26b07c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Wed, 27 Jul 2016 21:50:13 +0200 Subject: [PATCH 0688/1117] Deprecate 1 argument write. fix #17612 --- base/deprecated.jl | 2 ++ base/iostream.jl | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index ac4eb4e1c5347..50f2386804559 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -780,4 +780,6 @@ end const _oldstyle_array_vcat_ = false +@deprecate write(x) write(STDOUT::IO, x) + # End deprecations scheduled for 0.6 diff --git a/base/iostream.jl b/base/iostream.jl index bd675c345b75c..0a1521450d956 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -191,8 +191,6 @@ function takebuf_raw(s::IOStream) return buf, sz end -write(x) = write(STDOUT::IO, x) - function readuntil(s::IOStream, delim::UInt8) ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8), s.ios, delim) end From 3184730065ca31a4eb89ff7851f8f379c6077087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Thu, 28 Jul 2016 19:56:28 +0200 Subject: [PATCH 0689/1117] Fix Vararg interaction with keyword function lowering,Fix #13919 and #17031 --- src/julia-syntax.scm | 2 +- test/keywordargs.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index a3c39d62b15a7..882229a0cbde6 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -379,7 +379,7 @@ (ftype (decl-type (car pargl))) ;; 1-element list of vararg argument, or empty if none (vararg (let ((l (if (null? pargl) '() (last pargl)))) - (if (vararg? l) + (if (or (vararg? l) (varargexpr? l)) (list l) '()))) ;; positional args without vararg (pargl (if (null? vararg) pargl (butlast pargl))) diff --git a/test/keywordargs.jl b/test/keywordargs.jl index bf6ac51b5e762..7a582b73645f1 100644 --- a/test/keywordargs.jl +++ b/test/keywordargs.jl @@ -21,6 +21,13 @@ kwf2(x, rest...; y=1) = (x, y, rest) @test_throws MethodError kwf2(0, z=1) @test_throws MethodError kwf2(y=1) +# Test for #13919 +test13919(x::Vararg{Int}; key=100) = (x, key) +@test test13919(1, 1)[1] === (1, 1) +@test test13919(1, 1)[2] === 100 +@test test13919(1, 1, key=10)[1] === (1, 1) +@test test13919(1, 1, key=10)[2] === 10 + # keyword arg with declared type kwf3(x; y::Float64 = 1.0) = x + y @test kwf3(2) == 3.0 From ac758fd0e35591365403b7fb5faef40b0b38439f Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 28 Jul 2016 14:24:26 -0400 Subject: [PATCH 0690/1117] fix #17662, failure in nullable test due to type cache corruption --- src/builtins.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/builtins.c b/src/builtins.c index c16890238bc51..b64c01be9611c 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1053,6 +1053,10 @@ static uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v) jl_datatype_t *dt = (jl_datatype_t*)tv; if (dt == jl_datatype_type) { jl_datatype_t *dtv = (jl_datatype_t*)v; + // `name->primary` is cacheable even though it contains TypeVars + // that don't have stable IDs. + if (jl_egal(dtv->name->primary, v)) + return bitmix(~dtv->name->hash, 0xaa5566aa); return bitmix(~dtv->name->hash, hash_svec(dtv->parameters)); } if (dt == jl_typename_type) From 37c9d44df40944d61ef94c239ef958abb6f01f14 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Thu, 28 Jul 2016 21:25:25 +0200 Subject: [PATCH 0691/1117] Format code literals consistently Wrap all Julia and Lispy syntax in backticks in devdocs/ast.rst. --- doc/devdocs/ast.rst | 134 ++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/doc/devdocs/ast.rst b/doc/devdocs/ast.rst index 72f1ca4774a36..7cfe51bb4ac18 100644 --- a/doc/devdocs/ast.rst +++ b/doc/devdocs/ast.rst @@ -229,13 +229,13 @@ Calls ======================= ==================================== Input AST ======================= ==================================== -f(x) (call f x) -f(x, y=1, z=2) (call f x (kw y 1) (kw z 2)) -f(x; y=1) (call f (parameters (kw y 1)) x) -f(x...) (call f (... x)) +``f(x)`` ``(call f x)`` +``f(x, y=1, z=2)`` ``(call f x (kw y 1) (kw z 2))`` +``f(x; y=1)`` ``(call f (parameters (kw y 1)) x)`` +``f(x...)`` ``(call f (... x))`` ======================= ==================================== -``Do`` syntax:: +``do`` syntax:: f(x) do a,b body @@ -258,54 +258,54 @@ Finally, chains of comparisons have their own special expression structure. ======================= ==================================== Input AST ======================= ==================================== -x+y (call + x y) -a+b+c+d (call + a b c d) -2x (call * 2 x) -a&&b (&& a b) -x += 1 (+= x 1) -a ? 1 : 2 (if a 1 2) -a:b (: a b) -a:b:c (: a b c) -a,b (tuple a b) -a==b (comparison a == b) -1<i<=n (comparison 1 < i <= n) -a.b (. a (quote b)) -a.(b) (. a b) +``x+y`` ``(call + x y)`` +``a+b+c+d`` ``(call + a b c d)`` +``2x`` ``(call * 2 x)`` +``a&&b`` ``(&& a b)`` +``x += 1`` ``(+= x 1)`` +``a ? 1 : 2`` ``(if a 1 2)`` +``a:b`` ``(: a b)`` +``a:b:c`` ``(: a b c)`` +``a,b`` ``(tuple a b)`` +``a==b`` ``(comparison a == b)`` +``1<i<=n`` ``(comparison 1 < i <= n)`` +``a.b`` ``(. a (quote b))`` +``a.(b)`` ``(. a b)`` ======================= ==================================== Bracketed forms ~~~~~~~~~~~~~~~ -======================= ==================================== -Input AST -======================= ==================================== -a[i] (ref a i) -t[i;j] (typed_vcat t i j) -t[i j] (typed_hcat t i j) -t[a b; c d] (typed_vcat t (row a b) (row c d)) -a{b} (curly a b) -a{b;c} (curly a (parameters c) b) -[x] (vect x) -[x,y] (vect x y) -[x;y] (vcat x y) -[x y] (hcat x y) -[x y; z t] (vcat (row x y) (row z t)) -[x for y in z, a in b] (comprehension x (= y z) (= a b)) -T[x for y in z] (typed_comprehension T x (= y z)) -(a, b, c) (tuple a b c) -(a; b; c) (block a (block b c)) -======================= ==================================== +========================== ====================================== +Input AST +========================== ====================================== +``a[i]`` ``(ref a i)`` +``t[i;j]`` ``(typed_vcat t i j)`` +``t[i j]`` ``(typed_hcat t i j)`` +``t[a b; c d]`` ``(typed_vcat t (row a b) (row c d))`` +``a{b}`` ``(curly a b)`` +``a{b;c}`` ``(curly a (parameters c) b)`` +``[x]`` ``(vect x)`` +``[x,y]`` ``(vect x y)`` +``[x;y]`` ``(vcat x y)`` +``[x y]`` ``(hcat x y)`` +``[x y; z t]`` ``(vcat (row x y) (row z t))`` +``[x for y in z, a in b]`` ``(comprehension x (= y z) (= a b))`` +``T[x for y in z]`` ``(typed_comprehension T x (= y z))`` +``(a, b, c)`` ``(tuple a b c)`` +``(a; b; c)`` ``(block a (block b c))`` +========================== ====================================== Macros ~~~~~~ -======================= ==================================== +======================= ======================================= Input AST -======================= ==================================== -@m x y (macrocall @m x y) -Base.@m x y (macrocall (. Base (quote @m)) x y) -@Base.m x y (macrocall (. Base (quote @m)) x y) -======================= ==================================== +======================= ======================================= +``@m x y`` ``(macrocall @m x y)`` +``Base.@m x y`` ``(macrocall (. Base (quote @m)) x y)`` +``@Base.m x y`` ``(macrocall (. Base (quote @m)) x y)`` +======================= ======================================= Strings ~~~~~~~ @@ -313,12 +313,12 @@ Strings ======================= ==================================== Input AST ======================= ==================================== -"a" "a" -x"y" (macrocall @x_str "y") -x"y"z (macrocall @x_str "y" "z") -"x = $x" (string "x = " x) -\`a b c\` (macrocall @cmd "a b c") -x ~ distr (macrocall @~ x distr) +``"a"`` ``"a"`` +``x"y"`` ``(macrocall @x_str "y")`` +``x"y"z`` ``(macrocall @x_str "y" "z")`` +``"x = $x"`` ``(string "x = " x)`` +```a b c``` ``(macrocall @cmd "a b c")`` +``x ~ distr`` ``(macrocall @~ x distr)`` ======================= ==================================== Doc string syntax:: @@ -326,22 +326,22 @@ Doc string syntax:: "some docs" f(x) = x -parses as ``(macrocall (|.| Base '@doc) "some docs" (= (call f x) (block x)))`` +parses as ``(macrocall (|.| Core '@doc) "some docs" (= (call f x) (block x)))``. Imports and such ~~~~~~~~~~~~~~~~ -======================= ==================================== +======================= ============================================== Input AST -======================= ==================================== -import a (import a) -import a.b.c (import a b c) -import ...a (import . . . a) -import a.b, c.d (toplevel (import a b) (import c d)) -import Base: x (import Base x) -import Base: x, y (toplevel (import Base x) (import Base y)) -export a, b (export a b) -======================= ==================================== +======================= ============================================== +``import a`` ``(import a)`` +``import a.b.c`` ``(import a b c)`` +``import ...a`` ``(import . . . a)`` +``import a.b, c.d`` ``(toplevel (import a b) (import c d))`` +``import Base: x`` ``(import Base x)`` +``import Base: x, y`` ``(toplevel (import Base x) (import Base y))`` +``export a, b`` ``(export a b)`` +======================= ============================================== Numbers ~~~~~~~ @@ -349,13 +349,13 @@ Numbers Julia supports more number types than many scheme implementations, so not all numbers are represented directly as scheme numbers in the AST. -======================= ==================================== -Input AST -======================= ==================================== -11111111111111111111 (macrocall @int128_str "11111111111111111111") -0xfffffffffffffffff (macrocall @uint128_str "0xfffffffffffffffff") -1111...many digits... (macrocall @big_str "1111....") -======================= ==================================== +========================= ================================================== +Input AST +========================= ================================================== +``11111111111111111111`` ``(macrocall @int128_str "11111111111111111111")`` +``0xfffffffffffffffff`` ``(macrocall @uint128_str "0xfffffffffffffffff")`` +``1111...many digits...`` ``(macrocall @big_str "1111....")`` +========================= ================================================== Block forms ~~~~~~~~~~~ From 9fd82e16d6c6cb730245794247792f36498fa075 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Thu, 28 Jul 2016 21:28:02 +0200 Subject: [PATCH 0692/1117] Indent ordered list correctly Sphinx didn't parse the list correctly since it wasn't consistently indented to line up with the first line of each item. --- doc/manual/documentation.rst | 124 +++++++++++++++++------------------ 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index 4d38658912d20..f03f60a8b4ba2 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -50,95 +50,95 @@ As in the example above, we recommend following some simple conventions when wri documentation: 1. Always show the signature of a function at the top of the documentation, -with a four-space indent so that it is printed as Julia code. - - This can be identical to the signature present in the Julia code - (like ``mean(x::AbstractArray)``), or a simplified form. - Optional arguments should be represented with their default values (i.e. ``f(x, y=1)``) - when possible, following the actual Julia syntax. Optional arguments which - do not have a default value should be put in brackets (i.e. ``f(x[, y])`` and - ``f(x[, y[, z]])``). An alternative solution is to use several lines: one without - optional arguments, the other(s) with them. This solution can also be used to document - several related methods of a given function. When a function accepts many keyword - arguments, only include a ``<keyword arguments>`` placeholder in the signature (i.e. - ``f(x; <keyword arguments>)``), and give the complete list under an ``# Arguments`` - section (see point 4 below). + with a four-space indent so that it is printed as Julia code. + + This can be identical to the signature present in the Julia code + (like ``mean(x::AbstractArray)``), or a simplified form. + Optional arguments should be represented with their default values (i.e. ``f(x, y=1)``) + when possible, following the actual Julia syntax. Optional arguments which + do not have a default value should be put in brackets (i.e. ``f(x[, y])`` and + ``f(x[, y[, z]])``). An alternative solution is to use several lines: one without + optional arguments, the other(s) with them. This solution can also be used to document + several related methods of a given function. When a function accepts many keyword + arguments, only include a ``<keyword arguments>`` placeholder in the signature (i.e. + ``f(x; <keyword arguments>)``), and give the complete list under an ``# Arguments`` + section (see point 4 below). 2. Include a single one-line sentence describing what the function does or what the -object represents after the simplified signature block. If needed, provide more details -in a second paragraph, after a blank line. + object represents after the simplified signature block. If needed, provide more details + in a second paragraph, after a blank line. - The one-line sentence should use the imperative form ("Do this", "Return that") instead - of the third person (do not write "Returns the length...") when documenting functions. - It should end with a period. If the meaning of a function cannot be summarized easily, - splitting it into separate composable parts could be beneficial (this should not be - taken as an absolute requirement for every single case though). + The one-line sentence should use the imperative form ("Do this", "Return that") instead + of the third person (do not write "Returns the length...") when documenting functions. + It should end with a period. If the meaning of a function cannot be summarized easily, + splitting it into separate composable parts could be beneficial (this should not be + taken as an absolute requirement for every single case though). 3. Do not repeat yourself. - Since the function name is given by the signature, there is no need to - start the documentation with "The function ``bar``...": go straight to the point. - Similarly, if the signature specifies the types of the arguments, mentioning them - in the description is redundant. + Since the function name is given by the signature, there is no need to + start the documentation with "The function ``bar``...": go straight to the point. + Similarly, if the signature specifies the types of the arguments, mentioning them + in the description is redundant. 4. Only provide an argument list when really necessary. - For simple functions, it is often clearer to mention the role of the arguments directly - in the description of the function's purpose. An argument list would only repeat - information already provided elsewhere. However, providing an argument list can be a good - idea for complex functions with many arguments (in particular keyword arguments). - In that case, insert it after the general description of the function, under - an ``# Arguments`` header, with one ``*`` bullet for each argument. The list should - mention the types and default values (if any) of the arguments:: - - """ - ... - # Arguments - * `n::Integer`: the number of elements to compute. - * `dim::Integer=1`: the dimensions along which to perform the computation. - ... - """ + For simple functions, it is often clearer to mention the role of the arguments directly + in the description of the function's purpose. An argument list would only repeat + information already provided elsewhere. However, providing an argument list can be a good + idea for complex functions with many arguments (in particular keyword arguments). + In that case, insert it after the general description of the function, under + an ``# Arguments`` header, with one ``*`` bullet for each argument. The list should + mention the types and default values (if any) of the arguments:: + + """ + ... + # Arguments + * `n::Integer`: the number of elements to compute. + * `dim::Integer=1`: the dimensions along which to perform the computation. + ... + """ 5. Group examples under an ``# Examples`` section and use ````julia`` blocks instead of -standard text. + standard text. - Examples should consist of verbatim copies of the Julia REPL, including the ``julia>`` - prompt (see example above). This will be used in the future to allow running examples - automatically and checking that their actual output is consistent with that presented - in the documentation (a feature called *doctests*). This way, the code will be tested and - examples won't get out of date without notice. + Examples should consist of verbatim copies of the Julia REPL, including the ``julia>`` + prompt (see example above). This will be used in the future to allow running examples + automatically and checking that their actual output is consistent with that presented + in the documentation (a feature called *doctests*). This way, the code will be tested and + examples won't get out of date without notice. 6. Use backticks to identify code and equations. - Julia identifiers and code excerpts should always appear between backticks ````` - to enable highlighting. Equations in the LaTeX syntax can be inserted between - double backticks ``````. Use Unicode characters rather than their LaTeX escape sequence, - i.e. ````α = 1```` rather than :samp:`\`\`\\\\alpha = 1\`\``. + Julia identifiers and code excerpts should always appear between backticks ````` + to enable highlighting. Equations in the LaTeX syntax can be inserted between + double backticks ``````. Use Unicode characters rather than their LaTeX escape sequence, + i.e. ````α = 1```` rather than :samp:`\`\`\\\\alpha = 1\`\``. 7. Place the starting and ending ``"""`` characters on lines by themselves. - That is, write:: + That is, write:: - """ - ... + """ + ... - ... - """ - f(x, y) = ... + ... + """ + f(x, y) = ... - rather than:: + rather than:: - """... + """... - ...""" - f(x, y) = ... + ...""" + f(x, y) = ... - This makes it more clear where docstrings start and end. + This makes it more clear where docstrings start and end. 8. Respect the line length limit used in the surrounding code. - Docstrings are edited using the same tools as code. Therefore, the same conventions - should apply. It it advised to add line breaks after 92 characters. + Docstrings are edited using the same tools as code. Therefore, the same conventions + should apply. It it advised to add line breaks after 92 characters. Accessing Documentation ----------------------- From 0b6906558dc260a284a29eb04dad85aeebca17ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Thu, 28 Jul 2016 22:23:11 +0200 Subject: [PATCH 0693/1117] fix todo --- test/parse.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/parse.jl b/test/parse.jl index 6714351c4016d..665e2be48da62 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -276,15 +276,14 @@ parsehex(s) = parse(Int,s,16) @test parse(Int,"-0o1234") == -Int(0o1234) @test parse(Int,"-0b1011") == -Int(0b1011) -## FIXME: #4905, do these tests for Int128/UInt128! -for T in (Int8, Int16, Int32, Int64) +for T in (Int8, Int16, Int32, Int64, Int128) @test parse(T,string(typemin(T))) == typemin(T) @test parse(T,string(typemax(T))) == typemax(T) @test_throws OverflowError parse(T,string(big(typemin(T))-1)) @test_throws OverflowError parse(T,string(big(typemax(T))+1)) end -for T in (UInt8,UInt16,UInt32,UInt64) +for T in (UInt8,UInt16,UInt32,UInt64,UInt128) @test parse(T,string(typemin(T))) == typemin(T) @test parse(T,string(typemax(T))) == typemax(T) @test_throws ArgumentError parse(T,string(big(typemin(T))-1)) From a4dbfc3590390c15039945bd832ddc89e75d3edb Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Thu, 28 Jul 2016 16:23:45 -0400 Subject: [PATCH 0694/1117] Don't zero `usesshagent` `usesshagent` is just a flag to determine whether or not to use the agent, so zeroing it isn't necessary. Even worse, since it only gets a reference to one of the flag strings (stored in the AST), zeroing them globally changes those strings, disabling ssh-agent authentication entirely. Fixes the problem noted in #16041 --- base/libgit2/types.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index a3e9ecb3019ea..65f180e4d4c59 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -730,7 +730,6 @@ reset!(p::UserPasswordCredentials, cnt::Int=3) = (p.count = cnt) function securezero!(cred::UserPasswordCredentials) securezero!(cred.user) securezero!(cred.pass) - securezero!(cred.usesshagent) cred.count = 0 return cred end @@ -755,7 +754,6 @@ function securezero!(cred::SSHCredentials) securezero!(cred.pass) securezero!(cred.pubkey) securezero!(cred.prvkey) - securezero!(cred.usesshagent) return cred end From 786ac15d7122dc200c909115cf967f42411fe3fb Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 28 Jul 2016 16:24:06 -0400 Subject: [PATCH 0695/1117] null initialize jl_codectx_t struct and cleanup an unused function --- src/codegen.cpp | 6 +++--- src/julia.h | 1 - src/signal-handling.c | 7 ------- test/testdefs.jl | 1 + 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 74a39a26c1fd3..aca1f9f10d429 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3468,7 +3468,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t DebugLoc noDbg; builder.SetCurrentDebugLocation(noDbg); - jl_codectx_t ctx; + jl_codectx_t ctx = {}; ctx.f = cw; ctx.linfo = lam; ctx.sret = false; @@ -3826,7 +3826,7 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, Function *f, bool sre DebugLoc noDbg; builder.SetCurrentDebugLocation(noDbg); - jl_codectx_t ctx; + jl_codectx_t ctx = {}; ctx.f = w; ctx.linfo = lam; ctx.sret = false; @@ -3895,7 +3895,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func std::map<int, jl_arrayvar_t> arrayvars; std::map<int, BasicBlock*> labels; std::map<int, Value*> handlers; - jl_codectx_t ctx; + jl_codectx_t ctx = {}; ctx.arrayvars = &arrayvars; ctx.labels = &labels; ctx.handlers = &handlers; diff --git a/src/julia.h b/src/julia.h index 734361d28f2ee..3e3b2284300bf 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1404,7 +1404,6 @@ JL_DLLEXPORT void jl_yield(void); // async signal handling ------------------------------------------------------ -JL_DLLEXPORT void jl_sigint_action(void); JL_DLLEXPORT void jl_install_sigint_handler(void); JL_DLLEXPORT void jl_sigatomic_begin(void); JL_DLLEXPORT void jl_sigatomic_end(void); diff --git a/src/signal-handling.c b/src/signal-handling.c index 3c5a97cc1a673..cb2dda03ac7db 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -53,13 +53,6 @@ JL_DLLEXPORT void jl_exit_on_sigint(int on) exit_on_sigint = on; } -// what to do on SIGINT -JL_DLLEXPORT void jl_sigint_action(void) -{ - if (exit_on_sigint) jl_exit(130); // 128+SIGINT - jl_throw(jl_interrupt_exception); -} - #if defined(_WIN32) #include "signals-win.c" #else diff --git a/test/testdefs.jl b/test/testdefs.jl index 0c9a7cb4f002b..893670af8f74f 100644 --- a/test/testdefs.jl +++ b/test/testdefs.jl @@ -12,3 +12,4 @@ end # looking in . messes things up badly filter!(x->x!=".", LOAD_PATH) +nothing From 174e8cde4f32313e884cf023a26e4d762325d6fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Thu, 28 Jul 2016 22:29:40 +0200 Subject: [PATCH 0696/1117] Test for 10046 --- test/parse.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/parse.jl b/test/parse.jl index 665e2be48da62..9a802e90569fb 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -642,3 +642,9 @@ let nometh = expand(:(A15838.@f(1, 2))) @test e.f === getfield(A15838, Symbol("@f")) @test e.args === (1,2) end + +# issue 10046 +for op in ["+", "-", "\$", "|", ".+", ".-", "*", ".*"] + @test_throws ParseError parse("$op in [+, -]") +end + From ba20966c4ecd03c0c8ff6770fac252778cf34758 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 28 Jul 2016 19:40:02 -0400 Subject: [PATCH 0697/1117] NEWS for array concat syntax change. fixes #17672 [ci skip] --- NEWS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.md b/NEWS.md index f2982ce5516db..8d5d1126852fb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -81,6 +81,10 @@ Language changes Instead of adding methods to `call`, methods are added by type using the syntax `(::ftype)(...) = ...`. `call` is deprecated ([#13412]). + * Square brackets and commas (e.g. `[x, y]`) no longer concatenate arrays, and always + simply construct a vector of the provided values. If `x` and `y` are arrays, + `[x, y]` will be an array of arrays ([#3737], [#2488], [#8599]). + * `using` and `import` are now case-sensitive even on case-insensitive filesystems (common on Mac and Windows) ([#13542]). @@ -382,6 +386,8 @@ Deprecated or removed [#964]: https://github.com/JuliaLang/julia/issues/964 [#1090]: https://github.com/JuliaLang/julia/issues/1090 [#1765]: https://github.com/JuliaLang/julia/issues/1765 +[#2488]: https://github.com/JuliaLang/julia/issues/2488 +[#3737]: https://github.com/JuliaLang/julia/issues/3737 [#4163]: https://github.com/JuliaLang/julia/issues/4163 [#4211]: https://github.com/JuliaLang/julia/issues/4211 [#4470]: https://github.com/JuliaLang/julia/issues/4470 @@ -389,6 +395,7 @@ Deprecated or removed [#6190]: https://github.com/JuliaLang/julia/issues/6190 [#7258]: https://github.com/JuliaLang/julia/issues/7258 [#8036]: https://github.com/JuliaLang/julia/issues/8036 +[#8599]: https://github.com/JuliaLang/julia/issues/8599 [#8846]: https://github.com/JuliaLang/julia/issues/8846 [#9503]: https://github.com/JuliaLang/julia/issues/9503 [#9627]: https://github.com/JuliaLang/julia/issues/9627 From 788e01ecf83bbb50d62dcd27d8a1f49ed186a50d Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 28 Jul 2016 14:03:12 -0700 Subject: [PATCH 0698/1117] Make general concatenations (`cat`) of combinations of sparse matrices with sparse matrices or dense matrices/vectors yield sparse arrays (fixes #17680). --- base/abstractarray.jl | 2 +- base/array.jl | 3 +++ base/sparse/sparse.jl | 4 ++-- base/sparse/sparsematrix.jl | 6 ++++++ test/sparsedir/sparse.jl | 32 ++++++++++++++++++++++++-------- 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index ca46ba05d13c1..067cdd0093174 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -998,7 +998,7 @@ function cat_t(catdims, typeC::Type, X...) end end - C = similar(isa(X[1],AbstractArray) ? full(X[1]) : [X[1]], typeC, tuple(dimsC...)) + C = similar(isa(X[1],AbstractArray) ? X[1] : [X[1]], typeC, tuple(dimsC...)) if length(catdims)>1 fill!(C,0) end diff --git a/base/array.jl b/base/array.jl index 19e548054e338..4fc67b12ba356 100644 --- a/base/array.jl +++ b/base/array.jl @@ -699,6 +699,9 @@ hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Vector{T}...) = typed_hvcat(T, rows, xs.. hvcat(rows::Tuple{Vararg{Int}}, xs::Matrix...) = typed_hvcat(promote_eltype(xs...), rows, xs...) hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Matrix{T}...) = typed_hvcat(T, rows, xs...) +cat(catdims, xs::Union{Matrix,Vector}...) = Base.cat_t(catdims, promote_eltype(xs...), xs...) +cat{T}(catdims, xs::Union{Matrix{T},Vector{T}}...) = Base.cat_t(catdims, T, xs...) + ## find ## # returns the index of the next non-zero element, or 0 if all zeros diff --git a/base/sparse/sparse.jl b/base/sparse/sparse.jl index e303c0e0b8969..189a8562814df 100644 --- a/base/sparse/sparse.jl +++ b/base/sparse/sparse.jl @@ -21,10 +21,10 @@ import Base: @get!, acos, acosd, acot, acotd, acsch, asech, asin, asind, asinh, tand, tanh, trace, transpose!, tril!, triu!, trunc, vecnorm, abs, abs2, broadcast, ceil, complex, cond, conj, convert, copy, copy!, ctranspose, diagm, exp, expm1, factorize, find, findmax, findmin, findnz, float, full, getindex, - hcat, hvcat, imag, indmax, ishermitian, kron, length, log, log1p, max, min, + vcat, hcat, hvcat, cat, imag, indmax, ishermitian, kron, length, log, log1p, max, min, maximum, minimum, norm, one, promote_eltype, real, reinterpret, reshape, rot180, rotl90, rotr90, round, scale!, setindex!, similar, size, transpose, tril, - triu, vcat, vec, permute! + triu, vec, permute! import Base.Broadcast: broadcast_shape diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 876f1e74b0cae..821e04d4d18da 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -3253,6 +3253,12 @@ function hvcat(rows::Tuple{Vararg{Int}}, X::Union{Vector, Matrix, SparseMatrixCS vcat(tmp_rows...) end +function cat(catdims, Xin::Union{Vector, Matrix, SparseMatrixCSC}...) + X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] + T = promote_eltype(Xin...) + Base.cat_t(catdims, T, X...) +end + """ blkdiag(A...) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index fbe6e18f9a3b4..e6d4e9767e2b1 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -1554,14 +1554,30 @@ end @inferred sprand(1, 1, 1.0, rand, Float64) @inferred sprand(1, 1, 1.0, x->round(Int,rand(x)*100)) -# dense sparse concatenation -> sparse return type -@test issparse([sprand(10,10,.1) rand(10,10)]) -@test issparse([sprand(10,10,.1); rand(10,10)]) -@test issparse([sprand(10,10,.1) rand(10,10); rand(10,10) rand(10,10)]) -# --- -@test !issparse([rand(10,10) rand(10,10)]) -@test !issparse([rand(10,10); rand(10,10)]) -@test !issparse([rand(10,10) rand(10,10); rand(10,10) rand(10,10)]) +# Test that concatenations of combinations of sparse matrices with sparse matrices or dense +# matrices/vectors yield sparse arrays +let + N = 4 + densevec = ones(N) + densemat = diagm(ones(N)) + spmat = spdiagm(ones(N)) + # Test that concatenations of pairs of sparse matrices yield sparse arrays + @test issparse(vcat(spmat, spmat)) + @test issparse(hcat(spmat, spmat)) + @test issparse(hvcat((2,), spmat, spmat)) + @test issparse(cat((1,2), spmat, spmat)) + # Test that concatenations of a sparse matrice with a dense matrix/vector yield sparse arrays + @test issparse(vcat(spmat, densemat)) + @test issparse(vcat(densemat, spmat)) + for densearg in (densevec, densemat) + @test issparse(hcat(spmat, densearg)) + @test issparse(hcat(densearg, spmat)) + @test issparse(hvcat((2,), spmat, densearg)) + @test issparse(hvcat((2,), densearg, spmat)) + @test issparse(cat((1,2), spmat, densearg)) + @test issparse(cat((1,2), densearg, spmat)) + end +end # issue #14816 let m = 5 From ea860b9ccae468f9c36581a8d262d8f452f64018 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 28 Jul 2016 21:28:05 -0400 Subject: [PATCH 0699/1117] Fix #17462, break/continue in an at-testset for loop make the macro return a :for instead of a :comprehension --- base/test.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/base/test.jl b/base/test.jl index 2436a0800f5b9..0391180376ee3 100644 --- a/base/test.jl +++ b/base/test.jl @@ -732,7 +732,12 @@ function testset_forloop(args, testloop) pop_testset() finish(ts) end - Expr(:comprehension, blk, [esc(v) for v in loopvars]...) + quote + arr = Array{Any,1}(0) + $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), + :(push!(arr, $blk)))) + arr + end end """ From 931fc5d33cf8d6b46c7e97d5cda8915740dc01c3 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 28 Jul 2016 21:44:05 -0400 Subject: [PATCH 0700/1117] Add a test for #17462, break/continue in a testset for Add some counters to make sure things actually execute --- test/test.jl | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/test.jl b/test/test.jl index eb183681c82dd..39a33744ecc02 100644 --- a/test/test.jl +++ b/test/test.jl @@ -313,3 +313,28 @@ end @test @inferred(inferrable_kwtest(1; y=1)) == 2 @test @inferred(uninferrable_kwtest(1)) == 3 @test_throws ErrorException @inferred(uninferrable_kwtest(1; y=2)) == 2 + +# Issue #17462 +counter_17462_pre = 0 +counter_17462_post = 0 +@testset for x in [1,2,3,4] + counter_17462_pre += 1 + if x == 1 + @test counter_17462_pre == x + continue + @test false + elseif x == 3 + @test counter_17462_pre == x + break + @test false + elseif x == 4 + @test false + else + @test counter_17462_pre == x + @test x == 2 + @test counter_17462_post == 0 + end + counter_17462_post += 1 +end +@test counter_17462_pre == 3 +@test counter_17462_post == 1 From 4a733e702c760e31b6b893f54ebd0b3db6aece34 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 25 Jul 2016 15:25:01 -0400 Subject: [PATCH 0701/1117] ensure known_object_data is assigned before deserialize is called again also reduce code duplication for easier maintenance ref #16091 --- base/clusterserialize.jl | 52 +++++++++++++--------------------------- base/serialize.jl | 32 +++++++++++++++---------- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/base/clusterserialize.jl b/base/clusterserialize.jl index f8505ca9b7ae5..e265c34cebd10 100644 --- a/base/clusterserialize.jl +++ b/base/clusterserialize.jl @@ -9,45 +9,24 @@ type ClusterSerializer{I<:IO} <: AbstractSerializer counter::Int table::ObjectIdDict - sent_objects::Dict{UInt64, Bool} # used by serialize (track objects sent) + sent_objects::Set{UInt64} # used by serialize (track objects sent) - ClusterSerializer(io::I) = new(io, 0, ObjectIdDict(), Dict()) + ClusterSerializer(io::I) = new(io, 0, ObjectIdDict(), Set{UInt64}()) end ClusterSerializer(io::IO) = ClusterSerializer{typeof(io)}(io) function deserialize(s::ClusterSerializer, ::Type{TypeName}) - number, full_body_sent = deserialize(s) - makenew = false - known = haskey(known_object_data, number) + full_body_sent = deserialize(s) if !full_body_sent - if !known - error("Expected object in cache. Not found.") - else - tn = known_object_data[number]::TypeName + number = read(s.io, UInt64) + tn = get(known_object_data, number, nothing)::TypeName + if !haskey(object_numbers, tn) + # setup reverse mapping for serialize + object_numbers[tn] = number end + deserialize_cycle(s, tn) else - name = deserialize(s) - mod = deserialize(s) - if known - tn = known_object_data[number]::TypeName - elseif mod !== __deserialized_types__ && isdefined(mod, name) - tn = getfield(mod, name).name - # TODO: confirm somehow that the types match - #warn(mod, ".", name, " isdefined, need not have been serialized") - name = tn.name - mod = tn.module - else - name = gensym() - mod = __deserialized_types__ - tn = ccall(:jl_new_typename_in, Ref{TypeName}, (Any, Any), name, mod) - makenew = true - end - end - deserialize_cycle(s, tn) - full_body_sent && deserialize_typename_body(s, tn, number, name, mod, makenew) - !known && (known_object_data[number] = tn) - if !haskey(object_numbers, tn) - object_numbers[tn] = number + tn = invoke(deserialize, (AbstractSerializer, Type{TypeName}), s, TypeName) end return tn end @@ -57,15 +36,18 @@ function serialize(s::ClusterSerializer, t::TypeName) writetag(s.io, TYPENAME_TAG) identifier = object_number(t) - if !haskey(s.sent_objects, identifier) - serialize(s, (identifier, true)) + if !(identifier in s.sent_objects) + serialize(s, true) + write(s.io, identifier) serialize(s, t.name) serialize(s, t.module) serialize_typename_body(s, t) - s.sent_objects[identifier] = true + push!(s.sent_objects, identifier) # println(t.module, ":", t.name, ", id:", identifier, " sent") else - serialize(s, (identifier, false)) + serialize(s, false) + write(s.io, identifier) # println(t.module, ":", t.name, ", id:", identifier, " NOT sent") end + nothing end diff --git a/base/serialize.jl b/base/serialize.jl index f0e6d938fba49..d0d5e0cbe9d26 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -419,6 +419,7 @@ function serialize_typename_body(s::AbstractSerializer, t::TypeName) else writetag(s.io, UNDEFREF_TAG) end + nothing end # decide whether to send all data for a type (instead of just its name) @@ -729,29 +730,32 @@ end function deserialize(s::AbstractSerializer, ::Type{TypeName}) + # the deserialize_cycle call can be delayed, since neither + # Symbol nor Module will use the backref table number = read(s.io, UInt64) - name = deserialize(s) - mod = deserialize(s) - if haskey(known_object_data, number) - tn = known_object_data[number]::TypeName - name = tn.name - mod = tn.module + name = deserialize(s)::Symbol + mod = deserialize(s)::Module + tn = get(known_object_data, number, nothing) + if tn !== nothing makenew = false - elseif isdefined(mod, name) + elseif mod !== __deserialized_types__ && isdefined(mod, name) tn = getfield(mod, name).name # TODO: confirm somehow that the types match - name = tn.name - mod = tn.module makenew = false + known_object_data[number] = tn else name = gensym() mod = __deserialized_types__ tn = ccall(:jl_new_typename_in, Ref{TypeName}, (Any, Any), name, mod) makenew = true + known_object_data[number] = tn + end + if !haskey(object_numbers, tn) + # setup up reverse mapping for serialize + object_numbers[tn] = number end deserialize_cycle(s, tn) deserialize_typename_body(s, tn, number, name, mod, makenew) - makenew && (known_object_data[number] = tn) return tn end @@ -767,14 +771,18 @@ function deserialize_typename_body(s::AbstractSerializer, tn, number, name, mod, if makenew tn.names = names + # TODO: there's an unhanded cycle in the dependency graph at this point: + # while deserializing super and/or types, we may have encountered + # tn.primary and throw UndefRefException before we get to this point tn.primary = ccall(:jl_new_datatype, Any, (Any, Any, Any, Any, Any, Cint, Cint, Cint), tn, super, parameters, names, types, abstr, mutable, ninitialized) ty = tn.primary ccall(:jl_set_const, Void, (Any, Any, Any), mod, name, ty) - if !isdefined(ty,:instance) + if !isdefined(ty, :instance) if isempty(parameters) && !abstr && size == 0 && (!mutable || isempty(names)) - setfield!(ty, :instance, ccall(:jl_new_struct, Any, (Any,Any...), ty)) + # use setfield! directly to avoid `fieldtype` lowering expecting to see a Singleton object already on ty + Core.setfield!(ty, :instance, ccall(:jl_new_struct, Any, (Any, Any...), ty)) end end end From 47c3f8f2c0cdfa0cf6d685acb2681916d2ef6a4a Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 28 Jul 2016 14:29:48 -0400 Subject: [PATCH 0702/1117] fix anonymous function serialization logic should_send_whole_type was sending too many regular type that weren't functions and deserialize was trying to hide that error, but was then not deserializing things that were anonymous functions fix #16091 --- base/clusterserialize.jl | 23 ++++------- base/methodshow.jl | 1 + base/serialize.jl | 88 +++++++++++++++++++--------------------- test/parallel_exec.jl | 28 +++++++++++++ test/serialize.jl | 15 +++++++ 5 files changed, 95 insertions(+), 60 deletions(-) diff --git a/base/clusterserialize.jl b/base/clusterserialize.jl index e265c34cebd10..4f42a862c3ba9 100644 --- a/base/clusterserialize.jl +++ b/base/clusterserialize.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license import .Serializer: known_object_data, object_number, serialize_cycle, deserialize_cycle, writetag, - __deserialized_types__, serialize_typename_body, deserialize_typename_body, + __deserialized_types__, serialize_typename, deserialize_typename, TYPENAME_TAG, object_numbers type ClusterSerializer{I<:IO} <: AbstractSerializer @@ -17,8 +17,8 @@ ClusterSerializer(io::IO) = ClusterSerializer{typeof(io)}(io) function deserialize(s::ClusterSerializer, ::Type{TypeName}) full_body_sent = deserialize(s) + number = read(s.io, UInt64) if !full_body_sent - number = read(s.io, UInt64) tn = get(known_object_data, number, nothing)::TypeName if !haskey(object_numbers, tn) # setup reverse mapping for serialize @@ -26,7 +26,7 @@ function deserialize(s::ClusterSerializer, ::Type{TypeName}) end deserialize_cycle(s, tn) else - tn = invoke(deserialize, (AbstractSerializer, Type{TypeName}), s, TypeName) + tn = deserialize_typename(s, number) end return tn end @@ -36,18 +36,13 @@ function serialize(s::ClusterSerializer, t::TypeName) writetag(s.io, TYPENAME_TAG) identifier = object_number(t) - if !(identifier in s.sent_objects) - serialize(s, true) - write(s.io, identifier) - serialize(s, t.name) - serialize(s, t.module) - serialize_typename_body(s, t) + send_whole = !(identifier in s.sent_objects) + serialize(s, send_whole) + write(s.io, identifier) + if send_whole + serialize_typename(s, t) push!(s.sent_objects, identifier) -# println(t.module, ":", t.name, ", id:", identifier, " sent") - else - serialize(s, false) - write(s.io, identifier) -# println(t.module, ":", t.name, ", id:", identifier, " NOT sent") end +# println(t.module, ":", t.name, ", id:", identifier, send_whole ? " sent" : " NOT sent") nothing end diff --git a/base/methodshow.jl b/base/methodshow.jl index 79737b4f596f4..06a24df9377d3 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -79,6 +79,7 @@ function show(io::IO, m::Method; kwtype::Nullable{DataType}=Nullable{DataType}() decls = Any[(), ("...", "")] elseif ft <: Function && isdefined(ft.name.module, ft.name.mt.name) && + # TODO: more accurate test? (tn.name === "#" name) ft == typeof(getfield(ft.name.module, ft.name.mt.name)) print(io, ft.name.mt.name) elseif isa(ft, DataType) && is(ft.name, Type.name) && isleaftype(ft) diff --git a/base/serialize.jl b/base/serialize.jl index d0d5e0cbe9d26..39af8a55a7607 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -374,7 +374,7 @@ end function serialize(s::AbstractSerializer, g::GlobalRef) writetag(s.io, GLOBALREF_TAG) if g.mod === Main && isdefined(g.mod, g.name) && isconst(g.mod, g.name) - v = eval(g) + v = getfield(g.mod, g.name) if isa(v, DataType) && v === v.name.primary && should_send_whole_type(s, v) # handle references to types in Main by sending the whole type. # needed to be able to send nested functions (#15451). @@ -393,17 +393,16 @@ function serialize(s::AbstractSerializer, t::TypeName) serialize_cycle(s, t) && return writetag(s.io, TYPENAME_TAG) write(s.io, object_number(t)) - serialize(s, t.name) - serialize(s, t.module) - serialize_typename_body(s, t) + serialize_typename(s, t) end -function serialize_typename_body(s::AbstractSerializer, t::TypeName) +function serialize_typename(s::AbstractSerializer, t::TypeName) + serialize(s, t.name) serialize(s, t.names) serialize(s, t.primary.super) serialize(s, t.primary.parameters) serialize(s, t.primary.types) - serialize(s, t.primary.size) + serialize(s, isdefined(t.primary, :instance)) serialize(s, t.primary.abstract) serialize(s, t.primary.mutable) serialize(s, t.primary.ninitialized) @@ -423,24 +422,28 @@ function serialize_typename_body(s::AbstractSerializer, t::TypeName) end # decide whether to send all data for a type (instead of just its name) -function should_send_whole_type(s, t::ANY) +function should_send_whole_type(s, t::DataType) tn = t.name if isdefined(tn, :mt) # TODO improve somehow # send whole type for anonymous functions in Main - fname = tn.mt.name + name = tn.mt.name mod = tn.module - toplevel = isdefined(mod, fname) && isdefined(t, :instance) && - getfield(mod, fname) === t.instance - ishidden = unsafe_load(unsafe_convert(Ptr{UInt8}, fname))==UInt8('#') - return mod === __deserialized_types__ || (mod === Main && (ishidden || !toplevel)) + isanonfunction = mod === Main && # only Main + t.super === Function && # only Functions + unsafe_load(unsafe_convert(Ptr{UInt8}, tn.name)) == UInt8('#') && # hidden type + (!isdefined(mod, name) || t != typeof(getfield(mod, name))) # XXX: 95% accurate test for this being an inner function + # TODO: more accurate test? (tn.name !== "#" name) + #TODO: iskw = startswith(tn.name, "#kw#") && ??? + #TODO: iskw && return send-as-kwftype + return mod === __deserialized_types__ || isanonfunction end return false end # `type_itself` means we are serializing a type object. when it's false, we are # sending the type tag part of some other object's representation. -function serialize_type_data(s, t::ANY, type_itself::Bool) +function serialize_type_data(s, t::DataType, type_itself::Bool) whole = should_send_whole_type(s, t) form = type_itself ? UInt8(0) : UInt8(1) if whole @@ -725,28 +728,25 @@ function deserialize(s::AbstractSerializer, ::Type{Union}) Union{types...} end -module __deserialized_types__ -end - +module __deserialized_types__ end function deserialize(s::AbstractSerializer, ::Type{TypeName}) # the deserialize_cycle call can be delayed, since neither # Symbol nor Module will use the backref table number = read(s.io, UInt64) + return deserialize_typename(s, number) +end + +function deserialize_typename(s::AbstractSerializer, number) name = deserialize(s)::Symbol - mod = deserialize(s)::Module tn = get(known_object_data, number, nothing) if tn !== nothing makenew = false - elseif mod !== __deserialized_types__ && isdefined(mod, name) - tn = getfield(mod, name).name - # TODO: confirm somehow that the types match - makenew = false - known_object_data[number] = tn else - name = gensym() - mod = __deserialized_types__ - tn = ccall(:jl_new_typename_in, Ref{TypeName}, (Any, Any), name, mod) + # reuse the same name for the type, if possible, for nicer debugging + tn_name = isdefined(__deserialized_types__, name) ? gensym() : name + tn = ccall(:jl_new_typename_in, Ref{TypeName}, (Any, Any), + tn_name, __deserialized_types__) makenew = true known_object_data[number] = tn end @@ -755,19 +755,15 @@ function deserialize(s::AbstractSerializer, ::Type{TypeName}) object_numbers[tn] = number end deserialize_cycle(s, tn) - deserialize_typename_body(s, tn, number, name, mod, makenew) - return tn -end -function deserialize_typename_body(s::AbstractSerializer, tn, number, name, mod, makenew) - names = deserialize(s) - super = deserialize(s) - parameters = deserialize(s) - types = deserialize(s) - size = deserialize(s) - abstr = deserialize(s) - mutable = deserialize(s) - ninitialized = deserialize(s) + names = deserialize(s)::SimpleVector + super = deserialize(s)::Type + parameters = deserialize(s)::SimpleVector + types = deserialize(s)::SimpleVector + has_instance = deserialize(s)::Bool + abstr = deserialize(s)::Bool + mutable = deserialize(s)::Bool + ninitialized = deserialize(s)::Int32 if makenew tn.names = names @@ -778,12 +774,10 @@ function deserialize_typename_body(s::AbstractSerializer, tn, number, name, mod, tn, super, parameters, names, types, abstr, mutable, ninitialized) ty = tn.primary - ccall(:jl_set_const, Void, (Any, Any, Any), mod, name, ty) - if !isdefined(ty, :instance) - if isempty(parameters) && !abstr && size == 0 && (!mutable || isempty(names)) - # use setfield! directly to avoid `fieldtype` lowering expecting to see a Singleton object already on ty - Core.setfield!(ty, :instance, ccall(:jl_new_struct, Any, (Any, Any...), ty)) - end + ccall(:jl_set_const, Void, (Any, Any, Any), tn.module, tn.name, ty) + if has_instance && !isdefined(ty, :instance) + # use setfield! directly to avoid `fieldtype` lowering expecting to see a Singleton object already on ty + Core.setfield!(ty, :instance, ccall(:jl_new_struct, Any, (Any, Any...), ty)) end end @@ -791,9 +785,9 @@ function deserialize_typename_body(s::AbstractSerializer, tn, number, name, mod, if tag != UNDEFREF_TAG mtname = handle_deserialize(s, tag) defs = deserialize(s) - maxa = deserialize(s) + maxa = deserialize(s)::Int if makenew - tn.mt = ccall(:jl_new_method_table, Any, (Any, Any), name, mod) + tn.mt = ccall(:jl_new_method_table, Any, (Any, Any), name, tn.module) tn.mt.name = mtname tn.mt.defs = defs tn.mt.max_args = maxa @@ -806,6 +800,7 @@ function deserialize_typename_body(s::AbstractSerializer, tn, number, name, mod, end end end + return tn::TypeName end function deserialize_datatype(s::AbstractSerializer) @@ -853,7 +848,8 @@ function deserialize(s::AbstractSerializer, t::DataType) return ccall(:jl_new_struct, Any, (Any,Any...), t) elseif isbits(t) if nf == 1 - return ccall(:jl_new_struct, Any, (Any,Any...), t, deserialize(s)) + f1 = deserialize(s) + return ccall(:jl_new_struct, Any, (Any,Any...), t, f1) elseif nf == 2 f1 = deserialize(s) f2 = deserialize(s) diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 81d3d47a20507..6718c99a21eb0 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -1023,6 +1023,34 @@ end # issue #15451 @test remotecall_fetch(x->(y->2y)(x)+1, workers()[1], 3) == 7 +# issue #16091 +type T16091 end +wid = workers()[1] +@test try + remotecall_fetch(()->T16091, wid) + false +catch ex + ((ex::RemoteException).captured::CapturedException).ex === UndefVarError(:T16091) +end +@test try + remotecall_fetch(identity, wid, T16091) + false +catch ex + ((ex::RemoteException).captured::CapturedException).ex === UndefVarError(:T16091) +end + +f16091a() = 1 +remotecall_fetch(()->eval(:(f16091a() = 2)), wid) +@test remotecall_fetch(f16091a, wid) === 2 +@test remotecall_fetch((myid)->remotecall_fetch(f16091a, myid), wid, myid()) === 1 + +# these will only heisen-fail, since it depends on the gensym counter collisions: +f16091b = () -> 1 +remotecall_fetch(()->eval(:(f16091b = () -> 2)), wid) +@test remotecall_fetch(f16091b, 2) === 1 +@test remotecall_fetch((myid)->remotecall_fetch(f16091b, myid), wid, myid()) === 2 + + # issue #16451 rng=RandomDevice() retval = @parallel (+) for _ in 1:10 diff --git a/test/serialize.jl b/test/serialize.jl index d36622738d950..68c41ab89fc54 100644 --- a/test/serialize.jl +++ b/test/serialize.jl @@ -278,6 +278,21 @@ create_serialization_stream() do s # Base generic function @test deserialize(s)() === 1 end +# Anonymous Functions +create_serialization_stream() do s + local g() = :magic_token_anon_fun_test + serialize(s, g) + serialize(s, g) + + seekstart(s) + local g2 = deserialize(s) + @test g2 !== g + @test g2() == :magic_token_anon_fun_test + @test g2() == :magic_token_anon_fun_test + @test deserialize(s) === g2 +end + + # Task create_serialization_stream() do s # user-defined type array f = () -> begin task_local_storage(:v, 2); return 1+1 end From b908f51035219fd613d40713321b4fbd029152e1 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Fri, 29 Jul 2016 08:35:10 +0530 Subject: [PATCH 0703/1117] There have been no command line option changes, verified with a diff of the --help output on 0.4 and 0.5. Removing the empty section in the manual. [ci skip] --- NEWS.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8d5d1126852fb..23d2c164e4aad 100644 --- a/NEWS.md +++ b/NEWS.md @@ -108,9 +108,6 @@ Language changes When `x` is global, `x::T = ...` and `global x::T` used to mean type assertion, but this syntax is now reserved for type declaration ([#964]). -Command-line option changes ---------------------------- - Compiler/Runtime improvements ----------------------------- From 80584b571490a4ce9feeb2cad5e82fa34b57c162 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 28 Jul 2016 23:52:44 -0400 Subject: [PATCH 0704/1117] fix methodshow to be inline with its documentation --- base/docs/helpdb/Base.jl | 10 ---------- base/interactiveutil.jl | 15 +++++++++++++-- doc/stdlib/base.rst | 6 ++++-- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index f7c6d85fd6560..e914630b09a4f 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4561,16 +4561,6 @@ whichever shape has more dimensions. """ promote_shape -""" - methodswith(typ[, module or function][, showparents]) - -Return an array of methods with an argument of type `typ`. If optional `showparents` is -`true`, also return arguments with a parent type of `typ`, excluding type `Any`. - -The optional second argument restricts the search to a particular module or function. -""" -methodswith - """ foldr(op, v0, itr) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index c4301ce159264..0a757ee10b619 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -442,6 +442,17 @@ function type_close_enough(x::ANY, t::ANY) end # `methodswith` -- shows a list of methods using the type given +""" + methodswith(typ[, module or function][, showparents]) + +Return an array of methods with an argument of type `typ`. + +The optional second argument restricts the search to a particular module or function +(the default is all modules, starting from Main). + +If optional `showparents` is `true`, also return arguments with a parent type of `typ`, +excluding type `Any`. +""" function methodswith(t::Type, f::Function, showparents::Bool=false, meths = Method[]) for d in methods(f) if any(x -> (type_close_enough(x, t) || @@ -470,10 +481,10 @@ end function methodswith(t::Type, showparents::Bool=false) meths = Method[] - mainmod = current_module() + mainmod = Main # find modules in Main for nm in names(mainmod) - if isdefined(mainmod,nm) + if isdefined(mainmod, nm) mod = getfield(mainmod, nm) if isa(mod, Module) append!(meths, methodswith(t, mod, showparents)) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 40d0b40d307c3..def151edf1efb 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -202,9 +202,11 @@ Getting Around .. Docstring generated from Julia source - Return an array of methods with an argument of type ``typ``\ . If optional ``showparents`` is ``true``\ , also return arguments with a parent type of ``typ``\ , excluding type ``Any``\ . + Return an array of methods with an argument of type ``typ``\ . - The optional second argument restricts the search to a particular module or function. + The optional second argument restricts the search to a particular module or function (the default is all modules, starting from Main). + + If optional ``showparents`` is ``true``\ , also return arguments with a parent type of ``typ``\ , excluding type ``Any``\ . .. function:: @show From 53fca772c05d7dc4181a873108546d5b32267c0e Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Fri, 29 Jul 2016 00:16:20 -0400 Subject: [PATCH 0705/1117] hotfix for a latent bug exposed by #17592 re-extraction of a tarball doesn't properly re-apply patches for llvm this patch will force everyone to get a rebuild of a properly patched llvm --- deps/llvm.mk | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index efea7d7fcd9bd..d6c221b96e365 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -258,7 +258,7 @@ $(LLVM_LLDB_TAR): | $(SRCDIR)/srccache endif ifeq ($(BUILD_LLDB),1) $(LLVM_SRC_DIR)/tools/lldb: -$(LLVM_SRC_DIR)/CMakeLists.txt: $(LLVM_SRC_DIR)/tools/lldb +$(LLVM_SRC_DIR).extracted: $(LLVM_SRC_DIR)/tools/lldb endif # LLDB still relies on plenty of python 2.x infrastructure, without checking @@ -278,13 +278,13 @@ ifeq ($(USEICC),1) LIBCXX_EXTRA_FLAGS := -Bstatic -lirc -Bdynamic endif -$(LLVM_SRC_DIR)/projects/libcxx: $(LLVM_LIBCXX_TAR) | $(LLVM_SRC_DIR)/CMakeLists.txt +$(LLVM_SRC_DIR)/projects/libcxx: $(LLVM_LIBCXX_TAR) | $(LLVM_SRC_DIR).extracted ([ ! -d $@ ] && \ git clone $(LLVM_GIT_URL_LIBCXX) $@ ) || \ (cd $@ && \ git pull --ff-only) $(LLVM_SRC_DIR)/projects/libcxx/.git/HEAD: | $(LLVM_SRC_DIR)/projects/libcxx/.git/HEAD -$(LLVM_SRC_DIR)/projects/libcxxabi: $(LLVM_LIBCXXABI_TAR) | $(LLVM_SRC_DIR)/CMakeLists.txt +$(LLVM_SRC_DIR)/projects/libcxxabi: $(LLVM_LIBCXXABI_TAR) | $(LLVM_SRC_DIR).extracted ([ ! -d $@ ] && \ git clone $(LLVM_GIT_URL_LIBCXXABI) $@ ) || \ (cd $@ && \ @@ -320,7 +320,7 @@ LIBCXX_DEPENDENCY := $(build_libdir)/libc++abi.so.1.0 $(build_libdir)/libc++.so. get-llvm: get-libcxx get-libcxxabi endif -$(LLVM_SRC_DIR)/CMakeLists.txt: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) +$(LLVM_SRC_DIR).extracted: | $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) ifneq ($(LLVM_CLANG_TAR),) $(JLCHECKSUM) $(LLVM_CLANG_TAR) endif @@ -336,6 +336,7 @@ endif ifneq ($(LLVM_LLDB_TAR),) $(JLCHECKSUM) $(LLVM_LLDB_TAR) endif + -rm -rf $(LLVM_SRC_DIR) ifneq ($(LLVM_VER),svn) mkdir -p $(LLVM_SRC_DIR) $(TAR) -C $(LLVM_SRC_DIR) --strip-components 1 -xf $(LLVM_TAR) @@ -401,13 +402,16 @@ ifneq ($(LLVM_GIT_VER_POLLY),) endif # LLVM_GIT_VER_POLLY endif # USE_POLLY endif # LLVM_VER - touch -c $@ + # touch some extra files to ensure bisect works pretty well + touch -c $(LLVM_SRC_DIR)/configure + touch -c $(LLVM_SRC_DIR)/CMakeLists.txt + echo 1 > $@ # Apply version-specific LLVM patches LLVM_PATCH_PREV:= LLVM_PATCH_LIST:= define LLVM_PATCH -$$(LLVM_SRC_DIR)/$1.patch-applied: | $$(LLVM_SRC_DIR)/CMakeLists.txt $$(SRCDIR)/patches/$1.patch $$(LLVM_PATCH_PREV) +$$(LLVM_SRC_DIR)/$1.patch-applied: | $$(LLVM_SRC_DIR).extracted $$(SRCDIR)/patches/$1.patch $$(LLVM_PATCH_PREV) cd $$(LLVM_SRC_DIR) && patch -p1 < $$(SRCDIR)/patches/$1.patch echo 1 > $$@ LLVM_PATCH_PREV := $$(LLVM_SRC_DIR)/$1.patch-applied @@ -468,7 +472,7 @@ endif ifeq ($(LLVM_USE_CMAKE),1) -$(LLVM_BUILDDIR_withtype)/CMakeCache.txt: $(LLVM_SRC_DIR)/CMakeLists.txt $(LLVM_PATCH_LIST) | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) +$(LLVM_BUILDDIR_withtype)/CMakeCache.txt: $(LLVM_SRC_DIR).extracted $(LLVM_PATCH_LIST) | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) mkdir -p $(dir $@) cd $(dir $@) && \ export PATH=$(llvm_python_workaround):$$PATH && \ @@ -485,7 +489,7 @@ $(LLVM_OBJ_SOURCE): $(LLVM_BUILDDIR_withtype)/CMakeCache.txt | $(llvm_python_wor else -$(LLVM_BUILDDIR_withtype)/config.status: $(LLVM_SRC_DIR)/CMakeLists.txt $(LLVM_PATCH_LIST) | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) +$(LLVM_BUILDDIR_withtype)/config.status: $(LLVM_SRC_DIR).extracted $(LLVM_PATCH_LIST) | $(llvm_python_workaround) $(LIBCXX_DEPENDENCY) mkdir -p $(dir $@) cd $(dir $@) && \ export PATH=$(llvm_python_workaround):$$PATH && \ @@ -535,7 +539,7 @@ distclean-llvm: ifneq ($(LLVM_VER),svn) get-llvm: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) else -get-llvm: $(LLVM_SRC_DIR)/CMakeLists.txt +get-llvm: $(LLVM_SRC_DIR).extracted endif ifeq ($(LLVM_USE_CMAKE),1) configure-llvm: $(LLVM_BUILDDIR_withtype)/CMakeCache.txt From 0b374fbde031bd1195b0128257bdecf3f2a56ea0 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 28 Jul 2016 16:03:54 -0700 Subject: [PATCH 0706/1117] Make `hvcat`s of heterogeneous combinations of dense matrices and vectors yield dense arrays (fixes #17686). --- base/array.jl | 8 +++++--- test/arrayops.jl | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/base/array.jl b/base/array.jl index 4fc67b12ba356..89bec1acb5ee9 100644 --- a/base/array.jl +++ b/base/array.jl @@ -689,7 +689,6 @@ vcat{T}(A::Matrix{T}...) = typed_vcat(T, A...) hcat(A::Union{Matrix, Vector}...) = typed_hcat(promote_eltype(A...), A...) hcat{T}(A::Union{Matrix{T}, Vector{T}}...) = typed_hcat(T, A...) - vcat(A::Union{Matrix, Vector}...) = typed_vcat(promote_eltype(A...), A...) vcat{T}(A::Union{Matrix{T}, Vector{T}}...) = typed_vcat(T, A...) @@ -699,8 +698,11 @@ hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Vector{T}...) = typed_hvcat(T, rows, xs.. hvcat(rows::Tuple{Vararg{Int}}, xs::Matrix...) = typed_hvcat(promote_eltype(xs...), rows, xs...) hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Matrix{T}...) = typed_hvcat(T, rows, xs...) -cat(catdims, xs::Union{Matrix,Vector}...) = Base.cat_t(catdims, promote_eltype(xs...), xs...) -cat{T}(catdims, xs::Union{Matrix{T},Vector{T}}...) = Base.cat_t(catdims, T, xs...) +hvcat(rows::Tuple{Vararg{Int}}, xs::Union{Vector,Matrix}...) = typed_hvcat(promote_eltype(xs...), rows, xs...) +hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Union{Vector{T},Matrix{T}}...) = typed_hvcat(T, rows, xs...) + +cat(catdims, xs::Union{Vector,Matrix}...) = Base.cat_t(catdims, promote_eltype(xs...), xs...) +cat{T}(catdims, xs::Union{Vector{T},Matrix{T}}...) = Base.cat_t(catdims, T, xs...) ## find ## diff --git a/test/arrayops.jl b/test/arrayops.jl index 60cd54e437f50..92549881f665b 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1703,3 +1703,25 @@ for op in (:.+, :.*, :.÷, :.%, :.<<, :.>>, :.-, :./, :.\, :.//, :.^) end end + +# Test that concatenations of dense matrices/vectors yield dense matrices/vectors +let + N = 4 + densevec = ones(N) + densemat = diagm(ones(N)) + # Test that concatenations of homogeneous pairs of either dense matrices or dense vectors + # (i.e., Matrix-Matrix concatenations, and Vector-Vector concatenations) yield dense arrays + for densearray in (densevec, densemat) + @test isa(vcat(densearray, densearray), Array) + @test isa(hcat(densearray, densearray), Array) + @test isa(hvcat((2,), densearray, densearray), Array) + @test isa(cat((1,2), densearray, densearray), Array) + end + # Test that concatenations of heterogeneous Matrix-Vector pairs yield dense matrices + @test isa(hcat(densemat, densevec), Array) + @test isa(hcat(densevec, densemat), Array) + @test isa(hvcat((2,), densemat, densevec), Array) + @test isa(hvcat((2,), densevec, densemat), Array) + @test isa(cat((1,2), densemat, densevec), Array) + @test isa(cat((1,2), densevec, densemat), Array) +end From c753530b84106f8bd1d371d30cc359ab31aee351 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 28 Jul 2016 16:16:21 -0700 Subject: [PATCH 0707/1117] Remove seemingly redundant dense array concatenation methods and clean up the associated code. --- base/array.jl | 53 +++++++++++++++++++-------------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/base/array.jl b/base/array.jl index 89bec1acb5ee9..dc97409659f13 100644 --- a/base/array.jl +++ b/base/array.jl @@ -643,6 +643,26 @@ function reverse!(v::AbstractVector, s=1, n=length(v)) return v end + +# concatenations of combinations (homogeneous, heterogeneous) of dense matrices/vectors # +vcat{T}(A::Union{Vector{T},Matrix{T}}...) = typed_vcat(T, A...) +vcat(A::Union{Vector,Matrix}...) = typed_vcat(promote_eltype(A...), A...) +hcat{T}(A::Union{Vector{T},Matrix{T}}...) = typed_hcat(T, A...) +hcat(A::Union{Vector,Matrix}...) = typed_hcat(promote_eltype(A...), A...) +hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Union{Vector{T},Matrix{T}}...) = typed_hvcat(T, rows, xs...) +hvcat(rows::Tuple{Vararg{Int}}, xs::Union{Vector,Matrix}...) = typed_hvcat(promote_eltype(xs...), rows, xs...) +cat{T}(catdims, xs::Union{Vector{T},Matrix{T}}...) = Base.cat_t(catdims, T, xs...) +cat(catdims, xs::Union{Vector,Matrix}...) = Base.cat_t(catdims, promote_eltype(xs...), xs...) +# concatenations of homogeneous combinations of vectors, horizontal and vertical +function hcat{T}(V::Vector{T}...) + height = length(V[1]) + for j = 2:length(V) + if length(V[j]) != height + throw(DimensionMismatch("vectors must have same lengths")) + end + end + return [ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ] +end function vcat{T}(arrays::Vector{T}...) n = 0 for a in arrays @@ -670,39 +690,6 @@ function vcat{T}(arrays::Vector{T}...) return arr end -function hcat{T}(V::Vector{T}...) - height = length(V[1]) - for j = 2:length(V) - if length(V[j]) != height - throw(DimensionMismatch("vectors must have same lengths")) - end - end - return [ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ] -end - -hcat(A::Matrix...) = typed_hcat(promote_eltype(A...), A...) -hcat{T}(A::Matrix{T}...) = typed_hcat(T, A...) - -vcat(A::Matrix...) = typed_vcat(promote_eltype(A...), A...) -vcat{T}(A::Matrix{T}...) = typed_vcat(T, A...) - -hcat(A::Union{Matrix, Vector}...) = typed_hcat(promote_eltype(A...), A...) -hcat{T}(A::Union{Matrix{T}, Vector{T}}...) = typed_hcat(T, A...) - -vcat(A::Union{Matrix, Vector}...) = typed_vcat(promote_eltype(A...), A...) -vcat{T}(A::Union{Matrix{T}, Vector{T}}...) = typed_vcat(T, A...) - -hvcat(rows::Tuple{Vararg{Int}}, xs::Vector...) = typed_hvcat(promote_eltype(xs...), rows, xs...) -hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Vector{T}...) = typed_hvcat(T, rows, xs...) - -hvcat(rows::Tuple{Vararg{Int}}, xs::Matrix...) = typed_hvcat(promote_eltype(xs...), rows, xs...) -hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Matrix{T}...) = typed_hvcat(T, rows, xs...) - -hvcat(rows::Tuple{Vararg{Int}}, xs::Union{Vector,Matrix}...) = typed_hvcat(promote_eltype(xs...), rows, xs...) -hvcat{T}(rows::Tuple{Vararg{Int}}, xs::Union{Vector{T},Matrix{T}}...) = typed_hvcat(T, rows, xs...) - -cat(catdims, xs::Union{Vector,Matrix}...) = Base.cat_t(catdims, promote_eltype(xs...), xs...) -cat{T}(catdims, xs::Union{Vector{T},Matrix{T}}...) = Base.cat_t(catdims, T, xs...) ## find ## From 990171c7c8dc073e8bf3c02fa534a10aaf957677 Mon Sep 17 00:00:00 2001 From: Amit Murthy <amit.murthy@gmail.com> Date: Fri, 29 Jul 2016 14:41:48 +0530 Subject: [PATCH 0708/1117] remove LocalProcess and ProcessGroup from exports --- base/exports.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 12204a129e6e3..aaaf335426d36 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -70,7 +70,6 @@ export IOBuffer, IOStream, LinSpace, - LocalProcess, LowerTriangular, Irrational, Matrix, @@ -82,7 +81,6 @@ export Pair, PartialQuickSort, PollingFileWatcher, - ProcessGroup, QuickSort, Range, RangeIndex, From 22030610f9d23f054bc247f0461f780d0f04af62 Mon Sep 17 00:00:00 2001 From: Nathan Daly <NHDaly@gmail.com> Date: Fri, 29 Jul 2016 09:57:25 -0500 Subject: [PATCH 0709/1117] RFC: Restricts parameter type for randn & randexp to Float types for #17608. (#17627) * Fixes infinite recursion in randn for non-floats; fixes #17608. The functions already only supported Float types (Float16,Float32,Float64), but could be called with non-float types. Calling with Non-Float types, e.g. `Int`, could cause an infinite loop. Also adds tests verifying that randn throws a MethodError for types besides Float types. * s/dim0/dim1/ * removed other dimensions, fixed randn(T), added comment. --- base/random.jl | 19 ++++++++++--------- test/random.jl | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/base/random.jl b/base/random.jl index 2f9a6e8bd5e7f..c6cf2bfdeb2c0 100644 --- a/base/random.jl +++ b/base/random.jl @@ -1200,7 +1200,7 @@ let Floats = Union{Float16,Float32,Float64} @eval begin # scalars $randfun{T<:$Floats}(rng::AbstractRNG, ::Type{T}) = convert(T, $randfun(rng)) - $randfun{T<:$Floats}(::Type{T}) = $randfun(GLOBAL_RNG, T) + $randfun{T}(::Type{T}) = $randfun(GLOBAL_RNG, T) # filling arrays function $randfun!{T}(rng::AbstractRNG, A::AbstractArray{T}) @@ -1213,14 +1213,15 @@ let Floats = Union{Float16,Float32,Float64} $randfun!(A::AbstractArray) = $randfun!(GLOBAL_RNG, A) # generating arrays - $randfun{T}(rng::AbstractRNG, ::Type{T}, dims::Dims) = $randfun!(rng, Array{T}(dims)) - $randfun{T}(rng::AbstractRNG, ::Type{T}, dims::Integer...) = $randfun!(rng, Array{T}(dims...)) - $randfun{T}( ::Type{T}, dims::Dims) = $randfun(GLOBAL_RNG, T, dims) - $randfun{T}( ::Type{T}, dims::Integer...) = $randfun(GLOBAL_RNG, T, dims...) - $randfun( rng::AbstractRNG, dims::Dims) = $randfun(rng, Float64, dims) - $randfun( rng::AbstractRNG, dims::Integer...) = $randfun(rng, Float64, dims...) - $randfun( dims::Dims) = $randfun(GLOBAL_RNG, Float64, dims) - $randfun( dims::Integer...) = $randfun(GLOBAL_RNG, Float64, dims...) + $randfun{T}(rng::AbstractRNG, ::Type{T}, dims::Dims ) = $randfun!(rng, Array{T}(dims)) + # Note that this method explicitly does not define $randfun(rng, T), in order to prevent an infinite recursion. + $randfun{T}(rng::AbstractRNG, ::Type{T}, dim1::Integer, dims::Integer...) = $randfun!(rng, Array{T}(dim1, dims...)) + $randfun{T}( ::Type{T}, dims::Dims ) = $randfun(GLOBAL_RNG, T, dims) + $randfun{T}( ::Type{T}, dims::Integer... ) = $randfun(GLOBAL_RNG, T, dims...) + $randfun( rng::AbstractRNG, dims::Dims ) = $randfun(rng, Float64, dims) + $randfun( rng::AbstractRNG, dims::Integer... ) = $randfun(rng, Float64, dims...) + $randfun( dims::Dims ) = $randfun(GLOBAL_RNG, Float64, dims) + $randfun( dims::Integer... ) = $randfun(GLOBAL_RNG, Float64, dims...) end end end diff --git a/test/random.jl b/test/random.jl index 591725f1fded0..75a2a871d8a58 100644 --- a/test/random.jl +++ b/test/random.jl @@ -343,6 +343,23 @@ for rng in ([], [MersenneTwister()], [RandomDevice()]) bitrand(rng..., 2, 3) ::BitArray{2} rand!(rng..., BitArray(5)) ::BitArray{1} rand!(rng..., BitArray(2, 3)) ::BitArray{2} + + # Test that you cannot call randn or randexp with non-Float types. + for r in [randn, randexp, randn!, randexp!] + @test_throws MethodError r(Int) + @test_throws MethodError r(Int32) + @test_throws MethodError r(Bool) + @test_throws MethodError r(String) + @test_throws MethodError r(AbstractFloat) + # TODO(#17627): Consider adding support for randn(BigFloat) and removing this test. + @test_throws MethodError r(BigFloat) + + @test_throws MethodError r(Int64, (2,3)) + @test_throws MethodError r(String, 1) + + @test_throws MethodError r(rng..., Number, (2,3)) + @test_throws MethodError r(rng..., Any, 1) + end end function hist(X,n) From d76ee821cb2e505dcc7a4eaca6411df5b057ac19 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Sat, 30 Jul 2016 09:23:41 +0200 Subject: [PATCH 0710/1117] print homogeneous tuple types over length 3 compactly --- base/show.jl | 31 +++++++++++++++++++++---------- test/show.jl | 5 +++++ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/base/show.jl b/base/show.jl index 2915b8dc903ef..7af26d1d71ec0 100644 --- a/base/show.jl +++ b/base/show.jl @@ -190,24 +190,35 @@ function show_type_parameter(io::IO, p::ANY, has_tvar_env::Bool) end show(io::IO, x::DataType) = show_datatype(io, x) + function show_datatype(io::IO, x::DataType) - show(io, x.name) # tvar_env is a `::Vector{Any}` when we are printing a method signature # and `true` if we are printing type parameters outside a method signature. has_tvar_env = get(io, :tvar_env, false) !== false + if ((!isempty(x.parameters) || x.name === Tuple.name) && x !== Tuple && !(has_tvar_env && x.name.primary === x)) - # Do not print the type parameters for the primary type if we are - # printing a method signature or type parameter. - # Always print the type parameter if we are printing the type directly - # since this information is still useful. - print(io, '{') n = length(x.parameters) - for (i, p) in enumerate(x.parameters) - show_type_parameter(io, p, has_tvar_env) - i < n && print(io, ',') + + # Print homogeneous tuples with more than 3 elements compactly + # as NTuple{N, T} + if length(unique(x.parameters)) == 1 && n > 3 + show(io, "NTuple{", n, ',', x.parameters[1], "}") + else + show(io, x.name) + # Do not print the type parameters for the primary type if we are + # printing a method signature or type parameter. + # Always print the type parameter if we are printing the type directly + # since this information is still useful. + print(io, '{') + for (i, p) in enumerate(x.parameters) + show_type_parameter(io, p, has_tvar_env) + i < n && print(io, ',') + end + print(io, '}') end - print(io, '}') + else + show(io, x.name) end end diff --git a/test/show.jl b/test/show.jl index 5c5327b381147..b091e3277d73f 100644 --- a/test/show.jl +++ b/test/show.jl @@ -570,3 +570,8 @@ end @test repr(:(f.(X,Y))) == ":(f.(X,Y))" @test repr(:(f.(X))) == ":(f.(X))" @test repr(:(f.())) == ":(f.())" + +# Test compact printing of homogeneous tuples +@test repr(NTuple{7,Int64}) == "NTuple{7,Int64}" +@test repr(Tuple{Float64, Float64, Float64, Float64}) == "NTuple{4,Float64}" +@test repr(Tuple{Float32, Float32, Float32}) == "Tuple{Float32,Float32,Float32}" \ No newline at end of file From 17ab97421a2d9d67533c7c229f827fc51910ea5c Mon Sep 17 00:00:00 2001 From: Jim Garrison <jim@garrison.cc> Date: Sat, 30 Jul 2016 02:04:35 -0700 Subject: [PATCH 0711/1117] Fix typo in NEWS.md [ci skip] (#17718) --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 23d2c164e4aad..24ae9767d806e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -355,7 +355,7 @@ Deprecated or removed * `set_rounding` -> `setrounding` * `with_rounding` -> `setrounding` - * The method `A_ldiv_B!(SparseMatrixCSC, StrideVecOrMat)` has been deprecated + * The method `A_ldiv_B!(SparseMatrixCSC, StridedVecOrMat)` has been deprecated in favor of versions that require the matrix to be in factored form ([#13496]). From 3df357e4c5df351e86be29190f898430aff3e602 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 30 Jul 2016 06:11:30 -0500 Subject: [PATCH 0712/1117] Make atsign-inferred(A[i]) work --- base/test.jl | 3 +++ test/test.jl | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/base/test.jl b/base/test.jl index 0391180376ee3..586f5767906ef 100644 --- a/base/test.jl +++ b/base/test.jl @@ -918,6 +918,9 @@ julia> @inferred max(1,2) ``` """ macro inferred(ex) + if Meta.isexpr(ex, :ref) + ex = Expr(:call, :getindex, ex.args...) + end Meta.isexpr(ex, :call)|| error("@inferred requires a call expression") Base.remove_linenums!(quote diff --git a/test/test.jl b/test/test.jl index 39a33744ecc02..d168a38eeb859 100644 --- a/test/test.jl +++ b/test/test.jl @@ -297,6 +297,13 @@ end @test @inferred inferred_test_function() @test inferred_test_global == 1 +# Test that @inferred works with A[i] expressions +@test @inferred((1:3)[2]) == 2 +immutable SillyArray <: AbstractArray{Float64,1} end +Base.getindex(a::SillyArray, i) = rand() > 0.5 ? 0 : false +test_result = @test_throws ErrorException @inferred(SillyArray()[2]) +@test contains(test_result.value.msg, "Bool") + # Issue #14928 # Make sure abstract error type works. @test_throws Exception error("") From dcb6bfa3805c611ebac6541063cd875cd94fe21a Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Thu, 28 Jul 2016 20:46:19 -0400 Subject: [PATCH 0713/1117] Refactor libgit2 credential callback Cleanly separate the SSHCredentials and UserPasswordCredentials to only use each type when appropriate (the former for SSH keys, the latter for HTTPS for SSH password/keyboard-interactive authentication). Previously the types would sometimes get confused and SSHCredentials would be used for userpass authentication, since the field names happen to be a subset (though they have different meaning, since the `pass` field in SSHCredentials is the SSH key passphrase while the `pass` in UserPasswordCredentials is the remote user's password). Fix a couple issues in the process: - Allow externally passed in credentials to reach the right place - Don't skip using the SSH agent if there are two private packages --- base/libgit2/callbacks.jl | 316 +++++++++++++++++++++----------------- base/libgit2/types.jl | 74 +++------ test/libgit2.jl | 36 +---- test/misc.jl | 4 +- 4 files changed, 197 insertions(+), 233 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 40b24542d250a..d48899090b304 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -24,6 +24,155 @@ function mirror_callback(remote::Ptr{Ptr{Void}}, repo_ptr::Ptr{Void}, return Cint(0) end +function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, + username_ptr, schema, host) + isusedcreds = checkused!(creds) + + errcls, errmsg = Error.last_error() + if errcls != Error.None + # Check if we used ssh-agent + if creds.usesshagent == "U" + println("ERROR: $errmsg ssh-agent") + creds.usesshagent = "E" # reported ssh-agent error, disables ssh agent use for the future + else + println("ERROR: $errmsg") + end + flush(STDOUT) + end + + # first try ssh-agent if credentials support its usage + if creds.usesshagent === nothing || creds.usesshagent == "Y" || creds.usesshagent == "U" + err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring), libgit2credptr, username_ptr) + creds.usesshagent = "U" # used ssh-agent only one time + err == 0 && return Cint(0) + end + + # if username is not provided, then prompt for it + username = if username_ptr == Cstring(C_NULL) + uname = creds.user # check if credentials were already used + uname !== nothing && !isusedcreds ? uname : prompt("Username for '$schema$host'") + else + unsafe_string(username_ptr) + end + creds.user = username # save credentials + isempty(username) && return Cint(Error.EAUTH) + + # For SSH we need a private key location + privatekey = if haskey(ENV,"SSH_KEY_PATH") + ENV["SSH_KEY_PATH"] + else + keydefpath = creds.prvkey # check if credentials were already used + keydefpath === nothing && (keydefpath = "") + if !isempty(keydefpath) && !isusedcreds + keydefpath # use cached value + else + defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") + if isempty(keydefpath) && isfile(defaultkeydefpath) + keydefpath = defaultkeydefpath + else + keydefpath = + prompt("Private key location for '$schema$username@$host'", default=keydefpath) + end + end + end + + # If the private key changed, invalidate the cached public key + (privatekey != creds.prvkey) && + (creds.pubkey = "") + creds.prvkey = privatekey # save credentials + + # For SSH we need a public key location, look for environment vars SSH_* as well + publickey = if haskey(ENV,"SSH_PUB_KEY_PATH") + ENV["SSH_PUB_KEY_PATH"] + else + keydefpath = creds.pubkey # check if credentials were already used + if keydefpath !== nothing && !isusedcreds + keydefpath # use cached value + else + if keydefpath === nothing || isempty(keydefpath) + keydefpath = privatekey*".pub" + end + if isfile(keydefpath) + keydefpath + else + prompt("Public key location for '$schema$username@$host'", default=keydefpath) + end + end + end + creds.pubkey = publickey # save credentials + + passphrase_required = true + if !isfile(privatekey) + warn("Private key not found") + else + # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" + open(privatekey) do f + passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") + end + end + + passphrase = if haskey(ENV,"SSH_KEY_PASS") + ENV["SSH_KEY_PASS"] + else + passdef = creds.pass # check if credentials were already used + passdef === nothing && (passdef = "") + if passphrase_required && (isempty(passdef) || isusedcreds) + if is_windows() + passdef = Base.winprompt( + "Your SSH Key requires a password, please enter it now:", + "Passphrase required", privatekey; prompt_username = false) + isnull(passdef) && return Cint(Error.EAUTH) + passdef = Base.get(passdef)[2] + else + passdef = prompt("Passphrase for $privatekey", password=true) + end + end + passdef + end + creds.pass = passphrase + + err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), + libgit2credptr, username, publickey, privatekey, passphrase) + return err +end + +function authenticate_userpass(creds::UserPasswordCredentials, libgit2credptr::Ptr{Ptr{Void}}, + schema, host, urlusername) + isusedcreds = checkused!(creds) + + username = creds.user + userpass = creds.pass + if is_windows() + if username === nothing || userpass === nothing || isusedcreds + res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", + username === nothing || isempty(username) ? + urlusername : username; prompt_username = true) + isnull(res) && return Cint(Error.EAUTH) + username, userpass = Base.get(res) + end + else + if username === nothing || isusedcreds + username = prompt("Username for '$schema$host'", default = urlusername) + end + + if userpass === nothing || isusedcreds + userpass = prompt("Password for '$schema$username@$host'", password=true) + end + end + creds.user = username # save credentials + creds.pass = userpass # save credentials + + isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) + + err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, + (Ptr{Ptr{Void}}, Cstring, Cstring), + libgit2credptr, username, userpass) + err == 0 && return Cint(0) +end + + """Credentials callback function Function provides different credential acquisition functionality w.r.t. a connection protocol. @@ -50,7 +199,7 @@ Using credentials triggers a user prompt for (re)entering required information. `UserPasswordCredentials` and `CachedCredentials` are implemented using a call counting strategy that prevents repeated usage of faulty credentials. """ -function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, +function credentials_callback(libgit2credptr::Ptr{Ptr{Void}}, url_ptr::Cstring, username_ptr::Cstring, allowed_types::Cuint, payload_ptr::Ptr{Void}) err = 0 @@ -74,157 +223,42 @@ function credentials_callback(cred::Ptr{Ptr{Void}}, url_ptr::Cstring, creds_are_temp = false end end - isusedcreds = checkused!(creds) try # use ssh key or ssh-agent if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) - creds === nothing && (creds = SSHCredentials()) - credid = "ssh://$host" - - # first try ssh-agent if credentials support its usage - if creds[:usesshagent, credid] === nothing || creds[:usesshagent, credid] == "Y" - err = ccall((:git_cred_ssh_key_from_agent, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring), cred, username_ptr) - creds[:usesshagent, credid] = "U" # used ssh-agent only one time - err == 0 && return Cint(0) - end - - errcls, errmsg = Error.last_error() - if errcls != Error.None - # Check if we used ssh-agent - if creds[:usesshagent, credid] == "U" - println("ERROR: $errmsg ssh-agent") - creds[:usesshagent, credid] = "E" # reported ssh-agent error - else - println("ERROR: $errmsg") - end - flush(STDOUT) - end - - # if username is not provided, then prompt for it - username = if username_ptr == Cstring(C_NULL) - uname = creds[:user, credid] # check if credentials were already used - uname !== nothing && !isusedcreds ? uname : prompt("Username for '$schema$host'") - else - unsafe_string(username_ptr) - end - creds[:user, credid] = username # save credentials - - # For SSH we need a private key location - privatekey = if haskey(ENV,"SSH_KEY_PATH") - ENV["SSH_KEY_PATH"] - else - keydefpath = creds[:prvkey, credid] # check if credentials were already used - keydefpath === nothing && (keydefpath = "") - if !isempty(keydefpath) && !isusedcreds - keydefpath # use cached value - else - defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") - if isempty(keydefpath) && isfile(defaultkeydefpath) - keydefpath = defaultkeydefpath - else - keydefpath = - prompt("Private key location for '$schema$username@$host'", default=keydefpath) - end - end - end - - # If the private key changed, invalidate the cached public key - (privatekey != creds[:prvkey, credid]) && - (creds[:pubkey, credid] = "") - creds[:prvkey, credid] = privatekey # save credentials - - # For SSH we need a public key location, look for environment vars SSH_* as well - publickey = if haskey(ENV,"SSH_PUB_KEY_PATH") - ENV["SSH_PUB_KEY_PATH"] - else - keydefpath = creds[:pubkey, credid] # check if credentials were already used - if keydefpath !== nothing && !isusedcreds - keydefpath # use cached value - else - if keydefpath === nothing || isempty(keydefpath) - keydefpath = privatekey*".pub" - end - if isfile(keydefpath) - keydefpath - else - prompt("Public key location for '$schema$username@$host'", default=keydefpath) - end - end + sshcreds = get_creds!(creds, "ssh://$host", SSHCredentials()) + if isa(sshcreds, SSHCredentials) + creds = sshcreds # To make sure these get cleaned up below + err = authenticate_ssh(creds, libgit2credptr, username_ptr, schema, host) + err == 0 && return err end - creds[:pubkey, credid] = publickey # save credentials - - passphrase_required = true - if !isfile(privatekey) - warn("Private key not found") - else - # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" - open(privatekey) do f - passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") - end - end - - passphrase = if haskey(ENV,"SSH_KEY_PASS") - ENV["SSH_KEY_PASS"] - else - passdef = creds[:pass, credid] # check if credentials were already used - passdef === nothing && (passdef = "") - if passphrase_required && (isempty(passdef) || isusedcreds) - if is_windows() - passdef = Base.winprompt( - "Your SSH Key requires a password, please enter it now:", - "Passphrase required", privatekey; prompt_username = false) - isnull(passdef) && return Cint(Error.EAUTH) - passdef = Base.get(passdef)[2] - else - passdef = prompt("Passphrase for $privatekey", password=true) - end - end - passdef - end - creds[:pass, credid] = passphrase - - isempty(username) && return Cint(Error.EAUTH) - - err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), - cred, username, publickey, privatekey, passphrase) - err == 0 && return Cint(0) end if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) - creds === nothing && (creds = UserPasswordCredentials()) + defaultcreds = UserPasswordCredentials() credid = "$schema$host" - - username = creds[:user, credid] - userpass = creds[:pass, credid] - if is_windows() - if username === nothing || userpass === nothing || isusedcreds - res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", - username === nothing || isempty(username) ? - urlusername : username; prompt_username = true) - isnull(res) && return Cint(Error.EAUTH) - username, userpass = Base.get(res) - end - else - if username === nothing || isusedcreds - username = prompt("Username for '$schema$host'", default = urlusername) - end - - if userpass === nothing || isusedcreds - userpass = prompt("Password for '$schema$username@$host'", password=true) - end + upcreds = get_creds!(creds, credid, defaultcreds) + # If there were stored SSH credentials, but we ended up here that must + # mean that something went wrong. Replace the SSH credentials by user/pass + # credentials + if !isa(upcreds, UserPasswordCredentials) + upcreds = defaultcreds + isa(creds, CachedCredentials) && (creds.creds[credid] = upcreds) end - creds[:user, credid] = username # save credentials - creds[:pass, credid] = userpass # save credentials - - isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) + creds = upcreds # To make sure these get cleaned up below + return authenticate_userpass(creds, libgit2credptr, schema, host, urlusername) + end - err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, - (Ptr{Ptr{Void}}, Cstring, Cstring), - cred, username, userpass) - err == 0 && return Cint(0) + # No authentication method we support succeeded. The most likely cause is + # that explicit credentials were passed in, but said credentials are incompatible + # with the remote host. + if err == 0 + if (creds != nothing && !isa(creds, CachedCredentials)) + warn("The explicitly provided credentials were incompatible with " * + "the server's supported authentication methods") + end + err = Cint(Error.EAUTH) end finally # if credentials are not passed back to caller via payload, diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 65f180e4d4c59..2f643366f6a92 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -55,19 +55,6 @@ abstract AbstractPayload "Abstract credentials payload" abstract AbstractCredentials <: AbstractPayload -"Returns a credentials parameter" -function Base.getindex(p::AbstractCredentials, key, _=nothing) - ks = Symbol(key) - isdefined(p, ks) && return getfield(p, ks) - return nothing -end -"Sets credentials with `key` parameter to a value" -function Base.setindex!(p::AbstractCredentials, val, key, _=nothing) - ks = Symbol(key) - @assert isdefined(p, ks) - setfield!(p, ks, val) - return p -end "Checks if credentials were used" checkused!(p::AbstractCredentials) = true checkused!(p::Void) = false @@ -711,22 +698,15 @@ import Base.securezero! type UserPasswordCredentials <: AbstractCredentials user::String pass::String - usesshagent::String # used for ssh-agent authentication count::Int # authentication failure protection count function UserPasswordCredentials(u::AbstractString,p::AbstractString) - c = new(u,p,"Y",3) + c = new(u,p,3) finalizer(c, securezero!) return c end + UserPasswordCredentials() = UserPasswordCredentials("","") end -"Checks if credentials were used or failed authentication, see `LibGit2.credentials_callback`" -function checkused!(p::UserPasswordCredentials) - p.count <= 0 && return true - p.count -= 1 - return false -end -"Resets authentication failure protection count" -reset!(p::UserPasswordCredentials, cnt::Int=3) = (p.count = cnt) + function securezero!(cred::UserPasswordCredentials) securezero!(cred.user) securezero!(cred.pass) @@ -741,9 +721,10 @@ type SSHCredentials <: AbstractCredentials pubkey::String prvkey::String usesshagent::String # used for ssh-agent authentication + count::Int function SSHCredentials(u::AbstractString,p::AbstractString) - c = new(u,p,"","","Y") + c = new(u,p,"","","Y",3) finalizer(c, securezero!) return c end @@ -754,50 +735,31 @@ function securezero!(cred::SSHCredentials) securezero!(cred.pass) securezero!(cred.pubkey) securezero!(cred.prvkey) + cred.count = 0 return cred end "Credentials that support caching" type CachedCredentials <: AbstractCredentials - cred::Dict{String,SSHCredentials} + cred::Dict{String,AbstractCredentials} count::Int # authentication failure protection count - CachedCredentials() = new(Dict{String,SSHCredentials}(),3) -end -"Returns specific credential parameter value: first index is a credential -parameter name, second index is a host name (with schema)" -function Base.getindex(p::CachedCredentials, keys...) - length(keys) != 2 && return nothing - key, host = keys - if haskey(p.cred, host) - creds = p.cred[host] - if isdefined(creds,key) - kval = getfield(creds, key) - !isempty(kval) && return kval - end - end - return nothing -end -"Sets specific credential parameter value: first index is a credential -parameter name, second index is a host name (with schema)" -function Base.setindex!(p::CachedCredentials, val, keys...) - length(keys) != 2 && return nothing - key, host = keys - if !haskey(p.cred, host) - p.cred[host] = SSHCredentials() - end - creds = p.cred[host] - @assert isdefined(creds,key) - setfield!(creds, key, val) - return p + CachedCredentials() = new(Dict{String,AbstractCredentials}(),3) end + "Checks if credentials were used or failed authentication, see `LibGit2.credentials_callback`" -function checkused!(p::CachedCredentials) +function checkused!(p::Union{UserPasswordCredentials, SSHCredentials}) p.count <= 0 && return true p.count -= 1 return false end -"Resets authentication failure protection count" -reset!(p::CachedCredentials, cnt::Int=3) = (p.count = cnt) +reset!(p::Union{UserPasswordCredentials, SSHCredentials}, cnt::Int=3) = (p.count = cnt) +reset!(p::CachedCredentials) = foreach(reset!, values(p.cred)) + +"Obtained the cached credential for the given host+protocol (credid), or return and store the default if not found" +get_creds!(collection::CachedCredentials, credid, default) = get!(collection.cred, credid, default) +get_creds!(creds::AbstractCredentials, credid, default) = creds +get_creds!(creds::Void, credid, default) = default + function securezero!(p::CachedCredentials) foreach(securezero!, values(p.cred)) return p diff --git a/test/libgit2.jl b/test/libgit2.jl index 970ab90f92b0c..c5ba7a9082e9d 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -564,40 +564,8 @@ mktempdir() do dir @test !LibGit2.checkused!(creds) @test !LibGit2.checkused!(creds) @test LibGit2.checkused!(creds) - @test LibGit2.reset!(creds) == 3 - @test !LibGit2.checkused!(creds) - @test creds.count == 2 - @test creds[:user] == creds_user - @test creds[:pass] == creds_pass - @test creds[:pubkey] === nothing - @test creds[:user, "localhost"] == creds_user - @test creds[:pubkey, "localhost"] === nothing - @test creds[:usesshagent, "localhost"] == "Y" - creds[:usesshagent, "localhost"] = "E" - @test creds[:usesshagent, "localhost"] == "E" - - creds = LibGit2.CachedCredentials() - @test !LibGit2.checkused!(creds) - @test !LibGit2.checkused!(creds) - @test !LibGit2.checkused!(creds) - @test LibGit2.checkused!(creds) - @test LibGit2.reset!(creds) == 3 - @test !LibGit2.checkused!(creds) - @test creds.count == 2 - @test creds[:user, "localhost"] === nothing - @test creds[:pass, "localhost"] === nothing - @test creds[:pubkey, "localhost"] === nothing - @test creds[:prvkey, "localhost"] === nothing - @test creds[:usesshagent, "localhost"] === nothing - creds[:user, "localhost"] = creds_user - creds[:pass, "localhost"] = creds_pass - creds[:usesshagent, "localhost"] = "Y" - @test creds[:user] === nothing - @test creds[:user, "localhost2"] === nothing - @test creds[:user, "localhost"] == creds_user - @test creds[:pass, "localhost"] == creds_pass - @test creds[:pubkey, "localhost"] === nothing - @test creds[:usesshagent, "localhost"] == "Y" + @test creds.user == creds_user + @test creds.pass == creds_pass #end end diff --git a/test/misc.jl b/test/misc.jl index 7a2e5738089f8..252eed846ada9 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -406,7 +406,7 @@ let a = [1,2,3] @test a == [0,0,0] end let creds = Base.LibGit2.CachedCredentials() - creds[:pass, "foo"] = "bar" + LibGit2.get_creds!(creds, "foo", LibGit2.SSHCredentials()).pass = "bar" securezero!(creds) - @test creds[:pass, "foo"] == "\0\0\0" + @test LibGit2.get_creds!(creds, "foo", nothing).pass == "\0\0\0" end From 7c4435c06136d85644d89d4083e1461c1335bd32 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Sat, 30 Jul 2016 08:32:20 -0700 Subject: [PATCH 0714/1117] Improve documentation for ClusterManagers, tighten function types (#17688) * Improve documentation for ClusterManagers, tighten function types Cleaned up formatting and wording, documented `Base.process_messages` a little, made `incoming` a `Bool` (since `message_handler_loop` requires this anyway). * Moved docs out of HelpDB, got rid of references to AsyncStream. AsyncStream hasn't existed since #12839, nearly a year ago. Updated the parallel docs to reflect this fact and to correspond more closely with the current state of `multi.jl` and `managers.jl`. --- base/docs/helpdb/Base.jl | 168 ----------------------- base/managers.jl | 118 ++++++++++++++++ base/multi.jl | 39 +++++- doc/manual/parallel-computing.rst | 219 +++++++++++++++--------------- doc/stdlib/parallel.rst | 48 +++---- 5 files changed, 284 insertions(+), 308 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index e914630b09a4f..26225f4362986 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -646,16 +646,6 @@ A string giving the literal bit representation of a number. """ bits -""" - launch(manager::FooManager, params::Dict, launched::Vector{WorkerConfig}, launch_ntfy::Condition) - -Implemented by cluster managers. For every Julia worker launched by this function, it should -append a `WorkerConfig` entry to `launched` and notify `launch_ntfy`. The function MUST exit -once all workers, requested by `manager` have been launched. `params` is a dictionary of all -keyword arguments `addprocs` was called with. -""" -launch - """ invdigamma(x) @@ -1569,19 +1559,6 @@ Connect to the named pipe / UNIX domain socket at `path`. """ connect(path) -""" - connect(manager::FooManager, pid::Int, config::WorkerConfig) -> (instrm::AsyncStream, outstrm::AsyncStream) - -Implemented by cluster managers using custom transports. It should establish a logical -connection to worker with id `pid`, specified by `config` and return a pair of `AsyncStream` -objects. Messages from `pid` to current process will be read off `instrm`, while messages to -be sent to `pid` will be written to `outstrm`. The custom transport implementation must -ensure that messages are delivered and received completely and in order. -`Base.connect(manager::ClusterManager.....)` sets up TCP/IP socket connections in-between -workers. -""" -connect(manager, pid::Int, config::WorkerConfig) - """ mean(v[, region]) @@ -2918,109 +2895,6 @@ See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. """ Float64 -""" -``` -addprocs(n::Integer; exeflags=``) -> List of process identifiers -``` - -Launches workers using the in-built `LocalManager` which only launches workers on the -local host. This can be used to take advantage of multiple cores. `addprocs(4)` will add 4 -processes on the local machine. -""" -addprocs(n::Integer) - -""" - addprocs() -> List of process identifiers - -Equivalent to `addprocs(Sys.CPU_CORES)` - -Note that workers do not run a `.juliarc.jl` startup script, nor do they synchronize their -global state (such as global variables, new method definitions, and loaded modules) with any -of the other running processes. -""" -addprocs() - -""" -``` -addprocs(machines; keyword_args...) -> List of process identifiers -``` - -Add processes on remote machines via SSH. Requires `julia` to be installed in the same -location on each node, or to be available via a shared file system. - -`machines` is a vector of machine specifications. Worker are started for each specification. - -A machine specification is either a string `machine_spec` or a tuple - `(machine_spec, count)`. - -`machine_spec` is a string of the form `[user@]host[:port] [bind_addr[:port]]`. `user` defaults -to current user, `port` to the standard ssh port. If `[bind_addr[:port]]` is specified, other -workers will connect to this worker at the specified `bind_addr` and `port`. - -`count` is the number of workers to be launched on the specified host. If specified as `:auto` -it will launch as many workers as the number of cores on the specific host. - - -Keyword arguments: - -* `tunnel`: if `true` then SSH tunneling will be used to connect to the worker from the - master process. Default is `false`. - -* `sshflags`: specifies additional ssh options, e.g. - - ``` - sshflags=`-i /home/foo/bar.pem` - ``` - -* `max_parallel`: specifies the maximum number of workers connected to in parallel at a host. - Defaults to 10. - -* `dir`: specifies the working directory on the workers. Defaults to the host's current - directory (as found by `pwd()`) - -* `exename`: name of the `julia` executable. Defaults to `"\$JULIA_HOME/julia"` or - `"\$JULIA_HOME/julia-debug"` as the case may be. - -* `exeflags`: additional flags passed to the worker processes. - -* `topology`: Specifies how the workers connect to each other. Sending a message - between unconnected workers results in an error. - - + `topology=:all_to_all` : All processes are connected to each other. - This is the default. - - + `topology=:master_slave` : Only the driver process, i.e. pid 1 connects to the - workers. The workers do not connect to each other. - - + `topology=:custom` : The `launch` method of the cluster manager specifes the - connection topology via fields `ident` and `connect_idents` in - `WorkerConfig`. A worker with a cluster manager identity `ident` - will connect to all workers specified in `connect_idents`. - - -Environment variables : - -If the master process fails to establish a connection with a newly launched worker within -60.0 seconds, the worker treats it a fatal situation and terminates. This timeout can be -controlled via environment variable `JULIA_WORKER_TIMEOUT`. The value of -`JULIA_WORKER_TIMEOUT` on the master process, specifies the number of seconds a newly -launched worker waits for connection establishment. -""" -addprocs(machines) - -""" - addprocs(manager::ClusterManager; kwargs...) -> List of process identifiers - -Launches worker processes via the specified cluster manager. - -For example Beowulf clusters are supported via a custom cluster manager implemented in -package `ClusterManagers`. - -The number of seconds a newly launched worker waits for connection establishment from the -master can be specified via variable `JULIA_WORKER_TIMEOUT` in the worker process's -environment. Relevant only when using TCP/IP as transport. -""" -addprocs(manager::ClusterManager) - """ mkpath(path, [mode]) @@ -3946,15 +3820,6 @@ Send a signal to a process. The default is to terminate the process. """ kill(p::Process, signum=SIGTERM) -""" - kill(manager::FooManager, pid::Int, config::WorkerConfig) - -Implemented by cluster managers. It is called on the master process, by `rmprocs`. It should -cause the remote worker specified by `pid` to exit. `Base.kill(manager::ClusterManager.....)` -executes a remote `exit()` on `pid` -""" -kill(manager, pid::Int, config::WorkerConfig) - """ sylvester(A, B, C) @@ -4441,19 +4306,6 @@ Byte-swap an integer. """ bswap -""" - manage(manager::FooManager, pid::Int, config::WorkerConfig. op::Symbol) - -Implemented by cluster managers. It is called on the master process, during a worker's -lifetime, with appropriate `op` values: - -- with `:register`/`:deregister` when a worker is added / removed from the Julia worker pool. -- with `:interrupt` when `interrupt(workers)` is called. The [`ClusterManager`](:class:`ClusterManager`) - should signal the appropriate worker with an interrupt signal. -- with `:finalize` for cleanup purposes. -""" -manage - """ resize!(collection, n) -> collection @@ -4713,15 +4565,6 @@ Quit the program indicating that the processes completed successfully. This func """ quit -""" - init_worker(manager::FooManager) - -Called by cluster managers implementing custom transports. It initializes a newly launched -process as a worker. Command line argument `--worker` has the effect of initializing a -process as a worker using TCP/IP sockets for transport. -""" -init_worker - """ escape_string(io, str::AbstractString, esc::AbstractString) @@ -8077,17 +7920,6 @@ true """ applicable -""" - Base.process_messages(instrm::AsyncStream, outstrm::AsyncStream) - -Called by cluster managers using custom transports. It should be called when the custom -transport implementation receives the first message from a remote worker. The custom -transport must manage a logical connection to the remote worker and provide two -`AsyncStream` objects, one for incoming messages and the other for messages addressed to the -remote worker. -""" -Base.process_messages - """ RandomDevice() diff --git a/base/managers.jl b/base/managers.jl index 7164bee7f0721..68f05b4efa33b 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -48,6 +48,65 @@ end # to be mutually reachable without a tunnel, as is often the case in a cluster. # Default value of kw arg max_parallel is the default value of MaxStartups in sshd_config # A machine is either a <hostname> or a tuple of (<hostname>, count) +""" + addprocs(machines; tunnel=false, sshflags=\`\`, max_parallel=10, kwargs...) -> List of process identifiers + +Add processes on remote machines via SSH. Requires `julia` to be installed in the same +location on each node, or to be available via a shared file system. + +`machines` is a vector of machine specifications. Workers are started for each specification. + +A machine specification is either a string `machine_spec` or a tuple - `(machine_spec, count)`. + +`machine_spec` is a string of the form `[user@]host[:port] [bind_addr[:port]]`. `user` defaults +to current user, `port` to the standard ssh port. If `[bind_addr[:port]]` is specified, other +workers will connect to this worker at the specified `bind_addr` and `port`. + +`count` is the number of workers to be launched on the specified host. If specified as `:auto` +it will launch as many workers as the number of cores on the specific host. + +Keyword arguments: + +* `tunnel`: if `true` then SSH tunneling will be used to connect to the worker from the + master process. Default is `false`. + +* `sshflags`: specifies additional ssh options, e.g. + ```sshflags=\`-i /home/foo/bar.pem\` ``` + +* `max_parallel`: specifies the maximum number of workers connected to in parallel at a host. + Defaults to 10. + +* `dir`: specifies the working directory on the workers. Defaults to the host's current + directory (as found by `pwd()`) + +* `exename`: name of the `julia` executable. Defaults to `"\$JULIA_HOME/julia"` or + `"\$JULIA_HOME/julia-debug"` as the case may be. + +* `exeflags`: additional flags passed to the worker processes. + +* `topology`: Specifies how the workers connect to each other. Sending a message + between unconnected workers results in an error. + + + `topology=:all_to_all` : All processes are connected to each other. + This is the default. + + + `topology=:master_slave` : Only the driver process, i.e. pid 1 connects to the + workers. The workers do not connect to each other. + + + `topology=:custom` : The `launch` method of the cluster manager specifes the + connection topology via fields `ident` and `connect_idents` in + `WorkerConfig`. A worker with a cluster manager identity `ident` + will connect to all workers specified in `connect_idents`. + + +Environment variables : + +If the master process fails to establish a connection with a newly launched worker within +60.0 seconds, the worker treats it as a fatal situation and terminates. +This timeout can be controlled via environment variable `JULIA_WORKER_TIMEOUT`. +The value of JULIA_WORKER_TIMEOUT` on the master process specifies the number of seconds a +newly launched worker waits for connection establishment. +""" function addprocs(machines::AbstractVector; tunnel=false, sshflags=``, max_parallel=10, kwargs...) check_addprocs_args(kwargs) addprocs(SSHManager(machines); tunnel=tunnel, sshflags=sshflags, max_parallel=max_parallel, kwargs...) @@ -223,7 +282,25 @@ immutable LocalManager <: ClusterManager restrict::Bool # Restrict binding to 127.0.0.1 only end +""" + addprocs(; kwargs...) -> List of process identifiers + +Equivalent to `addprocs(Sys.CPU_CORES; kwargs...)` + +Note that workers do not run a `.juliarc.jl` startup script, nor do they synchronize their +global state (such as global variables, new method definitions, and loaded modules) with any +of the other running processes. +""" addprocs(; kwargs...) = addprocs(Sys.CPU_CORES; kwargs...) + +""" + addprocs(np::Integer; restrict=true, kwargs...) -> List of process identifiers + +Launches workers using the in-built `LocalManager` which only launches workers on the +local host. This can be used to take advantage of multiple cores. `addprocs(4)` will add 4 +processes on the local machine. If `restrict` is `true`, binding is restricted to +`127.0.0.1`. +""" function addprocs(np::Integer; restrict=true, kwargs...) check_addprocs_args(kwargs) addprocs(LocalManager(np, restrict); kwargs...) @@ -256,6 +333,28 @@ function manage(manager::LocalManager, id::Integer, config::WorkerConfig, op::Sy end end +""" + launch(manager::ClusterManager, params::Dict, launched::Array, launch_ntfy::Condition) + +Implemented by cluster managers. For every Julia worker launched by this function, it should +append a `WorkerConfig` entry to `launched` and notify `launch_ntfy`. The function MUST exit +once all workers, requested by `manager` have been launched. `params` is a dictionary of all +keyword arguments `addprocs` was called with. +""" +launch + +""" + manage(manager::ClusterManager, id::Integer, config::WorkerConfig. op::Symbol) + +Implemented by cluster managers. It is called on the master process, during a worker's +lifetime, with appropriate `op` values: + +- with `:register`/`:deregister` when a worker is added / removed from the Julia worker pool. +- with `:interrupt` when `interrupt(workers)` is called. The [`ClusterManager`](:class:`ClusterManager`) + should signal the appropriate worker with an interrupt signal. +- with `:finalize` for cleanup purposes. +""" +manage # DefaultClusterManager for the default TCP transport - used by both SSHManager and LocalManager @@ -264,6 +363,17 @@ end const tunnel_hosts_map = Dict{AbstractString, Semaphore}() +""" + connect(manager::ClusterManager, pid::Int, config::WorkerConfig) -> (instrm::IO, outstrm::IO) + +Implemented by cluster managers using custom transports. It should establish a logical +connection to worker with id `pid`, specified by `config` and return a pair of `IO` +objects. Messages from `pid` to current process will be read off `instrm`, while messages to +be sent to `pid` will be written to `outstrm`. The custom transport implementation must +ensure that messages are delivered and received completely and in order. +`Base.connect(manager::ClusterManager.....)` sets up TCP/IP socket connections in-between +workers. +""" function connect(manager::ClusterManager, pid::Int, config::WorkerConfig) if !isnull(config.connect_at) # this is a worker-to-worker setup call. @@ -389,6 +499,14 @@ function connect_to_worker(host::AbstractString, bind_addr::AbstractString, port (s, bind_addr) end + +""" + kill(manager::ClusterManager, pid::Int, config::WorkerConfig) + +Implemented by cluster managers. It is called on the master process, by `rmprocs`. It should +cause the remote worker specified by `pid` to exit. `Base.kill(manager::ClusterManager.....)` +executes a remote `exit()` on `pid` +""" function kill(manager::ClusterManager, pid::Int, config::WorkerConfig) remote_do(exit, pid) # For TCP based transports this will result in a close of the socket # at our end, which will result in a cleanup of the worker. diff --git a/base/multi.jl b/base/multi.jl index 2ccd2ac1a41d9..94fa66909098a 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -1046,11 +1046,11 @@ function deliver_result(sock::IO, msg, oid, value) end ## message event handlers ## -function process_messages(r_stream::TCPSocket, w_stream::TCPSocket, incoming=true) +function process_messages(r_stream::TCPSocket, w_stream::TCPSocket, incoming::Bool=true) @schedule process_tcp_streams(r_stream, w_stream, incoming) end -function process_tcp_streams(r_stream::TCPSocket, w_stream::TCPSocket, incoming) +function process_tcp_streams(r_stream::TCPSocket, w_stream::TCPSocket, incoming::Bool) disable_nagle(r_stream) wait_connected(r_stream) if r_stream != w_stream @@ -1060,7 +1060,19 @@ function process_tcp_streams(r_stream::TCPSocket, w_stream::TCPSocket, incoming) message_handler_loop(r_stream, w_stream, incoming) end -function process_messages(r_stream::IO, w_stream::IO, incoming=true) +""" + Base.process_messages(r_stream::IO, w_stream::IO, incoming::Bool=true) + +Called by cluster managers using custom transports. It should be called when the custom +transport implementation receives the first message from a remote worker. The custom +transport must manage a logical connection to the remote worker and provide two +`IO` objects, one for incoming messages and the other for messages addressed to the +remote worker. +If `incoming` is `true`, the remote peer initiated the connection. +Whichever of the pair initiates the connection sends the cluster cookie and its +Julia version number to perform the authentication handshake. +""" +function process_messages(r_stream::IO, w_stream::IO, incoming::Bool=true) @schedule message_handler_loop(r_stream, w_stream, incoming) end @@ -1374,6 +1386,14 @@ function parse_connection_info(str) end end +""" + init_worker(cookie::AbstractString, manager::ClusterManager=DefaultClusterManager()) + +Called by cluster managers implementing custom transports. It initializes a newly launched +process as a worker. Command line argument `--worker` has the effect of initializing a +process as a worker using TCP/IP sockets for transport. +`cookie` is a [`cluster_cookie`](:func:`cluster_cookie`). +""" function init_worker(cookie::AbstractString, manager::ClusterManager=DefaultClusterManager()) # On workers, the default cluster manager connects via TCP sockets. Custom # transports will need to call this function with their own manager. @@ -1403,6 +1423,19 @@ end # Only one addprocs can be in progress at any time # const worker_lock = ReentrantLock() + +""" + addprocs(manager::ClusterManager; kwargs...) -> List of process identifiers + +Launches worker processes via the specified cluster manager. + +For example Beowulf clusters are supported via a custom cluster manager implemented in +package `ClusterManagers.jl`. + +The number of seconds a newly launched worker waits for connection establishment from the +master can be specified via variable `JULIA_WORKER_TIMEOUT` in the worker process's +environment. Relevant only when using TCP/IP as transport. +""" function addprocs(manager::ClusterManager; kwargs...) lock(worker_lock) try diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 057c3f3e4c719..4710e863ee3d8 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -35,17 +35,17 @@ be used from any process to refer to an object stored on a particular process. A remote call is a request by one process to call a certain function on certain arguments on another (possibly the same) process. -Remote references come in two flavors -``Future`` and ``RemoteChannel``. +Remote references come in two flavors - :class:`Future` and :class:`RemoteChannel`. -A remote call returns a ``Future`` to its result. Remote calls +A remote call returns a :class:`Future` to its result. Remote calls return immediately; the process that made the call proceeds to its next operation while the remote call happens somewhere else. You can wait for a remote call to finish by calling :func:`wait` on the returned -`Future`, and you can obtain the full value of the result using +:class:`Future`, and you can obtain the full value of the result using :func:`fetch`. -On the other hand ``RemoteChannel`` s are rewritable. For example, multiple processes -can co-ordinate their processing by referencing the same remote ``Channel``\ . +On the other hand :class:`RemoteChannel` s are rewritable. For example, multiple processes +can co-ordinate their processing by referencing the same remote :class:`Channel`\ . Let's try this out. Starting with ``julia -p n`` provides ``n`` worker processes on the local machine. Generally it makes sense for ``n`` to @@ -113,15 +113,15 @@ Note that we used ``1 .+ fetch(r)`` instead of ``1 .+ r``. This is because we do not know where the code will run, so in general a :func:`fetch` might be required to move ``r`` to the process doing the addition. In this case, :obj:`@spawn` is smart enough to perform the computation on the -process that owns ``r``, so the :func:`fetch` will be a no-op. +process that owns ``r``, so the :func:`fetch` will be a no-op (no work is done). (It is worth noting that :obj:`@spawn` is not built-in but defined in Julia as a :ref:`macro <man-macros>`. It is possible to define your own such constructs.) -An important thing to remember is that, once fetched, a ``Future`` will cache its value -locally. Further ``fetch`` calls do not entail a network hop. Once all referencing Futures -have fetched, the remote stored value is deleted. +An important thing to remember is that, once fetched, a :class:`Future` will cache its value +locally. Further :func:`fetch` calls do not entail a network hop. Once all referencing +:class:`Future`s have fetched, the remote stored value is deleted. .. _man-parallel-computing-code-availability: @@ -191,7 +191,7 @@ A file can also be preloaded on multiple processes at startup, and a driver scri julia -p <n> -L file1.jl -L file2.jl driver.jl Each process has an associated identifier. The process providing the interactive Julia prompt -always has an id equal to 1, as would the Julia process running the driver script in the +always has an ``id`` equal to 1, as would the Julia process running the driver script in the example above. The processes used by default for parallel operations are referred to as "workers". When there is only one process, process 1 is considered a worker. Otherwise, workers are @@ -256,7 +256,7 @@ process needs matrix ``A`` then the first method might be better. Or, if computing ``A`` is expensive and only the current process has it, then moving it to another process might be unavoidable. Or, if the current process has very little to do between the :obj:`@spawn` and -``fetch(Bref)`` then it might be better to eliminate the parallelism +``fetch(Bref)``, it might be better to eliminate the parallelism altogether. Or imagine ``rand(1000,1000)`` is replaced with a more expensive operation. Then it might make sense to add another :obj:`@spawn` statement just for this step. @@ -455,44 +455,44 @@ Channels -------- Channels provide for a fast means of inter-task communication. A ``Channel{T}(n::Int)`` is a shared queue of maximum length ``n`` -holding objects of type ``T``. Multiple readers can read off the channel -via ``fetch`` and ``take!``. Multiple writers can add to the channel via -``put!``. ``isready`` tests for the presence of any object in -the channel, while ``wait`` waits for an object to become available. -``close`` closes a Channel. On a closed channel, ``put!`` will fail, -while ``take!`` and ``fetch`` successfully return any existing values +holding objects of type ``T``. Multiple readers can read off the :class:`Channel` +via :func:`fetch` and :func:`take!`. Multiple writers can add to the :class:`Channel` via +:func:`put!`. :func:`isready` tests for the presence of any object in +the channel, while :func:`wait` waits for an object to become available. +:func:`close` closes a :class:`Channel`. On a closed :class:`Channel`\, :func:`put!` will fail, +while :func:`take!` and :func:`fetch` successfully return any existing values till it is emptied. -A Channel can be used as an iterable object in a ``for`` loop, in which -case the loop runs as long as the channel has data or is open. The loop -variable takes on all values added to the channel. An empty, closed channel +A :class:`Channel` can be used as an iterable object in a ``for`` loop, in which +case the loop runs as long as the :class:`Channel` has data or is open. The loop +variable takes on all values added to the :class:`Channel`. An empty, closed :class:`Channel` causes the ``for`` loop to terminate. Remote references and AbstractChannels -------------------------------------- -Remote references always refer to an implementation of an ``AbstractChannel`` +Remote references always refer to an implementation of an :class:`AbstractChannel`. -A concrete implementation of an ``AbstractChannel`` (like ``Channel``), is required -to implement ``put!``\ , ``take!``\ , ``fetch``\ , ``isready`` and ``wait``\ . The remote object -referred to by a ``Future`` is stored in a ``Channel{Any}(1)``\ , i.e., a channel of size 1 -capable of holding objects of ``Any`` type. +A concrete implementation of an :class:`AbstractChannel` (like :class:`Channel`), is required +to implement :func:`put!`\ , :func:`take!`\ , :func:`fetch`\ , :func:`isready` and :func:`wait`\ . +The remote object referred to by a :class:`Future` is stored in a ``Channel{Any}(1)``\ , +i.e., a :class:`Channel` of size 1 capable of holding objects of ``Any`` type. -``RemoteChannel``\ , which is rewritable, can point to any type and size of channels, or any other -implementation of an ``AbstractChannel``\ . +:class:`RemoteChannel`\ , which is rewritable, can point to any type and size of channels, or any other +implementation of an :class:`AbstractChannel`\ . -The constructor ``RemoteChannel(f::Function, pid)`` allows us to construct references to channels holding +The constructor :func:`RemoteChannel(f::Function, pid)` allows us to construct references to channels holding more than one value of a specific type. ``f()`` is a function executed on ``pid`` and it must return -an ``AbstractChannel``\ . +an :class:`AbstractChannel`\ . For example, ``RemoteChannel(()->Channel{Int}(10), pid)``\ , will return a reference to a channel of type ``Int`` and size 10. The channel exists on worker ``pid``\ . -Methods ``put!``\ , ``take!``\ , ``fetch``\ , ``isready`` and ``wait`` on a ``RemoteChannel`` are proxied onto +Methods :func:`put!`\ , :func:`take!`\ , :func:`fetch`\ , :func:`isready` and :func:`wait` on a :class:`RemoteChannel` are proxied onto the backing store on the remote process. -``RemoteChannel`` can thus be used to refer to user implemented ``AbstractChannel`` objects. A simple +:class:`RemoteChannel` can thus be used to refer to user implemented :class:`AbstractChannel` objects. A simple example of this is provided in ``examples/dictchannel.jl`` which uses a dictionary as its remote store. @@ -503,20 +503,20 @@ Objects referred to by remote references can be freed only when *all* held refer are deleted. The node where the value is stored keeps track of which of the workers have a reference to it. -Every time a ``RemoteChannel`` or a (unfetched) ``Future`` is serialized to a worker, the node pointed -to by the reference is notified. And every time a ``RemoteChannel`` or a (unfetched) ``Future`` +Every time a :class:`RemoteChannel` or a (unfetched) :class:`Future` is serialized to a worker, the node pointed +to by the reference is notified. And every time a :class:`RemoteChannel` or a (unfetched) :class:`Future` is garbage collected locally, the node owning the value is again notified. The notifications are done via sending of "tracking" messages - an "add reference" message when a reference is serialized to a different process and a "delete reference" message when a reference is locally garbage collected. -Since ``Future``\ s are write-once and cached locally, the act of ``fetch``\ ing a ``Future`` also updates +Since :class:`Future`\ s are write-once and cached locally, the act of :func:`fetch`\ ing a :class:`Future` also updates reference tracking information on the node owning the value. The node which owns the value frees it once all references to it are cleared. -With ``Future``\ s, serializing an already fetched ``Future`` to a different node also sends the value +With :class:`Future`\ s, serializing an already fetched :class:`Future` to a different node also sends the value since the original remote store may have collected the value by this time. It is important to note that *when* an object is locally garbage collected depends @@ -524,9 +524,9 @@ on the size of the object and the current memory pressure in the system. In case of remote references, the size of the local reference object is quite small, while the value stored on the remote node may be quite large. Since the local object may not be collected immediately, it is -a good practice to explicitly call ``finalize`` on local instances of a ``RemoteChannel``, or on unfetched -``Future``\ s. Since calling ``fetch`` on a ``Future`` also removes its reference from the remote store, this -is not required on fetched ``Future``\ s. Explicitly calling ``finalize`` results in an immediate message sent to +a good practice to explicitly call :func:`finalize` on local instances of a :class:`RemoteChannel`, or on unfetched +:class:`Future`\ s. Since calling :func:`fetch` on a :class:`Future` also removes its reference from the remote store, this +is not required on fetched :class:`Future`\ s. Explicitly calling :func:`finalize` results in an immediate message sent to the remote node to go ahead and remove its reference to the value. Once finalized, a reference becomes invalid and cannot be used in any further calls. @@ -549,14 +549,14 @@ is available to the local process. Therefore, most algorithms work naturally on :class:`SharedArray`\ s, albeit in single-process mode. In cases where an algorithm insists on an :class:`Array` input, the underlying array can be retrieved from a :class:`SharedArray` by calling :func:`sdata`. -For other :class:`AbstractArray` types, ``sdata`` just returns the object -itself, so it's safe to use :func:`sdata` on any Array-type object. +For other :class:`AbstractArray` types, :func:`sdata` just returns the object +itself, so it's safe to use :func:`sdata` on any ``Array``\-type object. The constructor for a shared array is of the form:: SharedArray(T::Type, dims::NTuple; init=false, pids=Int[]) -which creates a shared array of a bitstype ``T`` and size ``dims`` +which creates a shared array of a bits type ``T`` and size ``dims`` across the processes specified by ``pids``. Unlike distributed arrays, a shared array is accessible only from those participating workers specified by the ``pids`` named argument (and the creating @@ -564,7 +564,7 @@ process too, if it is on the same host). If an ``init`` function, of signature ``initfn(S::SharedArray)``, is specified, it is called on all the participating workers. You can -arrange it so that each worker runs the ``init`` function on a +specify that each worker runs the ``init`` function on a distinct portion of the array, thereby parallelizing initialization. Here's a brief example: @@ -615,7 +615,7 @@ be careful not to set up conflicts. For example:: end end -would result in undefined behavior: because each process fills the +would result in undefined behavior. Because each process fills the *entire* array with its own ``pid``, whichever process is the last to execute (for any particular element of ``S``) will have its ``pid`` retained. @@ -730,37 +730,38 @@ ClusterManagers --------------- The launching, management and networking of Julia processes into a logical -cluster is done via cluster managers. A :obj:`ClusterManager` is responsible for +cluster is done via cluster managers. A :class:`ClusterManager` is responsible for - launching worker processes in a cluster environment - managing events during the lifetime of each worker -- optionally, a cluster manager can also provide data transport +- optionally, providing data transport A Julia cluster has the following characteristics: -- The initial Julia process, also called the ``master`` is special and has a id of 1. + +- The initial Julia process, also called the ``master``, is special and has an ``id`` of 1. - Only the ``master`` process can add or remove worker processes. - All processes can directly communicate with each other. Connections between workers (using the in-built TCP/IP transport) is established in the following manner: -- :func:`addprocs` is called on the master process with a :obj:`ClusterManager` object +- :func:`addprocs` is called on the master process with a :class:`ClusterManager` object - :func:`addprocs` calls the appropriate :func:`launch` method which spawns required number of worker processes on appropriate machines -- Each worker starts listening on a free port and writes out its host, port information to :const:`STDOUT` -- The cluster manager captures the stdout's of each worker and makes it available to the master process +- Each worker starts listening on a free port and writes out its host and port information to :const:`STDOUT` +- The cluster manager captures the :const:`STDOUT` of each worker and makes it available to the master process - The master process parses this information and sets up TCP/IP connections to each worker - Every worker is also notified of other workers in the cluster -- Each worker connects to all workers whose id is less than its own id +- Each worker connects to all workers whose ``id`` is less than the worker's own ``id`` - In this way a mesh network is established, wherein every worker is directly connected with every other worker -While the default transport layer uses plain TCP sockets, it is possible for a Julia cluster to provide +While the default transport layer uses plain :class:`TCPSocket`, it is possible for a Julia cluster to provide its own transport. Julia provides two in-built cluster managers: -- ``LocalManager``, used when :func:`addprocs` or :func:`addprocs(np::Integer) <addprocs>` are called -- ``SSHManager``, used when :func:`addprocs(hostnames::Array) <addprocs>` is called with a list of hostnames +- :class:`LocalManager`, used when :func:`addprocs` or :func:`addprocs(np::Integer) <addprocs>` are called +- :class:`SSHManager`, used when :func:`addprocs(hostnames::Array) <addprocs>` is called with a list of hostnames :class:`LocalManager` is used to launch additional workers on the same host, thereby leveraging multi-core and multi-processor hardware. @@ -769,7 +770,7 @@ Thus, a minimal cluster manager would need to: - be a subtype of the abstract :class:`ClusterManager` - implement :func:`launch`, a method responsible for launching new workers -- implement :func:`manage`, which is called at various events during a worker's lifetime +- implement :func:`manage`, which is called at various events during a worker's lifetime (for example, sending an interrupt signal) :func:`addprocs(manager::FooManager) <addprocs>` requires ``FooManager`` to implement:: @@ -808,13 +809,13 @@ signals that all requested workers have been launched. Hence the :func:`launch` as all the requested workers have been launched. Newly launched workers are connected to each other, and the master process, in a all-to-all manner. -Specifying command argument, ``--worker <cookie>`` results in the launched processes initializing themselves -as workers and connections being setup via TCP/IP sockets. Optionally ``--bind-to bind_addr[:port]`` +Specifying the command argument ``--worker <cookie>`` results in the launched processes initializing themselves +as workers and connections being setup via TCP/IP sockets. Optionally, ``--bind-to bind_addr[:port]`` may also be specified to enable other workers to connect to it at the specified ``bind_addr`` and ``port``. This is useful for multi-homed hosts. -For non-TCP/IP transports, for example, an implementation may choose to use MPI as the transport, -``--worker`` must NOT be specified. Instead newly launched workers should call ``init_worker(cookie)`` +As an example of a non-TCP/IP transport, an implementation may choose to use MPI, in which case +``--worker`` must NOT be specified. Instead, newly launched workers should call ``init_worker(cookie)`` before using any of the parallel constructs. For every worker launched, the :func:`launch` method must add a :class:`WorkerConfig` @@ -849,24 +850,21 @@ object (with appropriate fields initialized) to ``launched`` :: Most of the fields in :class:`WorkerConfig` are used by the inbuilt managers. Custom cluster managers would typically specify only ``io`` or ``host`` / ``port``: -If ``io`` is specified, it is used to read host/port information. A Julia -worker prints out its bind address and port at startup. This allows Julia -workers to listen on any free port available instead of requiring worker ports -to be configured manually. - -If ``io`` is not specified, ``host`` and ``port`` are used to connect. - -``count``, ``exename`` and ``exeflags`` are relevant for launching additional workers from a worker. -For example, a cluster manager may launch a single worker per node, and use that to launch -additional workers. ``count`` with an integer value ``n`` will launch a total of ``n`` workers, -while a value of ``:auto`` will launch as many workers as cores on that machine. -``exename`` is the name of the ``julia`` executable including the full path. -``exeflags`` should be set to the required command line arguments for new workers. - -``tunnel``, ``bind_addr``, ``sshflags`` and ``max_parallel`` are used when a ssh tunnel is -required to connect to the workers from the master process. - -``userdata`` is provided for custom cluster managers to store their own worker specific information. +- If ``io`` is specified, it is used to read host/port information. A Julia + worker prints out its bind address and port at startup. This allows Julia + workers to listen on any free port available instead of requiring worker ports + to be configured manually. +- If ``io`` is not specified, ``host`` and ``port`` are used to connect. +- ``count``, ``exename`` and ``exeflags`` are relevant for launching additional workers from a worker. + For example, a cluster manager may launch a single worker per node, and use that to launch + additional workers. + - ``count`` with an integer value ``n`` will launch a total of ``n`` workers. + - ``count`` with a value of ``:auto`` will launch as many workers as cores on that machine. + - ``exename`` is the name of the ``julia`` executable including the full path. + - ``exeflags`` should be set to the required command line arguments for new workers. +- ``tunnel``, ``bind_addr``, ``sshflags`` and ``max_parallel`` are used when a ssh tunnel is + required to connect to the workers from the master process. +- ``userdata`` is provided for custom cluster managers to store their own worker specific information. ``manage(manager::FooManager, id::Integer, config::WorkerConfig, op::Symbol)`` is called at different @@ -889,26 +887,26 @@ Each Julia process has as many communication tasks as the workers it is connecte - Each Julia process thus has 31 communication tasks - Each task handles all incoming messages from a single remote worker in a message processing loop -- The message processing loop waits on an ``AsyncStream`` object - for example, a TCP socket in the default implementation, reads an entire +- The message processing loop waits on an :class:`IO` object (for example, a :class:`TCPSocket` in the default implementation), reads an entire message, processes it and waits for the next one - Sending messages to a process is done directly from any Julia task - not just communication tasks - again, via the appropriate - ``AsyncStream`` object + :class:`IO` object Replacing the default transport involves the new implementation to setup connections to remote workers, and to provide appropriate -``AsyncStream`` objects that the message processing loops can wait on. The manager specific callbacks to be implemented are:: +:class:`IO` objects that the message processing loops can wait on. The manager specific callbacks to be implemented are:: connect(manager::FooManager, pid::Integer, config::WorkerConfig) kill(manager::FooManager, pid::Int, config::WorkerConfig) The default implementation (which uses TCP/IP sockets) is implemented as ``connect(manager::ClusterManager, pid::Integer, config::WorkerConfig)``. -``connect`` should return a pair of ``AsyncStream`` objects, one for reading data sent from worker ``pid``, -and the other to write data that needs to be sent to worker ``pid``. Custom cluster managers can use an in-memory ``BufferStream`` -as the plumbing to proxy data between the custom, possibly non-AsyncStream transport and Julia's in-built parallel infrastructure. +``connect`` should return a pair of :class:`IO` objects, one for reading data sent from worker ``pid``, +and the other to write data that needs to be sent to worker ``pid``. Custom cluster managers can use an in-memory :class:`BufferStream` +as the plumbing to proxy data between the custom, possibly non-:class:`IO` transport and Julia's in-built parallel infrastructure. -A ``BufferStream`` is an in-memory ``IOBuffer`` which behaves like an ``AsyncStream``. +A :class:`BufferStream` is an in-memory ``IOBuffer`` which behaves like an :class:`IO` - it is a stream which can be handled asynchronously. -Folder ``examples/clustermanager/0mq`` is an example of using ZeroMQ is connect Julia workers in a star network with a 0MQ broker in the middle. +Folder ``examples/clustermanager/0mq`` contains an example of using ZeroMQ to connect Julia workers in a star topology with a 0MQ broker in the middle. Note: The Julia processes are still all *logically* connected to each other - any worker can message any other worker directly without any awareness of 0MQ being used as the transport layer. @@ -916,37 +914,37 @@ When using custom transports: - Julia workers must NOT be started with ``--worker``. Starting with ``--worker`` will result in the newly launched workers defaulting to the TCP/IP socket transport implementation -- For every incoming logical connection with a worker, ``Base.process_messages(rd::AsyncStream, wr::AsyncStream)`` must be called. - This launches a new task that handles reading and writing of messages from/to the worker represented by the ``AsyncStream`` objects -- ``init_worker(cookie, manager::FooManager)`` MUST be called as part of worker process initializaton -- Field ``connect_at::Any`` in :class:`WorkerConfig` can be set by the cluster manager when ``launch`` is called. The value of - this field is passed in in all ``connect`` callbacks. Typically, it carries information on *how to connect* to a worker. For example, +- For every incoming logical connection with a worker, :func:`Base.process_messages(rd::IO, wr::IO)` must be called. + This launches a new task that handles reading and writing of messages from/to the worker represented by the :class:`IO` objects +- ``init_worker(cookie, manager::FooManager)`` MUST be called as part of worker process initialization +- Field ``connect_at::Any`` in :class:`WorkerConfig` can be set by the cluster manager when :func:`launch` is called. The value of + this field is passed in in all :func:`connect` callbacks. Typically, it carries information on *how to connect* to a worker. For example, the TCP/IP socket transport uses this field to specify the ``(host, port)`` tuple at which to connect to a worker ``kill(manager, pid, config)`` is called to remove a worker from the cluster. -On the master process, the corresponding ``AsyncStream`` objects must be closed by the implementation to ensure proper cleanup. The default -implementation simply executes an ``exit()`` call on the specified remote worker. +On the master process, the corresponding :class:`IO` objects must be closed by the implementation to ensure proper cleanup. +The default implementation simply executes an ``exit()`` call on the specified remote worker. -``examples/clustermanager/simple`` is an example that shows a simple implementation using unix domain sockets for cluster setup +``examples/clustermanager/simple`` is an example that shows a simple implementation using UNIX domain sockets for cluster setup. Network requirements for LocalManager and SSHManager ---------------------------------------------------- -Julia clusters are designed to be executed on already secured environments on infrastructure ranging from local laptops, -to departmental clusters or even on the cloud. This section covers network security requirements for the inbuilt ``LocalManager`` -and ``SSHManager``: +Julia clusters are designed to be executed on already secured environments on infrastructure such as local laptops, +departmental clusters, or even on the cloud. This section covers network security requirements for the inbuilt :class:`LocalManager` +and :class:`SSHManager`: - The master process does not listen on any port. It only connects out to the workers. -- Each worker binds to only one of the local interfaces and listens on the first free port starting from 9009. +- Each worker binds to only one of the local interfaces and listens on the first free port starting from ``9009``. -- ``LocalManager``, i.e. ``addprocs(N)``, by default binds only to the loopback interface. +- :class:`LocalManager`, i.e. ``addprocs(N)``, by default binds only to the loopback interface. This means that workers consequently started on remote hosts, or anyone with malicious intentions is unable to connect to the cluster. A ``addprocs(4)`` followed by a ``addprocs(["remote_host"])`` will fail. Some users may need to create a cluster comprised of their local system and a few remote systems. - This can be done by explicitly requesting ``LocalManager`` to bind to an external network interface via the + This can be done by explicitly requesting :class:`LocalManager` to bind to an external network interface via the ``restrict`` keyword argument. For example, ``addprocs(4; restrict=false)``. -- ``SSHManager``, i.e. ``addprocs(list_of_remote_hosts)`` launches workers on remote hosts via SSH. +- :class:`SSHManager`, i.e. ``addprocs(list_of_remote_hosts)`` launches workers on remote hosts via SSH. It is to be noted that by default SSH is only used to launch Julia workers. Subsequent master-worker and worker-worker connections use plain, unencrypted TCP/IP sockets. The remote hosts must have passwordless login enabled. Additional SSH flags or credentials may be specified via keyword @@ -955,7 +953,7 @@ and ``SSHManager``: - ``addprocs(list_of_remote_hosts; tunnel=true, sshflags=<ssh keys and other flags>)`` is useful when we wish to use SSH connections for master-worker too. A typical scenario for this is a local laptop running the Julia REPL (i.e., the master) with the rest of the cluster on the cloud, say on Amazon EC2. In this case only port 22 needs to be - opened at the remote cluster coupled with SSH client authenticated via PKI. + opened at the remote cluster coupled with SSH client authenticated via public key infrastructure (PKI). Authentication credentials can be supplied via ``sshflags``, for example ``sshflags=`-e <keyfile>```. Note that worker-worker connections are still plain TCP and the local security policy on the remote cluster @@ -968,17 +966,16 @@ Cluster cookie -------------- All processes in a cluster share the same cookie which, by default, is a randomly generated string on the master process: -- ``Base.cluster_cookie()`` returns the cookie, ``Base.cluster_cookie(cookie)`` sets it. +- :func:`Base.cluster_cookie()` returns the cookie, while :func:`Base.cluster_cookie(cookie)` sets it and returns the new cookie. - All connections are authenticated on both sides to ensure that only workers started by the master are allowed to connect to each other. - The cookie must be passed to the workers at startup via argument ``--worker <cookie>``. Custom ClusterManagers can retrieve the cookie on the master by calling - ``Base.cluster_cookie()``. Cluster managers not using the default TCP/IP transport (and hence not specifying ``--worker``) + :func:`Base.cluster_cookie()`. Cluster managers not using the default TCP/IP transport (and hence not specifying ``--worker``) must call ``init_worker(cookie, manager)`` with the same cookie as on the master. -It is to be noted that environments requiring higher levels of security (for example, cookies can be pre-shared and hence not -specified as a startup arg) can implement this via a custom ClusterManager. - +Note that environments requiring higher levels of security can implement this via a custom :class:`ClusterManager`. +For example, cookies can be pre-shared and hence not specified as a startup argument. Specifying network topology (Experimental) ------------------------------------------- @@ -987,19 +984,19 @@ Keyword argument ``topology`` to ``addprocs`` is used to specify how the workers - ``:all_to_all`` : is the default, where all workers are connected to each other. -- ``:master_slave`` : only the driver process, i.e. pid 1 has connections to the workers. +- ``:master_slave`` : only the driver process, i.e. `pid` 1 has connections to the workers. -- ``:custom`` : the ``launch`` method of the cluster manager specifes the connection topology. +- ``:custom`` : the ``launch`` method of the cluster manager specifies the connection topology. Fields ``ident`` and ``connect_idents`` in ``WorkerConfig`` are used to specify the same. - ``connect_idents`` is a list of ``ClusterManager`` provided identifiers to workers that worker + ``connect_idents`` is a list of :class:`ClusterManager` provided identifiers to workers that worker with identified by ``ident`` must connect to. -Currently sending a message between unconnected workers results in an error. This behaviour, as also the -functionality and interface should be considered experimental in nature and may change in future releases. +Currently, sending a message between unconnected workers results in an error. This behaviour, as with the +functionality and interface, should be considered experimental in nature and may change in future releases. Multi-threading (Experimental) ------------------------------- -In addition to tasks, remote calls and remote references, Julia from v0.5 will natively support +In addition to tasks, remote calls, and remote references, Julia from ``v0.5`` forwards will natively support multi-threading. Note that this section is experimental and the interfaces may change in the future. diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 93c5befca6d60..0cd92137b958b 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -143,27 +143,27 @@ Tasks General Parallel Computing Support ---------------------------------- -.. function:: addprocs(n::Integer; exeflags=``) -> List of process identifiers +.. function:: addprocs(np::Integer; restrict=true, kwargs...) -> List of process identifiers .. Docstring generated from Julia source - Launches workers using the in-built ``LocalManager`` which only launches workers on the local host. This can be used to take advantage of multiple cores. ``addprocs(4)`` will add 4 processes on the local machine. + Launches workers using the in-built ``LocalManager`` which only launches workers on the local host. This can be used to take advantage of multiple cores. ``addprocs(4)`` will add 4 processes on the local machine. If ``restrict`` is ``true``\ , binding is restricted to ``127.0.0.1``\ . -.. function:: addprocs() -> List of process identifiers +.. function:: addprocs(; kwargs...) -> List of process identifiers .. Docstring generated from Julia source - Equivalent to ``addprocs(Sys.CPU_CORES)`` + Equivalent to ``addprocs(Sys.CPU_CORES; kwargs...)`` Note that workers do not run a ``.juliarc.jl`` startup script, nor do they synchronize their global state (such as global variables, new method definitions, and loaded modules) with any of the other running processes. -.. function:: addprocs(machines; keyword_args...) -> List of process identifiers +.. function:: addprocs(machines; tunnel=false, sshflags=\`\`, max_parallel=10, kwargs...) -> List of process identifiers .. Docstring generated from Julia source Add processes on remote machines via SSH. Requires ``julia`` to be installed in the same location on each node, or to be available via a shared file system. - ``machines`` is a vector of machine specifications. Worker are started for each specification. + ``machines`` is a vector of machine specifications. Workers are started for each specification. A machine specification is either a string ``machine_spec`` or a tuple - ``(machine_spec, count)``\ . @@ -174,11 +174,7 @@ General Parallel Computing Support Keyword arguments: * ``tunnel``\ : if ``true`` then SSH tunneling will be used to connect to the worker from the master process. Default is ``false``\ . - * ``sshflags``\ : specifies additional ssh options, e.g. - - .. code-block:: julia - - sshflags=`-i /home/foo/bar.pem` + * ``sshflags``\ : specifies additional ssh options, e.g. ``sshflags=`-i /home/foo/bar.pem``` * ``max_parallel``\ : specifies the maximum number of workers connected to in parallel at a host. Defaults to 10. * ``dir``\ : specifies the working directory on the workers. Defaults to the host's current directory (as found by ``pwd()``\ ) * ``exename``\ : name of the ``julia`` executable. Defaults to ``"$JULIA_HOME/julia"`` or ``"$JULIA_HOME/julia-debug"`` as the case may be. @@ -191,7 +187,7 @@ General Parallel Computing Support Environment variables : - If the master process fails to establish a connection with a newly launched worker within 60.0 seconds, the worker treats it a fatal situation and terminates. This timeout can be controlled via environment variable ``JULIA_WORKER_TIMEOUT``\ . The value of ``JULIA_WORKER_TIMEOUT`` on the master process, specifies the number of seconds a newly launched worker waits for connection establishment. + If the master process fails to establish a connection with a newly launched worker within 60.0 seconds, the worker treats it as a fatal situation and terminates. This timeout can be controlled via environment variable ``JULIA_WORKER_TIMEOUT``\ . The value of JULIA_WORKER_TIMEOUT` on the master process specifies the number of seconds a newly launched worker waits for connection establishment. .. function:: addprocs(manager::ClusterManager; kwargs...) -> List of process identifiers @@ -199,7 +195,7 @@ General Parallel Computing Support Launches worker processes via the specified cluster manager. - For example Beowulf clusters are supported via a custom cluster manager implemented in package ``ClusterManagers``\ . + For example Beowulf clusters are supported via a custom cluster manager implemented in package ``ClusterManagers.jl``\ . The number of seconds a newly launched worker waits for connection establishment from the master can be specified via variable ``JULIA_WORKER_TIMEOUT`` in the worker process's environment. Relevant only when using TCP/IP as transport. @@ -285,6 +281,12 @@ General Parallel Computing Support Call a function asynchronously on the given arguments on the specified process. Returns a ``Future``\ . Keyword arguments, if any, are passed through to ``func``\ . +.. function:: Base.process_messages(r_stream::IO, w_stream::IO, incoming::Bool=true) + + .. Docstring generated from Julia source + + Called by cluster managers using custom transports. It should be called when the custom transport implementation receives the first message from a remote worker. The custom transport must manage a logical connection to the remote worker and provide two ``IO`` objects, one for incoming messages and the other for messages addressed to the remote worker. If ``incoming`` is ``true``\ , the remote peer initiated the connection. Whichever of the pair initiates the connection sends the cluster cookie and its Julia version number to perform the authentication handshake. + .. function:: Future() .. Docstring generated from Julia source @@ -808,13 +810,13 @@ LocalManager, for launching additional workers on the same host and SSHManager, hosts via ssh are present in Base. TCP/IP sockets are used to connect and transport messages between processes. It is possible for Cluster Managers to provide a different transport. -.. function:: launch(manager::FooManager, params::Dict, launched::Vector{WorkerConfig}, launch_ntfy::Condition) +.. function:: launch(manager::ClusterManager, params::Dict, launched::Array, launch_ntfy::Condition) .. Docstring generated from Julia source Implemented by cluster managers. For every Julia worker launched by this function, it should append a ``WorkerConfig`` entry to ``launched`` and notify ``launch_ntfy``\ . The function MUST exit once all workers, requested by ``manager`` have been launched. ``params`` is a dictionary of all keyword arguments ``addprocs`` was called with. -.. function:: manage(manager::FooManager, pid::Int, config::WorkerConfig. op::Symbol) +.. function:: manage(manager::ClusterManager, id::Integer, config::WorkerConfig. op::Symbol) .. Docstring generated from Julia source @@ -824,27 +826,21 @@ between processes. It is possible for Cluster Managers to provide a different tr * with ``:interrupt`` when ``interrupt(workers)`` is called. The :class:`ClusterManager` should signal the appropriate worker with an interrupt signal. * with ``:finalize`` for cleanup purposes. -.. function:: kill(manager::FooManager, pid::Int, config::WorkerConfig) +.. function:: kill(manager::ClusterManager, pid::Int, config::WorkerConfig) .. Docstring generated from Julia source Implemented by cluster managers. It is called on the master process, by ``rmprocs``\ . It should cause the remote worker specified by ``pid`` to exit. ``Base.kill(manager::ClusterManager.....)`` executes a remote ``exit()`` on ``pid`` -.. function:: init_worker(manager::FooManager) - - .. Docstring generated from Julia source - - Called by cluster managers implementing custom transports. It initializes a newly launched process as a worker. Command line argument ``--worker`` has the effect of initializing a process as a worker using TCP/IP sockets for transport. - -.. function:: connect(manager::FooManager, pid::Int, config::WorkerConfig) -> (instrm::AsyncStream, outstrm::AsyncStream) +.. function:: init_worker(cookie::AbstractString, manager::ClusterManager=DefaultClusterManager()) .. Docstring generated from Julia source - Implemented by cluster managers using custom transports. It should establish a logical connection to worker with id ``pid``\ , specified by ``config`` and return a pair of ``AsyncStream`` objects. Messages from ``pid`` to current process will be read off ``instrm``\ , while messages to be sent to ``pid`` will be written to ``outstrm``\ . The custom transport implementation must ensure that messages are delivered and received completely and in order. ``Base.connect(manager::ClusterManager.....)`` sets up TCP/IP socket connections in-between workers. + Called by cluster managers implementing custom transports. It initializes a newly launched process as a worker. Command line argument ``--worker`` has the effect of initializing a process as a worker using TCP/IP sockets for transport. ``cookie`` is a :func:`cluster_cookie`\ . -.. function:: Base.process_messages(instrm::AsyncStream, outstrm::AsyncStream) +.. function:: connect(manager::ClusterManager, pid::Int, config::WorkerConfig) -> (instrm::IO, outstrm::IO) .. Docstring generated from Julia source - Called by cluster managers using custom transports. It should be called when the custom transport implementation receives the first message from a remote worker. The custom transport must manage a logical connection to the remote worker and provide two ``AsyncStream`` objects, one for incoming messages and the other for messages addressed to the remote worker. + Implemented by cluster managers using custom transports. It should establish a logical connection to worker with id ``pid``\ , specified by ``config`` and return a pair of ``IO`` objects. Messages from ``pid`` to current process will be read off ``instrm``\ , while messages to be sent to ``pid`` will be written to ``outstrm``\ . The custom transport implementation must ensure that messages are delivered and received completely and in order. ``Base.connect(manager::ClusterManager.....)`` sets up TCP/IP socket connections in-between workers. From 5da915706fa98162a025a94e95fb80c05da3af93 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 30 Jul 2016 10:03:25 +0800 Subject: [PATCH 0715/1117] Improve REPL SIGINT user experience. * Add a SIGINT dead time after a force throw Fixes Jeff's issue in #17706 * Make the eval and print loop sigatomic to avoid sigint being delivered outside `try`-`catch` blocks. --- base/REPL.jl | 6 ++++++ src/signal-handling.c | 17 ++++++++++++++++- src/signals-unix.c | 3 ++- src/signals-win.c | 6 ++++-- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/base/REPL.jl b/base/REPL.jl index edc2109516dd5..0ad9968178ce8 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -52,8 +52,10 @@ end function eval_user_input(ast::ANY, backend::REPLBackend) iserr, lasterr = false, ((), nothing) + Base.sigatomic_begin() while true try + Base.sigatomic_end() if iserr put!(backend.response_channel, lasterr) iserr, lasterr = false, () @@ -74,6 +76,7 @@ function eval_user_input(ast::ANY, backend::REPLBackend) iserr, lasterr = true, (err, catch_backtrace()) end end + Base.sigatomic_end() end function start_repl_backend(repl_channel::Channel, response_channel::Channel) @@ -136,8 +139,10 @@ function print_response(repl::AbstractREPL, val::ANY, bt, show_value::Bool, have print_response(outstream(repl), val, bt, show_value, have_color, specialdisplay(repl)) end function print_response(errio::IO, val::ANY, bt, show_value::Bool, have_color::Bool, specialdisplay=nothing) + Base.sigatomic_begin() while true try + Base.sigatomic_end() if bt !== nothing display_error(errio, val, bt) println(errio) @@ -166,6 +171,7 @@ function print_response(errio::IO, val::ANY, bt, show_value::Bool, have_color::B bt = catch_backtrace() end end + Base.sigatomic_end() end # A reference to a backend diff --git a/src/signal-handling.c b/src/signal-handling.c index cb2dda03ac7db..adc91f7a70a03 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -24,6 +24,7 @@ JL_DLLEXPORT void jl_profile_stop_timer(void); JL_DLLEXPORT int jl_profile_start_timer(void); static uint64_t jl_last_sigint_trigger = 0; +static uint64_t jl_disable_sigint_time = 0; static void jl_clear_force_sigint(void) { jl_last_sigint_trigger = 0; @@ -44,7 +45,21 @@ static int jl_check_force_sigint(void) if (!isnormal(new_weight)) new_weight = 0; accum_weight = new_weight; - return new_weight > 1; + if (new_weight > 1) { + jl_disable_sigint_time = cur_time + (uint64_t)0.5e9; + return 1; + } + jl_disable_sigint_time = 0; + return 0; +} + +// Force sigint requires pressing `Ctrl-C` repeatedly. +// Ignore sigint for a short time after that to avoid rethrowing sigint too +// quickly again. (Code that has this issue is inherently racy but this is +// a interactive feature anyway.) +static int jl_ignore_sigint(void) +{ + return jl_disable_sigint_time && jl_disable_sigint_time > uv_hrtime(); } static int exit_on_sigint = 0; diff --git a/src/signals-unix.c b/src/signals-unix.c index 838b162cb5e45..9557dd0a3bd60 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -389,7 +389,8 @@ static void *signal_listener(void *arg) critical = 1; } else { - jl_try_deliver_sigint(); + if (!jl_ignore_sigint()) + jl_try_deliver_sigint(); continue; } } diff --git a/src/signals-win.c b/src/signals-win.c index 1f74c77d7f2c2..5388ac33aadf7 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -81,7 +81,8 @@ void __cdecl crt_sig_handler(int sig, int num) signal(SIGINT, (void (__cdecl *)(int))crt_sig_handler); if (exit_on_sigint) jl_exit(130); // 128 + SIGINT - jl_try_throw_sigint(); + if (!jl_ignore_sigint()) + jl_try_throw_sigint(); break; default: // SIGSEGV, (SSIGTERM, IGILL) memset(&Context, 0, sizeof(Context)); @@ -180,7 +181,8 @@ static BOOL WINAPI sigint_handler(DWORD wsig) //This needs winapi types to guara } if (exit_on_sigint) jl_exit(128 + sig); // 128 + SIGINT - jl_try_deliver_sigint(); + if (!jl_ignore_sigint()) + jl_try_deliver_sigint(); return 1; } From 2893c0256c3b317e3f4e7f244ce634fe718b3f98 Mon Sep 17 00:00:00 2001 From: Peter Colberg <peter@colberg.org> Date: Sat, 30 Jul 2016 16:55:13 -0400 Subject: [PATCH 0716/1117] Fix spelling mistakes --- base/docs/Docs.jl | 2 +- base/docs/helpdb/Base.jl | 2 +- base/error.jl | 2 +- base/inference.jl | 2 +- base/managers.jl | 2 +- base/multi.jl | 2 +- base/operators.jl | 2 +- base/subarray.jl | 2 +- doc/stdlib/base.rst | 2 +- doc/stdlib/math.rst | 2 +- doc/stdlib/parallel.rst | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 754c63607c310..76b1401500208 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -590,7 +590,7 @@ finddoc(λ, def) = false const FUNC_HEADS = [:function, :stagedfunction, :macro, :(=)] const BINDING_HEADS = [:typealias, :const, :global, :(=)] -# For the special `:@mac` / `:(Base.@mac)` syntax for documenting a macro after defintion. +# For the special `:@mac` / `:(Base.@mac)` syntax for documenting a macro after definition. isquotedmacrocall(x) = isexpr(x, :copyast, 1) && isa(x.args[1], QuoteNode) && diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 26225f4362986..80f726d0b53df 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -3788,7 +3788,7 @@ DimensionMismatch """ take!(RemoteChannel) -Fetch a value from a remote channel, also removing it in the processs. +Fetch a value from a remote channel, also removing it in the process. """ take!(::RemoteChannel) diff --git a/base/error.jl b/base/error.jl index 70a771d754545..9beef3b93bdd7 100644 --- a/base/error.jl +++ b/base/error.jl @@ -65,7 +65,7 @@ for exceptions of that type. If `retry_on` is a function The first retry happens after a gap of 50 milliseconds or `max_delay`, whichever is lower. Subsequently, the delays between retries are -exponentially increased with a random factor upto `max_delay`. +exponentially increased with a random factor up to `max_delay`. **Examples** ```julia diff --git a/base/inference.jl b/base/inference.jl index e4e5cfb08b269..90ae3d82356d9 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1631,7 +1631,7 @@ function typeinf_loop(frame) # end in_typeinf_loop = false catch ex - println("WARNING: An error occured during inference. Type inference is now partially disabled.") + println("WARNING: An error occurred during inference. Type inference is now partially disabled.") println(ex) ccall(:jlbacktrace, Void, ()) end diff --git a/base/managers.jl b/base/managers.jl index 68f05b4efa33b..53e0db798df3e 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -158,7 +158,7 @@ function launch_on_machine(manager::SSHManager, machine, cnt, params, launched, machine_def = split(machine_bind[1], ':') # if this machine def has a port number, add the port information to the ssh flags if length(machine_def) > 2 - throw(ArgumentError("invalid machine defintion format string: invalid port format \"$machine_def\"")) + throw(ArgumentError("invalid machine definition format string: invalid port format \"$machine_def\"")) end host = machine_def[1] portopt = `` diff --git a/base/multi.jl b/base/multi.jl index 94fa66909098a..4e8b5a5617350 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -1142,7 +1142,7 @@ function message_handler_loop(r_stream::IO, w_stream::IO, incoming::Bool) oldstate = werr.state set_worker_state(werr, W_TERMINATED) - # If unhandleable error occured talking to pid 1, exit + # If unhandleable error occurred talking to pid 1, exit if wpid == 1 if isopen(w_stream) print(STDERR, "fatal error on ", myid(), ": ") diff --git a/base/operators.jl b/base/operators.jl index 22febbc58a633..1998122216ba1 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -222,7 +222,7 @@ Unsigned right bit shift operator, `x >>> n`. For `n >= 0`, the result is `x` shifted right by `n` bits, where `n >= 0`, filling with `0`s. For `n < 0`, this is equivalent to `x [<<](:func:`<<`) -n`]. -For `Unsigned` integer types, this is eqivalent to [`>>`](:func:`>>`). For +For `Unsigned` integer types, this is equivalent to [`>>`](:func:`>>`). For `Signed` integer types, this is equivalent to `signed(unsigned(x) >> n)`. ```jldoctest diff --git a/base/subarray.jl b/base/subarray.jl index 4d4fa370410b6..1f1be5b260a72 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -300,7 +300,7 @@ end """ replace_ref_end!(ex) -Recursively replace occurences of the symbol :end in a "ref" expression (i.e. A[...]) `ex` +Recursively replace occurrences of the symbol :end in a "ref" expression (i.e. A[...]) `ex` with the appropriate function calls (`endof`, `size` or `trailingsize`). Replacement uses the closest enclosing ref, so diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index def151edf1efb..eba35ef40dfc1 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -1252,7 +1252,7 @@ Errors Returns a lambda that retries function ``f`` up to ``n`` times in the event of an exception. If ``retry_on`` is a ``Type`` then retry only for exceptions of that type. If ``retry_on`` is a function ``test_error(::Exception) -> Bool`` then retry only if it is true. - The first retry happens after a gap of 50 milliseconds or ``max_delay``\ , whichever is lower. Subsequently, the delays between retries are exponentially increased with a random factor upto ``max_delay``\ . + The first retry happens after a gap of 50 milliseconds or ``max_delay``\ , whichever is lower. Subsequently, the delays between retries are exponentially increased with a random factor up to ``max_delay``\ . **Examples** diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index f6e0fe5496fca..0d26212076d46 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -275,7 +275,7 @@ Mathematical Operators Unsigned right bit shift operator, ``x >>> n``\ . For ``n >= 0``\ , the result is ``x`` shifted right by ``n`` bits, where ``n >= 0``\ , filling with ``0``\ s. For ``n < 0``\ , this is equivalent to ``x [<<](:func:``\ <<``) -n``\ ]. - For ``Unsigned`` integer types, this is eqivalent to :func:`>>`\ . For ``Signed`` integer types, this is equivalent to ``signed(unsigned(x) >> n)``\ . + For ``Unsigned`` integer types, this is equivalent to :func:`>>`\ . For ``Signed`` integer types, this is equivalent to ``signed(unsigned(x) >> n)``\ . .. doctest:: diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 0cd92137b958b..804392667d439 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -381,7 +381,7 @@ General Parallel Computing Support .. Docstring generated from Julia source - Fetch a value from a remote channel, also removing it in the processs. + Fetch a value from a remote channel, also removing it in the process. .. function:: take!(Channel) From 29f2736ab862d355d7161629d0ffd647ed85f3c0 Mon Sep 17 00:00:00 2001 From: Carlo Baldassi <carlobaldassi@gmail.com> Date: Sat, 30 Jul 2016 03:09:01 +0200 Subject: [PATCH 0717/1117] Pkg: more aggressive dependency pruning helps resolve --- base/pkg/query.jl | 28 ++++++++++++++-------------- base/pkg/resolve.jl | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/base/pkg/query.jl b/base/pkg/query.jl index b44db7a68b3f1..6f605c52a5b13 100644 --- a/base/pkg/query.jl +++ b/base/pkg/query.jl @@ -197,11 +197,6 @@ end # the allowed range (checking for impossible ranges while at it). # This is a pre-pruning step, so it also creates some structures which are later used by pruning function filter_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}) - # To each version in each package, we associate a BitVector. - # It is going to hold a pattern such that all versions with - # the same pattern are equivalent. - vmask = Dict{String,Dict{VersionNumber, BitVector}}() - # Parse requirements and store allowed versions. allowed = Dict{String,Dict{VersionNumber, Bool}}() for (p,vs) in reqs @@ -228,7 +223,7 @@ function filter_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Av end end - return filtered_deps, allowed, vmask + return filtered_deps, allowed end # Reduce the number of versions by creating equivalence classes, and retaining @@ -239,7 +234,12 @@ end # 2) They have the same dependencies # Preliminarily calls filter_versions. function prune_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}) - filtered_deps, allowed, vmask = filter_versions(reqs, deps) + filtered_deps, allowed = filter_versions(reqs, deps) + + # To each version in each package, we associate a BitVector. + # It is going to hold a pattern such that all versions with + # the same pattern are equivalent. + vmask = Dict{String,Dict{VersionNumber, BitVector}}() # For each package, we examine the dependencies of its versions # and put together those which are equal. @@ -249,10 +249,10 @@ function prune_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Ava # Extract unique dependencies lists (aka classes), thereby # assigning an index to each class. - uniqdepssets = unique(values(fdepsp)) + uniqdepssets = unique(a.requires for a in values(fdepsp)) # Store all dependencies seen so far for later use - for a in uniqdepssets, (rp,rvs) in a.requires + for r in uniqdepssets, (rp,rvs) in r haskey(alldeps, rp) || (alldeps[rp] = Set{VersionSet}()) push!(alldeps[rp], rvs) end @@ -271,8 +271,8 @@ function prune_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Ava vmaskp[vn] = falses(luds) end for (vn,a) in fdepsp - vmind = findfirst(uniqdepssets, a) - @assert vmind >= 0 + vmind = findfirst(uniqdepssets, a.requires) + @assert vmind > 0 vm = vmaskp[vn] vm[vmind] = true end @@ -291,7 +291,7 @@ function prune_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Ava @assert haskey(vmask, p) vmaskp = vmask[p] for (vn,vm) in vmaskp - push!(vm, in(vn, vs)) + push!(vm, vn in vs) end end @@ -302,7 +302,7 @@ function prune_versions(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Ava for (p, vmaskp) in vmask vmask0_uniq = unique(values(vmaskp)) nc = length(vmask0_uniq) - classes = [ VersionNumber[] for c0 = 1:nc ] + classes = [VersionNumber[] for c0 = 1:nc] for (vn,vm) in vmaskp c0 = findfirst(vmask0_uniq, vm) push!(classes[c0], vn) @@ -452,7 +452,7 @@ end function filter_dependencies(reqs::Requires, deps::Dict{String,Dict{VersionNumber,Available}}) deps = dependencies_subset(deps, Set{String}(keys(reqs))) - deps, _, _ = filter_versions(reqs, deps) + deps, _ = filter_versions(reqs, deps) return deps end diff --git a/base/pkg/resolve.jl b/base/pkg/resolve.jl index 184ff24fd113c..1f07d66fb98e3 100644 --- a/base/pkg/resolve.jl +++ b/base/pkg/resolve.jl @@ -91,7 +91,7 @@ function sanity_check(deps::Dict{String,Dict{VersionNumber,Available}}, end sub_reqs = Dict{String,VersionSet}(p=>VersionSet([vn, nvn])) - sub_deps = Query.filter_dependencies(sub_reqs, deps) + sub_deps = Query.prune_dependencies(sub_reqs, deps) interface = Interface(sub_reqs, sub_deps) red_pkgs = interface.pkgs From d6c67c259022f53b3373fcc2340f0259b46798ba Mon Sep 17 00:00:00 2001 From: pabloferz <pabloferz@yahoo.com.mx> Date: Fri, 29 Jul 2016 23:14:09 +0200 Subject: [PATCH 0718/1117] Speed up ntuple --- base/tuple.jl | 17 ++++++++++------- test/tuple.jl | 8 ++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/base/tuple.jl b/base/tuple.jl index eabbce23e4f18..3942202d90d97 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -63,13 +63,16 @@ end ## mapping ## ntuple(f::Function, n::Integer) = - n<=0 ? () : - n==1 ? (f(1),) : - n==2 ? (f(1),f(2),) : - n==3 ? (f(1),f(2),f(3),) : - n==4 ? (f(1),f(2),f(3),f(4),) : - n==5 ? (f(1),f(2),f(3),f(4),f(5),) : - tuple(ntuple(f,n-5)..., f(n-4), f(n-3), f(n-2), f(n-1), f(n)) + n <= 0 ? () : + n == 1 ? (f(1),) : + n == 2 ? (f(1),f(2),) : + n == 3 ? (f(1),f(2),f(3),) : + n == 4 ? (f(1),f(2),f(3),f(4),) : + n == 5 ? (f(1),f(2),f(3),f(4),f(5),) : + n < 16 ? (ntuple(f,n-5)..., f(n-4), f(n-3), f(n-2), f(n-1), f(n)) : + _ntuple(f, n) + +_ntuple(f::Function, n::Integer) = (@_noinline_meta; ((f(i) for i = 1:n)...)) # inferrable ntuple function ntuple{F,N}(f::F, ::Type{Val{N}}) diff --git a/test/tuple.jl b/test/tuple.jl index 5d7240f15bbed..dbb7570273cba 100644 --- a/test/tuple.jl +++ b/test/tuple.jl @@ -162,3 +162,11 @@ end # issue #12854 @test_throws TypeError ntuple(identity, Val{1:2}) + +for n = 0:20 + t = ntuple(identity, n) + @test length(t) == n + for i = 1:n + @test t[i] == i + end +end From 60661785d3b2884bc66a7b8526302df8137a4407 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sun, 31 Jul 2016 20:56:44 -0400 Subject: [PATCH 0719/1117] acquire the codegen lock as a more frequent lock the codegen lock is being overused here, but otherwise, it is deadlock-prone, since the code design here is too arbitrarily recursive to ensure these are being acquired in the right order --- src/gf.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/gf.c b/src/gf.c index 3f0f1e3a43edb..2c18d7d2b228e 100644 --- a/src/gf.c +++ b/src/gf.c @@ -190,6 +190,7 @@ jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force) { JL_TIMING(INFERENCE); #ifdef ENABLE_INFERENCE + JL_LOCK(&codegen_lock); // use codegen lock to synchronize type-inference jl_module_t *mod = NULL; if (li->def != NULL) mod = li->def->module; @@ -199,7 +200,6 @@ jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force) if (jl_typeinf_func != NULL && (force || (mod != jl_gf_mtable(jl_typeinf_func)->module && (mod != jl_core_module || !lastIn)))) { // avoid any potential recursion in calling jl_typeinf_func on itself - JL_LOCK(&codegen_lock); // Might GC assert(li->inInference == 0); jl_value_t *fargs[2]; fargs[0] = (jl_value_t*)jl_typeinf_func; @@ -211,9 +211,9 @@ jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force) #endif li = (jl_lambda_info_t*)jl_apply(fargs, 2); assert(li->def || li->inInference == 0); // if this is toplevel expr, make sure inference finished - JL_UNLOCK(&codegen_lock); // Might GC } inInference = lastIn; + JL_UNLOCK(&codegen_lock); // Might GC (li might be rooted?) #endif return li; } @@ -614,7 +614,6 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_svec_t *sparams) { // caller must hold the mt->writelock - JL_LOCK(&codegen_lock); // Might GC jl_method_t *definition = m->func.method; jl_tupletype_t *decl = m->sig; jl_value_t *temp = NULL; @@ -798,7 +797,6 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_typemap_insert(cache, parent, origtype, jl_emptysvec, type, guardsigs, (jl_value_t*)newmeth, jl_cachearg_offset(mt), &lambda_cache, NULL); - JL_UNLOCK(&codegen_lock); // Might GC if (definition->traced && jl_method_tracer) jl_call_tracer(jl_method_tracer, (jl_value_t*)newmeth); JL_GC_POP(); @@ -1278,9 +1276,11 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) if (li->functionObjectsDecls.functionObject != NULL) return li; if (li->def) { + JL_LOCK(&codegen_lock); JL_LOCK(&li->def->writelock); if (li->functionObjectsDecls.functionObject != NULL) { JL_UNLOCK(&li->def->writelock); + JL_UNLOCK(&codegen_lock); return li; } if (li->inInference || li->inCompile) { @@ -1310,8 +1310,10 @@ jl_lambda_info_t *jl_compile_for_dispatch(jl_lambda_info_t *li) if (li->functionObjectsDecls.functionObject == NULL) { // check again, because jl_type_infer may have compiled it jl_compile_linfo(li); } - if (li->def) + if (li->def) { JL_UNLOCK(&li->def->writelock); + JL_UNLOCK(&codegen_lock); + } return li; } @@ -2286,7 +2288,12 @@ JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int return ml_matches(mt->defs, 0, types, lim, include_ambiguous); } -static jl_mutex_t typeinf_lock; +// TODO: separate the codegen and typeinf locks +// currently using a coarser lock seems like +// the best way to avoid acquisition priority +// ordering violations +//static jl_mutex_t typeinf_lock; +#define typeinf_lock codegen_lock JL_DLLEXPORT void jl_typeinf_begin(void) { From 7e3eb88519b8cc45994a0ca5f83fdb8f62c244dd Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 18:12:51 -0400 Subject: [PATCH 0720/1117] wait for stream to finish flush before returning from close fix #14434 --- base/process.jl | 8 ++++++-- base/socket.jl | 10 +--------- base/stream.jl | 31 ++++++++++++++++++++++++++----- src/jl_uv.c | 2 +- test/spawn.jl | 10 +++++++--- 5 files changed, 41 insertions(+), 20 deletions(-) diff --git a/base/process.jl b/base/process.jl index 55495765febe5..41c959286b49d 100644 --- a/base/process.jl +++ b/base/process.jl @@ -336,7 +336,9 @@ function uv_return_spawn(p::Ptr{Void}, exit_status::Int64, termsignal::Int32) proc = unsafe_pointer_to_objref(data)::Process proc.exitcode = exit_status proc.termsignal = termsignal - if isa(proc.exitcb, Function) proc.exitcb(proc, exit_status, termsignal) end + if isa(proc.exitcb, Function) + proc.exitcb(proc, exit_status, termsignal) + end ccall(:jl_close_uv, Void, (Ptr{Void},), proc.handle) notify(proc.exitnotify) nothing @@ -344,7 +346,9 @@ end function _uv_hook_close(proc::Process) proc.handle = C_NULL - if isa(proc.closecb, Function) proc.closecb(proc) end + if isa(proc.closecb, Function) + proc.closecb(proc) + end notify(proc.closenotify) end diff --git a/base/socket.jl b/base/socket.jl index 29dd8d6caa3aa..c336b12c26277 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -365,19 +365,11 @@ function UDPSocket() throw(UVError("failed to create udp socket",err)) end this.status = StatusInit - this + return this end show(io::IO, stream::UDPSocket) = print(io, typeof(stream), "(", uv_status_string(stream), ")") -function uvfinalize(uv::Union{TTY,PipeEndpoint,PipeServer,TCPServer,TCPSocket,UDPSocket}) - if (uv.status != StatusUninit && uv.status != StatusInit) - close(uv) - end - disassociate_julia_struct(uv) - uv.handle = C_NULL -end - function _uv_hook_close(sock::UDPSocket) sock.handle = C_NULL sock.status = StatusClosed diff --git a/base/stream.jl b/base/stream.jl index d649b88116736..733bba8d450ce 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -322,9 +322,26 @@ function wait_close(x::Union{LibuvStream, LibuvServer}) end function close(stream::Union{LibuvStream, LibuvServer}) - if isopen(stream) && stream.status != StatusClosing - ccall(:jl_close_uv,Void, (Ptr{Void},), stream.handle) - stream.status = StatusClosing + if isopen(stream) + if stream.status != StatusClosing + ccall(:jl_close_uv, Void, (Ptr{Void},), stream.handle) + stream.status = StatusClosing + end + if uv_handle_data(stream) != C_NULL + stream_wait(stream, stream.closenotify) + end + end + nothing +end + +function uvfinalize(uv::Union{LibuvStream, LibuvServer}) + if uv.handle != C_NULL + disassociate_julia_struct(uv.handle) # not going to call the usual close hooks + if uv.status != StatusUninit && uv.status != StatusInit + close(uv) + uv.handle = C_NULL + uv.status = StatusClosed + end end nothing end @@ -472,8 +489,10 @@ function uv_readcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}) stream.status = StatusEOF # libuv called stop_reading already notify(stream.readnotify) notify(stream.closenotify) - else - close(stream) + elseif stream.status != StatusClosing + # begin shutdown of the stream + ccall(:jl_close_uv, Void, (Ptr{Void},), stream.handle) + stream.status = StatusClosing end else # This is a fatal connection error. Shutdown requests as per the usual @@ -1019,6 +1038,8 @@ function close(s::BufferStream) notify(s.close_c; all=true) nothing end +uvfinalize(s::BufferStream) = nothing + read(s::BufferStream, ::Type{UInt8}) = (wait_readnb(s, 1); read(s.buffer, UInt8)) unsafe_read(s::BufferStream, a::Ptr{UInt8}, nb::UInt) = (wait_readnb(s, Int(nb)); unsafe_read(s.buffer, a, nb)) nb_available(s::BufferStream) = nb_available(s.buffer) diff --git a/src/jl_uv.c b/src/jl_uv.c index 3cdc6c61340c1..96857eb61fb87 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -230,7 +230,7 @@ JL_DLLEXPORT void jl_close_uv(uv_handle_t *handle) JL_DLLEXPORT void jl_forceclose_uv(uv_handle_t *handle) { - uv_close(handle,&jl_uv_closeHandle); + uv_close(handle, &jl_uv_closeHandle); } JL_DLLEXPORT void jl_uv_associate_julia_struct(uv_handle_t *handle, diff --git a/test/spawn.jl b/test/spawn.jl index fe59cd7148461..be1dfc6a92262 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -266,9 +266,9 @@ let bad = "bad\0name" end # issue #12829 -let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", readstring(STDIN))'`, ready = Condition() +let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", readstring(STDIN))'`, ready = Condition(), t @test_throws ArgumentError write(out, "not open error") - @async begin # spawn writer task + t = @async begin # spawn writer task open(echo, "w", out) do in1 open(echo, "w", out) do in2 notify(ready) @@ -283,10 +283,13 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test isreadable(out) @test iswritable(out) close(out.in) + @test !isopen(out.in) + is_windows() || @test !isopen(out.out) # it takes longer to propagate EOF through the Windows event system @test_throws ArgumentError write(out, "now closed error") @test isreadable(out) @test !iswritable(out) - @test isopen(out) + is_windows() && Base.process_events(false) # should be enough steps to fully propagate EOF now + @test !isopen(out) end wait(ready) # wait for writer task to be ready before using `out` @test nb_available(out) == 0 @@ -309,6 +312,7 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test isempty(read(out)) @test eof(out) @test desc == "Pipe(open => active, 0 bytes waiting)" + wait(t) end # issue #8529 From ab6c91284b637fb92fc23a15ccc5f5143dd9065c Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 20 Jul 2016 21:00:27 -0400 Subject: [PATCH 0721/1117] avoid leaving process zombies while investigating #14434, I noticed that the test code there is creating zombies to fix that, we need to avoid calling uv_close in the finalizer until after the process has exited (or during atexit) --- base/process.jl | 11 ++++++---- base/socket.jl | 4 ++-- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/libuv.version | 2 +- src/init.c | 9 +++++++- src/jl_uv.c | 21 +++++++++++++++---- 9 files changed, 37 insertions(+), 14 deletions(-) create mode 100644 deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 create mode 100644 deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 delete mode 100644 deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 delete mode 100644 deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 diff --git a/base/process.jl b/base/process.jl index 41c959286b49d..88d51cb4f1634 100644 --- a/base/process.jl +++ b/base/process.jl @@ -289,7 +289,7 @@ type Process <: AbstractPipe typemin(fieldtype(Process, :termsignal)), false, Condition(), false, Condition()) finalizer(this, uvfinalize) - this + return this end end pipe_reader(p::Process) = p.out @@ -325,9 +325,12 @@ function _jl_spawn(cmd, argv, loop::Ptr{Void}, pp::Process, end function uvfinalize(proc::Process) - proc.handle != C_NULL && ccall(:jl_close_uv, Void, (Ptr{Void},), proc.handle) - disassociate_julia_struct(proc) - proc.handle = C_NULL + if proc.handle != C_NULL + disassociate_julia_struct(proc.handle) + ccall(:jl_close_uv, Void, (Ptr{Void},), proc.handle) + proc.handle = C_NULL + end + nothing end function uv_return_spawn(p::Ptr{Void}, exit_status::Int64, termsignal::Int32) diff --git a/base/socket.jl b/base/socket.jl index c336b12c26277..8365f219fe711 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -281,7 +281,7 @@ function TCPSocket() throw(UVError("failed to create tcp socket",err)) end this.status = StatusInit - this + return this end type TCPServer <: LibuvServer @@ -312,7 +312,7 @@ function TCPServer() throw(UVError("failed to create tcp server",err)) end this.status = StatusInit - this + return this end isreadable(io::TCPSocket) = isopen(io) || nb_available(io) > 0 diff --git a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 new file mode 100644 index 0000000000000..793f153e26212 --- /dev/null +++ b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 @@ -0,0 +1 @@ +c6a019d79d20eabc39619a04961c9a3b diff --git a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 new file mode 100644 index 0000000000000..f3f1ec9dca483 --- /dev/null +++ b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 @@ -0,0 +1 @@ +478ab473244b01bef344892a75e09fef50da8fb1a7212e0257c53f3223de4fde5f6bd449eef34bc1f025481c7d9f854002acb6eb203b447a50a34bae4ad9dee4 diff --git a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 b/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 deleted file mode 100644 index 6dfbbd2a475fb..0000000000000 --- a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -7248e38acefd92761c54640622357b7b diff --git a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 b/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 deleted file mode 100644 index e04d388782eba..0000000000000 --- a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -2fad17bddc568125d50ce50b7f27aa2de4380400589043d74b180883c96070c2dcad93557f2f5f5416cc297de3a66b35ec8942bd33d8ef1bb22744dad60b760d diff --git a/deps/libuv.version b/deps/libuv.version index efe634777261b..51f79989c5fc3 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv1.9.0 -LIBUV_SHA1=ecbd6eddfac4940ab8db57c73166a7378563ebd3 +LIBUV_SHA1=cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68 diff --git a/src/init.c b/src/init.c index 53e2dd62b578f..20d4017942c83 100644 --- a/src/init.c +++ b/src/init.c @@ -217,6 +217,7 @@ static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdow void jl_init_timing(void); void jl_destroy_timing(void); +void jl_uv_call_close_callback(jl_value_t *val); JL_DLLEXPORT void jl_atexit_hook(int exitcode) { @@ -270,6 +271,13 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) continue; } switch(handle->type) { + case UV_PROCESS: + // cause Julia to forget about the Process object + if (handle->data) + jl_uv_call_close_callback((jl_value_t*)handle->data); + // and make libuv think it is already dead + ((uv_process_t*)handle)->pid = 0; + // fall-through case UV_TTY: case UV_UDP: case UV_TCP: @@ -283,7 +291,6 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) case UV_PREPARE: case UV_CHECK: case UV_SIGNAL: - case UV_PROCESS: case UV_FILE: // These will be shutdown as appropriate by jl_close_uv jl_close_uv(handle); diff --git a/src/jl_uv.c b/src/jl_uv.c index 96857eb61fb87..2c9f312ae682a 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -78,17 +78,17 @@ void jl_init_signal_async(void) } #endif -static void jl_uv_call_close_callback(jl_value_t *val) +void jl_uv_call_close_callback(jl_value_t *val) { jl_value_t *args[2]; args[0] = jl_get_global(jl_base_relative_to(((jl_datatype_t*)jl_typeof(val))->name->module), jl_symbol("_uv_hook_close")); // topmod(typeof(val))._uv_hook_close args[1] = val; assert(args[0]); - jl_apply(args, 2); + jl_apply(args, 2); // TODO: wrap in try-catch? } -JL_DLLEXPORT void jl_uv_closeHandle(uv_handle_t *handle) +static void jl_uv_closeHandle(uv_handle_t *handle) { // if the user killed a stdio handle, // revert back to direct stdio FILE* writes @@ -107,7 +107,7 @@ JL_DLLEXPORT void jl_uv_closeHandle(uv_handle_t *handle) free(handle); } -JL_DLLEXPORT void jl_uv_shutdownCallback(uv_shutdown_t *req, int status) +static void jl_uv_shutdownCallback(uv_shutdown_t *req, int status) { /* * This happens if the remote machine closes the connecition while we're @@ -180,8 +180,21 @@ JL_DLLEXPORT int jl_init_pipe(uv_pipe_t *pipe, int writable, int readable, return err; } +static void jl_proc_exit_cleanup(uv_process_t *process, int64_t exit_status, int term_signal) +{ + uv_close((uv_handle_t*)process, (uv_close_cb)&free); +} + JL_DLLEXPORT void jl_close_uv(uv_handle_t *handle) { + if (handle->type == UV_PROCESS && ((uv_process_t*)handle)->pid != 0) { + // take ownership of this handle, + // so we can waitpid for the resource to exit and avoid leaving zombies + assert(handle->data == NULL); // make sure Julia has forgotten about it already + ((uv_process_t*)handle)->exit_cb = jl_proc_exit_cleanup; + return; + } + if (handle->type == UV_FILE) { uv_fs_t req; jl_uv_file_t *fd = (jl_uv_file_t*)handle; From ca21575c48aaf4b84785ca71c4309bf497cf12d5 Mon Sep 17 00:00:00 2001 From: Ron Rock <rrock@uchicago.edu> Date: Fri, 29 Jul 2016 17:40:01 -0500 Subject: [PATCH 0722/1117] Fix tests to ignore the startup file To find the offending tests, it was helpful to use a .juliarc.jl with just `error("don't include me")`. --- .../clustermanager/simple/UnixDomainCM.jl | 2 +- test/boundscheck.jl | 6 ++--- test/cmdlineargs.jl | 25 +++++++++++-------- test/compile.jl | 2 +- test/examples.jl | 2 +- test/file.jl | 2 +- test/parallel.jl | 2 +- test/parallel_exec.jl | 2 +- test/runtests.jl | 4 +-- test/spawn.jl | 2 +- 10 files changed, 26 insertions(+), 23 deletions(-) diff --git a/examples/clustermanager/simple/UnixDomainCM.jl b/examples/clustermanager/simple/UnixDomainCM.jl index 8adc39cdd5555..b9519cde60caa 100644 --- a/examples/clustermanager/simple/UnixDomainCM.jl +++ b/examples/clustermanager/simple/UnixDomainCM.jl @@ -12,7 +12,7 @@ function launch(manager::UnixDomainCM, params::Dict, launched::Array, c::Conditi for i in 1:manager.np sockname = tempname() try - cmd = `$(params[:exename]) $(@__FILE__) udwrkr $sockname $cookie` + cmd = `$(params[:exename]) --startup-file=no $(@__FILE__) udwrkr $sockname $cookie` io, pobj = open(cmd, "r") wconfig = WorkerConfig() diff --git a/test/boundscheck.jl b/test/boundscheck.jl index d450f104f2dee..80181e21f734d 100644 --- a/test/boundscheck.jl +++ b/test/boundscheck.jl @@ -2,17 +2,17 @@ # run boundscheck tests on separate workers launched with --check-bounds={default,yes,no} -cmd = `$(Base.julia_cmd()) --depwarn=error boundscheck_exec.jl` +cmd = `$(Base.julia_cmd()) --depwarn=error --startup-file=no boundscheck_exec.jl` if !success(pipeline(cmd; stdout=STDOUT, stderr=STDERR)) error("boundscheck test failed, cmd : $cmd") end -cmd = `$(Base.julia_cmd()) --check-bounds=yes --depwarn=error boundscheck_exec.jl` +cmd = `$(Base.julia_cmd()) --check-bounds=yes --startup-file=no --depwarn=error boundscheck_exec.jl` if !success(pipeline(cmd; stdout=STDOUT, stderr=STDERR)) error("boundscheck test failed, cmd : $cmd") end -cmd = `$(Base.julia_cmd()) --check-bounds=no --depwarn=error boundscheck_exec.jl` +cmd = `$(Base.julia_cmd()) --check-bounds=no --startup-file=no --depwarn=error boundscheck_exec.jl` if !success(pipeline(cmd; stdout=STDOUT, stderr=STDERR)) error("boundscheck test failed, cmd : $cmd") end diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 07dcd3c5e06a5..db5d6583016e2 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -8,7 +8,7 @@ if is_windows() end end -let exename = `$(Base.julia_cmd()) --precompiled=yes` +let exename = `$(Base.julia_cmd()) --precompiled=yes --startup-file=no` # --version let v = split(readstring(`$exename -v`), "julia version ")[end] @test Base.VERSION_STRING == chomp(v) @@ -100,14 +100,6 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` @test readchomp(`$exename -E "Bool(Base.JLOptions().historyfile)" --history-file=no`) == "false" @test !success(`$exename --history-file=false`) - # --startup-file - let JL_OPTIONS_STARTUPFILE_ON = 1, - JL_OPTIONS_STARTUPFILE_OFF = 2 - @test parse(Int,readchomp(`$exename -E "Base.JLOptions().startupfile" --startup-file=yes`)) == JL_OPTIONS_STARTUPFILE_ON - @test parse(Int,readchomp(`$exename -E "Base.JLOptions().startupfile" --startup-file=no`)) == JL_OPTIONS_STARTUPFILE_OFF - end - @test !success(`$exename --startup-file=false`) - # --code-coverage @test readchomp(`$exename -E "Bool(Base.JLOptions().code_coverage)"`) == "false" @test readchomp(`$exename -E "Bool(Base.JLOptions().code_coverage)" --code-coverage=none`) == "false" @@ -277,6 +269,17 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes` end end +let exename = `$(Base.julia_cmd()) --precompiled=yes` + # --startup-file + let JL_OPTIONS_STARTUPFILE_ON = 1, + JL_OPTIONS_STARTUPFILE_OFF = 2 + # `HOME=/tmp` to avoid errors in the user .juliarc.jl, which hangs the tests. Issue #17642 + @test parse(Int,readchomp(setenv(`$exename -E "Base.JLOptions().startupfile" --startup-file=yes`, ["HOME=/tmp"]))) == JL_OPTIONS_STARTUPFILE_ON + @test parse(Int,readchomp(`$exename -E "Base.JLOptions().startupfile" --startup-file=no`)) == JL_OPTIONS_STARTUPFILE_OFF + end + @test !success(`$exename --startup-file=false`) +end + # Make sure `julia --lisp` doesn't break run(pipeline(DevNull, `$(joinpath(JULIA_HOME, Base.julia_exename())) --lisp`, DevNull)) @@ -284,14 +287,14 @@ run(pipeline(DevNull, `$(joinpath(JULIA_HOME, Base.julia_exename())) --lisp`, De @test_throws ErrorException run(pipeline(DevNull, pipeline(`$(joinpath(JULIA_HOME, Base.julia_exename())) -Cnative --lisp`, stderr=DevNull), DevNull)) # --precompiled={yes|no} -let exename = `$(Base.julia_cmd())` +let exename = `$(Base.julia_cmd()) --startup-file=no` @test readchomp(`$exename --precompiled=yes -E "Bool(Base.JLOptions().use_precompiled)"`) == "true" @test readchomp(`$exename --precompiled=no -E "Bool(Base.JLOptions().use_precompiled)"`) == "false" end # backtrace contains type and line number info (esp. on windows #17179) for precomp in ("yes", "no") - bt = readstring(pipeline(ignorestatus(`$(Base.julia_cmd()) --precompiled=$precomp + bt = readstring(pipeline(ignorestatus(`$(Base.julia_cmd()) --startup-file=no --precompiled=$precomp -E 'include("____nonexistent_file")'`), stderr=catcmd)) @test contains(bt, "in include_from_node1") if is_windows() && Sys.WORD_SIZE == 32 && precomp == "yes" diff --git a/test/compile.jl b/test/compile.jl index b0170a3015ccc..ce768211a96cc 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -197,7 +197,7 @@ let dir = mktempdir(), Base.compilecache(:Time4b3a94a1a081a8cb) end) - exename = `$(Base.julia_cmd()) --precompiled=yes` + exename = `$(Base.julia_cmd()) --precompiled=yes --startup-file=no` testcode = """ insert!(LOAD_PATH, 1, $(repr(dir))) diff --git a/test/examples.jl b/test/examples.jl index 08c6d13c3863e..aa0486e4bee96 100644 --- a/test/examples.jl +++ b/test/examples.jl @@ -38,7 +38,7 @@ include(joinpath(dir, "queens.jl")) # cluster manager example through a new Julia session. if is_unix() script = joinpath(dir, "clustermanager/simple/test_simple.jl") - cmd = `$(Base.julia_cmd()) $script` + cmd = `$(Base.julia_cmd()) --startup-file=no $script` if !success(pipeline(cmd; stdout=STDOUT, stderr=STDERR)) && ccall(:jl_running_on_valgrind,Cint,()) == 0 error("UnixDomainCM failed test, cmd : $cmd") end diff --git a/test/file.jl b/test/file.jl index 5da9ac7e89816..67b566f753327 100644 --- a/test/file.jl +++ b/test/file.jl @@ -1109,7 +1109,7 @@ function test_13559() run(`mkfifo $fn`) # use subprocess to write 127 bytes to FIFO writer_cmds = "x=open(\"$fn\", \"w\"); for i=1:127 write(x,0xaa); flush(x); sleep(0.1) end; close(x); quit()" - open(pipeline(`$(Base.julia_cmd()) -e $writer_cmds`, stderr=STDERR)) + open(pipeline(`$(Base.julia_cmd()) --startup-file=no -e $writer_cmds`, stderr=STDERR)) #quickly read FIFO, draining it and blocking but not failing with EOFError yet r = open(fn, "r") # 15 proper reads diff --git a/test/parallel.jl b/test/parallel.jl index 3d1cb906c6c24..2b30ad7f0f809 100644 --- a/test/parallel.jl +++ b/test/parallel.jl @@ -11,7 +11,7 @@ elseif Base.JLOptions().code_coverage == 2 cov_flag = `--code-coverage=all` end -cmd = `$(Base.julia_cmd()) $inline_flag $cov_flag --check-bounds=yes --depwarn=error parallel_exec.jl` +cmd = `$(Base.julia_cmd()) $inline_flag $cov_flag --check-bounds=yes --startup-file=no --depwarn=error parallel_exec.jl` if !success(pipeline(cmd; stdout=STDOUT, stderr=STDERR)) && ccall(:jl_running_on_valgrind,Cint,()) == 0 error("Parallel test failed, cmd : $cmd") diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 6718c99a21eb0..a5be3fa85915d 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -17,7 +17,7 @@ end 1 end -addprocs(4; exeflags=`$cov_flag $inline_flag --check-bounds=yes --depwarn=error`) +addprocs(4; exeflags=`$cov_flag $inline_flag --check-bounds=yes --startup-file=no --depwarn=error`) # Test remote() let diff --git a/test/runtests.jl b/test/runtests.jl index 4fef9b34cfbf2..4d91b652b1aac 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -27,7 +27,7 @@ cd(dirname(@__FILE__)) do n = 1 if net_on n = min(8, Sys.CPU_CORES, length(tests)) - n > 1 && addprocs(n; exeflags=`--check-bounds=yes --depwarn=error`) + n > 1 && addprocs(n; exeflags=`--check-bounds=yes --startup-file=no --depwarn=error`) BLAS.set_num_threads(1) end @@ -50,7 +50,7 @@ cd(dirname(@__FILE__)) do if (isa(resp, Integer) && (resp > max_worker_rss)) || isa(resp, Exception) if n > 1 rmprocs(p, waitfor=0.5) - p = addprocs(1; exeflags=`--check-bounds=yes --depwarn=error`)[1] + p = addprocs(1; exeflags=`--check-bounds=yes --startup-file=no --depwarn=error`)[1] remotecall_fetch(()->include("testdefs.jl"), p) else # single process testing, bail if mem limit reached, or, on an exception. diff --git a/test/spawn.jl b/test/spawn.jl index be1dfc6a92262..4bb442b2b08cd 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -330,7 +330,7 @@ let fname = tempname() run(cmd) end """ - @test success(pipeline(`$catcmd $fname`, `$exename -e $code`)) + @test success(pipeline(`$catcmd $fname`, `$exename --startup-file=no -e $code`)) rm(fname) end From 5037a125253f51486d7a67cd489336a436f480bc Mon Sep 17 00:00:00 2001 From: Iblis Lin <iblis@hs.ntnu.edu.tw> Date: Mon, 1 Aug 2016 15:40:59 +0800 Subject: [PATCH 0723/1117] Fix utf8proc include path for flisp close #17734 --- src/flisp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flisp/Makefile b/src/flisp/Makefile index 67d35ffe9ae09..5d7cf6fc51ccd 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -28,7 +28,7 @@ LIBS += -lpthread endif FLAGS := -I$(LLTDIR) $(CFLAGS) $(HFILEDIRS:%=-I%) \ - -I$(LIBUV_INC) -I$(build_includedir) $(LIBDIRS:%=-L%) \ + -I$(LIBUV_INC) -I$(UTF8PROC_INC) -I$(build_includedir) $(LIBDIRS:%=-L%) \ -DLIBRARY_EXPORTS -DUTF8PROC_EXPORTS ifneq ($(USEMSVC), 1) FLAGS += -Wall -Wno-strict-aliasing -DUSE_COMPUTED_GOTO -fvisibility=hidden -Wpointer-arith -Wundef From 19f97a583c764d7649b3d169a78fb5d11fd2d60d Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 1 Aug 2016 01:24:29 -0700 Subject: [PATCH 0724/1117] Indent and grammar fix [ci skip] --- base/libgit2/callbacks.jl | 2 +- base/libgit2/types.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index d48899090b304..81aa56c56d78a 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -113,7 +113,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, end passphrase = if haskey(ENV,"SSH_KEY_PASS") - ENV["SSH_KEY_PASS"] + ENV["SSH_KEY_PASS"] else passdef = creds.pass # check if credentials were already used passdef === nothing && (passdef = "") diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index 2f643366f6a92..c5ce76c79fb8c 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -755,7 +755,7 @@ end reset!(p::Union{UserPasswordCredentials, SSHCredentials}, cnt::Int=3) = (p.count = cnt) reset!(p::CachedCredentials) = foreach(reset!, values(p.cred)) -"Obtained the cached credential for the given host+protocol (credid), or return and store the default if not found" +"Obtain the cached credentials for the given host+protocol (credid), or return and store the default if not found" get_creds!(collection::CachedCredentials, credid, default) = get!(collection.cred, credid, default) get_creds!(creds::AbstractCredentials, credid, default) = creds get_creds!(creds::Void, credid, default) = default From a94b12728872d9ecf314cef1f2ede6b8d402e97c Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Mon, 1 Aug 2016 11:21:08 +0200 Subject: [PATCH 0725/1117] update from comments by vtjnash --- base/show.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/base/show.jl b/base/show.jl index 7af26d1d71ec0..06edde5a109e0 100644 --- a/base/show.jl +++ b/base/show.jl @@ -200,10 +200,9 @@ function show_datatype(io::IO, x::DataType) !(has_tvar_env && x.name.primary === x)) n = length(x.parameters) - # Print homogeneous tuples with more than 3 elements compactly - # as NTuple{N, T} - if length(unique(x.parameters)) == 1 && n > 3 - show(io, "NTuple{", n, ',', x.parameters[1], "}") + # Print homogeneous tuples with more than 3 elements compactly as NTuple{N, T} + if n > 3 && all(i -> is(x.parameters[1], i), x.parameters) + print(io, "NTuple{", n, ',', x.parameters[1], "}") else show(io, x.name) # Do not print the type parameters for the primary type if we are From 6c6de42d9e536f95095e6af6859dcf09f1d4e3b3 Mon Sep 17 00:00:00 2001 From: Matthias Reisinger <matthias.j.reisinger@gmail.com> Date: Tue, 14 Jun 2016 13:52:22 +0200 Subject: [PATCH 0726/1117] Reorganize pass-pipeline for Polly We now move Polly after LoopRotate since at this point the LLVM IR is more amenable to Polly. For example, a simple gemm kernel like @polly function gemm!(A,B,C) m,n = size(A) n,o = size(B) @inbounds for i=1:m, j=1:o, k=1:n C[i,j] += A[i,k] * B[k,j] end end could not be optimized before this change. However, this new position after LoopRotate also makes it necessary to rerun InstCombine before Polly to get rid of hindering phi instructions that are introduced by LCSSA. Parts of these changes have already been discussed in https://groups.google.com/forum/#!topic/polly-dev/P2RZUlKRo0I. The reason why they were held back was to see if further adaptions to the pass-pipeline will become apparent, but for now this configuration seems to be convenient. --- src/jitlayers.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 474f4e5feb59e..529fcb8b013fb 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -165,12 +165,15 @@ void addOptimizationPasses(PassManager *PM) PM->add(createEarlyCSEPass()); //// **** + PM->add(createLoopIdiomPass()); //// **** + PM->add(createLoopRotatePass()); // Rotate loops. #ifdef USE_POLLY + // LCSSA (which has already run at this point due to the dependencies of the + // above passes) introduces redundant phis that hinder Polly. Therefore we + // run InstCombine here to remove them. + PM->add(createInstructionCombiningPass()); polly::registerPollyPasses(*PM); #endif - - PM->add(createLoopIdiomPass()); //// **** - PM->add(createLoopRotatePass()); // Rotate loops. // LoopRotate strips metadata from terminator, so run LowerSIMD afterwards PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "simdloop" as LLVM parallel loop PM->add(createLICMPass()); // Hoist loop invariants From 06af3093fc57d76042c4c993acda31789a56f503 Mon Sep 17 00:00:00 2001 From: Matthias Reisinger <matthias.j.reisinger@gmail.com> Date: Mon, 1 Aug 2016 12:34:43 +0200 Subject: [PATCH 0727/1117] Add Polly's CodePreparation pass This pass performs a simple canonicalization needed by Polly. --- src/Makefile | 1 + src/jitlayers.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/Makefile b/src/Makefile index ebf8219150674..1f85e9169348b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -55,6 +55,7 @@ LLVM_LIBS := all ifeq ($(USE_POLLY),1) LLVMLINK += -lPolly -lPollyISL FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --src-root)/tools/polly/include +FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --obj-root)/tools/polly/include FLAGS += -DUSE_POLLY endif else diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 529fcb8b013fb..d81ea17ebc41a 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -23,6 +23,7 @@ #endif #if defined(USE_POLLY) #include <polly/RegisterPasses.h> +#include <polly/LinkAllPasses.h> #endif #include <llvm/Transforms/Scalar.h> @@ -172,6 +173,7 @@ void addOptimizationPasses(PassManager *PM) // above passes) introduces redundant phis that hinder Polly. Therefore we // run InstCombine here to remove them. PM->add(createInstructionCombiningPass()); + PM->add(polly::createCodePreparationPass()); polly::registerPollyPasses(*PM); #endif // LoopRotate strips metadata from terminator, so run LowerSIMD afterwards From 21ef560c7cf1ec3f960d2eaf3f7aac280d1317c9 Mon Sep 17 00:00:00 2001 From: Matthias Reisinger <matthias.j.reisinger@gmail.com> Date: Mon, 1 Aug 2016 12:42:24 +0200 Subject: [PATCH 0728/1117] Add Polly's CodegenCleanup pass Clean up the optimized code emitted by Polly. --- src/jitlayers.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index d81ea17ebc41a..4efb509e33348 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -24,6 +24,7 @@ #if defined(USE_POLLY) #include <polly/RegisterPasses.h> #include <polly/LinkAllPasses.h> +#include <polly/CodeGen/CodegenCleanup.h> #endif #include <llvm/Transforms/Scalar.h> @@ -175,6 +176,7 @@ void addOptimizationPasses(PassManager *PM) PM->add(createInstructionCombiningPass()); PM->add(polly::createCodePreparationPass()); polly::registerPollyPasses(*PM); + PM->add(polly::createCodegenCleanupPass()); #endif // LoopRotate strips metadata from terminator, so run LowerSIMD afterwards PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "simdloop" as LLVM parallel loop From 6fc4b1a511a23eebdd65fd6fd4a5b192c4890f14 Mon Sep 17 00:00:00 2001 From: Helge Eichhorn <git@helgeeichhorn.de> Date: Mon, 1 Aug 2016 12:50:12 +0200 Subject: [PATCH 0729/1117] Fix broken build after distclean-llvm. --- deps/llvm.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index d6c221b96e365..7a6e99bf2287c 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -534,7 +534,7 @@ clean-llvm: distclean-llvm: -rm -rf $(LLVM_TAR) $(LLVM_CLANG_TAR) \ $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) \ - $(LLVM_SRC_DIR) $(LLVM_BUILDDIR_withtype) + $(LLVM_SRC_DIR) $(LLVM_SRC_DIR).extracted $(LLVM_BUILDDIR_withtype) ifneq ($(LLVM_VER),svn) get-llvm: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR) From 7bb5f61a4296d27573d1c7bec867c315f56b0d3c Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Mon, 1 Aug 2016 13:13:07 +0200 Subject: [PATCH 0730/1117] add docs for NTuple [ci skip] --- base/tuple.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base/tuple.jl b/base/tuple.jl index 3942202d90d97..fe4c3fca4e80f 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -1,5 +1,12 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +# Document NTuple here where we have everything needed for the doc system +""" + NTuple{N, T} + +A compact way of representing the type for a tuple of length `N` where all elements are of type `T`.""" +NTuple + ## indexing ## length(t::Tuple) = nfields(t) From 2a058d7afd47e3b32096e2befd126f2ffeb20805 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sun, 24 Jul 2016 00:18:14 -0400 Subject: [PATCH 0731/1117] add islocked predicate and align API for task and thread locks --- base/docs/helpdb/Base.jl | 25 ------- base/exports.jl | 2 + base/lock.jl | 92 +++++++++++++++++++++++-- base/locks.jl | 144 ++++++++++++++++++++++++++++----------- doc/stdlib/parallel.rst | 117 ++++++++++++++++++++++++++----- test/misc.jl | 2 + test/threads.jl | 59 ++++++++++++---- 7 files changed, 339 insertions(+), 102 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 80f726d0b53df..a14860591ba11 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -3108,14 +3108,6 @@ lower-case. Returns a `String`. """ bytes2hex -""" - unlock(l::ReentrantLock) - -Releases ownership of the lock by the current task. If the lock had been acquired before, it -just decrements an internal counter and returns immediately. -""" -unlock - """ BigFloat(x) @@ -3852,14 +3844,6 @@ Return an iterator over all keys in a collection. `collect(keys(d))` returns an """ keys -""" - ReentrantLock() - -Creates a reentrant lock. The same task can acquire the lock as many times as required. Each -lock must be matched with an unlock. -""" -ReentrantLock - """ real(z) @@ -4517,15 +4501,6 @@ category Letter, i.e. a character whose category code begins with 'L'. """ isalpha -""" - lock(l::ReentrantLock) - -Associates `l` with the current task. If `l` is already locked by a different task, waits -for it to become available. The same task can acquire the lock multiple times. Each "lock" -must be matched by an "unlock" -""" -lock - """ transpose(A) diff --git a/base/exports.jl b/base/exports.jl index aaaf335426d36..b92dfe12af6b9 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -986,6 +986,7 @@ export Condition, consume, current_task, + islocked, istaskdone, istaskstarted, lock, @@ -994,6 +995,7 @@ export ReentrantLock, schedule, task_local_storage, + trylock, unlock, yield, yieldto, diff --git a/base/lock.jl b/base/lock.jl index 0c21ddecfc869..ea42b8e454dd2 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -1,6 +1,15 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # Advisory reentrant lock +""" + ReentrantLock() + +Creates a reentrant lock for synchronizing Tasks. +The same task can acquire the lock as many times as required. +Each `lock` must be matched with an `unlock`. + +This lock is NOT threadsafe. See `Threads.Mutex` for a threadsafe lock. +""" type ReentrantLock locked_by::Nullable{Task} cond_wait::Condition @@ -9,6 +18,48 @@ type ReentrantLock ReentrantLock() = new(nothing, Condition(), 0) end +""" + islocked(the_lock) -> Status (Boolean) + +Check whether the lock is held by any task/thread. +This should not be used for synchronization (see instead `trylock`). +""" +function islocked(rl::ReentrantLock) + return rl.reentrancy_cnt != 0 +end + +""" + trylock(the_lock) -> Success (Boolean) + +Acquires the lock if it is available, +returning `true` if successful. +If the lock is already locked by a different task/thread, +returns `false`. + +Each successful `trylock` must be matched by an `unlock`. +""" +function trylock(rl::ReentrantLock) + t = current_task() + if rl.reentrancy_cnt == 0 + rl.locked_by = t + rl.reentrancy_cnt = 1 + return true + elseif t == get(rl.locked_by) + rl.reentrancy_cnt += 1 + return true + end + return false +end + +""" + lock(the_lock) + +Acquires the lock when it becomes available. +If the lock is already locked by a different task/thread, +it waits for it to become available. + +Each `lock` must be matched by an `unlock`. +""" function lock(rl::ReentrantLock) t = current_task() while true @@ -24,25 +75,48 @@ function lock(rl::ReentrantLock) end end +""" + unlock(the_lock) + +Releases ownership of the lock. + +If this is a recursive lock which has been acquired before, it +just decrements an internal counter and returns immediately. +""" function unlock(rl::ReentrantLock) - rl.reentrancy_cnt = rl.reentrancy_cnt - 1 - if rl.reentrancy_cnt < 0 + if rl.reentrancy_cnt == 0 error("unlock count must match lock count") end + rl.reentrancy_cnt -= 1 if rl.reentrancy_cnt == 0 rl.locked_by = nothing notify(rl.cond_wait) end - return rl + return end +""" + Semaphore(sem_size) + +Creates a counting semaphore that allows at most `sem_size` +acquires to be in use at any time. +Each acquire must be mached with a release. + +This construct is NOT threadsafe. +""" type Semaphore sem_size::Int curr_cnt::Int cond_wait::Condition - Semaphore(sem_size) = new(sem_size, 0, Condition()) + Semaphore(sem_size) = sem_size > 0 ? new(sem_size, 0, Condition()) : throw(ArgumentError("Semaphore size must be > 0")) end +""" + acquire(s::Semaphore) + +Wait for one of the `sem_size` permits to be available, +blocking until one can be acquired. +""" function acquire(s::Semaphore) while true if s.curr_cnt < s.sem_size @@ -54,7 +128,15 @@ function acquire(s::Semaphore) end end +""" + release(s::Semaphore) + +Return one permit to the pool, +possibly allowing another task to acquire it +and resume execution. +""" function release(s::Semaphore) - s.curr_cnt = s.curr_cnt - 1 + @assert s.curr_cnt > 0 "release count must match acquire count" + s.curr_cnt -= 1 notify(s.cond_wait; all=false) end diff --git a/base/locks.jl b/base/locks.jl index 4865a0e0bc327..5f699c3f7507c 100644 --- a/base/locks.jl +++ b/base/locks.jl @@ -1,33 +1,60 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -import Base: _uv_hook_close, unsafe_convert +import Base: _uv_hook_close, unsafe_convert, + lock, trylock, unlock, islocked -export SpinLock, Mutex, - init_lock!, destroy_lock!, lock!, trylock!, unlock! +export SpinLock, RecursiveSpinLock, Mutex ########################################## # Atomic Locks ########################################## +""" + AbstractLock + +Abstract supertype describing types that +implement the thread-safe synchronization primitives: +`lock`, `trylock`, `unlock`, and `islocked` +""" abstract AbstractLock # Test-and-test-and-set spin locks are quickest up to about 30ish # contending threads. If you have more contention than that, perhaps # a lock is the wrong way to synchronize. +""" + TatasLock() + +See SpinLock. +""" immutable TatasLock <: AbstractLock handle::Atomic{Int} TatasLock() = new(Atomic{Int}(0)) end +""" + SpinLock() + +Creates a non-reentrant lock. +Recursive use will result in a deadlock. +Each `lock` must be matched with an `unlock`. + +Test-and-test-and-set spin locks are quickest up to about 30ish +contending threads. If you have more contention than that, perhaps +a lock is the wrong way to synchronize. + +See also RecursiveSpinLock for a version that permits recursion. + +See also Mutex for a more efficient version on one core or if the lock may be held for a considerable length of time. +""" typealias SpinLock TatasLock -function lock!(l::TatasLock) +function lock(l::TatasLock) while true if l.handle[] == 0 p = atomic_xchg!(l.handle, 1) if p == 0 - return 0 + return end end ccall(:jl_cpu_pause, Void, ()) @@ -36,39 +63,58 @@ function lock!(l::TatasLock) end end -function trylock!(l::TatasLock) +function trylock(l::TatasLock) if l.handle[] == 0 - return atomic_xchg!(l.handle, 1) + return atomic_xchg!(l.handle, 1) == 0 end - return 1 + return false end -function unlock!(l::TatasLock) +function unlock(l::TatasLock) l.handle[] = 0 ccall(:jl_cpu_wake, Void, ()) - return 0 + return end +function islocked(l::TatasLock) + return l.handle[] != 0 +end + + +""" + RecursiveTatasLock() -# Recursive test-and-test-and-set lock. Slower. +See RecursiveSpinLock. +""" immutable RecursiveTatasLock <: AbstractLock ownertid::Atomic{Int16} handle::Atomic{Int} RecursiveTatasLock() = new(Atomic{Int16}(0), Atomic{Int}(0)) end +""" + RecursiveSpinLock() + +Creates a reentrant lock. +The same thread can acquire the lock as many times as required. +Each `lock` must be matched with an `unlock`. + +See also SpinLock for a slightly faster version. + +See also Mutex for a more efficient version on one core or if the lock may be held for a considerable length of time. +""" typealias RecursiveSpinLock RecursiveTatasLock -function lock!(l::RecursiveTatasLock) +function lock(l::RecursiveTatasLock) if l.ownertid[] == threadid() l.handle[] += 1 - return 0 + return end while true if l.handle[] == 0 if atomic_cas!(l.handle, 0, 1) == 0 l.ownertid[] = threadid() - return 0 + return end end ccall(:jl_cpu_pause, Void, ()) @@ -77,25 +123,24 @@ function lock!(l::RecursiveTatasLock) end end -function trylock!(l::RecursiveTatasLock) +function trylock(l::RecursiveTatasLock) if l.ownertid[] == threadid() l.handle[] += 1 - return 0 + return true end if l.handle[] == 0 if atomic_cas!(l.handle, 0, 1) == 0 l.ownertid[] = threadid() - return 0 + return true end - return 1 + return false end - return 1 + return false end -function unlock!(l::RecursiveTatasLock) - if l.ownertid[] != threadid() - return 1 - end +function unlock(l::RecursiveTatasLock) + @assert(l.ownertid[] == threadid(), "unlock from wrong thread") + @assert(l.handle[] != 0, "unlock count must match lock count") if l.handle[] == 1 l.ownertid[] = 0 l.handle[] = 0 @@ -103,7 +148,11 @@ function unlock!(l::RecursiveTatasLock) else l.handle[] -= 1 end - return 0 + return +end + +function islocked(l::RecursiveTatasLock) + return l.handle[] != 0 end @@ -111,8 +160,7 @@ end # System Mutexes ########################################## -# These are mutexes from libuv, which abstract pthread mutexes and -# Windows critical sections. We're doing some error checking (and +# These are mutexes from libuv. We're doing some error checking (and # paying for it in overhead), but regardless, in some situations, # passing a bad parameter will cause an abort. @@ -121,6 +169,16 @@ end const UV_MUTEX_SIZE = ccall(:jl_sizeof_uv_mutex, Cint, ()) +""" + Mutex() + +These are standard system mutexes for locking critical sections of logic. + +On Windows, this is a critical section object, +on pthreads, this is a `pthread_mutex_t`. + +See also SpinLock for a lighter-weight lock. +""" type Mutex <: AbstractLock ownertid::Int16 handle::Ptr{Void} @@ -136,15 +194,17 @@ unsafe_convert(::Type{Ptr{Void}}, m::Mutex) = m.handle function _uv_hook_close(x::Mutex) h = x.handle - x.handle = C_NULL - ccall(:uv_mutex_destroy, Void, (Ptr{Void},), h) - Libc.free(h) - nothing + if h != C_NULL + x.handle = C_NULL + ccall(:uv_mutex_destroy, Void, (Ptr{Void},), h) + Libc.free(h) + nothing + end end -function lock!(m::Mutex) +function lock(m::Mutex) if m.ownertid == threadid() - return 0 + return end # Temporary solution before we have gc transition support in codegen. # This could mess up gc state when we add codegen support. @@ -152,25 +212,27 @@ function lock!(m::Mutex) ccall(:uv_mutex_lock, Void, (Ptr{Void},), m) ccall(:jl_gc_safe_leave, Void, (Int8,), gc_state) m.ownertid = threadid() - return 0 + return end -function trylock!(m::Mutex) +function trylock(m::Mutex) if m.ownertid == threadid() - return 0 + return true end r = ccall(:uv_mutex_trylock, Cint, (Ptr{Void},), m) if r == 0 m.ownertid = threadid() end - return r + return r == 0 end -function unlock!(m::Mutex) - if m.ownertid != threadid() - return Base.UV_EPERM - end +function unlock(m::Mutex) + @assert(m.ownertid == threadid(), "unlock from wrong thread") m.ownertid = 0 ccall(:uv_mutex_unlock, Void, (Ptr{Void},), m) - return 0 + return +end + +function islocked(m::Mutex) + return m.ownertid != 0 end diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 804392667d439..5b4f7e9c73465 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -111,24 +111,6 @@ Tasks Block the current task for a specified number of seconds. The minimum sleep time is 1 millisecond or input of ``0.001``\ . -.. function:: ReentrantLock() - - .. Docstring generated from Julia source - - Creates a reentrant lock. The same task can acquire the lock as many times as required. Each lock must be matched with an unlock. - -.. function:: lock(l::ReentrantLock) - - .. Docstring generated from Julia source - - Associates ``l`` with the current task. If ``l`` is already locked by a different task, waits for it to become available. The same task can acquire the lock multiple times. Each "lock" must be matched by an "unlock" - -.. function:: unlock(l::ReentrantLock) - - .. Docstring generated from Julia source - - Releases ownership of the lock by the current task. If the lock had been acquired before, it just decrements an internal counter and returns immediately. - .. function:: Channel{T}(sz::Int) .. Docstring generated from Julia source @@ -802,6 +784,105 @@ will) change in the future. For further details, see LLVM's ``fence`` instruction. +Synchronization Primitives +-------------------------- + +.. data:: AbstractLock + + .. Docstring generated from Julia source + + Abstract supertype describing types that implement the thread-safe synchronization primitives: ``lock``\ , ``trylock``\ , ``unlock``\ , and ``islocked`` + +.. function:: lock(the_lock) + + .. Docstring generated from Julia source + + Acquires the lock when it becomes available. If the lock is already locked by a different task/thread, it waits for it to become available. + + Each ``lock`` must be matched by an ``unlock``\ . + +.. function:: unlock(the_lock) + + .. Docstring generated from Julia source + + Releases ownership of the lock. + + If this is a recursive lock which has been acquired before, it just decrements an internal counter and returns immediately. + +.. function:: trylock(the_lock) -> Success (Boolean) + + .. Docstring generated from Julia source + + Acquires the lock if it is available, returning ``true`` if successful. If the lock is already locked by a different task/thread, returns ``false``\ . + + Each successful ``trylock`` must be matched by an ``unlock``\ . + +.. function:: islocked(the_lock) -> Status (Boolean) + + .. Docstring generated from Julia source + + Check whether the lock is held by any task/thread. This should not be used for synchronization (see instead ``trylock``\ ). + +.. function:: ReentrantLock() + + .. Docstring generated from Julia source + + Creates a reentrant lock for synchronizing Tasks. The same task can acquire the lock as many times as required. Each ``lock`` must be matched with an ``unlock``\ . + + This lock is NOT threadsafe. See ``Threads.Mutex`` for a threadsafe lock. + +.. function:: Mutex() + + .. Docstring generated from Julia source + + These are standard system mutexes for locking critical sections of logic. + + On Windows, this is a critical section object, on pthreads, this is a ``pthread_mutex_t``\ . + + See also SpinLock for a lighter-weight lock. + +.. function:: SpinLock() + + .. Docstring generated from Julia source + + Creates a non-reentrant lock. Recursive use will result in a deadlock. Each ``lock`` must be matched with an ``unlock``\ . + + Test-and-test-and-set spin locks are quickest up to about 30ish contending threads. If you have more contention than that, perhaps a lock is the wrong way to synchronize. + + See also RecursiveSpinLock for a version that permits recursion. + + See also Mutex for a more efficient version on one core or if the lock may be held for a considerable length of time. + +.. function:: RecursiveSpinLock() + + .. Docstring generated from Julia source + + Creates a reentrant lock. The same thread can acquire the lock as many times as required. Each ``lock`` must be matched with an ``unlock``\ . + + See also SpinLock for a slightly faster version. + + See also Mutex for a more efficient version on one core or if the lock may be held for a considerable length of time. + +.. function:: Semaphore(sem_size) + + .. Docstring generated from Julia source + + Creates a counting semaphore that allows at most ``sem_size`` acquires to be in use at any time. Each acquire must be mached with a release. + + This construct is NOT threadsafe. + +.. function:: acquire(s::Semaphore) + + .. Docstring generated from Julia source + + Wait for one of the ``sem_size`` permits to be available, blocking until one can be acquired. + +.. function:: release(s::Semaphore) + + .. Docstring generated from Julia source + + Return one permit to the pool, possibly allowing another task to acquire it and resume execution. + Cluster Manager Interface ------------------------- diff --git a/test/misc.jl b/test/misc.jl index 252eed846ada9..633576319e108 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -138,6 +138,8 @@ end # lock / unlock let l = ReentrantLock() lock(l) + @test trylock(l) + unlock(l) unlock(l) @test_throws ErrorException unlock(l) end diff --git a/test/threads.jl b/test/threads.jl index 4fb50d6ba080a..5b4d99d917712 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -64,37 +64,70 @@ test_threaded_atomic_minmax(Int16(-5000),Int16(5000)) test_threaded_atomic_minmax(UInt16(27000),UInt16(37000)) function threaded_add_locked{LockT}(::Type{LockT}, x, n) - lock = LockT() + critical = LockT() @threads for i = 1:n - lock!(lock) + @test lock(critical) === nothing + @test islocked(critical) x = x + 1 - unlock!(lock) + @test unlock(critical) === nothing end + @test !islocked(critical) + nentered = 0 + nfailed = Atomic() + @threads for i = 1:n + if trylock(critical) + @test islocked(critical) + nentered += 1 + @test unlock(critical) === nothing + else + atomic_add!(nfailed, 1) + end + end + @test 0 < nentered <= n + @test nentered + nfailed[] == n + @test !islocked(critical) return x end @test threaded_add_locked(SpinLock, 0, 10000) == 10000 -@test threaded_add_locked(Threads.RecursiveSpinLock, 0, 10000) == 10000 +@test threaded_add_locked(RecursiveSpinLock, 0, 10000) == 10000 @test threaded_add_locked(Mutex, 0, 10000) == 10000 # Check if the recursive lock can be locked and unlocked correctly. -let lock = Threads.RecursiveSpinLock() - @test lock!(lock) == 0 - @test lock!(lock) == 0 - @test unlock!(lock) == 0 - @test unlock!(lock) == 0 - @test unlock!(lock) == 1 +let critical = RecursiveSpinLock() + @test !islocked(critical) + @test_throws AssertionError unlock(critical) + @test lock(critical) === nothing + @test islocked(critical) + @test lock(critical) === nothing + @test trylock(critical) == true + @test islocked(critical) + @test unlock(critical) === nothing + @test islocked(critical) + @test unlock(critical) === nothing + @test islocked(critical) + @test unlock(critical) === nothing + @test !islocked(critical) + @test_throws AssertionError unlock(critical) + @test trylock(critical) == true + @test islocked(critical) + @test unlock(critical) === nothing + @test !islocked(critical) + @test_throws AssertionError unlock(critical) + @test !islocked(critical) end # Make sure doing a GC while holding a lock doesn't cause dead lock # PR 14190. (This is only meaningful for threading) function threaded_gc_locked{LockT}(::Type{LockT}) - lock = LockT() + critical = LockT() @threads for i = 1:20 - lock!(lock) + @test lock(critical) === nothing + @test islocked(critical) gc(false) - unlock!(lock) + @test unlock(critical) === nothing end + @test !islocked(critical) end threaded_gc_locked(SpinLock) From 02d68fb1f6fa87b71e6cea78b04d2a78003833eb Mon Sep 17 00:00:00 2001 From: ranjanan <benditlikeranjan@gmail.com> Date: Mon, 1 Aug 2016 17:30:33 +0530 Subject: [PATCH 0732/1117] Fix all doctests [ci skip] --- doc/devdocs/reflection.rst | 5 +--- doc/devdocs/types.rst | 6 ++-- doc/manual/complex-and-rational-numbers.rst | 2 +- doc/manual/constructors.rst | 8 ++--- doc/manual/control-flow.rst | 2 +- doc/manual/functions.rst | 2 +- doc/manual/methods.rst | 33 +++++++++++---------- doc/manual/stacktraces.rst | 1 - doc/manual/strings.rst | 4 +-- doc/manual/types.rst | 4 +-- doc/stdlib/collections.rst | 2 +- 11 files changed, 34 insertions(+), 35 deletions(-) diff --git a/doc/devdocs/reflection.rst b/doc/devdocs/reflection.rst index 161ec02b61566..22d053413e362 100644 --- a/doc/devdocs/reflection.rst +++ b/doc/devdocs/reflection.rst @@ -106,10 +106,7 @@ variable assignments: julia> expand( :(f() = 1) ) :(begin $(Expr(:method, :f)) - $(Expr(:method, :f, :((Core.svec)((Core.apply_type)(Tuple,(Core.Typeof)(f)),(Core.svec)())), Toplevel LambdaInfo thunk - :(begin # none, line 1: - return 1 - end), false)) + $(Expr(:method, :f, :((Core.svec)((Core.apply_type)(Tuple,(Core.Typeof)(f)),(Core.svec)())), Toplevel LambdaInfo thunk, false)) return f end) diff --git a/doc/devdocs/types.rst b/doc/devdocs/types.rst index b97d9ffb6e7b5..68ceba060c623 100644 --- a/doc/devdocs/types.rst +++ b/doc/devdocs/types.rst @@ -228,7 +228,7 @@ These therefore print identically, but they have very different behavior: julia> candid([1],3.2) ERROR: MethodError: no method matching candid(::Array{Int64,1}, ::Float64) Closest candidates are: - candid{T}(::Array{T,N}, !Matched::T) + candid{T}(::Array{T,N}, !Matched::T) at none:1 ... julia> sneaky([1],3.2) @@ -319,9 +319,11 @@ the type, which is an object of type :obj:`TypeName`: primary: Array{T,N} <: DenseArray{T,N} cache: SimpleVector ... + linearcache: SimpleVector ... - uid: Int64 47 + + uid: Int64 -7900426068641098781 mt: MethodTable name: Symbol Array defs: Void nothing diff --git a/doc/manual/complex-and-rational-numbers.rst b/doc/manual/complex-and-rational-numbers.rst index 4a102e31407f8..00ab37d26a3b4 100644 --- a/doc/manual/complex-and-rational-numbers.rst +++ b/doc/manual/complex-and-rational-numbers.rst @@ -159,7 +159,7 @@ versus ``-1 + 0im`` even though ``-1 == -1 + 0im``: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:149 + in sqrt(::Int64) at ./math.jl:154 ... julia> sqrt(-1 + 0im) diff --git a/doc/manual/constructors.rst b/doc/manual/constructors.rst index aef8a8f437a76..94b944669ebb5 100644 --- a/doc/manual/constructors.rst +++ b/doc/manual/constructors.rst @@ -327,8 +327,8 @@ types of the arguments given to the constructor. Here are some examples: julia> Point(1,2.5) ERROR: MethodError: no method matching Point{T<:Real}(::Int64, ::Float64) Closest candidates are: - Point{T<:Real}{T<:Real}(::T<:Real, !Matched::T<:Real) - Point{T<:Real}{T}(::Any) + Point{T<:Real}{T<:Real}(::T<:Real, !Matched::T<:Real) at none:2 + Point{T<:Real}{T}(::Any) at sysimg.jl:53 ... ## explicit T ## @@ -426,8 +426,8 @@ However, other similar calls still don't work: julia> Point(1.5,2) ERROR: MethodError: no method matching Point{T<:Real}(::Float64, ::Int64) Closest candidates are: - Point{T<:Real}{T<:Real}(::T<:Real, !Matched::T<:Real) - Point{T<:Real}{T}(::Any) + Point{T<:Real}{T<:Real}(::T<:Real, !Matched::T<:Real) at none:2 + Point{T<:Real}{T}(::Any) at sysimg.jl:53 ... For a much more general way of making all such calls work sensibly, see diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index eb430778eb2ed..0e5b8913a3805 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -666,7 +666,7 @@ negative real value: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:149 + in sqrt(::Int64) at ./math.jl:154 ... You may define your own exceptions in the following way: diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 6fbe105171586..87414c31fc169 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -412,7 +412,7 @@ function (although it often is): julia> baz(args...) ERROR: MethodError: no method matching baz(::Int64, ::Int64, ::Int64) Closest candidates are: - baz(::Any, ::Any) + baz(::Any, ::Any) at none:1 ... As you can see, if the wrong number of elements are in the spliced diff --git a/doc/manual/methods.rst b/doc/manual/methods.rst index 55a2870fdb008..ec4ac2d15f9ff 100644 --- a/doc/manual/methods.rst +++ b/doc/manual/methods.rst @@ -95,24 +95,24 @@ Applying it to any other types of arguments will result in a :exc:`MethodError`: julia> f(2.0, 3) ERROR: MethodError: no method matching f(::Float64, ::Int64) Closest candidates are: - f(::Float64, !Matched::Float64) - ... + f(::Float64, !Matched::Float64) at none:1 + ... julia> f(Float32(2.0), 3.0) ERROR: MethodError: no method matching f(::Float32, ::Float64) Closest candidates are: - f(!Matched::Float64, ::Float64) - ... + f(!Matched::Float64, ::Float64) at none:1 + ... julia> f(2.0, "3.0") ERROR: MethodError: no method matching f(::Float64, ::String) Closest candidates are: - f(::Float64, !Matched::Float64) - ... + f(::Float64, !Matched::Float64) at none:1 + ... julia> f("2.0", "3.0") ERROR: MethodError: no method matching f(::String, ::String) - ... + ... As you can see, the arguments must be precisely of type :obj:`Float64`. Other numeric types, such as integers or 32-bit floating-point values, @@ -179,15 +179,16 @@ function ``f`` remains undefined, and applying it will still result in a julia> f("foo", 3) ERROR: MethodError: no method matching f(::String, ::Int64) Closest candidates are: - f(!Matched::Number, ::Number) - ... + f(!Matched::Number, ::Number) at none:1 + ... julia> f() ERROR: MethodError: no method matching f() Closest candidates are: - f(!Matched::Float64, !Matched::Float64) - f(!Matched::Number, !Matched::Number) - ... + f(!Matched::Float64, !Matched::Float64) at none:1 + f(!Matched::Number, !Matched::Number) at none:1 + ... + You can easily see which methods exist for a function by entering the function object itself in an interactive session: @@ -379,8 +380,8 @@ signature: julia> myappend([1,2,3],2.5) ERROR: MethodError: no method matching myappend(::Array{Int64,1}, ::Float64) Closest candidates are: - myappend{T}(::Array{T,1}, !Matched::T) - ... + myappend{T}(::Array{T,1}, !Matched::T) at none:1 + ... julia> myappend([1.0,2.0,3.0],4.0) 4-element Array{Float64,1}: @@ -392,8 +393,8 @@ signature: julia> myappend([1.0,2.0,3.0],4) ERROR: MethodError: no method matching myappend(::Array{Float64,1}, ::Int64) Closest candidates are: - myappend{T}(::Array{T,1}, !Matched::T) - ... + myappend{T}(::Array{T,1}, !Matched::T) at none:1 + ... As you can see, the type of the appended element must match the element type of the vector it is appended to, or else a :exc:`MethodError` is raised. diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index e6c55ba6f4c61..a581d4155906a 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -90,7 +90,6 @@ Each :obj:`StackFrame` contains the function name, file name, line number, lambd julia> top_frame.linfo Nullable{LambdaInfo}(LambdaInfo for eval(::Module, ::Any)) - ... julia> top_frame.inlined false diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index 674ee174973a8..a1838fd57e903 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -553,8 +553,8 @@ contained in a string: julia> contains("Xylophon", 'o') ERROR: MethodError: no method matching contains(::String, ::Char) Closest candidates are: - contains(!Matched::Function, ::Any, !Matched::Any) - contains(::AbstractString, !Matched::AbstractString) + contains(!Matched::Function, ::Any, !Matched::Any) at reduce.jl:402 + contains(::AbstractString, !Matched::AbstractString) at strings/search.jl:310 ... The last error is because ``'o'`` is a character literal, and :func:`contains` diff --git a/doc/manual/types.rst b/doc/manual/types.rst index 22504cc76fc88..8d40d7fb37e86 100644 --- a/doc/manual/types.rst +++ b/doc/manual/types.rst @@ -769,8 +769,8 @@ each field: julia> Point{Float64}(1.0,2.0,3.0) ERROR: MethodError: no method matching Point{Float64}(::Float64, ::Float64, ::Float64) Closest candidates are: - Point{Float64}{T}(::Any, ::Any) - Point{Float64}{T}(::Any) + Point{Float64}{T}(::Any, ::Any) at none:3 + Point{Float64}{T}(::Any) at sysimg.jl:53 ... Only one default constructor is generated for parametric types, since diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index a033f8716f556..6df9f2c837cdf 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1162,7 +1162,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:534 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:537 ... .. function:: splice!(collection, index, [replacement]) -> item From 9ac7323aadaa533c6a9f976936705a09fbe2967d Mon Sep 17 00:00:00 2001 From: pabloferz <pabloferz@yahoo.com.mx> Date: Wed, 13 Jul 2016 01:00:36 +0200 Subject: [PATCH 0733/1117] Make promote_op rely on Core.Inference.return_type --- base/abstractarray.jl | 12 ++++----- base/arraymath.jl | 46 ++++++++++++++++++---------------- base/bitarray.jl | 33 ++++++++++++++++-------- base/broadcast.jl | 4 +-- base/char.jl | 4 --- base/complex.jl | 3 ++- base/dates/arithmetic.jl | 18 ------------- base/float.jl | 2 ++ base/int.jl | 3 +++ base/irrationals.jl | 7 ------ base/linalg/matmul.jl | 31 ++++++++++++----------- base/number.jl | 12 --------- base/pkg/resolve/fieldvalue.jl | 1 - base/promotion.jl | 39 ++++++++++++++++++++++------ doc/devdocs/promote-op.rst | 36 -------------------------- test/arrayops.jl | 2 -- test/broadcast.jl | 19 +++++++++++--- test/linalg/matmul.jl | 4 +-- test/numbers.jl | 34 ++++++++++++------------- 19 files changed, 143 insertions(+), 167 deletions(-) delete mode 100644 doc/devdocs/promote-op.rst diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 067cdd0093174..7a61baf7185df 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1407,12 +1407,12 @@ end # These are needed because map(eltype, As) is not inferrable promote_eltype_op(::Any) = (@_pure_meta; Bottom) -promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T)) -promote_eltype_op{T}(op, ::T ) = (@_pure_meta; promote_op(op, T)) -promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S)) -promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::S) = (@_pure_meta; promote_op(op, R, S)) -promote_eltype_op{R,S}(op, ::R, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S)) -promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_op(op, eltype(A), promote_eltype_op(op, B, C, D...))) +promote_eltype_op(op, A) = (@_pure_meta; _promote_op(op, eltype(A))) +promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; _promote_op(op, T)) +promote_eltype_op{T}(op, ::AbstractArray{T}, A) = (@_pure_meta; _promote_op(op, T, eltype(A))) +promote_eltype_op{T}(op, A, ::AbstractArray{T}) = (@_pure_meta; _promote_op(op, eltype(A), T)) +promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; _promote_op(op, R, S)) +promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_eltype_op(op, promote_eltype_op(op, A, B), C, D...)) ## 1 argument map!{F}(f::F, A::AbstractArray) = map!(f, A, A) diff --git a/base/arraymath.jl b/base/arraymath.jl index 696ab37b871ab..516fbe2af98ee 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -35,30 +35,26 @@ function !(A::AbstractArray{Bool}) end ## Binary arithmetic operators ## -@pure promote_array_type{S<:Number, A<:AbstractArray}(F, ::Type{S}, ::Type{A}) = - promote_array_type(F, S, eltype(A), promote_op(F, S, eltype(A))) -@pure promote_array_type{S<:Number, A<:AbstractArray}(F, ::Type{A}, ::Type{S}) = - promote_array_type(F, S, eltype(A), promote_op(F, eltype(A), S)) -@pure promote_array_type{S, A, P}(F, ::Type{S}, ::Type{A}, ::Type{P}) = P -@pure promote_array_type{S<:Real, A<:AbstractFloat, P}(F, ::Type{S}, ::Type{A}, ::Type{P}) = A -@pure promote_array_type{S<:Integer, A<:Integer, P}(F::typeof(./), ::Type{S}, ::Type{A}, ::Type{P}) = P -@pure promote_array_type{S<:Integer, A<:Integer, P}(F::typeof(.\), ::Type{S}, ::Type{A}, ::Type{P}) = P -@pure promote_array_type{S<:Integer, A<:Integer, P}(F, ::Type{S}, ::Type{A}, ::Type{P}) = A -@pure promote_array_type{S<:Integer, P}(F::typeof(./), ::Type{S}, ::Type{Bool}, ::Type{P}) = P -@pure promote_array_type{S<:Integer, P}(F::typeof(.\), ::Type{S}, ::Type{Bool}, ::Type{P}) = P -@pure promote_array_type{S<:Integer, P}(F, ::Type{S}, ::Type{Bool}, ::Type{P}) = P +promote_array_type(F, ::Type, ::Type, T::Type) = T +promote_array_type{S<:Real, A<:AbstractFloat}(F, ::Type{S}, ::Type{A}, ::Type) = A +promote_array_type{S<:Integer, A<:Integer}(F, ::Type{S}, ::Type{A}, ::Type) = A +promote_array_type{S<:Integer, A<:Integer}(::typeof(./), ::Type{S}, ::Type{A}, T::Type) = T +promote_array_type{S<:Integer, A<:Integer}(::typeof(.\), ::Type{S}, ::Type{A}, T::Type) = T +promote_array_type{S<:Integer}(::typeof(./), ::Type{S}, ::Type{Bool}, T::Type) = T +promote_array_type{S<:Integer}(::typeof(.\), ::Type{S}, ::Type{Bool}, T::Type) = T +promote_array_type{S<:Integer}(F, ::Type{S}, ::Type{Bool}, T::Type) = T for f in (:+, :-, :div, :mod, :&, :|, :$) - @eval ($f){S,T}(A::AbstractArray{S}, B::AbstractArray{T}) = - _elementwise($f, A, B, promote_eltype_op($f, A, B)) + @eval ($f)(A::AbstractArray, B::AbstractArray) = + _elementwise($f, promote_op($f, eltype(A), eltype(B)), A, B) end -function _elementwise{S,T}(op, A::AbstractArray{S}, B::AbstractArray{T}, ::Type{Any}) - promote_shape(A,B) # check size compatibility +function _elementwise(op, ::Type{Any}, A::AbstractArray, B::AbstractArray) + promote_shape(A, B) # check size compatibility return broadcast(op, A, B) end -function _elementwise{S,T,R}(op, A::AbstractArray{S}, B::AbstractArray{T}, ::Type{R}) - F = similar(A, R, promote_shape(A,B)) +function _elementwise(op, T::Type, A::AbstractArray, B::AbstractArray) + F = similar(A, T, promote_shape(A, B)) for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) @inbounds F[iF] = op(A[iA], B[iB]) end @@ -67,15 +63,21 @@ end for f in (:.+, :.-, :.*, :./, :.\, :.^, :.÷, :.%, :.<<, :.>>, :div, :mod, :rem, :&, :|, :$) @eval begin - function ($f){T}(A::Number, B::AbstractArray{T}) - F = similar(B, promote_array_type($f,typeof(A),typeof(B))) + function ($f)(A::Number, B::AbstractArray) + P = promote_op($f, typeof(A), eltype(B)) + T = promote_array_type($f, typeof(A), eltype(B), P) + T === Any && return [($f)(A, b) for b in B] + F = similar(B, T) for (iF, iB) in zip(eachindex(F), eachindex(B)) @inbounds F[iF] = ($f)(A, B[iB]) end return F end - function ($f){T}(A::AbstractArray{T}, B::Number) - F = similar(A, promote_array_type($f,typeof(A),typeof(B))) + function ($f)(A::AbstractArray, B::Number) + P = promote_op($f, eltype(A), typeof(B)) + T = promote_array_type($f, typeof(B), eltype(A), P) + T === Any && return [($f)(a, B) for a in A] + F = similar(A, T) for (iF, iA) in zip(eachindex(F), eachindex(A)) @inbounds F[iF] = ($f)(A[iA], B) end diff --git a/base/bitarray.jl b/base/bitarray.jl index 0fc33f46a1e5b..3963c16474255 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1035,18 +1035,29 @@ for f in (:+, :-) return r end end + for (f) in (:.+, :.-) - for (arg1, arg2, T, fargs) in ((:(B::BitArray), :(x::Bool) , Int , :(b, x)), - (:(B::BitArray), :(x::Number) , :(promote_array_type($f, BitArray, typeof(x))), :(b, x)), - (:(x::Bool) , :(B::BitArray), Int , :(x, b)), - (:(x::Number) , :(B::BitArray), :(promote_array_type($f, typeof(x), BitArray)), :(x, b))) + for (arg1, arg2, T, t) in ((:(B::BitArray), :(x::Bool) , Int , (:b, :x)), + (:(B::BitArray), :(x::Number) , :(Bool, typeof(x)), (:b, :x)), + (:(x::Bool) , :(B::BitArray), Int , (:x, :b)), + (:(x::Number) , :(B::BitArray), :(typeof(x), Bool), (:x, :b))) @eval function ($f)($arg1, $arg2) - r = Array{$T}(size(B)) + $(if T === Int + quote + r = Array{Int}(size(B)) + end + else + quote + T = promote_op($f, $(T.args[1]), $(T.args[2])) + T === Any && return [($f)($(t[1]), $(t[2])) for b in B] + r = Array{T}(size(B)) + end + end) bi = start(B) ri = 1 while !done(B, bi) b, bi = next(B, bi) - @inbounds r[ri] = ($f)($fargs...) + @inbounds r[ri] = ($f)($(t[1]), $(t[2])) ri += 1 end return r @@ -1078,9 +1089,8 @@ function div(x::Bool, B::BitArray) end function div(x::Number, B::BitArray) all(B) || throw(DivideError()) - pt = promote_array_type(div, typeof(x), BitArray) y = div(x, true) - reshape(pt[ y for i = 1:length(B) ], size(B)) + return fill(y, size(B)) end function mod(A::BitArray, B::BitArray) @@ -1099,15 +1109,16 @@ function mod(x::Bool, B::BitArray) end function mod(x::Number, B::BitArray) all(B) || throw(DivideError()) - pt = promote_array_type(mod, typeof(x), BitArray) y = mod(x, true) - reshape(pt[ y for i = 1:length(B) ], size(B)) + return fill(y, size(B)) end for f in (:div, :mod) @eval begin function ($f)(B::BitArray, x::Number) - F = Array{promote_array_type($f, BitArray, typeof(x))}(size(B)) + T = promote_op($f, Bool, typeof(x)) + T === Any && return [($f)(b, x) for b in B] + F = Array{T}(size(B)) for i = 1:length(F) F[i] = ($f)(B[i], x) end diff --git a/base/broadcast.jl b/base/broadcast.jl index 937d66375bd9a..e42f2a67edfbc 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -3,7 +3,7 @@ module Broadcast using Base.Cartesian -using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, tail, OneTo, to_shape +using Base: promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, tail, OneTo, to_shape import Base: .+, .-, .*, ./, .\, .//, .==, .<, .!=, .<=, .÷, .%, .<<, .>>, .^ export broadcast, broadcast!, bitbroadcast, dotview export broadcast_getindex, broadcast_setindex! @@ -299,7 +299,7 @@ end ## elementwise operators ## for op in (:÷, :%, :<<, :>>, :-, :/, :\, ://, :^) - @eval $(Symbol(:., op))(A::AbstractArray, B::AbstractArray) = broadcast($(op), A, B) + @eval $(Symbol(:., op))(A::AbstractArray, B::AbstractArray) = broadcast($op, A, B) end .+(As::AbstractArray...) = broadcast(+, As...) .*(As::AbstractArray...) = broadcast(*, As...) diff --git a/base/char.jl b/base/char.jl index 52aa1492e2f86..58481c6d39c2a 100644 --- a/base/char.jl +++ b/base/char.jl @@ -40,10 +40,6 @@ hash(x::Char, h::UInt) = hash_uint64(((UInt64(x)+hashchar_seed)<<32) $ UInt64(h) +(x::Char, y::Integer) = Char(Int32(x) + Int32(y)) +(x::Integer, y::Char) = y + x -Base.promote_op{I<:Integer}(::typeof(-), ::Type{Char}, ::Type{I}) = Char -Base.promote_op{I<:Integer}(::typeof(+), ::Type{Char}, ::Type{I}) = Char -Base.promote_op{I<:Integer}(::typeof(+), ::Type{I}, ::Type{Char}) = Char - bswap(x::Char) = Char(bswap(UInt32(x))) print(io::IO, c::Char) = (write(io, c); nothing) diff --git a/base/complex.jl b/base/complex.jl index da644f9412331..7be18a036f951 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -804,7 +804,8 @@ big{T<:AbstractFloat,N}(A::AbstractArray{Complex{T},N}) = convert(AbstractArray{ ## promotion to complex ## -promote_array_type{S<:Union{Complex, Real}, AT<:AbstractFloat, P}(F, ::Type{S}, ::Type{Complex{AT}}, ::Type{P}) = Complex{AT} +_default_type(T::Type{Complex}) = Complex{Int} +promote_array_type{S<:Union{Complex, Real}, T<:AbstractFloat}(F, ::Type{S}, ::Type{Complex{T}}, ::Type) = Complex{T} function complex{S<:Real,T<:Real}(A::AbstractArray{S}, B::AbstractArray{T}) if size(A) != size(B); throw(DimensionMismatch()); end diff --git a/base/dates/arithmetic.jl b/base/dates/arithmetic.jl index 4a94fc13df911..94752ffe2390a 100644 --- a/base/dates/arithmetic.jl +++ b/base/dates/arithmetic.jl @@ -94,21 +94,3 @@ end # AbstractArray{TimeType}, AbstractArray{TimeType} (-){T<:TimeType}(x::OrdinalRange{T}, y::OrdinalRange{T}) = collect(x) - collect(y) (-){T<:TimeType}(x::Range{T}, y::Range{T}) = collect(x) - collect(y) - -# promotion rules - -for op in (:+, :-, :.+, :.-) - @eval begin - Base.promote_op{P<:Period}(::typeof($op), ::Type{P}, ::Type{P}) = P - Base.promote_op{P1<:Period,P2<:Period}(::typeof($op), ::Type{P1}, ::Type{P2}) = CompoundPeriod - Base.promote_op{D<:Date}(::typeof($op), ::Type{D}, ::Type{D}) = Day - Base.promote_op{D<:DateTime}(::typeof($op), ::Type{D}, ::Type{D}) = Millisecond - end -end - -for op in (:/, :%, :div, :mod, :./, :.%) - @eval begin - Base.promote_op{P<:Period}(::typeof($op), ::Type{P}, ::Type{P}) = typeof($op(1,1)) - Base.promote_op{P<:Period,R<:Real}(::typeof($op), ::Type{P}, ::Type{R}) = P - end -end diff --git a/base/float.jl b/base/float.jl index 20b7e9a2513ca..01767dd3d4123 100644 --- a/base/float.jl +++ b/base/float.jl @@ -230,6 +230,8 @@ promote_rule(::Type{Float64}, ::Type{Float32}) = Float64 widen(::Type{Float16}) = Float32 widen(::Type{Float32}) = Float64 +_default_type(T::Union{Type{Real},Type{AbstractFloat}}) = Float64 + ## floating point arithmetic ## -(x::Float32) = box(Float32,neg_float(unbox(Float32,x))) -(x::Float64) = box(Float64,neg_float(unbox(Float64,x))) diff --git a/base/int.jl b/base/int.jl index ca3242cc682f5..79cfb0ddafb0b 100644 --- a/base/int.jl +++ b/base/int.jl @@ -305,6 +305,9 @@ promote_rule{T<:BitSigned64}(::Type{UInt64}, ::Type{T}) = UInt64 promote_rule{T<:Union{UInt32, UInt64}}(::Type{T}, ::Type{Int128}) = Int128 promote_rule{T<:BitSigned}(::Type{UInt128}, ::Type{T}) = UInt128 +_default_type(T::Type{Unsigned}) = UInt +_default_type(T::Union{Type{Integer},Type{Signed}}) = Int + ## traits ## typemin(::Type{Int8 }) = Int8(-128) diff --git a/base/irrationals.jl b/base/irrationals.jl index e76f40760c0b9..a5e091f5ca1e3 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -10,13 +10,6 @@ promote_rule{s}(::Type{Irrational{s}}, ::Type{Float32}) = Float32 promote_rule{s,t}(::Type{Irrational{s}}, ::Type{Irrational{t}}) = Float64 promote_rule{s,T<:Number}(::Type{Irrational{s}}, ::Type{T}) = promote_type(Float64,T) -promote_op{S<:Irrational,T<:Irrational}(op::Any, ::Type{S}, ::Type{T}) = - promote_op(op, Float64, Float64) -promote_op{S<:Irrational,T<:Number}(op::Any, ::Type{S}, ::Type{T}) = - promote_op(op, Float64, T) -promote_op{S<:Irrational,T<:Number}(op::Any, ::Type{T}, ::Type{S}) = - promote_op(op, T, Float64) - convert(::Type{AbstractFloat}, x::Irrational) = Float64(x) convert(::Type{Float16}, x::Irrational) = Float16(Float32(x)) convert{T<:Real}(::Type{Complex{T}}, x::Irrational) = convert(Complex{T}, convert(T,x)) diff --git a/base/linalg/matmul.jl b/base/linalg/matmul.jl index 6c9117a3cd06e..aa713321a42bb 100644 --- a/base/linalg/matmul.jl +++ b/base/linalg/matmul.jl @@ -76,11 +76,11 @@ At_mul_B{T<:BlasComplex}(x::StridedVector{T}, y::StridedVector{T}) = [BLAS.dotu( # Matrix-vector multiplication function (*){T<:BlasFloat,S}(A::StridedMatrix{T}, x::StridedVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_B!(similar(x, TS, size(A,1)), A, convert(AbstractVector{TS}, x)) end function (*){T,S}(A::AbstractMatrix{T}, x::AbstractVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_B!(similar(x,TS,size(A,1)),A,x) end (*)(A::AbstractVector, B::AbstractMatrix) = reshape(A,length(A),1)*B @@ -99,22 +99,22 @@ end A_mul_B!(y::AbstractVector, A::AbstractVecOrMat, x::AbstractVector) = generic_matvecmul!(y, 'N', A, x) function At_mul_B{T<:BlasFloat,S}(A::StridedMatrix{T}, x::StridedVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) At_mul_B!(similar(x,TS,size(A,2)), A, convert(AbstractVector{TS}, x)) end function At_mul_B{T,S}(A::AbstractMatrix{T}, x::AbstractVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) At_mul_B!(similar(x,TS,size(A,2)), A, x) end At_mul_B!{T<:BlasFloat}(y::StridedVector{T}, A::StridedVecOrMat{T}, x::StridedVector{T}) = gemv!(y, 'T', A, x) At_mul_B!(y::AbstractVector, A::AbstractVecOrMat, x::AbstractVector) = generic_matvecmul!(y, 'T', A, x) function Ac_mul_B{T<:BlasFloat,S}(A::StridedMatrix{T}, x::StridedVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) Ac_mul_B!(similar(x,TS,size(A,2)),A,convert(AbstractVector{TS},x)) end function Ac_mul_B{T,S}(A::AbstractMatrix{T}, x::AbstractVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) Ac_mul_B!(similar(x,TS,size(A,2)), A, x) end @@ -142,14 +142,14 @@ end A_mul_B!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'N', 'N', A, B) function At_mul_B{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) - TS = promote_op(*,arithtype(T), arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) At_mul_B!(similar(B, TS, (size(A,2), size(B,2))), A, B) end At_mul_B!{T<:BlasFloat}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = is(A,B) ? syrk_wrapper!(C, 'T', A) : gemm_wrapper!(C, 'T', 'N', A, B) At_mul_B!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'T', 'N', A, B) function A_mul_Bt{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) - TS = promote_op(*,arithtype(T), arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_Bt!(similar(B, TS, (size(A,1), size(B,1))), A, B) end A_mul_Bt!{T<:BlasFloat}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = is(A,B) ? syrk_wrapper!(C, 'N', A) : gemm_wrapper!(C, 'N', 'T', A, B) @@ -166,7 +166,7 @@ end A_mul_Bt!(C::AbstractVecOrMat, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'N', 'T', A, B) function At_mul_Bt{T,S}(A::AbstractMatrix{T}, B::AbstractVecOrMat{S}) - TS = promote_op(*,arithtype(T), arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) At_mul_Bt!(similar(B, TS, (size(A,2), size(B,1))), A, B) end At_mul_Bt!{T<:BlasFloat}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = gemm_wrapper!(C, 'T', 'T', A, B) @@ -175,7 +175,7 @@ At_mul_Bt!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generi Ac_mul_B{T<:BlasReal}(A::StridedMatrix{T}, B::StridedMatrix{T}) = At_mul_B(A, B) Ac_mul_B!{T<:BlasReal}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = At_mul_B!(C, A, B) function Ac_mul_B{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) - TS = promote_op(*,arithtype(T), arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) Ac_mul_B!(similar(B, TS, (size(A,2), size(B,2))), A, B) end Ac_mul_B!{T<:BlasComplex}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = is(A,B) ? herk_wrapper!(C,'C',A) : gemm_wrapper!(C,'C', 'N', A, B) @@ -184,13 +184,14 @@ Ac_mul_B!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic A_mul_Bc{T<:BlasFloat,S<:BlasReal}(A::StridedMatrix{T}, B::StridedMatrix{S}) = A_mul_Bt(A, B) A_mul_Bc!{T<:BlasFloat,S<:BlasReal}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{S}) = A_mul_Bt!(C, A, B) function A_mul_Bc{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_Bc!(similar(B,TS,(size(A,1),size(B,1))),A,B) end A_mul_Bc!{T<:BlasComplex}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = is(A,B) ? herk_wrapper!(C, 'N', A) : gemm_wrapper!(C, 'N', 'C', A, B) A_mul_Bc!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'N', 'C', A, B) -Ac_mul_Bc{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) = Ac_mul_Bc!(similar(B, promote_op(*,arithtype(T), arithtype(S)), (size(A,2), size(B,1))), A, B) +Ac_mul_Bc{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) = + Ac_mul_Bc!(similar(B, promote_op(*, arithtype(T), arithtype(S)), (size(A,2), size(B,1))), A, B) Ac_mul_Bc!{T<:BlasFloat}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = gemm_wrapper!(C, 'C', 'C', A, B) Ac_mul_Bc!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'C', 'C', A, B) Ac_mul_Bt!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'C', 'T', A, B) @@ -423,7 +424,7 @@ end function generic_matmatmul{T,S}(tA, tB, A::AbstractVecOrMat{T}, B::AbstractMatrix{S}) mA, nA = lapack_size(tA, A) mB, nB = lapack_size(tB, B) - C = similar(B, promote_op(*,arithtype(T),arithtype(S)), mA, nB) + C = similar(B, promote_op(*, arithtype(T), arithtype(S)), mA, nB) generic_matmatmul!(C, tA, tB, A, B) end @@ -617,7 +618,7 @@ end # multiply 2x2 matrices function matmul2x2{T,S}(tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) - matmul2x2!(similar(B, promote_op(*,T,S), 2, 2), tA, tB, A, B) + matmul2x2!(similar(B, promote_op(*, T, S), 2, 2), tA, tB, A, B) end function matmul2x2!{T,S,R}(C::AbstractMatrix{R}, tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) @@ -646,7 +647,7 @@ end # Multiply 3x3 matrices function matmul3x3{T,S}(tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) - matmul3x3!(similar(B, promote_op(*,T,S), 3, 3), tA, tB, A, B) + matmul3x3!(similar(B, promote_op(*, T, S), 3, 3), tA, tB, A, B) end function matmul3x3!{T,S,R}(C::AbstractMatrix{R}, tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) diff --git a/base/number.jl b/base/number.jl index d2d3270f1cfc4..42ffb708736ee 100644 --- a/base/number.jl +++ b/base/number.jl @@ -63,16 +63,4 @@ zero{T<:Number}(::Type{T}) = convert(T,0) one(x::Number) = oftype(x,1) one{T<:Number}(::Type{T}) = convert(T,1) -promote_op{R,S<:Number}(::Type{R}, ::Type{S}) = (@_pure_meta; R) # to fix ambiguities -function promote_op{T<:Number}(op, ::Type{T}) - S = typeof(op(one(T))) - # preserve the most general (abstract) type when possible - return isleaftype(T) ? S : typejoin(S, T) -end -function promote_op{R<:Number,S<:Number}(op, ::Type{R}, ::Type{S}) - T = typeof(op(one(R), one(S))) - # preserve the most general (abstract) type when possible - return isleaftype(R) && isleaftype(S) ? T : typejoin(R, S, T) -end - factorial(x::Number) = gamma(x + 1) # fallback for x not Integer diff --git a/base/pkg/resolve/fieldvalue.jl b/base/pkg/resolve/fieldvalue.jl index a3666388d35a7..fe175ce356b0b 100644 --- a/base/pkg/resolve/fieldvalue.jl +++ b/base/pkg/resolve/fieldvalue.jl @@ -42,7 +42,6 @@ Base.typemin(::Type{FieldValue}) = (x=typemin(Int); y=typemin(VersionWeight); Fi Base.:-(a::FieldValue, b::FieldValue) = FieldValue(a.l0-b.l0, a.l1-b.l1, a.l2-b.l2, a.l3-b.l3, a.l4-b.l4) Base.:+(a::FieldValue, b::FieldValue) = FieldValue(a.l0+b.l0, a.l1+b.l1, a.l2+b.l2, a.l3+b.l3, a.l4+b.l4) -Base.promote_op(::Union{typeof(+), typeof(-)}, ::Type{FieldValue}, ::Type{FieldValue}) = FieldValue function Base.isless(a::FieldValue, b::FieldValue) a.l0 < b.l0 && return true diff --git a/base/promotion.jl b/base/promotion.jl index 778de13c004c9..a3be3c27829d4 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -217,13 +217,38 @@ max(x::Real, y::Real) = max(promote(x,y)...) min(x::Real, y::Real) = min(promote(x,y)...) minmax(x::Real, y::Real) = minmax(promote(x, y)...) -# "Promotion" that takes a function into account. You can override this -# as needed. For example, if you need to provide a custom result type -# for the multiplication of two types, -# promote_op{R<:MyType,S<:MyType}(::typeof(*), ::Type{R}, ::Type{S}) = MyType{multype(R,S)} -promote_op(::Any) = (@_pure_meta; Bottom) -promote_op(::Any, ::Any, ::Any...) = (@_pure_meta; Any) -promote_op{T}(::Type{T}, ::Any) = (@_pure_meta; T) +# "Promotion" that takes a function into account. These are meant to be +# used mainly by broadcast methods, so it is adviced against overriding them +if isdefined(Core, :Inference) + function _promote_op(op, T::Type) + G = Tuple{Generator{Tuple{T},typeof(op)}} + R = Core.Inference.return_type(first, G) + return isleaftype(R) ? R : Any + end + function _promote_op(op, R::Type, S::Type) + F = typeof(a -> op(a...)) + G = Tuple{Generator{Zip2{Tuple{R},Tuple{S}},F}} + T = Core.Inference.return_type(first, G) + return isleaftype(T) ? T : Any + end +else + _promote_op(::Any...) = (@_pure_meta; Any) +end +_default_type(T::Type) = (@_pure_meta; T) + +promote_op(::Any...) = (@_pure_meta; Any) +promote_op(T::Type, ::Any) = (@_pure_meta; T) +promote_op(T::Type, ::Type) = (@_pure_meta; T) # To handle ambiguities +# Promotion that tries to preserve non-concrete types +function promote_op(f, S::Type) + T = _promote_op(f, _default_type(S)) + return isleaftype(S) ? T : typejoin(S, T) +end +function promote_op(f, R::Type, S::Type) + T = _promote_op(f, _default_type(R), _default_type(S)) + isleaftype(R) && return isleaftype(S) ? T : typejoin(S, T) + return isleaftype(S) ? typejoin(R, T) : typejoin(R, S, T) +end ## catch-alls to prevent infinite recursion when definitions are missing ## diff --git a/doc/devdocs/promote-op.rst b/doc/devdocs/promote-op.rst deleted file mode 100644 index 53a5a93768549..0000000000000 --- a/doc/devdocs/promote-op.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. currentmodule:: Base - -.. _devdocs-promote-op: - -Operator-sensitive promotion -============================ - -In certain cases, the :ref:`simple rules for promotion -<man-promotion-rules>` may not be sufficient. For example, consider a -type that can represent an object with physical units, here restricted -to a single unit like "meter":: - - immutable MeterUnits{T,P} <: Number - val::T - end - MeterUnits{T}(val::T, pow::Int) = MeterUnits{T,pow}(val) - - m = MeterUnits(1.0, 1) # 1.0 meter, i.e. units of length - m2 = MeterUnits(1.0, 2) # 1.0 meter^2, i.e. units of area - -Now let's define the operations ``+`` and ``*`` for these objects: -``m+m`` should have the type of ``m`` but ``m*m`` should have the type -of ``m2``. When the result type depends on the operation, and not -just the input types, ``promote_rule`` will be inadequate. - -Fortunately, it's possible to provide such definitions via ``promote_op``:: - - Base.promote_op{R,S}(::typeof(+), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),1} - Base.promote_op{R,S}(::typeof(*), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),2} - Base.promote_op{R,S}(::typeof(.*), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),2} - -The first one defines the promotion rule for ``+``, and the second one -for ``*``. - -It's worth noting that as julia's internal representation of functions -evolves, this interface may change in a future version of Julia. diff --git a/test/arrayops.jl b/test/arrayops.jl index 92549881f665b..b52f04e4a0bda 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1424,7 +1424,6 @@ b = rand(6,7) @test_throws ArgumentError copy!(a,2:3,1:3,b,1:5,2:7) @test_throws ArgumentError Base.copy_transpose!(a,2:3,1:3,b,1:5,2:7) -# return type declarations (promote_op) module RetTypeDecl using Base.Test import Base: +, *, .*, convert @@ -1442,7 +1441,6 @@ module RetTypeDecl (*){T}(x::MeterUnits{T,1}, y::MeterUnits{T,1}) = MeterUnits{T,2}(x.val*y.val) (.*){T}(x::MeterUnits{T,1}, y::MeterUnits{T,1}) = MeterUnits{T,2}(x.val*y.val) convert{T,pow}(::Type{MeterUnits{T,pow}}, y::Real) = MeterUnits{T,pow}(convert(T,y)) - Base.promote_op{R,S}(::typeof(*), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),2} @test @inferred(m+[m,m]) == [m+m,m+m] @test @inferred([m,m]+m) == [m+m,m+m] diff --git a/test/broadcast.jl b/test/broadcast.jl index 70b2b39beb3fc..c4ff2cc43f98a 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -224,12 +224,11 @@ let x = sin.(1:10) @test sin.(atan2.(x, 3.7)) == broadcast(x -> sin(atan2(x,3.7)), x) @test atan2.(x, 3.7) == broadcast(x -> atan2(x,3.7), x) == broadcast(atan2, x, 3.7) end -# Use side effects to check for loop fusion. Note that, due to #17314, -# a broadcasted function is currently called an extra time with an argument 1. +# Use side effects to check for loop fusion. let g = Int[] - f17300(x) = begin; push!(g, x); x+1; end + f17300(x) = begin; push!(g, x); x+2; end f17300.(f17300.(f17300.(1:3))) - @test g == [1,2,3, 1,2,3, 2,3,4, 3,4,5] + @test g == [1,3,5, 2,4,6, 3,5,7] end # fusion with splatted args: let x = sin.(1:10), a = [x] @@ -294,3 +293,15 @@ end let foo = [[1,2,3],[4,5,6],[7,8,9]] @test max.(foo...) == broadcast(max, foo...) == [7,8,9] end + +# Issue 17314 +@test broadcast(x->log(log(log(x))), [1000]) == [log(log(log(1000)))] +let f17314 = x -> x < 0 ? false : x + @test eltype(broadcast(f17314, 1:3)) === Int + @test eltype(broadcast(f17314, -1:1)) === Integer + @test eltype(broadcast(f17314, Int[])) === Union{} +end +let io = IOBuffer() + broadcast(x->print(io,x), 1:5) # broadcast with side effects + @test takebuf_array(io) == [0x31,0x32,0x33,0x34,0x35] +end diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 44bb9a6afa85b..4755f1d9ddd64 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -320,10 +320,10 @@ end immutable RootInt i::Int end -import Base: *, transpose, promote_op +import Base: *, transpose (*)(x::RootInt, y::RootInt) = x.i*y.i transpose(x::RootInt) = x -promote_op(::typeof(*), ::Type{RootInt}, ::Type{RootInt}) = Int +@test Base.promote_op(*, RootInt, RootInt) === Int a = [RootInt(3)] C = [0] diff --git a/test/numbers.jl b/test/numbers.jl index 8ef819c0685ae..a0ccc138e776f 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2791,21 +2791,21 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Complex{Int}, Complex{UInt}, Complex32, Complex64, Complex128) for S in types for op in (+, -) - T = @inferred Base.promote_op(op, S) + T = @inferred Base._promote_op(op, S) t = @inferred op(one(S)) @test T === typeof(t) end - end - - @test @inferred(Base.promote_op(!, Bool)) === Bool - for R in types, S in types - for op in (+, -, *, /, ^) - T = @inferred Base.promote_op(op, R, S) - t = @inferred op(one(R), one(S)) - @test T === typeof(t) + for R in types + for op in (+, -, *, /, ^) + T = @inferred Base._promote_op(op, S, R) + t = @inferred op(one(S), one(R)) + @test T === typeof(t) + end end end + + @test @inferred(Base._promote_op(!, Bool)) === Bool end let types = (Base.BitInteger_types..., BigInt, Bool, @@ -2813,23 +2813,23 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Float16, Float32, Float64, BigFloat) for S in types, T in types for op in (<, >, <=, >=, (==)) - @test @inferred(Base.promote_op(op, S, T)) === Bool + @test @inferred(Base._promote_op(op, S, T)) === Bool end end end let types = (Base.BitInteger_types..., BigInt, Bool) for S in types - T = @inferred Base.promote_op(~, S) + T = @inferred Base._promote_op(~, S) t = @inferred ~one(S) @test T === typeof(t) - end - for S in types, T in types - for op in (&, |, <<, >>, (>>>), %, ÷) - T = @inferred Base.promote_op(op, S, T) - t = @inferred op(one(S), one(T)) - @test T === typeof(t) + for R in types + for op in (&, |, <<, >>, (>>>), %, ÷) + T = @inferred Base._promote_op(op, S, R) + t = @inferred op(one(S), one(R)) + @test T === typeof(t) + end end end end From a0270fce09348bda8404dc5268e71c65443a6a44 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Thu, 28 Jul 2016 13:25:01 -0400 Subject: [PATCH 0734/1117] NEWS: move tooling towards end; delete empty CL options section --- NEWS.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/NEWS.md b/NEWS.md index 24ae9767d806e..bf27a15c151b2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,22 +1,6 @@ Julia v0.5.0 Release Notes ========================== -Language tooling improvements ------------------------------ - - * The [Julia debugger](https://github.com/Keno/Gallium.jl) makes its debut - with this release. Install it with `Pkg.add("Gallium")`, and the - [documentation](https://github.com/Keno/Gallium.jl#gallium) should - get you going. The [JuliaCon - talk](https://www.youtube.com/watch?v=e6-hcOHO0tc&list=PLP8iPy9hna6SQPwZUDtAM59-wPzCPyD_S&index=5) - on Gallium shows off various features of the debugger. - - * The [Juno IDE](http://junolab.org) has matured significantly, and now - also includes support for plotting and debugging. - - * [Cxx.jl](https://github.com/Keno/Cxx.jl) provides a convenient FFI for - calling C++ code from Julia. - New language features --------------------- @@ -342,6 +326,22 @@ Library improvements * Display properties can now be passed among output functions (e.g. `show`) using an `IOContext` object ([#13825]). +Language tooling improvements +----------------------------- + + * The [Julia debugger](https://github.com/Keno/Gallium.jl) makes its debut + with this release. Install it with `Pkg.add("Gallium")`, and the + [documentation](https://github.com/Keno/Gallium.jl#gallium) should + get you going. The [JuliaCon + talk](https://www.youtube.com/watch?v=e6-hcOHO0tc&list=PLP8iPy9hna6SQPwZUDtAM59-wPzCPyD_S&index=5) + on Gallium shows off various features of the debugger. + + * The [Juno IDE](http://junolab.org) has matured significantly, and now + also includes support for plotting and debugging. + + * [Cxx.jl](https://github.com/Keno/Cxx.jl) provides a convenient FFI for + calling C++ code from Julia. + Deprecated or removed --------------------- From 940f60e06352cef1a88f37e8d9ca49ff618bca39 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Thu, 28 Jul 2016 13:58:50 -0400 Subject: [PATCH 0735/1117] NEWS: reorganize for better flow and structure --- NEWS.md | 337 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 174 insertions(+), 163 deletions(-) diff --git a/NEWS.md b/NEWS.md index bf27a15c151b2..f290fddea7cb3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -46,9 +46,9 @@ Experimental language features ------------------------------ * Support for - [multi-threading](http://docs.julialang.org/en/latest/manual/parallel-computing/#multi-threading-experimental). Loops - with independent iterations can be easily parallelized with the - `Threads.@threads` macro. + [multi-threading](http://docs.julialang.org/en/latest/manual/parallel-computing/#multi-threading-experimental). + Loops with independent iterations can be easily parallelized with the + `Threads.@threads` macro. * Support for arrays with indexing starting at values different from 1. The array types are expected to be defined in packages, but now @@ -59,11 +59,14 @@ Language changes ---------------- * Each function and closure now has its own type. The captured variables of a closure - are fields of its type. `Function` is now an abstract type, and is the default supertype - of functions and closures. All functions, including anonymous functions, - are generic and support all features (e.g. keyword arguments). - Instead of adding methods to `call`, methods are added by type using the syntax - `(::ftype)(...) = ...`. `call` is deprecated ([#13412]). + are fields of its type. `Function` is now an abstract type, and is the default + supertype of functions and closures. All functions, including anonymous functions, + are generic and support all features (e.g. keyword arguments). Instead of adding + methods to `call`, methods are added by type using the syntax + `(::ftype)(...) = ...`. `call` is deprecated ([#13412]). A significant result of + this language change is that higher order functions can be specialized on their + function arguments, leading to much faster functional programming, typically as + fast as if function arguments were manually inlined. See below for details. * Square brackets and commas (e.g. `[x, y]`) no longer concatenate arrays, and always simply construct a vector of the provided values. If `x` and `y` are arrays, @@ -75,8 +78,8 @@ Language changes * Relational algebra symbols are now allowed as infix operators ([#8036]): `⨝`, `⟕`, `⟖`, `⟗` for joins and `▷` for anti-join. - * A warning is always given when a method is overwritten (previously, this was done - only when the new and old definitions were in separate modules) ([#14759]). + * A warning is always given when a method is overwritten; previously, this was done + only when the new and old definitions were in separate modules ([#14759]). * The `if` keyword cannot be followed immediately by a line break ([#15763]). @@ -92,77 +95,6 @@ Language changes When `x` is global, `x::T = ...` and `global x::T` used to mean type assertion, but this syntax is now reserved for type declaration ([#964]). -Compiler/Runtime improvements ------------------------------ - - * Machine SIMD types can be represented in Julia as a homogeneous tuple of `VecElement` ([#15244]). - - * The performance of higher-order and anonymous functions has been greatly improved. - For example, `map(x->2x, A)` performs as well as `2.*A`([#13412]). - - * On windows, a DLL of standard library code is now precompiled and used by default, - improving startup time ([#16953]). - - * LLVM has been upgraded to version 3.7.1, improving the quality of generated - code and debug info. However compile times may be slightly longer ([#14623]). - -New architectures ------------------ - - This release greatly improves support for ARM, and introduces support for Power. - - * [ARM](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Aarm): - [#14194], [#14519], [#16645], [#16621] - - * [Power](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Apower): - [#16455], [#16404] - -Breaking changes ----------------- - -This section lists changes that do not have deprecation warnings. - - * The assignment operations `.+=`, `.*=` and so on now generate calls - to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side - if the latter is an indexing expression, e.g. `a[...]`). This means that they will fail - if the left-hand side is immutable (or does not support `view`), and will otherwise - change the left-hand side in-place ([#17510], [#17546]). - - * Method ambiguities no longer generate warnings when files are loaded, - nor do they dispatch to an arbitrarily-chosen method; instead, a call that - cannot be resolved to a single method results in a `MethodError` at run time, - rather than the previous definition-time warning. ([#6190]) - - * Array comprehensions preserve the dimensions of the input ranges. For example, - `[2x for x in A]` will have the same dimensions as `A` ([#16622]). - - * The result type of an array comprehension depends only on the types of elements - computed, instead of using type inference ([#7258]). If the result is empty, then - type inference is still used to determine the element type. - - * `reshape` is now defined to always share data with the original array. - If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of - the desired shape ([#4211]). - - * `mapslices` now re-uses temporary storage. Recipient functions that expect - input slices to be persistent should copy data to other storage ([#17266]). - All usages of `mapslices` should be carefully audited since this change can cause - silent, incorrect behavior, rather than failing noisily. - - * Local variables and arguments are represented in lowered code as numbered `Slot` - objects instead of as symbols ([#15609]). - - * The information that used to be in the `ast` field of the `LambdaStaticData` type - is now divided among the fields `code`, `slotnames`, `slottypes`, `slotflags`, - `gensymtypes`, `rettype`, `nargs`, and `isva` in the `LambdaInfo` type ([#15609]). - - * `A <: B` is parsed as `Expr(:(<:), :A, :B)` in all cases ([#9503]). - This also applies to the `>:` operator. - - * Simple 2-argument comparisons like `A < B` are parsed as calls instead of using the - `:comparison` expression type ([#15524]). The `:comparison` expression type is still - produced in ASTs when comparisons are chained (e.g. `A < B ≤ C`). - Library improvements -------------------- @@ -201,41 +133,6 @@ Library improvements * Support for Unicode 9 ([#17402]). - * Packages: - - * The package system (`Pkg`) is now based on the `libgit2` library, rather - than running the `git` program, increasing performance (especially on - Windows) ([#11196]). - - * Package-development functions like `Pkg.tag` and `Pkg.publish` - have been moved to an external [PkgDev] package ([#13387]). - - * Updating only a subset of the packages is now supported, - e.g. `Pkg.update("Example")` ([#17132]) - - * The `Base.Test` module now has a `@testset` feature to bundle - tests together and delay throwing an error until the end ([#13062]). - - * The new features are mirrored in the - [BaseTestNext](https://github.com/IainNZ/BaseTestNext.jl) - package for users who would like to use the new functionality on Julia v0.4. - - * The [BaseTestDeprecated](https://github.com/IainNZ/BaseTestDeprecated.jl) - package provides the old-style `handler` functionality, for compatibility - with code that needs to support both Julia v0.4 and v0.5. - - * Most of the combinatorics functions have been moved from `Base` - to the [Combinatorics.jl package](https://github.com/JuliaLang/Combinatorics.jl) ([#13897]). - - * `pmap` keyword arguments `err_retry=true` and `err_stop=false` are deprecated. - Action to be taken on errors can be specified via the `on_error` keyword argument. - Retry is specified via `retry_n`, `retry_on` and `retry_max_delay` ([#15409], [#15975], [#16663]). - - * The functions `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the - function argument as the first argument to allow for do-block syntax ([#13338]). - - * `cov` and `cor` don't use keyword arguments anymore and are therefore now type stable ([#13465]). - * Arrays and linear algebra: * All dimensions indexed by scalars are now dropped, whereas previously only @@ -270,13 +167,42 @@ Library improvements on zero assignment. To drop stored entries from sparse matrices and vectors, use `Base.SparseArrays.dropstored!` ([#17404]). - * New `foreach` function for calling a function on every element of a collection when - the results are not needed ([#13774]). As compared to `map(f, v)`, which allocates and - returns a result array, `foreach(f, v)` calls `f` on each element of `v`, returning nothing. + * Concatenating dense and sparse matrices now returns a sparse matrix ([#15172]). + + * Files and I/O: + + * The `open` function now respects `umask` on UNIX when creating files ([#16466], [#16502]). + + * A new function `walkdir()` returns an iterator that walks the directory tree of a directory. ([#1765]) + + ``` + for (root, dirs, files) in walkdir(expanduser("~/.julia/v0.5/Plots/src")) + println("$(length(files)) \t files in $root") + end + 19 files in /Users/me/.julia/v0.5/Plots/src + 15 files in /Users/me/.julia/v0.5/Plots/src/backends + 4 files in /Users/me/.julia/v0.5/Plots/src/deprecated + ``` + + * A new function `chown()` changes the ownership of files. ([#15007]) + + * Display properties can now be passed among output functions (e.g. `show`) + using an `IOContext` object ([#13825]). + + * `Cmd(cmd; ...)` now accepts new Windows-specific options `windows_verbatim` + (to alter Windows command-line generation) and `windows_hide` (to + suppress creation of new console windows) ([#13780]). - * `Cmd(cmd; ...)` now accepts new Windows-specific options `windows_verbatim` - (to alter Windows command-line generation) and `windows_hide` (to - suppress creation of new console windows) ([#13780]). + * There is now a default no-op `flush(io)` function for all `IO` types ([#16403]). + + * Parellel computing: + + * `pmap` keyword arguments `err_retry=true` and `err_stop=false` are deprecated. + Action to be taken on errors can be specified via the `on_error` keyword argument. + Retry is specified via `retry_n`, `retry_on` and `retry_max_delay` ([#15409], [#15975], [#16663]). + + * The functions `remotecall`, `remotecall_fetch`, and `remotecall_wait` now have the + function argument as the first argument to allow for do-block syntax ([#13338]). * Statistics: @@ -284,63 +210,133 @@ Library improvements * `extrema` can now operate over a region ([#15550]). - * The new `Base.StackTraces` module makes stack traces easier to use programmatically ([#14469]). + * `cov` and `cor` don't use keyword arguments anymore and are therefore now type stable ([#13465]). + + * Testing: + + * The `Base.Test` module now has a `@testset` feature to bundle + tests together and delay throwing an error until the end ([#13062]). + + * The new features are mirrored in the + [BaseTestNext](https://github.com/IainNZ/BaseTestNext.jl) + package for users who would like to use the new functionality on Julia v0.4. + + * The [BaseTestDeprecated](https://github.com/IainNZ/BaseTestDeprecated.jl) + package provides the old-style `handler` functionality, for compatibility + with code that needs to support both Julia v0.4 and v0.5. - * There is now a default no-op `flush(io)` function for all `IO` types ([#16403]). + * Package management: - * Concatenating dense and sparse matrices now returns a sparse matrix ([#15172]). + * The package system (`Pkg`) is now based on the `libgit2` library, rather + than running the `git` program, increasing performance (especially on + Windows) ([#11196]). - * The `libjulia` library is now properly versioned and installed to the public `<prefix>/lib` - directory, instead of the private `<prefix>/lib/julia` directory ([#16362]). + * Package-development functions like `Pkg.tag` and `Pkg.publish` + have been moved to an external [PkgDev] package ([#13387]). - * System reflection is now more consistently exposed from Sys and not Base. - `OS_NAME` has been replaced by `Sys.KERNEL` and always reports the name of the - kernel (as reported by `uname`). The `@windows_only` and `@osx` family of macros - have been replaced with functions such as `is_windows()` and `is_apple()`. - There is now also a `@static` macro that will evaluate the condition of an - if-statement at compile time, for when a static branch is required ([#16219]). + * Updating only a subset of the packages is now supported, + e.g. `Pkg.update("Example")` ([#17132]) - * Prime number related functions have been moved from `Base` to the - [Primes.jl package](https://github.com/JuliaMath/Primes.jl) ([#16481]). + * Miscellanous: - * `Date` and `DateTime` values can now be rounded to a specified resolution (e.g., 1 month or - 15 minutes) with `floor`, `ceil`, and `round` ([#17037]). + * Prime number related functions have been moved from `Base` to the + [Primes.jl package](https://github.com/JuliaMath/Primes.jl) ([#16481]). - * File handling: + * Most of the combinatorics functions have been moved from `Base` + to the [Combinatorics.jl package](https://github.com/JuliaLang/Combinatorics.jl) ([#13897]). - * The `open` function now respects `umask` on UNIX when creating files ([#16466], [#16502]). + * New `foreach` function for calling a function on every element of a collection when + the results are not needed ([#13774]). Compared to `map(f, v)`, which allocates and + returns a result array, `foreach(f, v)` calls `f` on each element of `v`, returning + nothing. - * A new function `walkdir()` returns an iterator that walks the directory tree of a directory. ([#1765]) + * The new `Base.StackTraces` module makes stack traces easier to use programmatically ([#14469]). - ``` - for (root, dirs, files) in walkdir(expanduser("~/.julia/v0.5/Plots/src")) - println("$(length(files)) \t files in $root") - end - 19 files in /Users/me/.julia/v0.5/Plots/src - 15 files in /Users/me/.julia/v0.5/Plots/src/backends - 4 files in /Users/me/.julia/v0.5/Plots/src/deprecated - ``` + * The `libjulia` library is now properly versioned and installed to the public `<prefix>/lib` + directory, instead of the private `<prefix>/lib/julia` directory ([#16362]). - * A new function `chown()` changes the ownership of files. ([#15007]) + * System reflection is now more consistently exposed from Sys and not Base. + `OS_NAME` has been replaced by `Sys.KERNEL` and always reports the name of the + kernel (as reported by `uname`). The `@windows_only` and `@osx` family of macros + have been replaced with functions such as `is_windows()` and `is_apple()`. + There is now also a `@static` macro that will evaluate the condition of an + if-statement at compile time, for when a static branch is required ([#16219]). - * Display properties can now be passed among output functions (e.g. `show`) - using an `IOContext` object ([#13825]). + * `Date` and `DateTime` values can now be rounded to a specified resolution (e.g., 1 month or + 15 minutes) with `floor`, `ceil`, and `round` ([#17037]). -Language tooling improvements +[PkgDev]: https://github.com/JuliaLang/PkgDev.jl + +Compiler/Runtime improvements ----------------------------- - * The [Julia debugger](https://github.com/Keno/Gallium.jl) makes its debut - with this release. Install it with `Pkg.add("Gallium")`, and the - [documentation](https://github.com/Keno/Gallium.jl#gallium) should - get you going. The [JuliaCon - talk](https://www.youtube.com/watch?v=e6-hcOHO0tc&list=PLP8iPy9hna6SQPwZUDtAM59-wPzCPyD_S&index=5) - on Gallium shows off various features of the debugger. + * Machine SIMD types can be represented in Julia as a homogeneous tuple of `VecElement` ([#15244]). - * The [Juno IDE](http://junolab.org) has matured significantly, and now - also includes support for plotting and debugging. + * The performance of higher-order and anonymous functions has been greatly improved. + For example, `map(x->2x, A)` performs as well as `2.*A`([#13412]). - * [Cxx.jl](https://github.com/Keno/Cxx.jl) provides a convenient FFI for - calling C++ code from Julia. + * On windows, a DLL of standard library code is now precompiled and used by default, + improving startup time ([#16953]). + + * LLVM has been upgraded to version 3.7.1, improving the quality of generated + code and debug info. However compile times may be slightly longer ([#14623]). + +New architectures +----------------- + + This release greatly improves support for ARM, and introduces support for Power. + + * [ARM](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Aarm): + [#14194], [#14519], [#16645], [#16621] + + * [Power](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Apower): + [#16455], [#16404] + +Breaking changes +---------------- + +This section lists changes that do not have deprecation warnings. + + * The assignment operations `.+=`, `.*=` and so on now generate calls + to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side + if the latter is an indexing expression, e.g. `a[...]`). This means that they will fail + if the left-hand side is immutable (or does not support `view`), and will otherwise + change the left-hand side in-place ([#17510], [#17546]). + + * Method ambiguities no longer generate warnings when files are loaded, + nor do they dispatch to an arbitrarily-chosen method; instead, a call that + cannot be resolved to a single method results in a `MethodError` at run time, + rather than the previous definition-time warning. ([#6190]) + + * Array comprehensions preserve the dimensions of the input ranges. For example, + `[2x for x in A]` will have the same dimensions as `A` ([#16622]). + + * The result type of an array comprehension depends only on the types of elements + computed, instead of using type inference ([#7258]). If the result is empty, then + type inference is still used to determine the element type. + + * `reshape` is now defined to always share data with the original array. + If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of + the desired shape ([#4211]). + + * `mapslices` now re-uses temporary storage. Recipient functions that expect + input slices to be persistent should copy data to other storage ([#17266]). + All usages of `mapslices` should be carefully audited since this change can cause + silent, incorrect behavior, rather than failing noisily. + + * Local variables and arguments are represented in lowered code as numbered `Slot` + objects instead of as symbols ([#15609]). + + * The information that used to be in the `ast` field of the `LambdaStaticData` type + is now divided among the fields `code`, `slotnames`, `slottypes`, `slotflags`, + `gensymtypes`, `rettype`, `nargs`, and `isva` in the `LambdaInfo` type ([#15609]). + + * `A <: B` is parsed as `Expr(:(<:), :A, :B)` in all cases ([#9503]). + This also applies to the `>:` operator. + + * Simple 2-argument comparisons like `A < B` are parsed as calls instead of using the + `:comparison` expression type ([#15524]). The `:comparison` expression type is still + produced in ASTs when comparisons are chained (e.g. `A < B ≤ C`). Deprecated or removed --------------------- @@ -377,7 +373,22 @@ Deprecated or removed * `writemime` is deprecated, and output methods specifying a MIME type are now methods of `show` ([#14052]). -[PkgDev]: https://github.com/JuliaLang/PkgDev.jl +Language tooling improvements +----------------------------- + + * The [Julia debugger](https://github.com/Keno/Gallium.jl) makes its debut + with this release. Install it with `Pkg.add("Gallium")`, and the + [documentation](https://github.com/Keno/Gallium.jl#gallium) should + get you going. The [JuliaCon + talk](https://www.youtube.com/watch?v=e6-hcOHO0tc&list=PLP8iPy9hna6SQPwZUDtAM59-wPzCPyD_S&index=5) + on Gallium shows off various features of the debugger. + + * The [Juno IDE](http://junolab.org) has matured significantly, and now + also includes support for plotting and debugging. + + * [Cxx.jl](https://github.com/Keno/Cxx.jl) provides a convenient FFI for + calling C++ code from Julia. + <!--- generated by NEWS-update.jl: --> [#550]: https://github.com/JuliaLang/julia/issues/550 [#964]: https://github.com/JuliaLang/julia/issues/964 From 4cc1e1b2157cd9b9d7584e90010ac7a4577442da Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Thu, 28 Jul 2016 14:06:39 -0400 Subject: [PATCH 0736/1117] NEWS: some copy improvements for Strings [ci skip] --- NEWS.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/NEWS.md b/NEWS.md index f290fddea7cb3..f6cc8619e2b76 100644 --- a/NEWS.md +++ b/NEWS.md @@ -109,20 +109,21 @@ Library improvements * The `UTF16String` and `UTF32String` types and corresponding `utf16` and `utf32` converter functions have been removed from the standard library. If you need these types, they have been moved to the - [LegacyStrings](https://github.com/JuliaArchive/LegacyStrings.jl) + [`LegacyStrings`](https://github.com/JuliaArchive/LegacyStrings.jl) package. In the future, more robust Unicode string support will be provided - by the `StringEncodings` package. If you only need these types to call wide - string APIs (UTF-16 on Windows, UTF-32 on UNIX), consider using the new - `transcode` function (see below) or the `Cwstring` type as a `ccall` argument - type, which also ensures correct NUL termination of string data. + by the [`StringEncodings`](https://github.com/nalimilan/StringEncodings.jl) + package. If you only need these types to call wide string APIs (UTF-16 on + Windows, UTF-32 on UNIX), consider using the new `transcode` function (see + below) or the `Cwstring` type as a `ccall` argument type, which also ensures + correct NUL termination of string data. + + * A `transcode(T, src)` function is now exported for converting data + between UTF-xx Unicode encodings ([#17323]). * The basic string construction routines are now `string(args...)`, `String(s)`, `unsafe_string(ptr)` (formerly `bytestring(ptr)`), and `unsafe_wrap(String, ptr)` (formerly `pointer_to_string`) ([#16731]). - * A `transcode(T, src)` function is now exported for converting data - between UTF-xx Unicode encodings ([#17323]). - * Comparisons between `Char`s and `Integer`s are now deprecated ([#16024]): `'x' == 120` now produces a warning but still evaluates to `true`. In the future it may evaluate to `false` or the comparison may be an error. To From e2a5046fd8e5386c0f6ea56ec84787d5a070f737 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Mon, 1 Aug 2016 12:10:20 -0400 Subject: [PATCH 0737/1117] NEWS: fix misspelling of "parallel" --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index f6cc8619e2b76..de8036da48a8f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -196,7 +196,7 @@ Library improvements * There is now a default no-op `flush(io)` function for all `IO` types ([#16403]). - * Parellel computing: + * Parallel computing: * `pmap` keyword arguments `err_retry=true` and `err_stop=false` are deprecated. Action to be taken on errors can be specified via the `on_error` keyword argument. From 55d3a5500a74b3dfc00f0a22c354eb6fa462f57a Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Sun, 31 Jul 2016 22:06:36 +0200 Subject: [PATCH 0738/1117] ASAN/MSAN: move preprocessor checks to a JL_ definition. --- src/codegen.cpp | 4 +--- src/dlload.c | 7 +------ src/init.c | 4 ++-- src/jitlayers.cpp | 8 +++----- src/julia_internal.h | 14 ++++++++++++++ src/options.h | 37 ++++++++++++++++++++++++++----------- src/sys.c | 8 ++------ 7 files changed, 49 insertions(+), 33 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index aca1f9f10d429..55b8aafba0118 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -989,10 +989,8 @@ void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) // it generally helps to have define KEEP_BODIES if you plan on using this extern "C" JL_DLLEXPORT void *jl_function_ptr_by_llvm_name(char *name) { -#ifdef __has_feature -#if __has_feature(memory_sanitizer) +#ifdef JL_MSAN_ENABLED __msan_unpoison_string(name); -#endif #endif return (void*)(intptr_t)jl_ExecutionEngine->FindFunctionNamed(name); } diff --git a/src/dlload.c b/src/dlload.c index da290a1f32a3c..1966c92803f58 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -54,11 +54,6 @@ static int endswith_extension(const char *path) extern char *julia_home; #define JL_RTLD(flags, FLAG) (flags & JL_RTLD_ ## FLAG ? RTLD_ ## FLAG : 0) -#ifdef __has_feature -# if __has_feature(address_sanitizer) -# define SANITIZE_ADDRESS -# endif -#endif static void JL_NORETURN jl_dlerror(const char *fmt, const char *sym) { @@ -95,7 +90,7 @@ JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) #ifdef RTLD_NOLOAD | JL_RTLD(flags, NOLOAD) #endif -#if defined(RTLD_DEEPBIND) && !defined(SANITIZE_ADDRESS) +#if defined(RTLD_DEEPBIND) && !defined(JL_ASAN_ENABLED) | JL_RTLD(flags, DEEPBIND) #endif #ifdef RTLD_FIRST diff --git a/src/init.c b/src/init.c index 20d4017942c83..1f92be310b7a1 100644 --- a/src/init.c +++ b/src/init.c @@ -161,8 +161,8 @@ void jl_init_stack_limits(int ismaster) static void jl_find_stack_bottom(void) { -#if !defined(_OS_WINDOWS_) && defined(__has_feature) -#if __has_feature(memory_sanitizer) || __has_feature(address_sanitizer) +#if !defined(_OS_WINDOWS_) +#if defined(JL_ASAN_ENABLED) || defined(JL_MSAN_ENABLED) struct rlimit rl; // When using the sanitizers, increase stack size because they bloat diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 4efb509e33348..aa653c333507b 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -104,17 +104,15 @@ void addOptimizationPasses(PassManager *PM) PM->add(createVerifierPass()); #endif -#ifdef __has_feature -# if __has_feature(address_sanitizer) +#if defined(JL_ASAN_ENABLED) # if defined(LLVM37) && !defined(LLVM38) // LLVM 3.7 BUG: ASAN pass doesn't properly initialize its dependencies initializeTargetLibraryInfoWrapperPassPass(*PassRegistry::getPassRegistry()); # endif PM->add(createAddressSanitizerFunctionPass()); -# endif -# if __has_feature(memory_sanitizer) +#endif +#if defined(JL_MSAN_ENABLED) PM->add(llvm::createMemorySanitizerPass(true)); -# endif #endif if (jl_options.opt_level == 0) { PM->add(createLowerPTLSPass(imaging_mode, tbaa_const)); diff --git a/src/julia_internal.h b/src/julia_internal.h index d967de1ff00cb..2c7105b62e53c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -12,6 +12,20 @@ #define sleep(x) Sleep(1000*x) #endif +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define JL_ASAN_ENABLED // Clang flavor +#endif +#elif defined(__SANITIZE_ADDRESS__) +#define JL_ASAN_ENABLED // GCC flavor +#endif + +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#define JL_MSAN_ENABLED +#endif +#endif + #ifdef __cplusplus extern "C" { #endif diff --git a/src/options.h b/src/options.h index a6445b9f1f7e8..1f5af35b02edd 100644 --- a/src/options.h +++ b/src/options.h @@ -67,6 +67,9 @@ // OBJPROFILE counts objects by type // #define OBJPROFILE +// Automatic Instrumenting Profiler +//#define ENABLE_TIMINGS + // method dispatch profiling -------------------------------------------------- @@ -93,6 +96,7 @@ #define COPY_STACKS #endif + // threading options ---------------------------------------------------------- // controls for when threads sleep @@ -109,21 +113,32 @@ #define MACHINE_EXCLUSIVE_NAME "JULIA_EXCLUSIVE" #define DEFAULT_MACHINE_EXCLUSIVE 0 + // sanitizer defaults --------------------------------------------------------- -// Automatically enable MEMDEBUG and KEEP_BODIES for the sanitizers +// XXX: these macros are duplicated from julia_internal.h #if defined(__has_feature) -# if __has_feature(address_sanitizer) || __has_feature(memory_sanitizer) -# define MEMDEBUG -# define KEEP_BODIES -# endif -// Memory sanitizer needs TLS, which llvm only supports for the small memory model -# if __has_feature(memory_sanitizer) - // todo: fix the llvm MemoryManager to work with small memory model -# endif +#if __has_feature(address_sanitizer) +#define JL_ASAN_ENABLED +#endif +#elif defined(__SANITIZE_ADDRESS__) +#define JL_ASAN_ENABLED +#endif +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#define JL_MSAN_ENABLED +#endif #endif -// Automatic Instrumenting Profiler -//#define ENABLE_TIMINGS +// Automatically enable MEMDEBUG and KEEP_BODIES for the sanitizers +#if defined(JL_ASAN_ENABLED) || defined(JL_MSAN_ENABLED) +#define MEMDEBUG +#define KEEP_BODIES +#endif + +// Memory sanitizer needs TLS, which llvm only supports for the small memory model +#if defined(JL_MSAN_ENABLED) +// todo: fix the llvm MemoryManager to work with small memory model +#endif #endif diff --git a/src/sys.c b/src/sys.c index 3406a8a7e11dc..0723b3dc60bf1 100644 --- a/src/sys.c +++ b/src/sys.c @@ -48,11 +48,9 @@ #include <intrin.h> #endif -#ifdef __has_feature -#if __has_feature(memory_sanitizer) +#ifdef JL_MSAN_ENABLED #include <sanitizer/msan_interface.h> #endif -#endif #ifdef __cplusplus extern "C" { @@ -663,14 +661,12 @@ JL_DLLEXPORT const char *jl_pathname_for_handle(void *handle) struct link_map *map; dlinfo(handle, RTLD_DI_LINKMAP, &map); -#ifdef __has_feature -#if __has_feature(memory_sanitizer) +#ifdef JL_MSAN_ENABLED __msan_unpoison(&map,sizeof(struct link_map*)); if (map) { __msan_unpoison(map, sizeof(struct link_map)); __msan_unpoison_string(map->l_name); } -#endif #endif if (map) return map->l_name; From 406cd7b4c2152ad8df9523a3ff0770f236fffb8a Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Sun, 31 Jul 2016 22:07:47 +0200 Subject: [PATCH 0739/1117] ASAN: warn if our SIGSEGV handler will be blocked. SIGSEGV can be benign, but ASAN will die so we need to be able to handle the signal ourselves (see #17706). --- src/init.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/init.c b/src/init.c index 1f92be310b7a1..a2d74fe60cc91 100644 --- a/src/init.c +++ b/src/init.c @@ -628,6 +628,17 @@ void _julia_init(JL_IMAGE_SEARCH rel) } #endif + +#ifdef JL_ASAN_ENABLED + const char *asan_options = getenv("ASAN_OPTIONS"); + if (!asan_options || !(strstr(asan_options, "allow_user_segv_handler=1") || + strstr(asan_options, "handle_segv=0"))) { + jl_printf(JL_STDERR,"WARNING: ASAN overrides Julia's SIGSEGV handler; " + "disable SIGSEGV handling or allow custom handlers.\n"); + } + +#endif + jl_init_threading(); jl_gc_init(); From 53ef6d748e2413bb95f6181716c1fe2695ffac30 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Mon, 1 Aug 2016 21:51:43 +0200 Subject: [PATCH 0740/1117] fix fallback for scaling blas number with array --- base/linalg/dense.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index 2a68cc7473c6d..47f74e3b6666d 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -10,6 +10,8 @@ const ASUM_CUTOFF = 32 const NRM2_CUTOFF = 32 function scale!{T<:BlasFloat}(X::Array{T}, s::T) + s == 0 && return fill!(X, zero(T)) + s == 1 && return X if length(X) < SCAL_CUTOFF generic_scale!(X, s) else @@ -18,6 +20,8 @@ function scale!{T<:BlasFloat}(X::Array{T}, s::T) X end +scale!{T<:BlasFloat}(s::T, X::Array{T}) = scale!(X, s) + scale!{T<:BlasFloat}(X::Array{T}, s::Number) = scale!(X, convert(T, s)) function scale!{T<:BlasComplex}(X::Array{T}, s::Real) R = typeof(real(zero(T))) From 1aeaf30c7b394fc45191d2dffe34fc63bcc8a262 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 1 Aug 2016 16:53:25 -0400 Subject: [PATCH 0741/1117] NEWS updates for Dict comprehensions and `map` of Dict. ref #17714 [ci skip] --- NEWS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.md b/NEWS.md index de8036da48a8f..d4fc9a2ee0aae 100644 --- a/NEWS.md +++ b/NEWS.md @@ -95,6 +95,9 @@ Language changes When `x` is global, `x::T = ...` and `global x::T` used to mean type assertion, but this syntax is now reserved for type declaration ([#964]). + * Dictionary comprehension syntax `[ a=>b for x in y ]` is deprecated. + Use `Dict(a=>b for x in y)` instead ([#16510]). + Library improvements -------------------- @@ -339,6 +342,9 @@ This section lists changes that do not have deprecation warnings. `:comparison` expression type ([#15524]). The `:comparison` expression type is still produced in ASTs when comparisons are chained (e.g. `A < B ≤ C`). + * `map` on a dictionary now expects a function that expects and returns a `Pair`. + The result is now another dictionary instead of an array ([#16622]). + Deprecated or removed --------------------- @@ -469,6 +475,7 @@ Language tooling improvements [#16466]: https://github.com/JuliaLang/julia/issues/16466 [#16481]: https://github.com/JuliaLang/julia/issues/16481 [#16502]: https://github.com/JuliaLang/julia/issues/16502 +[#16510]: https://github.com/JuliaLang/julia/issues/16510 [#16621]: https://github.com/JuliaLang/julia/issues/16621 [#16622]: https://github.com/JuliaLang/julia/issues/16622 [#16645]: https://github.com/JuliaLang/julia/issues/16645 From 5e5f490e366a55e6b9093e836a9e6d65be0f0dfc Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 1 Aug 2016 16:38:43 -0400 Subject: [PATCH 0742/1117] fix #17701, better error message for `i == 3 && i+=1` --- src/ast.scm | 2 +- src/julia-syntax.scm | 3 +++ test/parse.jl | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ast.scm b/src/ast.scm index 354fb0e770b3f..fc01cb05d35ff 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -60,7 +60,7 @@ ""))) ((comparison) (apply string (map deparse (cdr e)))) ((in) (string (deparse (cadr e)) " in " (deparse (caddr e)))) - ((ssavalue) (string "SSAValue(" (cdr e) ")")) + ((ssavalue) (string "SSAValue(" (cadr e) ")")) ((line) (if (length= e 2) (string "# line " (cadr e)) (string "# " (caddr e) ", line " (cadr e)))) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 882229a0cbde6..40eb5fc33a139 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1469,6 +1469,9 @@ `(block ,@(cdr e) ,(expand-update-operator op op= (car e) rhs T)))) (else + (if (and (pair? lhs) + (not (memq (car lhs) '(|.| tuple vcat typed_hcat typed_vcat)))) + (error (string "invalid assignment location \"" (deparse lhs) "\""))) (expand-update-operator- op op= lhs rhs declT)))) (define (lower-update-op e) diff --git a/test/parse.jl b/test/parse.jl index 9a802e90569fb..87a735eff1177 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -648,3 +648,5 @@ for op in ["+", "-", "\$", "|", ".+", ".-", "*", ".*"] @test_throws ParseError parse("$op in [+, -]") end +# issue #17701 +@test expand(:(i==3 && i+=1)) == Expr(:error, "invalid assignment location \"==(i,3)&&i\"") From c5d222202169cd7cd064a61afe864f70db6ab1a4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 1 Aug 2016 16:56:05 -0400 Subject: [PATCH 0743/1117] fix #17668, deprecation for `for ( ... )` --- NEWS.md | 3 +++ src/julia-parser.scm | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index d4fc9a2ee0aae..f5c7b0cd3d0a3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -98,6 +98,9 @@ Language changes * Dictionary comprehension syntax `[ a=>b for x in y ]` is deprecated. Use `Dict(a=>b for x in y)` instead ([#16510]). + * Parentheses are no longer allowed around iteration specifications, e.g. + `for (i = 1:n)` ([#17668]). + Library improvements -------------------- diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 3bad336c80bdc..0e21544ff3904 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1415,7 +1415,8 @@ ;; as above, but allows both "i=r" and "i in r" (define (parse-iteration-spec s) - (let* ((lhs (parse-pipes s)) + (let* ((paren? (eqv? (require-token s) #\()) + (lhs (parse-pipes s)) (t (peek-token s))) (cond ((memq t '(= in ∈)) (take-token s) @@ -1428,6 +1429,13 @@ `(= ,lhs ,rhs))) ((and (eq? lhs ':) (closing-token? t)) ':) + ((and paren? (length= lhs 4) (eq? (car lhs) 'call) + (memq (cadr lhs) '(in ∈))) + (syntax-deprecation s "for (...)" "for ...") + `(= ,@(cddr lhs))) + ((and paren? (length= lhs 3) (eq? (car lhs) '=)) + (syntax-deprecation s "for (...)" "for ...") + lhs) (else (error "invalid iteration specification"))))) (define (parse-comma-separated-iters s) From 9a517890046728508ca38cf8b3a94315706fe572 Mon Sep 17 00:00:00 2001 From: "Jane E. Herriman" <jane@caltech.edu> Date: Sun, 31 Jul 2016 14:28:53 -0700 Subject: [PATCH 0744/1117] Move doc strings for time-related macros/functions inline from docs/helpdb/Base.jl. --- base/docs/helpdb/Base.jl | 68 ---------------------------------------- base/util.jl | 56 +++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 71 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index a14860591ba11..0f915a2571284 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2,15 +2,6 @@ # Base -""" - @time - -A macro to execute an expression, printing the time it took to execute, the number of -allocations, and the total number of bytes its execution caused to be allocated, before -returning the value of the expression. -""" -:@time - """ systemerror(sysfunc, iftrue) @@ -4804,13 +4795,6 @@ Compile the given function `f` for the argument tuple (of types) `args`, but do """ precompile -""" - toc() - -Print and return the time elapsed since the last [`tic`](:func:`tic`). -""" -toc - """ asinh(x) @@ -5211,15 +5195,6 @@ Open a file and read its contents. `args` is passed to `read`: this is equivalen """ read(filename, args...) -""" - @timev - -This is a verbose version of the `@time` macro. It first prints the same information as -`@time`, then any non-zero memory allocation counters, and then returns the value of the -expression. -""" -:@timev - """ isopen(object) -> Bool @@ -5288,15 +5263,6 @@ after the end of the string. """ nextind -""" - @timed - -A macro to execute an expression, and return the value of the expression, elapsed time, -total bytes allocated, garbage collection time, and an object with various memory allocation -counters. -""" -:@timed - """ symdiff(s1,s2...) @@ -5979,13 +5945,6 @@ Get the number of fields of a `DataType`. """ nfields -""" - toq() - -Return, but do not print, the time elapsed since the last [`tic`](:func:`tic`). -""" -toq - """ show(stream, mime, x) @@ -6581,17 +6540,6 @@ by `show` generally includes Julia-specific formatting and type information. """ show(x) -""" - @allocated - -A macro to evaluate an expression, discarding the resulting value, instead returning the -total number of bytes allocated during evaluation of the expression. Note: the expression is -evaluated inside a local function, instead of the current context, in order to eliminate the -effects of compilation, however, there still may be some allocations due to JIT compilation. -This also makes the results inconsistent with the `@time` macros, which do not try to adjust -for the effects of compilation. -""" -:@allocated """ Array(dims) @@ -8038,14 +7986,6 @@ Return the index of the last element of `A` for which `predicate` returns `true` """ findlast(::Function, A) -""" - @elapsed - -A macro to evaluate an expression, discarding the resulting value, instead returning the -number of seconds it took to execute as a floating-point number. -""" -:@elapsed - """ findnext(A, i) @@ -8088,14 +8028,6 @@ Compute the phase angle in radians of a complex number `z`. """ angle -""" - tic() - -Set a timer to be read by the next call to [`toc`](:func:`toc`) or [`toq`](:func:`toq`). The -macro call `@time expr` can also be used to time evaluation. -""" -tic - """ LoadError(file::AbstractString, line::Int, error) diff --git a/base/util.jl b/base/util.jl index 6a75c35f660ed..8f8d9b6dc2b28 100644 --- a/base/util.jl +++ b/base/util.jl @@ -66,12 +66,23 @@ gc_time_ns() = ccall(:jl_gc_total_hrtime, UInt64, ()) # total number of bytes allocated so far gc_bytes() = ccall(:jl_gc_total_bytes, Int64, ()) +""" + tic() + +Set a timer to be read by the next call to [`toc`](:func:`toc`) or [`toq`](:func:`toq`). The +macro call `@time expr` can also be used to time evaluation. +""" function tic() t0 = time_ns() task_local_storage(:TIMERS, (t0, get(task_local_storage(), :TIMERS, ()))) return t0 end +""" + toq() + +Return, but do not print, the time elapsed since the last [`tic`](:func:`tic`). +""" function toq() t1 = time_ns() timers = get(task_local_storage(), :TIMERS, ()) @@ -83,6 +94,11 @@ function toq() (t1-t0)/1e9 end +""" + toc() + +Print and return the time elapsed since the last [`tic`](:func:`tic`). +""" function toc() t = toq() println("elapsed time: ", t, " seconds") @@ -149,6 +165,13 @@ function timev_print(elapsedtime, diff::GC_Diff) padded_nonzero_print(diff.full_sweep, "full collections") end +""" + @time + +A macro to execute an expression, printing the time it took to execute, the number of +allocations, and the total number of bytes its execution caused to be allocated, before +returning the value of the expression. +""" macro time(ex) quote local stats = gc_num() @@ -162,6 +185,13 @@ macro time(ex) end end +""" + @timev + +This is a verbose version of the `@time` macro. It first prints the same information as +`@time`, then any non-zero memory allocation counters, and then returns the value of the +expression. +""" macro timev(ex) quote local stats = gc_num() @@ -173,7 +203,12 @@ macro timev(ex) end end -# print nothing, return elapsed time +""" + @elapsed + +A macro to evaluate an expression, discarding the resulting value, instead returning the +number of seconds it took to execute as a floating-point number. +""" macro elapsed(ex) quote local t0 = time_ns() @@ -188,7 +223,16 @@ end # like: @allocated y = foo() # will not work correctly, because it will set y in the context of # the local function made by the macro, not the current function - +""" + @allocated + +A macro to evaluate an expression, discarding the resulting value, instead returning the +total number of bytes allocated during evaluation of the expression. Note: the expression is +evaluated inside a local function, instead of the current context, in order to eliminate the +effects of compilation, however, there still may be some allocations due to JIT compilation. +This also makes the results inconsistent with the `@time` macros, which do not try to adjust +for the effects of compilation. +""" macro allocated(ex) quote let @@ -203,7 +247,13 @@ macro allocated(ex) end end -# print nothing, return value, elapsed time, bytes allocated & gc time +""" + @timed + +A macro to execute an expression, and return the value of the expression, elapsed time, +total bytes allocated, garbage collection time, and an object with various memory allocation +counters. +""" macro timed(ex) quote local stats = gc_num() From facab090c2fd7176b4bce6cc60cd7407e7d5adfd Mon Sep 17 00:00:00 2001 From: "Jane E. Herriman" <jane@caltech.edu> Date: Sun, 31 Jul 2016 21:41:00 -0700 Subject: [PATCH 0745/1117] Add cross references to the doc strings of timing macros and functions. --- base/util.jl | 21 +++++++++++++++++++-- doc/stdlib/base.rst | 14 ++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/base/util.jl b/base/util.jl index 8f8d9b6dc2b28..b6799a6dd5faa 100644 --- a/base/util.jl +++ b/base/util.jl @@ -81,7 +81,8 @@ end """ toq() -Return, but do not print, the time elapsed since the last [`tic`](:func:`tic`). +Return, but do not print, the time elapsed since the last [`tic`](:func:`tic`). The +macro calls `@timed expr` and `@elapsed expr` also return evaluation time. """ function toq() t1 = time_ns() @@ -97,7 +98,8 @@ end """ toc() -Print and return the time elapsed since the last [`tic`](:func:`tic`). +Print and return the time elapsed since the last [`tic`](:func:`tic`). The macro call +`@time expr` can also be used to time evaluation. """ function toc() t = toq() @@ -171,6 +173,9 @@ end A macro to execute an expression, printing the time it took to execute, the number of allocations, and the total number of bytes its execution caused to be allocated, before returning the value of the expression. + +See also [`@timev`](:ref:@timev), [`@timed`](:ref:@timed), [`@elapsed`](:ref:@elapsed), and +[`@allocated`](:ref:@allocated). """ macro time(ex) quote @@ -191,6 +196,9 @@ end This is a verbose version of the `@time` macro. It first prints the same information as `@time`, then any non-zero memory allocation counters, and then returns the value of the expression. + +See also [`@time`](:ref:@time), [`@timed`](:ref:@timed), [`@elapsed`](:ref:@elapsed), and +[`@allocated`](:ref:@allocated). """ macro timev(ex) quote @@ -208,6 +216,9 @@ end A macro to evaluate an expression, discarding the resulting value, instead returning the number of seconds it took to execute as a floating-point number. + +See also [`@time`](:ref:@time), [`@timev`](:ref:@timev), [`@timed`](:ref:@timed), +and [`@allocated`](:ref:@allocated). """ macro elapsed(ex) quote @@ -232,6 +243,9 @@ evaluated inside a local function, instead of the current context, in order to e effects of compilation, however, there still may be some allocations due to JIT compilation. This also makes the results inconsistent with the `@time` macros, which do not try to adjust for the effects of compilation. + +See also [`@time`](:ref:@time), [`@timev`](:ref:@timev), [`@timed`](:ref:@timed), +and [`@elapsed`](:ref:@elapsed). """ macro allocated(ex) quote @@ -253,6 +267,9 @@ end A macro to execute an expression, and return the value of the expression, elapsed time, total bytes allocated, garbage collection time, and an object with various memory allocation counters. + +See also [`@time`](:ref:@time), [`@timev`](:ref:@timev), [`@elapsed`](:ref:@elapsed), and +[`@allocated`](:ref:@allocated). """ macro timed(ex) quote diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index eba35ef40dfc1..ff8ed6282532a 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -957,13 +957,13 @@ System .. Docstring generated from Julia source - Print and return the time elapsed since the last :func:`tic`\ . + Print and return the time elapsed since the last :func:`tic`\ . The macro call ``@time expr`` can also be used to time evaluation. .. function:: toq() .. Docstring generated from Julia source - Return, but do not print, the time elapsed since the last :func:`tic`\ . + Return, but do not print, the time elapsed since the last :func:`tic`\ . The macro calls ``@timed expr`` and ``@elapsed expr`` also return evaluation time. .. function:: @time @@ -971,30 +971,40 @@ System A macro to execute an expression, printing the time it took to execute, the number of allocations, and the total number of bytes its execution caused to be allocated, before returning the value of the expression. + See also ```@timev`` <:ref:@timev>`_\ , ```@timed`` <:ref:@timed>`_\ , ```@elapsed`` <:ref:@elapsed>`_\ , and ```@allocated`` <:ref:@allocated>`_\ . + .. function:: @timev .. Docstring generated from Julia source This is a verbose version of the ``@time`` macro. It first prints the same information as ``@time``\ , then any non-zero memory allocation counters, and then returns the value of the expression. + See also ```@time`` <:ref:@time>`_\ , ```@timed`` <:ref:@timed>`_\ , ```@elapsed`` <:ref:@elapsed>`_\ , and ```@allocated`` <:ref:@allocated>`_\ . + .. function:: @timed .. Docstring generated from Julia source A macro to execute an expression, and return the value of the expression, elapsed time, total bytes allocated, garbage collection time, and an object with various memory allocation counters. + See also ```@time`` <:ref:@time>`_\ , ```@timev`` <:ref:@timev>`_\ , ```@elapsed`` <:ref:@elapsed>`_\ , and ```@allocated`` <:ref:@allocated>`_\ . + .. function:: @elapsed .. Docstring generated from Julia source A macro to evaluate an expression, discarding the resulting value, instead returning the number of seconds it took to execute as a floating-point number. + See also ```@time`` <:ref:@time>`_\ , ```@timev`` <:ref:@timev>`_\ , ```@timed`` <:ref:@timed>`_\ , and ```@allocated`` <:ref:@allocated>`_\ . + .. function:: @allocated .. Docstring generated from Julia source A macro to evaluate an expression, discarding the resulting value, instead returning the total number of bytes allocated during evaluation of the expression. Note: the expression is evaluated inside a local function, instead of the current context, in order to eliminate the effects of compilation, however, there still may be some allocations due to JIT compilation. This also makes the results inconsistent with the ``@time`` macros, which do not try to adjust for the effects of compilation. + See also ```@time`` <:ref:@time>`_\ , ```@timev`` <:ref:@timev>`_\ , ```@timed`` <:ref:@timed>`_\ , and ```@elapsed`` <:ref:@elapsed>`_\ . + .. function:: EnvHash() -> EnvHash .. Docstring generated from Julia source From 1ec7c31ac8ca69d00c642a3abdc1c770fa2299bc Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sun, 31 Jul 2016 23:40:19 +0800 Subject: [PATCH 0746/1117] Fix GC stack scrub when the GC is running on a worker thread Also scrub the stacks of all tasks instead of only the currently running one. --- src/gc-debug.c | 61 +++++++++++++++++++++++++++++++++++++++++++------- src/gc.c | 10 ++++----- src/gc.h | 14 +++++++----- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index 44400d17f451a..3536eaaf75095 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -257,7 +257,6 @@ JL_DLLEXPORT jl_gc_debug_env_t jl_gc_debug_env = { {0, UINT64_MAX, 0, 0, 0, {0, 0, 0}}, {0, UINT64_MAX, 0, 0, 0, {0, 0, 0}} }; -static char *gc_stack_lo; static void gc_debug_alloc_setnext(jl_alloc_num_t *num) { @@ -345,11 +344,26 @@ void gc_debug_print(void) gc_debug_print_status(); } -static void gc_scrub_range(char *stack_lo, char *stack_hi) +// a list of tasks for conservative stack scan during gc_scrub +static arraylist_t jl_gc_debug_tasks; + +void gc_scrub_record_task(jl_task_t *t) +{ + arraylist_push(&jl_gc_debug_tasks, t); +} + +static void gc_scrub_range(char *low, char *high) { - stack_lo = (char*)((uintptr_t)stack_lo & ~(uintptr_t)15); - for (char **stack_p = (char**)stack_lo; - stack_p > (char**)stack_hi;stack_p--) { + jl_ptls_t ptls = jl_get_ptls_states(); + jl_jmp_buf *old_buf = ptls->safe_restore; + jl_jmp_buf buf; + if (jl_setjmp(buf, 0)) { + ptls->safe_restore = old_buf; + return; + } + ptls->safe_restore = &buf; + low = (char*)((uintptr_t)low & ~(uintptr_t)15); + for (char **stack_p = ((char**)high) - 1; stack_p > (char**)low; stack_p--) { char *p = *stack_p; size_t osize; jl_taggedvalue_t *tag = jl_gc_find_taggedvalue_pool(p, &osize); @@ -371,11 +385,42 @@ static void gc_scrub_range(char *stack_lo, char *stack_hi) // set mark to GC_MARKED (young and marked) tag->bits.gc = GC_MARKED; } + ptls->safe_restore = old_buf; +} + +static void gc_scrub_task(jl_task_t *ta) +{ + int16_t tid = ta->tid; + jl_ptls_t ptls = jl_get_ptls_states(); + jl_ptls_t ptls2 = jl_all_tls_states[tid]; + if (ptls == ptls2 && ta == ptls2->current_task) { + // scan up to current `sp` for current thread and task + char *low = (char*)jl_get_frame_addr(); +#ifdef COPY_STACKS + gc_scrub_range(low, ptls2->stack_hi); +#else + gc_scrub_range(low, (char*)ta->stkbuf + ta->ssize); +#endif + return; + } + // The task that owns/is running on the threads's stack. +#ifdef COPY_STACKS + jl_task_t *thread_task = ptls2->current_task; +#else + jl_task_t *thread_task = ptls2->root_task; +#endif + if (ta == thread_task) + gc_scrub_range(ptls2->stack_lo, ptls2->stack_hi); + if (ta->stkbuf == (void*)(intptr_t)(-1) || !ta->stkbuf) + return; + gc_scrub_range((char*)ta->stkbuf, (char*)ta->stkbuf + ta->ssize); } -void gc_scrub(char *stack_hi) +void gc_scrub(void) { - gc_scrub_range(gc_stack_lo, stack_hi); + for (size_t i = 0; i < jl_gc_debug_tasks.len; i++) + gc_scrub_task((jl_task_t*)jl_gc_debug_tasks.items[i]); + jl_gc_debug_tasks.len = 0; } #else void gc_debug_critical_error(void) @@ -694,7 +739,6 @@ void gc_time_sweep_pause(uint64_t gc_end_t, int64_t actual_allocd, void gc_debug_init(void) { #ifdef GC_DEBUG_ENV - gc_stack_lo = (char*)gc_get_stack_ptr(); char *env = getenv("JULIA_GC_NO_GENERATIONAL"); if (env && strcmp(env, "0") != 0) jl_gc_debug_env.always_full = 1; @@ -703,6 +747,7 @@ void gc_debug_init(void) gc_debug_alloc_init(&jl_gc_debug_env.pool, "POOL"); gc_debug_alloc_init(&jl_gc_debug_env.other, "OTHER"); gc_debug_alloc_init(&jl_gc_debug_env.print, "PRINT"); + arraylist_new(&jl_gc_debug_tasks, 0); #endif #ifdef GC_VERIFY diff --git a/src/gc.c b/src/gc.c index d8b8f15963c92..b5ac77e6a7c4d 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1255,6 +1255,7 @@ static void gc_mark_stack(jl_ptls_t ptls, jl_value_t *ta, jl_gcframe_t *s, static void gc_mark_task_stack(jl_ptls_t ptls, jl_task_t *ta, int d) { + gc_scrub_record_task(ta); int stkbuf = (ta->stkbuf != (void*)(intptr_t)-1 && ta->stkbuf != NULL); int16_t tid = ta->tid; jl_ptls_t ptls2 = jl_all_tls_states[tid]; @@ -1649,7 +1650,7 @@ void jl_gc_sync_total_bytes(void) {last_gc_total_bytes = jl_gc_total_bytes();} #define MIN_SCAN_BYTES 1024*1024 // Only one thread should be running in this function -static void _jl_gc_collect(jl_ptls_t ptls, int full, char *stack_hi) +static void _jl_gc_collect(jl_ptls_t ptls, int full) { JL_TIMING(GC); uint64_t t0 = jl_hrtime(); @@ -1785,7 +1786,7 @@ static void _jl_gc_collect(jl_ptls_t ptls, int full, char *stack_hi) // 5. start sweeping sweep_weak_refs(); gc_sweep_other(ptls, sweep_full); - gc_scrub(stack_hi); + gc_scrub(); gc_sweep_pool(sweep_full); // sweeping is over // 6. if it is a quick sweep, put back the remembered objects in queued state @@ -1822,7 +1823,7 @@ static void _jl_gc_collect(jl_ptls_t ptls, int full, char *stack_hi) gc_num.freed = 0; if (recollect) { - _jl_gc_collect(ptls, 0, stack_hi); + _jl_gc_collect(ptls, 0); } } @@ -1834,7 +1835,6 @@ JL_DLLEXPORT void jl_gc_collect(int full) gc_num.allocd = -(int64_t)gc_num.interval; return; } - char *stack_hi = (char*)gc_get_stack_ptr(); gc_debug_print(); int8_t old_state = jl_gc_state(ptls); @@ -1851,7 +1851,7 @@ JL_DLLEXPORT void jl_gc_collect(int full) if (!jl_gc_disable_counter) { JL_LOCK_NOGC(&finalizers_lock); - _jl_gc_collect(ptls, full, stack_hi); + _jl_gc_collect(ptls, full); JL_UNLOCK_NOGC(&finalizers_lock); } diff --git a/src/gc.h b/src/gc.h index 47eeb594df0ef..8051e1bc7424f 100644 --- a/src/gc.h +++ b/src/gc.h @@ -392,7 +392,8 @@ JL_DLLEXPORT extern jl_gc_debug_env_t jl_gc_debug_env; int gc_debug_check_other(void); int gc_debug_check_pool(void); void gc_debug_print(void); -void gc_scrub(char *stack_hi); +void gc_scrub_record_task(jl_task_t *ta); +void gc_scrub(void); #else #define gc_sweep_always_full 0 static inline int gc_debug_check_other(void) @@ -406,9 +407,12 @@ static inline int gc_debug_check_pool(void) static inline void gc_debug_print(void) { } -static inline void gc_scrub(char *stack_hi) +static inline void gc_scrub_record_task(jl_task_t *ta) +{ + (void)ta; +} +static inline void gc_scrub(void) { - (void)stack_hi; } #endif @@ -431,8 +435,8 @@ static inline void objprofile_reset(void) #endif #ifdef MEMPROFILE -static void gc_stats_all_pool(void); -static void gc_stats_big_obj(void); +void gc_stats_all_pool(void); +void gc_stats_big_obj(void); #else #define gc_stats_all_pool() #define gc_stats_big_obj() From 6e65287ef21151055b114c5e5ec85bc1df15b23e Mon Sep 17 00:00:00 2001 From: ranjanan <benditlikeranjan@gmail.com> Date: Tue, 2 Aug 2016 09:57:09 +0530 Subject: [PATCH 0747/1117] Fix auto-gen doc at source --- base/docs/helpdb/Base.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 0f915a2571284..fc0b97a5c8e4f 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -5165,7 +5165,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:534 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:537 ... ``` """ From f5d6c3c996d04b81863594ffc8c6ec7b86729c55 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 31 Jul 2016 13:13:19 -0500 Subject: [PATCH 0748/1117] Implement generic `summary` and fix `show_vector` for non-1 indices --- base/abstractarray.jl | 1 + base/show.jl | 19 ++++++++++--------- test/offsetarray.jl | 21 ++++++++++++++++++++- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 7a61baf7185df..eedcf03ac8238 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -50,6 +50,7 @@ end # comes up in other applications. indices1{T}(A::AbstractArray{T,0}) = OneTo(1) indices1{T}(A::AbstractArray{T}) = (@_inline_meta; indices(A)[1]) +indices1(iter) = OneTo(length(iter)) unsafe_indices(A) = indices(A) unsafe_indices(r::Range) = (OneTo(unsafe_length(r)),) # Ranges use checked_sub for size diff --git a/base/show.jl b/base/show.jl index 06edde5a109e0..0e0ce69c0557a 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1486,13 +1486,16 @@ e.g. `10-element Array{Int64,1}`. summary(x) = string(typeof(x)) # e.g. Int64 # sizes such as 0-dimensional, 4-dimensional, 2x3 -dims2string(d) = isempty(d) ? "0-dimensional" : - length(d) == 1 ? "$(d[1])-element" : - join(map(string,d), '×') +dims2string(d::Dims) = isempty(d) ? "0-dimensional" : + length(d) == 1 ? "$(d[1])-element" : + join(map(string,d), '×') + +inds2string(inds::Indices) = join(map(string,inds), '×') # anything array-like gets summarized e.g. 10-element Array{Int64,1} -summary(a::AbstractArray) = - string(dims2string(size(a)), " ", typeof(a)) +summary(a::AbstractArray) = _summary(a, to_shape(indices(a))) +_summary(a, dims::Dims) = string(dims2string(dims), " ", typeof(a)) +_summary(a, inds) = string(typeof(a), " with indices ", inds2string(inds)) # n-dimensional arrays function show_nd(io::IO, a::AbstractArray, print_matrix, label_slices) @@ -1667,8 +1670,8 @@ function show_vector(io::IO, v, opn, cls) io = IOContext(io, :compact => compact) end print(io, prefix) - if limited && length(v) > 20 - inds = _indices1(v) + if limited && _length(v) > 20 + inds = indices1(v) show_delim_array(io, v, opn, ",", "", false, inds[1], inds[1]+9) print(io, " \u2026 ") show_delim_array(io, v, "", ",", cls, false, inds[end-9], inds[end]) @@ -1676,8 +1679,6 @@ function show_vector(io::IO, v, opn, cls) show_delim_array(io, v, opn, ",", cls, false) end end -_indices1(v::AbstractArray) = indices(v,1) -_indices1(iter) = 1:length(iter) # printing BitArrays diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 4e83a3299226d..870e008ccb9ef 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -34,7 +34,6 @@ Base.size(A::OffsetArray) = errmsg(A) Base.size(A::OffsetArray, d) = errmsg(A) Base.eachindex(::LinearSlow, A::OffsetArray) = CartesianRange(indices(A)) Base.eachindex(::LinearFast, A::OffsetVector) = indices(A, 1) -Base.summary(A::OffsetArray) = string(typeof(A))*" with indices "*string(indices(A)) # Implementations of indices and indices1. Since bounds-checking is # performance-critical and relies on indices, these are usually worth @@ -195,6 +194,9 @@ show(io, v) str = takebuf_string(io) show(io, parent(v)) @test str == takebuf_string(io) +smry = summary(v) +@test contains(smry, "OffsetArray{Float64,1") +@test contains(smry, "with indices -1:1") function cmp_showf(printfunc, io, A) ioc = IOContext(io, limit=true, compact=true) printfunc(ioc, A) @@ -207,6 +209,23 @@ cmp_showf(Base.print_matrix, io, OffsetArray(rand(5,5), (10,-9))) # rows&c cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,5), (10,-9))) # columns fit cmp_showf(Base.print_matrix, io, OffsetArray(rand(5,10^3), (10,-9))) # rows fit cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,10^3), (10,-9))) # neither fits +targets1 = ["0-dimensional OAs.OffsetArray{Float64,0,Array{Float64,0}}:\n1.0", + "OAs.OffsetArray{Float64,1,Array{Float64,1}} with indices 2:2:\n 1.0", + "OAs.OffsetArray{Float64,2,Array{Float64,2}} with indices 2:2×3:3:\n 1.0", + "OAs.OffsetArray{Float64,3,Array{Float64,3}} with indices 2:2×3:3×4:4:\n[:, :, 4] =\n 1.0", + "OAs.OffsetArray{Float64,4,Array{Float64,4}} with indices 2:2×3:3×4:4×5:5:\n[:, :, 4, 5] =\n 1.0"] +targets2 = ["(1.0,1.0)", + "([1.0],[1.0])", + "(\n[1.0],\n\n[1.0])", + "(\n[1.0],\n\n[1.0])", + "(\n[1.0],\n\n[1.0])"] +for n = 0:4 + a = OffsetArray(ones(Float64,ntuple(d->1,n)), ntuple(identity,n)) + show(IOContext(io, limit=true), MIME("text/plain"), a) + @test takebuf_string(io) == targets1[n+1] + show(IOContext(io, limit=true), MIME("text/plain"), (a,a)) + @test takebuf_string(io) == targets2[n+1] +end # Similar B = similar(A, Float32) From 8732bccfac6a9601a28b6c6eb3dcdba71abe12c2 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 31 Jul 2016 18:49:30 -0500 Subject: [PATCH 0749/1117] Define indices for more containers and fix map --- base/abstractarray.jl | 3 ++- base/array.jl | 7 ++++--- base/generator.jl | 1 + test/offsetarray.jl | 4 ++++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index eedcf03ac8238..abe4f1fecd401 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -40,7 +40,7 @@ indices{T,N}(A::AbstractArray{T,N}, d) = d <= N ? indices(A)[d] : OneTo(1) Returns the tuple of valid indices for array `A`. """ -function indices{T,N}(A::AbstractArray{T,N}) +function indices(A) @_inline_pure_meta map(s->OneTo(s), size(A)) end @@ -77,6 +77,7 @@ ndims{T,N}(::Type{AbstractArray{T,N}}) = N ndims{T<:AbstractArray}(::Type{T}) = ndims(supertype(T)) length(t::AbstractArray) = prod(size(t)) _length(A::AbstractArray) = prod(map(unsafe_length, indices(A))) # circumvent missing size +_length(A) = length(A) endof(a::AbstractArray) = length(a) first(a::AbstractArray) = a[first(eachindex(a))] diff --git a/base/array.jl b/base/array.jl index dc97409659f13..f308d7ff53ffc 100644 --- a/base/array.jl +++ b/base/array.jl @@ -220,7 +220,7 @@ end # make a collection similar to `c` and appropriate for collecting `itr` _similar_for(c::AbstractArray, T, itr, ::SizeUnknown) = similar(c, T, 0) _similar_for(c::AbstractArray, T, itr, ::HasLength) = similar(c, T, Int(length(itr)::Integer)) -_similar_for(c::AbstractArray, T, itr, ::HasShape) = similar(c, T, convert(Dims,size(itr))) +_similar_for(c::AbstractArray, T, itr, ::HasShape) = similar(c, T, indices(itr)) _similar_for(c, T, itr, isz) = similar(c, T) """ @@ -286,8 +286,9 @@ function _collect(c, itr, ::EltypeUnknown, isz::Union{HasLength,HasShape}) end function collect_to_with_first!(dest::AbstractArray, v1, itr, st) - dest[1] = v1 - return collect_to!(dest, itr, 2, st) + i1 = first(linearindices(dest)) + dest[i1] = v1 + return collect_to!(dest, itr, i1+1, st) end function collect_to_with_first!(dest, v1, itr, st) diff --git a/base/generator.jl b/base/generator.jl index e598f9318c29f..fce99b73e31b7 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -88,6 +88,7 @@ iteratorsize{T<:AbstractArray}(::Type{T}) = HasShape() iteratorsize{I,F}(::Type{Generator{I,F}}) = iteratorsize(I) length(g::Generator) = length(g.iter) size(g::Generator) = size(g.iter) +indices(g::Generator) = indices(g.iter) ndims(g::Generator) = ndims(g.iter) iteratoreltype{I,T}(::Type{Generator{I,T}}) = EltypeUnknown() diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 870e008ccb9ef..2f26454826c7a 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -322,6 +322,10 @@ map!(+, dest, am, am) @test dest[1,8] == 4 @test dest[1,9] == -2 +am = map(identity, a) +@test isa(am, OffsetArray) +@test am == a + A = OffsetArray(rand(4,4), (-3,5)) @test maximum(A) == maximum(parent(A)) @test minimum(A) == minimum(parent(A)) From 7b51de96083b069bcdf4ef79c324ac7d05352cc6 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 1 Aug 2016 03:48:18 -0500 Subject: [PATCH 0750/1117] Improve performance of size(A) and drop `pure` on indices --- base/abstractarray.jl | 2 +- base/array.jl | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index abe4f1fecd401..e74181d6428d1 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -41,7 +41,7 @@ indices{T,N}(A::AbstractArray{T,N}, d) = d <= N ? indices(A)[d] : OneTo(1) Returns the tuple of valid indices for array `A`. """ function indices(A) - @_inline_pure_meta + @_inline_meta map(s->OneTo(s), size(A)) end diff --git a/base/array.jl b/base/array.jl index f308d7ff53ffc..e4d98d799b3c8 100644 --- a/base/array.jl +++ b/base/array.jl @@ -17,9 +17,12 @@ import Core: arraysize, arrayset, arrayref size(a::Array, d) = arraysize(a, d) size(a::Vector) = (arraysize(a,1),) size(a::Matrix) = (arraysize(a,1), arraysize(a,2)) -size(a::Array) = _size((), a) +size(a::Array) = (@_inline_meta; _size((), a)) _size{_,N}(out::NTuple{N}, A::Array{_,N}) = out -_size{_,M,N}(out::NTuple{M}, A::Array{_,N}) = _size((out..., size(A,M+1)), A) +function _size{_,M,N}(out::NTuple{M}, A::Array{_,N}) + @_inline_meta + _size((out..., size(A,M+1)), A) +end asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...) From 4a838b47c2e8d7294b081f0bdec2e86cd32e1954 Mon Sep 17 00:00:00 2001 From: Alex Arslan <ararslan@comcast.net> Date: Tue, 2 Aug 2016 06:32:13 -0700 Subject: [PATCH 0751/1117] RFC: Make isapprox compare element-wise when the vector norm is infinite (#17658) * Make isapprox compare elementwise when the vector norm is infinite * Test change to isapprox for arrays * Documented isapprox fallback behavior * Document change to isapprox fallback when norm(x,y) is infinite * Put docs in the right place hopefully * Commit the auto-generated change * Remove use of map, reword explanation of behavior --- base/docs/helpdb/Base.jl | 19 ------------------- base/floatfuncs.jl | 19 +++++++++++++++++++ base/linalg/generic.jl | 7 ++++++- doc/stdlib/math.rst | 2 +- test/linalg/generic.jl | 3 +++ 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index fc0b97a5c8e4f..80b3843981870 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1188,25 +1188,6 @@ Return ``\\exp(iz)``. """ cis -""" - isapprox(x, y; rtol::Real=sqrt(eps), atol::Real=0) - -Inexact equality comparison: `true` if `norm(x-y) <= atol + rtol*max(norm(x), norm(y))`. The -default `atol` is zero and the default `rtol` depends on the types of `x` and `y`. - -For real or complex floating-point values, `rtol` defaults to -`sqrt(eps(typeof(real(x-y))))`. This corresponds to requiring equality of about half of the -significand digits. For other types, `rtol` defaults to zero. - -`x` and `y` may also be arrays of numbers, in which case `norm` defaults to `vecnorm` but -may be changed by passing a `norm::Function` keyword argument. (For numbers, `norm` is the -same thing as `abs`.) - -The binary operator `≈` is equivalent to `isapprox` with the default arguments, and `x ≉ y` -is equivalent to `!isapprox(x,y)`. -""" -isapprox - """ sinh(x) diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index 45eb2e6e3c6fa..e6a02b176759b 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -228,6 +228,25 @@ for f in (:round, :ceil, :floor, :trunc) end # isapprox: approximate equality of numbers +""" + isapprox(x, y; rtol::Real=sqrt(eps), atol::Real=0) + +Inexact equality comparison: `true` if `norm(x-y) <= atol + rtol*max(norm(x), norm(y))`. The +default `atol` is zero and the default `rtol` depends on the types of `x` and `y`. + +For real or complex floating-point values, `rtol` defaults to +`sqrt(eps(typeof(real(x-y))))`. This corresponds to requiring equality of about half of the +significand digits. For other types, `rtol` defaults to zero. + +`x` and `y` may also be arrays of numbers, in which case `norm` defaults to `vecnorm` but +may be changed by passing a `norm::Function` keyword argument. (For numbers, `norm` is the +same thing as `abs`.) When `x` and `y` are arrays, if `norm(x-y)` is not finite (i.e. `±Inf` +or `NaN`), the comparison falls back to checking whether all elements of `x` and `y` are +approximately equal component-wise. + +The binary operator `≈` is equivalent to `isapprox` with the default arguments, and `x ≉ y` +is equivalent to `!isapprox(x,y)`. +""" function isapprox(x::Number, y::Number; rtol::Real=rtoldefault(x,y), atol::Real=0) x == y || (isfinite(x) && isfinite(y) && abs(x-y) <= atol + rtol*max(abs(x), abs(y))) end diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 5a076989d7f6a..3451999961ba7 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -588,7 +588,12 @@ end # isapprox: approximate equality of arrays [like isapprox(Number,Number)] function isapprox{T<:Number,S<:Number}(x::AbstractArray{T}, y::AbstractArray{S}; rtol::Real=Base.rtoldefault(T,S), atol::Real=0, norm::Function=vecnorm) d = norm(x - y) - return isfinite(d) ? d <= atol + rtol*max(norm(x), norm(y)) : x == y + if isfinite(d) + return d <= atol + rtol*max(norm(x), norm(y)) + else + # Fall back to a component-wise approximate comparison + return all(ab -> isapprox(ab[1], ab[2]; rtol=rtol, atol=atol), zip(x, y)) + end end """ diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 0d26212076d46..6ff3d36e6afce 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -495,7 +495,7 @@ Mathematical Functions For real or complex floating-point values, ``rtol`` defaults to ``sqrt(eps(typeof(real(x-y))))``\ . This corresponds to requiring equality of about half of the significand digits. For other types, ``rtol`` defaults to zero. - ``x`` and ``y`` may also be arrays of numbers, in which case ``norm`` defaults to ``vecnorm`` but may be changed by passing a ``norm::Function`` keyword argument. (For numbers, ``norm`` is the same thing as ``abs``\ .) + ``x`` and ``y`` may also be arrays of numbers, in which case ``norm`` defaults to ``vecnorm`` but may be changed by passing a ``norm::Function`` keyword argument. (For numbers, ``norm`` is the same thing as ``abs``\ .) When ``x`` and ``y`` are arrays, if ``norm(x-y)`` is not finite (i.e. ``±Inf`` or ``NaN``\ ), the comparison falls back to checking whether all elements of ``x`` and ``y`` are approximately equal component-wise. The binary operator ``≈`` is equivalent to ``isapprox`` with the default arguments, and ``x ≉ y`` is equivalent to ``!isapprox(x,y)``\ . diff --git a/test/linalg/generic.jl b/test/linalg/generic.jl index af2e73e525fcf..5e8f3b515a6ff 100644 --- a/test/linalg/generic.jl +++ b/test/linalg/generic.jl @@ -282,3 +282,6 @@ end @test det([true false; false true]) == det(eye(Int, 2)) @test_throws ArgumentError Base.LinAlg.char_uplo(:Z) + +# Issue 17650 +@test [0.01311489462160816, Inf] ≈ [0.013114894621608135, Inf] From 68aa0520117afb62e8a9c3d430bfa9749c3d4adb Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Tue, 2 Aug 2016 16:50:05 +0200 Subject: [PATCH 0752/1117] Documentation formatting fixes Macro reference syntax uses `:func:` rather than `:ref:`, which is just for section titles. Add space between a backtick and next character to correctly parse reference. Correct indent of a sublist which sphinx parsed as a list within a blockquote within another list. --- base/util.jl | 20 ++++++++++---------- doc/manual/parallel-computing.rst | 12 +++++++----- doc/stdlib/base.rst | 10 +++++----- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/base/util.jl b/base/util.jl index b6799a6dd5faa..0c2013b22b7d6 100644 --- a/base/util.jl +++ b/base/util.jl @@ -174,8 +174,8 @@ A macro to execute an expression, printing the time it took to execute, the numb allocations, and the total number of bytes its execution caused to be allocated, before returning the value of the expression. -See also [`@timev`](:ref:@timev), [`@timed`](:ref:@timed), [`@elapsed`](:ref:@elapsed), and -[`@allocated`](:ref:@allocated). +See also [`@timev`](:func:`@timev`), [`@timed`](:func:`@timed`), [`@elapsed`](:func:`@elapsed`), and +[`@allocated`](:func:`@allocated`). """ macro time(ex) quote @@ -197,8 +197,8 @@ This is a verbose version of the `@time` macro. It first prints the same informa `@time`, then any non-zero memory allocation counters, and then returns the value of the expression. -See also [`@time`](:ref:@time), [`@timed`](:ref:@timed), [`@elapsed`](:ref:@elapsed), and -[`@allocated`](:ref:@allocated). +See also [`@time`](:func:`@time`), [`@timed`](:func:`@timed`), [`@elapsed`](:func:`@elapsed`), and +[`@allocated`](:func:`@allocated`). """ macro timev(ex) quote @@ -217,8 +217,8 @@ end A macro to evaluate an expression, discarding the resulting value, instead returning the number of seconds it took to execute as a floating-point number. -See also [`@time`](:ref:@time), [`@timev`](:ref:@timev), [`@timed`](:ref:@timed), -and [`@allocated`](:ref:@allocated). +See also [`@time`](:func:`@time`), [`@timev`](:func:`@timev`), [`@timed`](:func:`@timed`), +and [`@allocated`](:func:`@allocated`). """ macro elapsed(ex) quote @@ -244,8 +244,8 @@ effects of compilation, however, there still may be some allocations due to JIT This also makes the results inconsistent with the `@time` macros, which do not try to adjust for the effects of compilation. -See also [`@time`](:ref:@time), [`@timev`](:ref:@timev), [`@timed`](:ref:@timed), -and [`@elapsed`](:ref:@elapsed). +See also [`@time`](:func:`@time`), [`@timev`](:func:`@timev`), [`@timed`](:func:`@timed`), +and [`@elapsed`](:func:`@elapsed`). """ macro allocated(ex) quote @@ -268,8 +268,8 @@ A macro to execute an expression, and return the value of the expression, elapse total bytes allocated, garbage collection time, and an object with various memory allocation counters. -See also [`@time`](:ref:@time), [`@timev`](:ref:@timev), [`@elapsed`](:ref:@elapsed), and -[`@allocated`](:ref:@allocated). +See also [`@time`](:func:`@time`), [`@timev`](:func:`@timev`), [`@elapsed`](:func:`@elapsed`), and +[`@allocated`](:func:`@allocated`). """ macro timed(ex) quote diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 4710e863ee3d8..0317171a1a9c3 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -121,7 +121,7 @@ own such constructs.) An important thing to remember is that, once fetched, a :class:`Future` will cache its value locally. Further :func:`fetch` calls do not entail a network hop. Once all referencing -:class:`Future`s have fetched, the remote stored value is deleted. +:class:`Future`\ s have fetched, the remote stored value is deleted. .. _man-parallel-computing-code-availability: @@ -858,10 +858,12 @@ Custom cluster managers would typically specify only ``io`` or ``host`` / ``port - ``count``, ``exename`` and ``exeflags`` are relevant for launching additional workers from a worker. For example, a cluster manager may launch a single worker per node, and use that to launch additional workers. - - ``count`` with an integer value ``n`` will launch a total of ``n`` workers. - - ``count`` with a value of ``:auto`` will launch as many workers as cores on that machine. - - ``exename`` is the name of the ``julia`` executable including the full path. - - ``exeflags`` should be set to the required command line arguments for new workers. + + - ``count`` with an integer value ``n`` will launch a total of ``n`` workers. + - ``count`` with a value of ``:auto`` will launch as many workers as cores on that machine. + - ``exename`` is the name of the ``julia`` executable including the full path. + - ``exeflags`` should be set to the required command line arguments for new workers. + - ``tunnel``, ``bind_addr``, ``sshflags`` and ``max_parallel`` are used when a ssh tunnel is required to connect to the workers from the master process. - ``userdata`` is provided for custom cluster managers to store their own worker specific information. diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index ff8ed6282532a..f55d2a3e5dbb6 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -971,7 +971,7 @@ System A macro to execute an expression, printing the time it took to execute, the number of allocations, and the total number of bytes its execution caused to be allocated, before returning the value of the expression. - See also ```@timev`` <:ref:@timev>`_\ , ```@timed`` <:ref:@timed>`_\ , ```@elapsed`` <:ref:@elapsed>`_\ , and ```@allocated`` <:ref:@allocated>`_\ . + See also :func:`@timev`\ , :func:`@timed`\ , :func:`@elapsed`\ , and :func:`@allocated`\ . .. function:: @timev @@ -979,7 +979,7 @@ System This is a verbose version of the ``@time`` macro. It first prints the same information as ``@time``\ , then any non-zero memory allocation counters, and then returns the value of the expression. - See also ```@time`` <:ref:@time>`_\ , ```@timed`` <:ref:@timed>`_\ , ```@elapsed`` <:ref:@elapsed>`_\ , and ```@allocated`` <:ref:@allocated>`_\ . + See also :func:`@time`\ , :func:`@timed`\ , :func:`@elapsed`\ , and :func:`@allocated`\ . .. function:: @timed @@ -987,7 +987,7 @@ System A macro to execute an expression, and return the value of the expression, elapsed time, total bytes allocated, garbage collection time, and an object with various memory allocation counters. - See also ```@time`` <:ref:@time>`_\ , ```@timev`` <:ref:@timev>`_\ , ```@elapsed`` <:ref:@elapsed>`_\ , and ```@allocated`` <:ref:@allocated>`_\ . + See also :func:`@time`\ , :func:`@timev`\ , :func:`@elapsed`\ , and :func:`@allocated`\ . .. function:: @elapsed @@ -995,7 +995,7 @@ System A macro to evaluate an expression, discarding the resulting value, instead returning the number of seconds it took to execute as a floating-point number. - See also ```@time`` <:ref:@time>`_\ , ```@timev`` <:ref:@timev>`_\ , ```@timed`` <:ref:@timed>`_\ , and ```@allocated`` <:ref:@allocated>`_\ . + See also :func:`@time`\ , :func:`@timev`\ , :func:`@timed`\ , and :func:`@allocated`\ . .. function:: @allocated @@ -1003,7 +1003,7 @@ System A macro to evaluate an expression, discarding the resulting value, instead returning the total number of bytes allocated during evaluation of the expression. Note: the expression is evaluated inside a local function, instead of the current context, in order to eliminate the effects of compilation, however, there still may be some allocations due to JIT compilation. This also makes the results inconsistent with the ``@time`` macros, which do not try to adjust for the effects of compilation. - See also ```@time`` <:ref:@time>`_\ , ```@timev`` <:ref:@timev>`_\ , ```@timed`` <:ref:@timed>`_\ , and ```@elapsed`` <:ref:@elapsed>`_\ . + See also :func:`@time`\ , :func:`@timev`\ , :func:`@timed`\ , and :func:`@elapsed`\ . .. function:: EnvHash() -> EnvHash From d561a87327f3556375a1d03467f7e446173b16d1 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 2 Aug 2016 11:21:47 -0400 Subject: [PATCH 0753/1117] fix #17712, `show` of generator exprs with >1 range --- base/show.jl | 9 +++++---- test/show.jl | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/base/show.jl b/base/show.jl index 0e0ce69c0557a..66574576eb59e 100644 --- a/base/show.jl +++ b/base/show.jl @@ -663,6 +663,10 @@ function show_generator(io, ex, indent) show_unquoted(io, ex.args[1], indent) print(io, " for ") show_unquoted(io, ex.args[2], indent) + for i = 3:length(ex.args) + print(io, ", ") + show_unquoted(io, ex.args[i], indent) + end end end @@ -802,7 +806,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) show_generator(io, args[1], indent) print(io, ']') - elseif head === :generator && length(args) == 2 + elseif (head === :generator && length(args) >= 2) || (head === :flatten && length(args) == 1) print(io, '(') show_generator(io, ex, indent) print(io, ')') @@ -812,9 +816,6 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) print(io, " if ") show_unquoted(io, args[1], indent) - elseif head === :flatten && length(args) == 1 - show_generator(io, ex, indent) - elseif is(head, :ccall) show_unquoted(io, :ccall, indent) show_enclosed_list(io, '(', args, ",", ')', indent) diff --git a/test/show.jl b/test/show.jl index b091e3277d73f..4179be60746f9 100644 --- a/test/show.jl +++ b/test/show.jl @@ -557,10 +557,11 @@ end @test repr(Core.svec(1,2)) == "svec(1,2)" # showing generator and comprehension expressions -@test repr(:(x for x in y for z in w)) == ":(x for x = y for z = w)" -@test repr(:(x for x in y if aa for z in w if bb)) == ":(x for x = y if aa for z = w if bb)" +@test repr(:(x for x in y for z in w)) == ":((x for x = y for z = w))" +@test repr(:(x for x in y if aa for z in w if bb)) == ":((x for x = y if aa for z = w if bb))" @test repr(:([x for x = y])) == ":([x for x = y])" @test repr(:([x for x = y if z])) == ":([x for x = y if z])" +@test repr(:(z for z = 1:5, y = 1:5)) == ":((z for z = 1:5, y = 1:5))" for op in (:(.=), :(.+=), :(.&=)) @test repr(parse("x $op y")) == ":(x $op y)" From 585ed717df829d16021e5ae31c0793ad3d5ccfe1 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 2 Aug 2016 12:15:59 -0400 Subject: [PATCH 0754/1117] fix #17720, showcompact shows Float32s with `e` instead of `f` --- base/grisu/grisu.jl | 2 +- test/show.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/grisu/grisu.jl b/base/grisu/grisu.jl index 04043bb8191da..2f93ff0b54f1a 100644 --- a/base/grisu/grisu.jl +++ b/base/grisu/grisu.jl @@ -86,7 +86,7 @@ function _show(io::IO, x::AbstractFloat, mode, n::Int, typed, nanstr, infstr) else write(io, '0') end - write(io, typed && isa(x,Float32) ? 'f' : 'e') + write(io, isa(x,Float32) ? 'f' : 'e') write(io, dec(pt-1)) typed && isa(x,Float16) && write(io, ")") return diff --git a/test/show.jl b/test/show.jl index 4179be60746f9..2d185d76fb379 100644 --- a/test/show.jl +++ b/test/show.jl @@ -5,6 +5,7 @@ replstr(x) = sprint((io,x) -> show(IOContext(io, limit=true), MIME("text/plain") @test replstr(Array{Any}(2)) == "2-element Array{Any,1}:\n #undef\n #undef" @test replstr(Array{Any}(2,2)) == "2×2 Array{Any,2}:\n #undef #undef\n #undef #undef" @test replstr(Array{Any}(2,2,2)) == "2×2×2 Array{Any,3}:\n[:, :, 1] =\n #undef #undef\n #undef #undef\n\n[:, :, 2] =\n #undef #undef\n #undef #undef" +@test replstr([1f10]) == "1-element Array{Float32,1}:\n 1.0f10" immutable T5589 names::Vector{String} From 6b8180982a2a31dc28872741e1f2f0bf72b347cd Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 2 Aug 2016 13:53:58 -0400 Subject: [PATCH 0755/1117] fix #17705, `2e3_x` should parse as `2e3 * _x` --- src/julia-parser.scm | 50 ++++++++++++++++++++------------------------ test/parse.jl | 12 +++++++++++ 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 0e21544ff3904..0e6d319f56c79 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -202,23 +202,21 @@ (error (string "invalid operator \"" str "\""))) (string->symbol str)))) -(define (accum-digits c pred port lz) - (if (and (not lz) (eqv? c #\_)) - (cons "_" #f) - (let loop ((str '()) - (c c)) - (if (eqv? c #\_) +(define (accum-digits c pred port _-digit-sep) + (let loop ((str '()) + (c c)) + (if (and _-digit-sep (eqv? c #\_)) + (begin (read-char port) + (let ((c (peek-char port))) + (if (and (not (eof-object? c)) (pred c)) + (loop str c) + (begin + (io.ungetc port #\_) + (list->string (reverse str)))))) + (if (and (not (eof-object? c)) (pred c)) (begin (read-char port) - (let ((c (peek-char port))) - (if (and (not (eof-object? c)) (pred c)) - (loop str c) - (begin - (io.ungetc port #\_) - (cons (list->string (reverse str)) #t))))) - (if (and (not (eof-object? c)) (pred c)) - (begin (read-char port) - (loop (cons c str) (peek-char port))) - (cons (list->string (reverse str)) #t)))))) + (loop (cons c str) (peek-char port))) + (list->string (reverse str)))))) (define (char-hex? c) (or (char-numeric? c) @@ -263,14 +261,12 @@ (io.ungetc port #\.) (error (string "invalid numeric constant \"" (get-output-string str) #\. "\"")))))) - (define (read-digs lz) - (let ((D (accum-digits (peek-char port) pred port lz))) - (let ((d (car D)) - (ok (cdr D))) - (if (not ok) - (begin (display d str) - (error (string "invalid numeric constant \"" - (get-output-string str) "\"")))) + (define (read-digs lz _-digit-sep) + (let ((c (peek-char port))) + (if (and (not lz) _-digit-sep (eqv? c #\_)) + (error (string "invalid numeric constant \"" + (get-output-string str) c "\""))) + (let ((d (accum-digits c pred port _-digit-sep))) (and (not (equal? d "")) (not (eof-object? d)) (display d str) @@ -291,13 +287,13 @@ (begin (set! leadingzero #f) (set! pred char-bin?))))) (allow #\.))) - (read-digs leadingzero) + (read-digs leadingzero #t) (if (eqv? (peek-char port) #\.) (begin (read-char port) (if (dot-opchar? (peek-char port)) (io.ungetc port #\.) (begin (write-char #\. str) - (read-digs #f) + (read-digs #f #t) (if (eq? pred char-hex?) (set! is-hex-float-literal #t)) (disallow-dot))))) @@ -314,7 +310,7 @@ (set! is-hex-float-literal ispP) (write-char c str) (write-char (read-char port) str) - (read-digs #f) + (read-digs #t #f) (disallow-dot)) (io.ungetc port c)))) ;; disallow digits after binary or octal literals, e.g., 0b12 diff --git a/test/parse.jl b/test/parse.jl index 87a735eff1177..f33e55273118e 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -260,6 +260,18 @@ parsehex(s) = parse(Int,s,16) @test_throws ArgumentError parse(Int,"2x") @test_throws ArgumentError parse(Int,"-") +# parsing numbers with _ and . +@test parse("1_2.3_4") == 12.34 +@test_throws ParseError parse("1._") +@test_throws ParseError parse("1._5") +@test_throws ParseError parse("1e.3") +@test_throws ParseError parse("1e3.") +@test parse("2e_1") == Expr(:call, :*, 2, :e_1) +# issue #17705 +@test parse("2e3_") == Expr(:call, :*, 2e3, :_) +@test parse("2e-3_") == Expr(:call, :*, 2e-3, :_) +@test parse("2e3_\"x\"") == Expr(:call, :*, 2e3, Expr(:macrocall, Symbol("@__str"), "x")) + # multibyte spaces @test parse(Int, "3\u2003\u202F") == 3 @test_throws ArgumentError parse(Int, "3\u2003\u202F,") From e53d242d829e2ecf4823b0747eb0427af7ad35b8 Mon Sep 17 00:00:00 2001 From: Pablo Zubieta <pabloferz@yahoo.com.mx> Date: Tue, 2 Aug 2016 20:28:55 +0200 Subject: [PATCH 0756/1117] fix #17741 (return type of sets built from generators) (#17757) --- base/set.jl | 5 +++++ test/sets.jl | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/base/set.jl b/base/set.jl index d30b9bcf07801..cafa9b66eeac1 100644 --- a/base/set.jl +++ b/base/set.jl @@ -8,6 +8,11 @@ type Set{T} <: AbstractSet{T} end Set() = Set{Any}() Set(itr) = Set{eltype(itr)}(itr) +function Set(g::Generator) + T = _default_eltype(typeof(g)) + T === Union{} && return grow_to!(Set{T}(), g) + return Set{T}(g) +end eltype{T}(::Type{Set{T}}) = T similar{T}(s::Set{T}) = Set{T}() diff --git a/test/sets.jl b/test/sets.jl index 8b830b9b2ee36..a00db78f41ba2 100644 --- a/test/sets.jl +++ b/test/sets.jl @@ -11,6 +11,12 @@ data_out = collect(s) @test is(typeof(data_out), Array{Any,1}) @test all(map(d->in(d,data_out), data_in)) @test length(data_out) == length(data_in) +let f17741 = x -> x < 0 ? false : 1 + @test isa(Set(x for x = 1:3), Set{Int}) + @test isa(Set(sin(x) for x = 1:3), Set{Float64}) + @test isa(Set(f17741(x) for x = 1:3), Set{Int}) + @test isa(Set(f17741(x) for x = -1:1), Set{Integer}) +end # hash s1 = Set(["bar", "foo"]) From 84ab20469ddc051f58b14e4da5231d49a9e9ac74 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Tue, 2 Aug 2016 14:31:15 -0400 Subject: [PATCH 0757/1117] make flipdim require an in-bounds dimension argument (fixes #17752) --- base/abstractarraymath.jl | 6 +++--- base/arraymath.jl | 6 ++---- base/bitarray.jl | 3 ++- doc/stdlib/arrays.rst | 1 - test/arrayops.jl | 6 +++--- test/bitarray.jl | 2 +- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 349b5533927a8..61b95287155ad 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -67,14 +67,14 @@ function slicedim(A::AbstractArray, d::Integer, i) end function flipdim(A::AbstractVector, d::Integer) - d > 0 || throw(ArgumentError("dimension to flip must be positive")) - d == 1 || return copy(A) + d == 1 || throw(ArgumentError("dimension to flip must be 1")) reverse(A) end function flipdim(A::AbstractArray, d::Integer) nd = ndims(A) - if d > nd || isempty(A) + 1 ≤ d ≤ nd || throw(ArgumentError("dimension $d is not 1 ≤ $d ≤ $nd")) + if isempty(A) return copy(A) end inds = indices(A) diff --git a/base/arraymath.jl b/base/arraymath.jl index 516fbe2af98ee..41cd74e843bb7 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -99,11 +99,9 @@ end ## data movement ## function flipdim{T}(A::Array{T}, d::Integer) - if d < 1 - throw(ArgumentError("dimension d must be ≥ 1")) - end nd = ndims(A) - sd = d > nd ? 1 : size(A, d) + 1 ≤ d ≤ nd || throw(ArgumentError("dimension $d is not 1 ≤ $d ≤ $nd")) + sd = size(A, d) if sd == 1 || isempty(A) return copy(A) end diff --git a/base/bitarray.jl b/base/bitarray.jl index 3963c16474255..81ba017866ee9 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1284,7 +1284,8 @@ end function flipdim(A::BitArray, d::Integer) nd = ndims(A) - sd = d > nd ? 1 : size(A, d) + 1 ≤ d ≤ nd || throw(ArgumentError("dimension $d is not 1 ≤ $d ≤ $nd")) + sd = size(A, d) if sd == 1 return copy(A) end diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 448dd55c0caa9..8d6bb1b0ad45e 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1086,4 +1086,3 @@ dense counterparts. The following functions are specific to sparse arrays. For additional (algorithmic) information, and for versions of these methods that forgo argument checking, see (unexported) parent methods :func:`Base.SparseArrays.unchecked_noalias_permute!` and :func:`Base.SparseArrays.unchecked_aliasing_permute!`\ . See also: :func:`Base.SparseArrays.permute` - diff --git a/test/arrayops.jl b/test/arrayops.jl index b52f04e4a0bda..0d735436338c3 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1019,12 +1019,12 @@ end # flipdim @test isequal(flipdim([2,3,1], 1), [1,3,2]) -@test isequal(flipdim([2,3,1], 2), [2,3,1]) +@test_throws ArgumentError flipdim([2,3,1], 2) @test isequal(flipdim([2 3 1], 1), [2 3 1]) @test isequal(flipdim([2 3 1], 2), [1 3 2]) @test_throws ArgumentError flipdim([2,3,1], -1) @test isequal(flipdim(1:10, 1), 10:-1:1) -@test isequal(flipdim(1:10, 2), 1:10) +@test_throws ArgumentError flipdim(1:10, 2) @test_throws ArgumentError flipdim(1:10, -1) @test isequal(flipdim(Array{Int}(0,0),1), Array{Int}(0,0)) # issue #5872 @@ -1415,7 +1415,7 @@ ctranspose!(a,b) # flipdim a = rand(5,3) @test flipdim(flipdim(a,2),2) == a -@test flipdim(a,3) == a +@test_throws ArgumentError flipdim(a,3) # bounds checking for copy! a = rand(5,3) diff --git a/test/bitarray.jl b/test/bitarray.jl index d401baa93ccda..e591173a5b5bb 100644 --- a/test/bitarray.jl +++ b/test/bitarray.jl @@ -861,7 +861,7 @@ for d = 1 : 4 #end @check_bit_operation flipdim(b1, d) BitArray{4} end -@check_bit_operation flipdim(b1, 5) BitArray{4} +@test_throws ArgumentError flipdim(b1, 5) b1 = bitrand(n1, n2) for k = 1 : 4 From aa5fe191e0f0156de999ace0096c0790ba6ddf53 Mon Sep 17 00:00:00 2001 From: Waldir Pimenta <waldyrious@gmail.com> Date: Wed, 3 Aug 2016 01:44:06 +0100 Subject: [PATCH 0758/1117] adjust line breaks to fix markdown rendering --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index f5c7b0cd3d0a3..1ac83bf4b12ce 100644 --- a/NEWS.md +++ b/NEWS.md @@ -50,8 +50,8 @@ Experimental language features Loops with independent iterations can be easily parallelized with the `Threads.@threads` macro. - * Support for arrays with indexing starting at values different from - 1. The array types are expected to be defined in packages, but now + * Support for arrays with indexing starting at values different from 1. + The array types are expected to be defined in packages, but now Julia provides an API for writing generic algorithms for arbitrary indexing schemes ([#16260]). From 8c5e13b9da5d05600ad0f9728382113040b7cbb8 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Tue, 2 Aug 2016 22:25:14 -0400 Subject: [PATCH 0759/1117] restore syntax deprecation for foo.(1) etc. --- src/julia-syntax.scm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 40eb5fc33a139..f456a68bf1ea4 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1682,7 +1682,9 @@ (else (cf (cdr old-fargs) (cdr old-args) (cons farg new-fargs) (cons arg new-args) renames varfarg vararg)))))) - (cf (cdadr f) args '() '() '() '() '())) + (if (and (= (length args) 1) (integer? (car args))) + e ; hack: don't compress e.g. f.(1) so that deprecation for getfield works + (cf (cdadr f) args '() '() '() '() '()))) e)) ; (not (fuse? e)) (let ((e (compress-fuse (dot-to-fuse rhs))) ; an expression '(fuse func args) if expr is a dot call (lhs-view (ref-to-view lhs))) ; x[...] expressions on lhs turn in to view(x, ...) to update x in-place From cff6ea30b1a77ccda7a4e0e7ee018e39942e932c Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Tue, 2 Aug 2016 22:30:27 -0400 Subject: [PATCH 0760/1117] make sure literal folding in f.(args...) is still tested --- test/broadcast.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/broadcast.jl b/test/broadcast.jl index c4ff2cc43f98a..7cc1ceffa500f 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -174,6 +174,7 @@ rt = Base.return_types(broadcast!, Tuple{Function, Array{Float64, 3}, Array{Floa let x = [1,3.2,4.7], y = [3.5, pi, 1e-4], α = 0.2342 @test sin.(x) == broadcast(sin, x) @test sin.(α) == broadcast(sin, α) + @test sin.(3.2) == broadcast(sin, 3.2) == sin(3.2) @test factorial.(3) == broadcast(factorial, 3) @test atan2.(x, y) == broadcast(atan2, x, y) @test atan2.(x, y') == broadcast(atan2, x, y') From 3ac1841c02b415e914bfbe40e3669894cf7c75ba Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Tue, 2 Aug 2016 09:23:59 -0700 Subject: [PATCH 0761/1117] Fix #17070, make keyword arg docs of read more accurate, move docs out of HelpDB too. Fixed the type signatures. Made `nb` reflect what's actually in the code. --- base/docs/helpdb/Base.jl | 12 ------------ base/iostream.jl | 10 ++++++++++ doc/stdlib/io-network.rst | 4 ++-- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 80b3843981870..3a316a93cc7bd 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -8736,18 +8736,6 @@ Letter: Lowercase. """ islower -""" - read(stream::IO, nb=typemax(Int); all=true) - -Read at most `nb` bytes from `stream`, returning a `Vector{UInt8}` of the bytes read. - -If `all` is `true` (the default), this function will block repeatedly trying to read all -requested bytes, until an error or end-of-file occurs. If `all` is `false`, at most one -`read` call is performed, and the amount of data returned is device-dependent. Note that not -all stream types support the `all` option. -""" -read - """ eig(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> D, V diff --git a/base/iostream.jl b/base/iostream.jl index 0a1521450d956..d355bd259c3a6 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -244,6 +244,16 @@ function read(s::IOStream) resize!(b, nr) end +""" + read(s::IOStream, nb::Integer; all=true) + +Read at most `nb` bytes from `s`, returning a `Vector{UInt8}` of the bytes read. + +If `all` is `true` (the default), this function will block repeatedly trying to read all +requested bytes, until an error or end-of-file occurs. If `all` is `false`, at most one +`read` call is performed, and the amount of data returned is device-dependent. Note that not +all stream types support the `all` option. +""" function read(s::IOStream, nb::Integer; all::Bool=true) b = Array{UInt8}(nb) nr = readbytes!(s, b, nb, all=all) diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index cb1ecfdb361c8..97e8c7c0c286a 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -168,11 +168,11 @@ General I/O See ``read`` for a description of the ``all`` option. -.. function:: read(stream::IO, nb=typemax(Int); all=true) +.. function:: read(s::IOStream, nb::Integer; all=true) .. Docstring generated from Julia source - Read at most ``nb`` bytes from ``stream``\ , returning a ``Vector{UInt8}`` of the bytes read. + Read at most ``nb`` bytes from ``s``\ , returning a ``Vector{UInt8}`` of the bytes read. If ``all`` is ``true`` (the default), this function will block repeatedly trying to read all requested bytes, until an error or end-of-file occurs. If ``all`` is ``false``\ , at most one ``read`` call is performed, and the amount of data returned is device-dependent. Note that not all stream types support the ``all`` option. From f4168a0c6920530c3ffb24f09fb308ac1d70831d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@debian.org> Date: Wed, 17 Sep 2014 17:22:59 +0200 Subject: [PATCH 0762/1117] Fix exit status of install.sh script Otherwise it can return a non-zero status even in case of success, and make the build fail. --- contrib/install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/install.sh b/contrib/install.sh index 9335c801bed10..702b207fbb5aa 100755 --- a/contrib/install.sh +++ b/contrib/install.sh @@ -27,3 +27,5 @@ for SRC in $ARGS; do # TODO: Test if it's a symlink instead of having to redirect stderr to /dev/null chmod $PERMS $DESTFILE 2>/dev/null done + +exit 0 From 0d11317622397cf08fedb81502ddc43d395a69e9 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Wed, 13 Jul 2016 11:03:02 -0400 Subject: [PATCH 0763/1117] checkout a remote branch correctly [fix #17364] --- base/libgit2/libgit2.jl | 12 ++++++++++-- test/pkg.jl | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index 80d4daf7d9036..de791786da527 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -201,10 +201,17 @@ function branch!(repo::GitRepo, branch_name::AbstractString, # try to lookup branch first branch_ref = force ? nothing : lookup_branch(repo, branch_name) if branch_ref === nothing + branch_rmt_ref = isempty(track) ? nothing : lookup_branch(repo, "$track/$branch_name", true) # if commit is empty get head commit oid commit_id = if isempty(commit) - with(head(repo)) do head_ref - with(peel(GitCommit, head_ref)) do hrc + if branch_rmt_ref === nothing + with(head(repo)) do head_ref + with(peel(GitCommit, head_ref)) do hrc + Oid(hrc) + end + end + else + with(peel(GitCommit, branch_rmt_ref)) do hrc Oid(hrc) end end @@ -218,6 +225,7 @@ function branch!(repo::GitRepo, branch_name::AbstractString, finally finalize(cmt) end + branch_rmt_ref !== nothing && finalize(branch_rmt_ref) end try #TODO: what if branch tracks other then "origin" remote diff --git a/test/pkg.jl b/test/pkg.jl index 946612d22604f..eaf77456b1c3b 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -141,6 +141,11 @@ temp_pkg_dir() do @test endswith(str, string(Pkg.installed("Example"))) @test isempty(Pkg.dependents("Example")) + Pkg.checkout("Example", "test-branch") #17364 + LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example")) do repo + @test LibGit2.head_oid(repo) == LibGit2.Oid("ba3888212e30a7974ac6803a89e64c7098f4865e") + end + # adding a package with unsatisfiable julia version requirements (REPL.jl) errors try Pkg.add("REPL") From 9c3c681267511b3457a423b4426b3fd13f72fe51 Mon Sep 17 00:00:00 2001 From: wildart <wildart@gmail.com> Date: Wed, 27 Jul 2016 15:21:45 -0400 Subject: [PATCH 0764/1117] create a branch on a cloned package and used it as remote for testing --- base/libgit2/libgit2.jl | 10 +++++++--- test/pkg.jl | 22 +++++++++++++++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index de791786da527..fe9b08c87594c 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -211,21 +211,25 @@ function branch!(repo::GitRepo, branch_name::AbstractString, end end else - with(peel(GitCommit, branch_rmt_ref)) do hrc + tmpcmt = with(peel(GitCommit, branch_rmt_ref)) do hrc Oid(hrc) end + finalize(branch_rmt_ref) + tmpcmt end else Oid(commit) end iszero(commit_id) && return cmt = get(GitCommit, repo, commit_id) + new_branch_ref = nothing try - branch_ref = create_branch(repo, branch_name, cmt, force=force) + new_branch_ref = create_branch(repo, branch_name, cmt, force=force) finally finalize(cmt) + new_branch_ref === nothing && throw(GitError(Error.Object, Error.ERROR, "cannot create branch `$branch_name` with `$commit_id`")) + branch_ref = new_branch_ref end - branch_rmt_ref !== nothing && finalize(branch_rmt_ref) end try #TODO: what if branch tracks other then "origin" remote diff --git a/test/pkg.jl b/test/pkg.jl index eaf77456b1c3b..5d59572889324 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -141,9 +141,25 @@ temp_pkg_dir() do @test endswith(str, string(Pkg.installed("Example"))) @test isempty(Pkg.dependents("Example")) - Pkg.checkout("Example", "test-branch") #17364 - LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example")) do repo - @test LibGit2.head_oid(repo) == LibGit2.Oid("ba3888212e30a7974ac6803a89e64c7098f4865e") + + #------- + # 17364 + #------- + let branch_name = "test-branch", + branch_commit = "ba3888212e30a7974ac6803a89e64c7098f4865e" + + # create a branch in Example package + LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example")) do repo + LibGit2.branch!(repo, branch_name, branch_commit, set_head=false) + end + + Pkg.clone(Pkg.dir("Example"), Pkg.dir("Example2")) + + Pkg.checkout("Example2", branch_name) + + LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example2")) do repo + @test LibGit2.head_oid(repo) == LibGit2.Oid(branch_commit) + end end # adding a package with unsatisfiable julia version requirements (REPL.jl) errors From e1871923df76e3d37b416fd0e3a751b06ef6ab21 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 3 Aug 2016 05:14:22 -0400 Subject: [PATCH 0765/1117] Add a test with a local commit before doing checkout --- test/pkg.jl | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 5d59572889324..77c7c152d61c1 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -141,10 +141,7 @@ temp_pkg_dir() do @test endswith(str, string(Pkg.installed("Example"))) @test isempty(Pkg.dependents("Example")) - - #------- - # 17364 - #------- + # 17364, Pkg.checkout with specific branch let branch_name = "test-branch", branch_commit = "ba3888212e30a7974ac6803a89e64c7098f4865e" @@ -153,13 +150,26 @@ temp_pkg_dir() do LibGit2.branch!(repo, branch_name, branch_commit, set_head=false) end - Pkg.clone(Pkg.dir("Example"), Pkg.dir("Example2")) + Pkg.clone(Pkg.dir("Example"), "Example2") + Pkg.clone(Pkg.dir("Example"), "Example3") + open(Pkg.dir("Example3", "README.md"), "w") do f + println(f, "overwritten") + end + LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example3")) do repo + LibGit2.add!(repo, "README.md") + test_sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time(), 0), 0) + LibGit2.commit(repo, "testmsg"; author=test_sig, committer=test_sig) + end Pkg.checkout("Example2", branch_name) + Pkg.checkout("Example3", branch_name) LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example2")) do repo @test LibGit2.head_oid(repo) == LibGit2.Oid(branch_commit) end + LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example3")) do repo + @test LibGit2.head_oid(repo) == LibGit2.Oid(branch_commit) + end end # adding a package with unsatisfiable julia version requirements (REPL.jl) errors From cc27979fd85a9ff693d1d51d23d51715810bffed Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 3 Aug 2016 10:04:39 -0400 Subject: [PATCH 0766/1117] mention x.(i) deprecation for getfield --- NEWS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 1ac83bf4b12ce..520d7cf5f0522 100644 --- a/NEWS.md +++ b/NEWS.md @@ -30,7 +30,8 @@ New language features * The `PROGRAM_FILE` global is now available for determining the name of the running script ([#14114]). - * The syntax `x.:sym` (e.g. `Base.:+`) is now supported, and `x.(:sym)` is deprecated ([#15032]). + * The syntax `x.:sym` (e.g. `Base.:+`) is now supported, while using `x.(:sym)` + or `x.(i)` for field access are deprecated in favor of `getfield` ([#15032]). * Function return type syntax `function f()::T` has been added ([#1090]). Values returned from a function with such a declaration will be converted to the specified type `T`. From b7d01ffb91b9b8e206b60043437dfd519c7dc3cf Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 2 Aug 2016 18:28:22 -0400 Subject: [PATCH 0767/1117] fix #17764, type problem due to object_id collision after `workspace()` In the type cache, make ordered comparison of types not depend on object_id. Clarify help for `object_id`, since it technically can have collisions. While I'm at it, remove unnecessary call to re-sort `tn->linearcache` (it is not in sorted order). --- base/docs/helpdb/Base.jl | 2 +- doc/stdlib/base.rst | 2 +- src/dump.c | 1 - src/jltypes.c | 34 ++++++++++++++++++++++++++++------ test/workspace.jl | 9 +++++++++ 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 3a316a93cc7bd..e1c0eac115f33 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -6391,7 +6391,7 @@ ctranspose! """ object_id(x) -Get a unique integer id for `x`. `object_id(x)==object_id(y)` if and only if `is(x,y)`. +Get a hash value for `x` based on object identity. `object_id(x)==object_id(y)` if `x === y`. """ object_id diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index f55d2a3e5dbb6..30288f57fbdb1 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -309,7 +309,7 @@ All Objects .. Docstring generated from Julia source - Get a unique integer id for ``x``\ . ``object_id(x)==object_id(y)`` if and only if ``is(x,y)``\ . + Get a hash value for ``x`` based on object identity. ``object_id(x)==object_id(y)`` if ``x === y``\ . .. function:: hash(x[, h::UInt]) diff --git a/src/dump.c b/src/dump.c index f422c9a1bf189..d0e3f88f5c8a3 100644 --- a/src/dump.c +++ b/src/dump.c @@ -2033,7 +2033,6 @@ static void jl_restore_system_image_from_stream(ios_t *f) tn->cache = (jl_svec_t*)jl_deserialize_value(&s, NULL); jl_gc_wb(tn, tn->cache); tn->linearcache = (jl_svec_t*)jl_deserialize_value(&s, NULL); jl_gc_wb(tn, tn->linearcache); jl_resort_type_cache(tn->cache); - jl_resort_type_cache(tn->linearcache); } jl_core_module = (jl_module_t*)jl_get_global(jl_main_module, diff --git a/src/jltypes.c b/src/jltypes.c index c35060cba9b38..d4977719145b9 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1891,14 +1891,36 @@ static int typekey_compare(jl_datatype_t *tt, jl_value_t **key, size_t n) for(j=0; j < n; j++) { jl_value_t *kj = key[j], *tj = jl_svecref(tt->parameters,j); if (tj != kj) { - int dtt = jl_is_datatype(tj); int dtk = jl_is_datatype(kj); - if (!dtt && !dtk && jl_egal(tj, kj)) + if (!jl_is_datatype(tj)) { + if (!dtk) { + if (jl_egal(tj, kj)) + continue; + return (jl_object_id(kj) < jl_object_id(tj) ? -1 : 1); + } + else { + return 1; + } + } + else if (!dtk) { + return -1; + } + jl_datatype_t *dt = (jl_datatype_t*)tj; + jl_datatype_t *dk = (jl_datatype_t*)kj; + if (dk->uid != dt->uid) { + return dk->uid < dt->uid ? -1 : 1; + } + else if (dk->uid != 0) { continue; - uintptr_t tid = (dtt && ((jl_datatype_t*)tj)->uid ? ((jl_datatype_t*)tj)->uid : jl_object_id(tj)); - uintptr_t kid = (dtk && ((jl_datatype_t*)kj)->uid ? ((jl_datatype_t*)kj)->uid : jl_object_id(kj)); - if (kid != tid) - return kid < tid ? -1 : 1; + } + else if (dk->name->hash != dt->name->hash) { + return dk->name->hash < dt->name->hash ? -1 : 1; + } + else { + int cmp = typekey_compare(dt, jl_svec_data(dk->parameters), jl_nparams(dk)); + if (cmp != 0) + return cmp; + } } } return 0; diff --git a/test/workspace.jl b/test/workspace.jl index f4d43c0b8dd18..d4d9c4168158f 100644 --- a/test/workspace.jl +++ b/test/workspace.jl @@ -15,3 +15,12 @@ show(io, Pair) """ exename = Base.julia_cmd() run(`$exename --startup-file=no -e $script`) + +# issue #17764 +script2 = """ +type Foo end +workspace() +type Foo end +@assert Tuple{Type{LastMain.Foo}} !== Tuple{Type{Main.Foo}} +""" +run(`$exename --startup-file=no -e $script2`) From b019b0b97a990b5d5fb98f8d4772ede9c5bc80a0 Mon Sep 17 00:00:00 2001 From: Katie Hyatt <kshyatt@physics.ucsb.edu> Date: Thu, 28 Jul 2016 13:02:55 -0700 Subject: [PATCH 0768/1117] Moved parallel docs out of helpDB, updated and improved them Moved a bunch of docstrings inline. Changed things to reflect the code that's actually in `multi.jl`. Documented `RemoteException`. --- base/channels.jl | 25 +++ base/docs/helpdb/Base.jl | 305 ------------------------------------- base/multi.jl | 318 ++++++++++++++++++++++++++++++++++++++- doc/stdlib/parallel.rst | 126 +++++++++++----- 4 files changed, 425 insertions(+), 349 deletions(-) diff --git a/base/channels.jl b/base/channels.jl index 1461f14285b5d..cf932d2c55bac 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -21,6 +21,15 @@ end Channel(sz::Int = DEF_CHANNEL_SZ) = Channel{Any}(sz) closed_exception() = InvalidStateException("Channel is closed.", :closed) + +""" + close(c::Channel) + +Closes a channel. An exception is thrown by: + +* `put!` on a closed channel. +* `take!` and `fetch` on an empty, closed channel. +""" function close(c::Channel) c.state = :closed notify_error(c::Channel, closed_exception()) @@ -33,6 +42,11 @@ type InvalidStateException <: Exception state::Symbol end +""" + put!(c::Channel, v) + +Appends an item `v` to the channel `c`. Blocks if the channel is full. +""" function put!(c::Channel, v) !isopen(c) && throw(closed_exception()) while length(c.data) == c.sz_max @@ -50,6 +64,11 @@ function fetch(c::Channel) c.data[1] end +""" + take!(c::Channel) + +Removes and returns a value from a `Channel`. Blocks till data is available. +""" function take!(c::Channel) wait(c) v = shift!(c.data) @@ -59,6 +78,12 @@ end shift!(c::Channel) = take!(c) +""" + isready(c::Channel) + +Determine whether a `Channel` has a value stored to it. +`isready` on `Channel`s is non-blocking. +""" isready(c::Channel) = n_avail(c) > 0 function wait(c::Channel) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index e1c0eac115f33..4f6cb0b6dc26c 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -105,26 +105,6 @@ functionality instead. """ download -""" - @everywhere - -Execute an expression on all processes. Errors on any of the processes are collected into a -`CompositeException` and thrown. For example : - - @everywhere bar=1 - -will define `bar` under module `Main` on all processes. - -Unlike `@spawn` and `@spawnat`, `@everywhere` does not capture any local variables. Prefixing -`@everywhere` with `@eval` allows us to broadcast local variables using interpolation : - - foo = 1 - @eval @everywhere bar=\$foo - - -""" -:@everywhere - """ lstrip(string, [chars]) @@ -210,13 +190,6 @@ If `types` is specified, returns an array of methods whose types match. """ methods -""" - workers() - -Returns a list of all worker process identifiers. -""" -workers - """ isinteger(x) -> Bool @@ -381,45 +354,6 @@ values in matrix `A`, and `V` is a vector of the non-zero values. """ findnz -""" - Future() - -Create a `Future` on the local machine. -""" -Future() - -""" - Future(n) - -Create a `Future` on process `n`. -""" -Future(::Integer) - -""" - RemoteChannel() - -Make an reference to a `Channel{Any}(1)` on the local machine. -""" -RemoteChannel() - -""" - RemoteChannel(n) - -Make an reference to a `Channel{Any}(1)` on process `n`. -""" -RemoteChannel(::Integer) - -""" - RemoteChannel(f::Function, pid) - -Create references to remote channels of a specific size and type. `f()` is a function that -when executed on `pid` must return an implementation of an `AbstractChannel`. - -For example, `RemoteChannel(()->Channel{Int}(10), pid)`, will return a reference to a -channel of type `Int` and size 10 on `pid`. -""" -RemoteChannel(f::Function, pid) - """ foldl(op, v0, itr) @@ -708,14 +642,6 @@ Compute the hyperbolic secant of `x` """ sech -""" - nworkers() - -Get the number of available worker processes. This is one less than `nprocs()`. Equal to -`nprocs()` if `nprocs() == 1`. -""" -nworkers - """ filemode(file) @@ -1058,13 +984,6 @@ Scaled modified Bessel function of the second kind of order `nu`, ``K_\\nu(x) e^ """ besselkx -""" - myid() - -Get the id of the current process. -""" -myid - """ oct(n, [pad]) @@ -1072,14 +991,6 @@ Convert an integer to an octal string, optionally specifying a number of digits """ oct -""" - timedwait(testcb::Function, secs::Float64; pollint::Float64=0.1) - -Waits till `testcb` returns `true` or for `secs` seconds, whichever is earlier. `testcb` is -polled every `pollint` seconds. -""" -timedwait - """ sizeof(T) @@ -1652,16 +1563,6 @@ Close an I/O stream. Performs a `flush` first. """ close(stream::IO) -""" - close(Channel) - -Closes a channel. An exception is thrown by: - -* `put!` on a closed channel. -* `take!` and `fetch` on an empty, closed channel. -""" -close(::Channel) - """ cospi(x) @@ -2490,13 +2391,6 @@ mapping the SharedArray """ indexpids -""" - remotecall_wait(func, id, args...; kwargs...) - -Perform `wait(remotecall(...))` in one message. Keyword arguments, if any, are passed through to `func`. -""" -remotecall_wait - """ append!(collection, collection2) -> collection. @@ -3749,20 +3643,6 @@ descriptive error string. """ DimensionMismatch -""" - take!(RemoteChannel) - -Fetch a value from a remote channel, also removing it in the process. -""" -take!(::RemoteChannel) - -""" - take!(Channel) - -Removes and returns a value from a `Channel`. Blocks till data is available. -""" -take!(::Channel) - """ sort!(v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) @@ -4399,14 +4279,6 @@ Delete the mapping for the given key in a collection, and return the collection. """ delete! -""" - interrupt([pids...]) - -Interrupt the current executing task on the specified workers. This is equivalent to -pressing Ctrl-C on the local machine. If no arguments are given, all workers are interrupted. -""" -interrupt - """ std(v[, region]) @@ -4747,14 +4619,6 @@ value is a range of indexes where the matching sequence is found, such that `s[s """ search -""" - remotecall_fetch(func, id, args...; kwargs...) - -Perform `fetch(remotecall(...))` in one message. Keyword arguments, if any, are passed through to `func`. -Any remote exceptions are captured in a `RemoteException` and thrown. -""" -remotecall_fetch - """ contains(haystack, needle) @@ -5215,13 +5079,6 @@ julia> A """ shift! -""" - @fetch - -Equivalent to `fetch(@spawn expr)`. -""" -:@fetch - """ spawn(command) @@ -5622,30 +5479,6 @@ Return the minimum of the arguments. Operates elementwise over arrays. """ min -""" - isready(r::RemoteChannel) - -Determine whether a `RemoteChannel` has a value stored to it. Note that this function can -cause race conditions, since by the time you receive its result it may no longer be true. -However, it can be safely used on a `Future` since they are assigned only once. -""" -isready - -""" - isready(r::Future) - -Determine whether a `Future` has a value stored to it. - -If the argument `Future` is owned by a different node, this call will block to wait for the -answer. It is recommended to wait for `r` in a separate task instead, or to use a local -`Channel` as a proxy: - - c = Channel(1) - @async put!(c, remotecall_fetch(long_computation, p)) - isready(c) # will not block -""" - isready(r::Future) - """ InexactError() @@ -6342,13 +6175,6 @@ Test whether a number is infinite. """ isinf -""" - @fetchfrom - -Equivalent to `fetch(@spawnat p expr)`. -""" -:@fetchfrom - """ secd(x) @@ -6695,30 +6521,6 @@ An iterator that cycles through `iter` forever. """ cycle -""" - put!(RemoteChannel, value) - -Store a value to the remote channel. If the channel is full, blocks until space is available. -Returns its first argument. -""" -put!(::RemoteChannel, value) - -""" - put!(Future, value) - -Store a value to a future. Future's are write-once remote references. A `put!` on an already -set `Future` throws an Exception. All asynchronous remote calls return `Future`s and set the -value to the return value of the call upon completion. -""" -put!(::Future, value) - -""" - put!(Channel, value) - -Appends an item to the channel. Blocks if the channel is full. -""" -put!(::Channel, value) - """ operm(file) @@ -6736,13 +6538,6 @@ output (e.g. to avoid overflow). """ cumsum -""" - rmprocs(pids...) - -Removes the specified workers. -""" -rmprocs - """ rpad(string, n, p) @@ -6838,13 +6633,6 @@ Get the system time in seconds since the epoch, with fairly high (typically, mic """ time() -""" - procs() - -Returns a list of all process identifiers. -""" -procs - """ procs(S::SharedArray) @@ -7501,13 +7289,6 @@ retrieved by accessing `m.match` and the captured sequences can be retrieved by """ match -""" - nprocs() - -Get the number of available processes. -""" -nprocs - """ Ac_mul_B(A, B) @@ -7585,14 +7366,6 @@ result is a `Vector{UInt8,1}`. """ readavailable -""" - remotecall(func, id, args...; kwargs...) - -Call a function asynchronously on the given arguments on the specified process. Returns a `Future`. -Keyword arguments, if any, are passed through to `func`. -""" -remotecall - """ slicedim(A, d, i) @@ -7616,14 +7389,6 @@ Less-than-or-equals comparison operator. """ Base.:(<=) -""" - ProcessExitedException() - -After a client Julia process has exited, further attempts to reference the dead child will -throw this exception. -""" -ProcessExitedException - """ unsafe_load(p::Ptr{T}, [i::Integer=1]) @@ -7988,20 +7753,6 @@ Find the next index >= `i` of an element of `A` equal to `v` (using `==`), or `0 """ findnext(A,v,i) -""" - fetch(x) - -Waits and fetches a value from `x` depending on the type of `x`. Does not remove the item fetched: - -* `Future`: Wait for and get the value of a Future. The fetched value is cached locally. - Further calls to `fetch` on the same reference return the cached value. If the remote value - is an exception, throws a `RemoteException` which captures the remote exception and backtrace. -* `RemoteChannel`: Wait for and get the value of a remote reference. Exceptions raised are - same as for a `Future` . -* `Channel` : Wait for and get the first available item from the channel. -""" -fetch - """ angle(z) @@ -8187,29 +7938,6 @@ julia> map(+, [1, 2, 3], [10, 20, 30]) """ map -""" - @parallel - -A parallel for loop of the form : - - @parallel [reducer] for var = range - body - end - -The specified range is partitioned and locally executed across all workers. In case an -optional reducer function is specified, `@parallel` performs local reductions on each worker -with a final reduction on the calling process. - -Note that without a reducer function, `@parallel` executes asynchronously, i.e. it spawns -independent tasks on all available workers and returns immediately without waiting for -completion. To wait for completion, prefix the call with `@sync`, like : - - @sync @parallel for var = range - body - end -""" -:@parallel - """ throw(e) @@ -9281,36 +9009,3 @@ Base.:$(x, y) Get the IP address and the port that the given TCP socket is connected to (or bound to, in the case of TCPServer). """ getsockname - -""" - Base.remoteref_id(r::AbstractRemoteRef) -> (whence, id) - -A low-level API which returns the unique identifying tuple for a remote reference. A -reference id is a tuple of two elements - pid where the reference was created from and a -one-up number from that node. -""" -Base.remoteref_id - -""" - Base.channel_from_id(refid) -> c - -A low-level API which returns the backing AbstractChannel for an id returned by -`remoteref_id`. The call is valid only on the node where the backing channel exists. -""" -Base.channel_from_id - -""" - Base.worker_id_from_socket(s::IO) -> pid - -A low-level API which given a `IO` connection, returns the pid of the worker it is connected -to. This is useful when writing custom `serialize` methods for a type, which optimizes the -data written out depending on the receiving process id. -""" -Base.worker_id_from_socket - -""" - Base.cluster_cookie([cookie]) -> cookie - -Returns the cluster cookie. If a cookie is passed, also sets it as the cluster cookie. -""" -Base.cluster_cookie diff --git a/base/multi.jl b/base/multi.jl index 4e8b5a5617350..fd44466114a8e 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -368,7 +368,19 @@ const LPROC = LocalProcess() const HDR_VERSION_LEN=16 const HDR_COOKIE_LEN=16 + +""" + Base.cluster_cookie() -> cookie + +Returns the cluster cookie, by default the `LocalProcess` cookie. +""" cluster_cookie() = LPROC.cookie + +""" + Base.cluster_cookie(cookie) -> cookie + +Sets the passed cookie as the cluster cookie, then returns it. +""" function cluster_cookie(cookie) # The cookie must be an ASCII string with length <= HDR_COOKIE_LEN assert(isascii(cookie)) @@ -424,15 +436,44 @@ function get_bind_addr(w::Worker) get(w.config.bind_addr) end +""" + myid() + +Get the id of the current process. +""" myid() = LPROC.id +""" + nprocs() + +Get the number of available processes. +""" nprocs() = length(PGRP.workers) + +""" + nworkers() + +Get the number of available worker processes. This is one less than `nprocs()`. Equal to +`nprocs()` if `nprocs() == 1`. +""" function nworkers() n = nprocs() n == 1 ? 1 : n-1 end +""" + procs() + +Returns a list of all process identifiers. +""" procs() = Int[x.id for x in PGRP.workers] + +""" + procs(pid::Integer) + +Returns a list of all process identifiers visible to worker `pid`. +The result depends on the topology of the workgroup. +""" function procs(pid::Integer) if myid() == 1 if (pid == 1) || (isa(map_pid_wrkr[pid].manager, LocalManager)) @@ -446,6 +487,11 @@ function procs(pid::Integer) end end +""" + workers() + +Returns a list of all worker process identifiers. +""" function workers() allp = procs() if nprocs() == 1 @@ -455,7 +501,17 @@ function workers() end end -function rmprocs(args...; waitfor = 0.0) +""" + rmprocs(pids...; waitfor=0.0) + +Removes the specified workers. Note that only +process 1 can add or remove workers - if another +worker tries to call `rmprocs`, an error will be +thrown. The optional argument `waitfor` determines +how long the first process will wait for the workers +to shut down. +""" +function rmprocs(pids...; waitfor = 0.0) # Only pid 1 can add and remove processes if myid() != 1 error("only process 1 can add and remove processes") @@ -464,7 +520,7 @@ function rmprocs(args...; waitfor = 0.0) lock(worker_lock) try rmprocset = [] - for i in vcat(args...) + for i in vcat(pids...) if i == 1 warn("rmprocs: process 1 not removed") else @@ -492,7 +548,13 @@ function rmprocs(args...; waitfor = 0.0) end end +""" + ProcessExitedException() +After a client Julia process has exited, further attempts to reference the dead child will +throw this exception. +""" +ProcessExitedException() type ProcessExitedException <: Exception end worker_from_id(i) = worker_from_id(PGRP, i) @@ -512,6 +574,14 @@ function worker_from_id(pg::ProcessGroup, i) w end +""" + Base.worker_id_from_socket(s) -> pid + +A low-level API which given a `IO` connection or a `Worker`, +returns the pid of the worker it is connected to. +This is useful when writing custom `serialize` methods for a type, +which optimizes the data written out depending on the receiving process id. +""" function worker_id_from_socket(s) w = get(map_sock_wrkr, s, nothing) if isa(w,Worker) @@ -650,12 +720,47 @@ function finalize_ref(r::AbstractRemoteRef) return r end +""" + Future(w::LocalProcess) + +Create a `Future` on the local machine. +""" Future(w::LocalProcess) = Future(w.id) + +""" + Future(w::Worker) + +Create a `Future` on the (possibly remote) worker `w`. +""" Future(w::Worker) = Future(w.id) + +""" + Future(pid::Integer=myid()) + +Create a `Future` on process `pid`. +The default `pid` is the current process. +""" Future(pid::Integer=myid()) = Future(pid, RRID()) +""" + RemoteChannel(pid::Integer=myid()) + +Make a reference to a `Channel{Any}(1)` on process `pid`. +The default `pid` is the current process. +""" RemoteChannel(pid::Integer=myid()) = RemoteChannel{Channel{Any}}(pid, RRID()) +""" + RemoteChannel(f::Function, pid::Integer=myid()) + +Create references to remote channels of a specific size and type. `f()` is a function that +when executed on `pid` must return an implementation of an `AbstractChannel`. + +For example, `RemoteChannel(()->Channel{Int}(10), pid)`, will return a reference to a +channel of type `Int` and size 10 on `pid`. + +The default `pid` is the current process. +""" function RemoteChannel(f::Function, pid::Integer=myid()) remotecall_fetch(pid, f, RRID()) do f, rrid rv=lookup_ref(rrid, f) @@ -666,7 +771,21 @@ end hash(r::AbstractRemoteRef, h::UInt) = hash(r.whence, hash(r.id, h)) ==(r::AbstractRemoteRef, s::AbstractRemoteRef) = (r.whence==s.whence && r.id==s.id) +""" + Base.remoteref_id(r::AbstractRemoteRef) -> (whence, id) + +A low-level API which returns the unique identifying tuple for a remote reference. A +reference id is a tuple of two elements - `pid` where the reference was created from and a +one-up number from that node. +""" remoteref_id(r::AbstractRemoteRef) = RRID(r.whence, r.id) + +""" + Base.channel_from_id(id) -> c + +A low-level API which returns the backing `AbstractChannel` for an `id` returned by +`remoteref_id`. The call is valid only on the node where the backing channel exists. +""" function channel_from_id(id) rv = get(PGRP.refs, id, false) if rv === false @@ -686,6 +805,20 @@ function lookup_ref(pg, rrid, f) end rv end + +""" + isready(rr::Future) + +Determine whether a `Future` has a value stored to it. + +If the argument `Future` is owned by a different node, this call will block to wait for the answer. +It is recommended to wait for `rr` in a separate task instead +or to use a local `Channel` as a proxy: + + c = Channel(1) + @async put!(c, remotecall_fetch(long_computation, p)) + isready(c) # will not block +""" function isready(rr::Future) !isnull(rr.v) && return true @@ -697,6 +830,14 @@ function isready(rr::Future) end end +""" + isready(rr::RemoteChannel, args...) + +Determine whether a `RemoteChannel` has a value stored to it. +Note that this function can cause race conditions, since by the +time you receive its result it may no longer be true. However, +it can be safely used on a `Future` since they are assigned only once. +""" function isready(rr::RemoteChannel, args...) rid = remoteref_id(rr) if rr.where == myid() @@ -832,6 +973,14 @@ type RemoteException <: Exception captured::CapturedException end +""" + RemoteException(captured) + +Contains the `CapturedException` `captured`, which was +generated on a worker and can be passed back to the first +process as a result of `pmap` or other calls to workers +(e.g. `remotecall_fetch`). +""" RemoteException(captured) = RemoteException(myid(), captured) function showerror(io::IO, re::RemoteException) (re.pid != myid()) && print(io, "On worker ", re.pid, ":\n") @@ -871,12 +1020,26 @@ function local_remotecall_thunk(f, args, kwargs) return ()->f(args...; kwargs...) end +""" + remotecall(f, w::LocalProcess, args...; kwargs...) -> Future + +Call a function `f` asynchronously on the given arguments on the specified `LocalProcess`. +Returns a `Future`. +Keyword arguments, if any, are passed through to `f`. +""" function remotecall(f, w::LocalProcess, args...; kwargs...) rr = Future(w) schedule_call(remoteref_id(rr), local_remotecall_thunk(f, args, kwargs)) rr end +""" + remotecall(f, w::Worker, args...; kwargs...) -> Future + +Call a function `f` asynchronously on the given arguments on the specified `Worker`. +Returns a `Future`. +Keyword arguments, if any, are passed through to `f`. +""" function remotecall(f, w::Worker, args...; kwargs...) rr = Future(w) #println("$(myid()) asking for $rr") @@ -884,14 +1047,34 @@ function remotecall(f, w::Worker, args...; kwargs...) rr end +""" + remotecall(f, id::Integer, args...; kwargs...) -> Future + +Call a function `f` asynchronously on the given arguments on the specified process. +Returns a `Future`. +Keyword arguments, if any, are passed through to `f`. +""" remotecall(f, id::Integer, args...; kwargs...) = remotecall(f, worker_from_id(id), args...; kwargs...) -# faster version of fetch(remotecall(...)) +""" + remotecall_fetch(f, w::LocalProcess, args...; kwargs...) + +Perform a faster `fetch(remotecall(...))` in one message. +Keyword arguments, if any, are passed through to `f`. +Any remote exceptions are captured in a `RemoteException` and thrown. +""" function remotecall_fetch(f, w::LocalProcess, args...; kwargs...) v=run_work_thunk(local_remotecall_thunk(f,args, kwargs), false) isa(v, RemoteException) ? throw(v) : v end +""" + remotecall_fetch(f, w::Worker, args...; kwargs...) + +Perform a faster `fetch(remotecall(...))` in one message. +Keyword arguments, if any, are passed through to `f`. +Any remote exceptions are captured in a `RemoteException` and thrown. +""" function remotecall_fetch(f, w::Worker, args...; kwargs...) # can be weak, because the program will have no way to refer to the Ref # itself, it only gets the result. @@ -904,12 +1087,30 @@ function remotecall_fetch(f, w::Worker, args...; kwargs...) isa(v, RemoteException) ? throw(v) : v end +""" + remotecall_fetch(f, id::Integer, args...; kwargs...) + +Perform `fetch(remotecall(...))` in one message. +Keyword arguments, if any, are passed through to `f`. +Any remote exceptions are captured in a `RemoteException` and thrown. +""" remotecall_fetch(f, id::Integer, args...; kwargs...) = remotecall_fetch(f, worker_from_id(id), args...; kwargs...) -# faster version of wait(remotecall(...)) +""" + remotecall_wait(f, w::LocalProcess, args...; kwargs...) + +Perform `wait(remotecall(...))` in one message on `LocalProcess` `w`. +Keyword arguments, if any, are passed through to `f`. +""" remotecall_wait(f, w::LocalProcess, args...; kwargs...) = wait(remotecall(f, w, args...; kwargs...)) +""" + remotecall_wait(f, w::Worker, args...; kwargs...) + +Perform a faster `wait(remotecall(...))` in one message on `Worker` `w`. +Keyword arguments, if any, are passed through to `f`. +""" function remotecall_wait(f, w::Worker, args...; kwargs...) prid = RRID() rv = lookup_ref(prid) @@ -922,6 +1123,12 @@ function remotecall_wait(f, w::Worker, args...; kwargs...) rr end +""" + remotecall_wait(f, id::Integer, args...; kwargs...) + +Perform a faster `wait(remotecall(...))` in one message on the `Worker` specified by worker id `id`. +Keyword arguments, if any, are passed through to `f`. +""" remotecall_wait(f, id::Integer, args...; kwargs...) = remotecall_wait(f, worker_from_id(id), args...; kwargs...) @@ -980,9 +1187,31 @@ end fetch_ref(rid, args...) = fetch(lookup_ref(rid).c, args...) fetch(r::RemoteChannel, args...) = call_on_owner(fetch_ref, r, args...) + +""" + fetch(x) + +Waits and fetches a value from `x` depending on the type of `x`. Does not remove the item fetched: + +* `Future`: Wait for and get the value of a Future. The fetched value is cached locally. + Further calls to `fetch` on the same reference return the cached value. If the remote value + is an exception, throws a `RemoteException` which captures the remote exception and backtrace. +* `RemoteChannel`: Wait for and get the value of a remote reference. Exceptions raised are + same as for a `Future` . +* `Channel` : Wait for and get the first available item from the channel. +""" fetch(x::ANY) = x isready(rv::RemoteValue, args...) = isready(rv.c, args...) + +""" + put!(rr::Future, v) + +Store a value to a `Future` `rr`. `Future`s are write-once remote references. +A `put!` on an already set `Future` throws an `Exception`. +All asynchronous remote calls return `Future`s and set the +value to the return value of the call upon completion. +""" function put!(rr::Future, v) !isnull(rr.v) && error("Future can be set only once") call_on_owner(put_future, rr, v, myid()) @@ -1001,6 +1230,14 @@ end put!(rv::RemoteValue, args...) = put!(rv.c, args...) put_ref(rid, args...) = (put!(lookup_ref(rid), args...); nothing) + +""" + put!(rr::RemoteChannel, args...) + +Store a set of values to the `RemoteChannel`. +If the channel is full, blocks until space is available. +Returns its first argument. +""" put!(rr::RemoteChannel, args...) = (call_on_owner(put_ref, rr, args...); rr) # take! is not supported on Future @@ -1011,6 +1248,12 @@ function take_ref(rid, callee, args...) isa(v, RemoteException) && (myid() == callee) && throw(v) v end + +""" + take!(rr::RemoteChannel, args...) + +Fetch value(s) from a remote channel, removing the value(s) in the processs. +""" take!(rr::RemoteChannel, args...) = call_on_owner(take_ref, rr, myid(), args...) # close is not supported on Future @@ -1692,6 +1935,11 @@ macro spawnat(p, expr) :(spawnat($(esc(p)), $expr)) end +""" + @fetch + +Equivalent to `fetch(@spawn expr)`. +""" macro fetch(expr) expr = localize_vars(esc(:(()->($expr))), false) quote @@ -1700,11 +1948,34 @@ macro fetch(expr) end end +""" + @fetchfrom + +Equivalent to `fetch(@spawnat p expr)`. +""" macro fetchfrom(p, expr) expr = localize_vars(esc(:(()->($expr))), false) :(remotecall_fetch($expr, $(esc(p)))) end +""" + @everywhere + +Execute an expression on all processes. Errors on any of the processes are collected into a +`CompositeException` and thrown. For example : + + @everywhere bar=1 + +will define `bar` under module `Main` on all processes. + +Unlike `@spawn` and `@spawnat`, `@everywhere` does not capture any local variables. Prefixing +`@everywhere` with `@eval` allows us to broadcast local variables using interpolation : + + foo = 1 + @eval @everywhere bar=\$foo + + +""" macro everywhere(ex) quote sync_begin() @@ -1785,6 +2056,27 @@ function make_pfor_body(var, body) end end +""" + @parallel + +A parallel for loop of the form : + + @parallel [reducer] for var = range + body + end + +The specified range is partitioned and locally executed across all workers. In case an +optional reducer function is specified, `@parallel` performs local reductions on each worker +with a final reduction on the calling process. + +Note that without a reducer function, `@parallel` executes asynchronously, i.e. it spawns +independent tasks on all available workers and returns immediately without waiting for +completion. To wait for completion, prefix the call with `@sync`, like : + + @sync @parallel for var = range + body + end +""" macro parallel(args...) na = length(args) if na==1 @@ -1830,7 +2122,12 @@ function check_master_connect() end end +""" + timedwait(testcb::Function, secs::Float64; pollint::Float64=0.1) +Waits till `testcb` returns `true` or for `secs` seconds, whichever is earlier. +`testcb` is polled every `pollint` seconds. +""" function timedwait(testcb::Function, secs::Float64; pollint::Float64=0.1) pollint > 0 || throw(ArgumentError("cannot set pollint to $pollint seconds")) start = time() @@ -1866,8 +2163,21 @@ function interrupt(pid::Integer) manage(w.manager, w.id, w.config, :interrupt) end end + +""" + interrupt(pids::Integer...) + +Interrupt the current executing task on the specified workers. This is equivalent to +pressing Ctrl-C on the local machine. If no arguments are given, all workers are interrupted. +""" interrupt(pids::Integer...) = interrupt([pids...]) +""" + interrupt(pids::AbstractVector=workers()) + +Interrupt the current executing task on the specified workers. This is equivalent to +pressing Ctrl-C on the local machine. If no arguments are given, all workers are interrupted. +""" function interrupt(pids::AbstractVector=workers()) assert(myid() == 1) @sync begin diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 5b4f7e9c73465..649ed5ba29cbf 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -205,13 +205,19 @@ General Parallel Computing Support Returns a list of all worker process identifiers. -.. function:: rmprocs(pids...) +.. function:: rmprocs(pids...; waitfor=0.0) .. Docstring generated from Julia source - Removes the specified workers. + Removes the specified workers. Note that only process 1 can add or remove workers - if another worker tries to call ``rmprocs``\ , an error will be thrown. The optional argument ``waitfor`` determines how long the first process will wait for the workers to shut down. -.. function:: interrupt([pids...]) +.. function:: interrupt(pids::AbstractVector=workers()) + + .. Docstring generated from Julia source + + Interrupt the current executing task on the specified workers. This is equivalent to pressing Ctrl-C on the local machine. If no arguments are given, all workers are interrupted. + +.. function:: interrupt(pids::Integer...) .. Docstring generated from Julia source @@ -257,43 +263,51 @@ General Parallel Computing Support * ``pmap(f, c; retry_n=1)`` and ``asyncmap(retry(remote(f)),c)`` * ``pmap(f, c; retry_n=1, on_error=e->e)`` and ``asyncmap(x->try retry(remote(f))(x) catch e; e end, c)`` -.. function:: remotecall(func, id, args...; kwargs...) +.. function:: remotecall(f, w::LocalProcess, args...; kwargs...) -> Future .. Docstring generated from Julia source - Call a function asynchronously on the given arguments on the specified process. Returns a ``Future``\ . Keyword arguments, if any, are passed through to ``func``\ . + Call a function ``f`` asynchronously on the given arguments on the specified ``LocalProcess``\ . Returns a ``Future``\ . Keyword arguments, if any, are passed through to ``f``\ . -.. function:: Base.process_messages(r_stream::IO, w_stream::IO, incoming::Bool=true) +.. function:: remotecall(f, w::Worker, args...; kwargs...) -> Future .. Docstring generated from Julia source - Called by cluster managers using custom transports. It should be called when the custom transport implementation receives the first message from a remote worker. The custom transport must manage a logical connection to the remote worker and provide two ``IO`` objects, one for incoming messages and the other for messages addressed to the remote worker. If ``incoming`` is ``true``\ , the remote peer initiated the connection. Whichever of the pair initiates the connection sends the cluster cookie and its Julia version number to perform the authentication handshake. + Call a function ``f`` asynchronously on the given arguments on the specified ``Worker``\ . Returns a ``Future``\ . Keyword arguments, if any, are passed through to ``f``\ . -.. function:: Future() +.. function:: remotecall(f, id::Integer, args...; kwargs...) -> Future .. Docstring generated from Julia source - Create a ``Future`` on the local machine. + Call a function ``f`` asynchronously on the given arguments on the specified process. Returns a ``Future``\ . Keyword arguments, if any, are passed through to ``f``\ . -.. function:: Future(n) +.. function:: Base.process_messages(r_stream::IO, w_stream::IO, incoming::Bool=true) .. Docstring generated from Julia source - Create a ``Future`` on process ``n``\ . + Called by cluster managers using custom transports. It should be called when the custom transport implementation receives the first message from a remote worker. The custom transport must manage a logical connection to the remote worker and provide two ``IO`` objects, one for incoming messages and the other for messages addressed to the remote worker. If ``incoming`` is ``true``\ , the remote peer initiated the connection. Whichever of the pair initiates the connection sends the cluster cookie and its Julia version number to perform the authentication handshake. -.. function:: RemoteChannel() +.. function:: Future() + +.. function:: RemoteException(captured) .. Docstring generated from Julia source - Make an reference to a ``Channel{Any}(1)`` on the local machine. + Contains the ``CapturedException`` ``captured``\ , which was generated on a worker and can be passed back to the first process as a result of ``pmap`` or other calls to workers (e.g. ``remotecall_fetch``\ ). -.. function:: RemoteChannel(n) +.. function:: Future(pid::Integer=myid()) .. Docstring generated from Julia source - Make an reference to a ``Channel{Any}(1)`` on process ``n``\ . + Create a ``Future`` on process ``pid``\ . The default ``pid`` is the current process. -.. function:: RemoteChannel(f::Function, pid) +.. function:: RemoteChannel(pid::Integer=myid()) + + .. Docstring generated from Julia source + + Make a reference to a ``Channel{Any}(1)`` on process ``pid``\ . The default ``pid`` is the current process. + +.. function:: RemoteChannel(f::Function, pid::Integer=myid()) .. Docstring generated from Julia source @@ -301,6 +315,8 @@ General Parallel Computing Support For example, ``RemoteChannel(()->Channel{Int}(10), pid)``\ , will return a reference to a channel of type ``Int`` and size 10 on ``pid``\ . + The default ``pid`` is the current process. + .. function:: wait([x]) .. Docstring generated from Julia source @@ -329,61 +345,85 @@ General Parallel Computing Support * ``RemoteChannel``\ : Wait for and get the value of a remote reference. Exceptions raised are same as for a ``Future`` . * ``Channel`` : Wait for and get the first available item from the channel. -.. function:: remotecall_wait(func, id, args...; kwargs...) +.. function:: remotecall_wait(f, w::LocalProcess, args...; kwargs...) .. Docstring generated from Julia source - Perform ``wait(remotecall(...))`` in one message. Keyword arguments, if any, are passed through to ``func``\ . + Perform ``wait(remotecall(...))`` in one message on ``LocalProcess`` ``w``\ . Keyword arguments, if any, are passed through to ``f``\ . -.. function:: remotecall_fetch(func, id, args...; kwargs...) +.. function:: remotecall_wait(f, w::Worker, args...; kwargs...) .. Docstring generated from Julia source - Perform ``fetch(remotecall(...))`` in one message. Keyword arguments, if any, are passed through to ``func``\ . Any remote exceptions are captured in a ``RemoteException`` and thrown. + Perform a faster ``wait(remotecall(...))`` in one message on ``Worker`` ``w``\ . Keyword arguments, if any, are passed through to ``f``\ . -.. function:: put!(RemoteChannel, value) +.. function:: remotecall_wait(f, id::Integer, args...; kwargs...) .. Docstring generated from Julia source - Store a value to the remote channel. If the channel is full, blocks until space is available. Returns its first argument. + Perform a faster ``wait(remotecall(...))`` in one message on the ``Worker`` specified by worker id ``id``\ . Keyword arguments, if any, are passed through to ``f``\ . -.. function:: put!(Future, value) +.. function:: remotecall_fetch(f, w::LocalProcess, args...; kwargs...) .. Docstring generated from Julia source - Store a value to a future. Future's are write-once remote references. A ``put!`` on an already set ``Future`` throws an Exception. All asynchronous remote calls return ``Future``\ s and set the value to the return value of the call upon completion. + Perform a faster ``fetch(remotecall(...))`` in one message. Keyword arguments, if any, are passed through to ``f``\ . Any remote exceptions are captured in a ``RemoteException`` and thrown. -.. function:: put!(Channel, value) +.. function:: remotecall_fetch(f, w::Worker, args...; kwargs...) .. Docstring generated from Julia source - Appends an item to the channel. Blocks if the channel is full. + Perform a faster ``fetch(remotecall(...))`` in one message. Keyword arguments, if any, are passed through to ``f``\ . Any remote exceptions are captured in a ``RemoteException`` and thrown. -.. function:: take!(RemoteChannel) +.. function:: remotecall_fetch(f, id::Integer, args...; kwargs...) .. Docstring generated from Julia source - Fetch a value from a remote channel, also removing it in the process. + Perform ``fetch(remotecall(...))`` in one message. Keyword arguments, if any, are passed through to ``f``\ . Any remote exceptions are captured in a ``RemoteException`` and thrown. -.. function:: take!(Channel) +.. function:: put!(rr::RemoteChannel, args...) + + .. Docstring generated from Julia source + + Store a set of values to the ``RemoteChannel``\ . If the channel is full, blocks until space is available. Returns its first argument. + +.. function:: put!(rr::Future, v) + + .. Docstring generated from Julia source + + Store a value to a ``Future`` ``rr``\ . ``Future``\ s are write-once remote references. A ``put!`` on an already set ``Future`` throws an ``Exception``\ . All asynchronous remote calls return ``Future``\ s and set the value to the return value of the call upon completion. + +.. function:: put!(c::Channel, v) + + .. Docstring generated from Julia source + + Appends an item ``v`` to the channel ``c``\ . Blocks if the channel is full. + +.. function:: take!(rr::RemoteChannel, args...) + + .. Docstring generated from Julia source + + Fetch value(s) from a remote channel, removing the value(s) in the processs. + +.. function:: take!(c::Channel) .. Docstring generated from Julia source Removes and returns a value from a ``Channel``\ . Blocks till data is available. -.. function:: isready(r::RemoteChannel) +.. function:: isready(rr::RemoteChannel, args...) .. Docstring generated from Julia source Determine whether a ``RemoteChannel`` has a value stored to it. Note that this function can cause race conditions, since by the time you receive its result it may no longer be true. However, it can be safely used on a ``Future`` since they are assigned only once. -.. function:: isready(r::Future) +.. function:: isready(rr::Future) .. Docstring generated from Julia source Determine whether a ``Future`` has a value stored to it. - If the argument ``Future`` is owned by a different node, this call will block to wait for the answer. It is recommended to wait for ``r`` in a separate task instead, or to use a local ``Channel`` as a proxy: + If the argument ``Future`` is owned by a different node, this call will block to wait for the answer. It is recommended to wait for ``rr`` in a separate task instead or to use a local ``Channel`` as a proxy: .. code-block:: julia @@ -391,7 +431,7 @@ General Parallel Computing Support @async put!(c, remotecall_fetch(long_computation, p)) isready(c) # will not block -.. function:: close(Channel) +.. function:: close(c::Channel) .. Docstring generated from Julia source @@ -551,25 +591,31 @@ General Parallel Computing Support .. Docstring generated from Julia source - A low-level API which returns the unique identifying tuple for a remote reference. A reference id is a tuple of two elements - pid where the reference was created from and a one-up number from that node. + A low-level API which returns the unique identifying tuple for a remote reference. A reference id is a tuple of two elements - ``pid`` where the reference was created from and a one-up number from that node. + +.. function:: Base.channel_from_id(id) -> c + + .. Docstring generated from Julia source + + A low-level API which returns the backing ``AbstractChannel`` for an ``id`` returned by ``remoteref_id``\ . The call is valid only on the node where the backing channel exists. -.. function:: Base.channel_from_id(refid) -> c +.. function:: Base.worker_id_from_socket(s) -> pid .. Docstring generated from Julia source - A low-level API which returns the backing AbstractChannel for an id returned by ``remoteref_id``\ . The call is valid only on the node where the backing channel exists. + A low-level API which given a ``IO`` connection or a ``Worker``\ , returns the pid of the worker it is connected to. This is useful when writing custom ``serialize`` methods for a type, which optimizes the data written out depending on the receiving process id. -.. function:: Base.worker_id_from_socket(s::IO) -> pid +.. function:: Base.cluster_cookie() -> cookie .. Docstring generated from Julia source - A low-level API which given a ``IO`` connection, returns the pid of the worker it is connected to. This is useful when writing custom ``serialize`` methods for a type, which optimizes the data written out depending on the receiving process id. + Returns the cluster cookie, by default the ``LocalProcess`` cookie. -.. function:: Base.cluster_cookie([cookie]) -> cookie +.. function:: Base.cluster_cookie(cookie) -> cookie .. Docstring generated from Julia source - Returns the cluster cookie. If a cookie is passed, also sets it as the cluster cookie. + Sets the passed cookie as the cluster cookie, then returns it. Shared Arrays ------------- From f66eb607238e2a0789b18f4d15542db443afd5ee Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Fri, 29 Jul 2016 22:23:08 -0700 Subject: [PATCH 0769/1117] Remove docs for unexported types --- base/multi.jl | 79 +++++++++++------------------------------ doc/stdlib/parallel.rst | 52 ++++++++------------------- 2 files changed, 35 insertions(+), 96 deletions(-) diff --git a/base/multi.jl b/base/multi.jl index fd44466114a8e..f8b479de39861 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -372,7 +372,7 @@ const HDR_COOKIE_LEN=16 """ Base.cluster_cookie() -> cookie -Returns the cluster cookie, by default the `LocalProcess` cookie. +Returns the cluster cookie. """ cluster_cookie() = LPROC.cookie @@ -720,18 +720,7 @@ function finalize_ref(r::AbstractRemoteRef) return r end -""" - Future(w::LocalProcess) - -Create a `Future` on the local machine. -""" Future(w::LocalProcess) = Future(w.id) - -""" - Future(w::Worker) - -Create a `Future` on the (possibly remote) worker `w`. -""" Future(w::Worker) = Future(w.id) """ @@ -772,11 +761,25 @@ hash(r::AbstractRemoteRef, h::UInt) = hash(r.whence, hash(r.id, h)) ==(r::AbstractRemoteRef, s::AbstractRemoteRef) = (r.whence==s.whence && r.id==s.id) """ - Base.remoteref_id(r::AbstractRemoteRef) -> (whence, id) + Base.remoteref_id(r::AbstractRemoteRef) -> RRID + +`Future`s and `RemoteChannel`s are identified by fields: + +`where` - refers to the node where the underlying object/storage +referred to by the reference actually exists. + +`whence` - refers to the node the remote reference was created from. + Note that this is different from the node where the underlying object + referred to actually exists. For example calling `RemoteChannel(2)` + from the master process would result in a `where` value of 2 and + a `whence` value of 1. -A low-level API which returns the unique identifying tuple for a remote reference. A -reference id is a tuple of two elements - `pid` where the reference was created from and a -one-up number from that node. + `id` is unique across all references created from the worker specified by `whence`. + + Taken together, `whence` and `id` uniquely identify a reference across all workers. + + `Base.remoteref_id` is a low-level API which returns a `Base.RRID` + object that wraps `whence` and `id` values of a remote reference. """ remoteref_id(r::AbstractRemoteRef) = RRID(r.whence, r.id) @@ -784,7 +787,8 @@ remoteref_id(r::AbstractRemoteRef) = RRID(r.whence, r.id) Base.channel_from_id(id) -> c A low-level API which returns the backing `AbstractChannel` for an `id` returned by -`remoteref_id`. The call is valid only on the node where the backing channel exists. +[`remoteref_id`](:func:`Base.remoteref_id`). +The call is valid only on the node where the backing channel exists. """ function channel_from_id(id) rv = get(PGRP.refs, id, false) @@ -1020,29 +1024,14 @@ function local_remotecall_thunk(f, args, kwargs) return ()->f(args...; kwargs...) end -""" - remotecall(f, w::LocalProcess, args...; kwargs...) -> Future - -Call a function `f` asynchronously on the given arguments on the specified `LocalProcess`. -Returns a `Future`. -Keyword arguments, if any, are passed through to `f`. -""" function remotecall(f, w::LocalProcess, args...; kwargs...) rr = Future(w) schedule_call(remoteref_id(rr), local_remotecall_thunk(f, args, kwargs)) rr end -""" - remotecall(f, w::Worker, args...; kwargs...) -> Future - -Call a function `f` asynchronously on the given arguments on the specified `Worker`. -Returns a `Future`. -Keyword arguments, if any, are passed through to `f`. -""" function remotecall(f, w::Worker, args...; kwargs...) rr = Future(w) - #println("$(myid()) asking for $rr") send_msg(w, MsgHeader(remoteref_id(rr)), CallMsg{:call}(f, args, kwargs)) rr end @@ -1056,25 +1045,11 @@ Keyword arguments, if any, are passed through to `f`. """ remotecall(f, id::Integer, args...; kwargs...) = remotecall(f, worker_from_id(id), args...; kwargs...) -""" - remotecall_fetch(f, w::LocalProcess, args...; kwargs...) - -Perform a faster `fetch(remotecall(...))` in one message. -Keyword arguments, if any, are passed through to `f`. -Any remote exceptions are captured in a `RemoteException` and thrown. -""" function remotecall_fetch(f, w::LocalProcess, args...; kwargs...) v=run_work_thunk(local_remotecall_thunk(f,args, kwargs), false) isa(v, RemoteException) ? throw(v) : v end -""" - remotecall_fetch(f, w::Worker, args...; kwargs...) - -Perform a faster `fetch(remotecall(...))` in one message. -Keyword arguments, if any, are passed through to `f`. -Any remote exceptions are captured in a `RemoteException` and thrown. -""" function remotecall_fetch(f, w::Worker, args...; kwargs...) # can be weak, because the program will have no way to refer to the Ref # itself, it only gets the result. @@ -1097,20 +1072,8 @@ Any remote exceptions are captured in a `RemoteException` and thrown. remotecall_fetch(f, id::Integer, args...; kwargs...) = remotecall_fetch(f, worker_from_id(id), args...; kwargs...) -""" - remotecall_wait(f, w::LocalProcess, args...; kwargs...) - -Perform `wait(remotecall(...))` in one message on `LocalProcess` `w`. -Keyword arguments, if any, are passed through to `f`. -""" remotecall_wait(f, w::LocalProcess, args...; kwargs...) = wait(remotecall(f, w, args...; kwargs...)) -""" - remotecall_wait(f, w::Worker, args...; kwargs...) - -Perform a faster `wait(remotecall(...))` in one message on `Worker` `w`. -Keyword arguments, if any, are passed through to `f`. -""" function remotecall_wait(f, w::Worker, args...; kwargs...) prid = RRID() rv = lookup_ref(prid) diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 649ed5ba29cbf..58333f2364e0c 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -199,6 +199,12 @@ General Parallel Computing Support Returns a list of all process identifiers. +.. function:: procs(pid::Integer) + + .. Docstring generated from Julia source + + Returns a list of all process identifiers visible to worker ``pid``\ . The result depends on the topology of the workgroup. + .. function:: workers() .. Docstring generated from Julia source @@ -263,18 +269,6 @@ General Parallel Computing Support * ``pmap(f, c; retry_n=1)`` and ``asyncmap(retry(remote(f)),c)`` * ``pmap(f, c; retry_n=1, on_error=e->e)`` and ``asyncmap(x->try retry(remote(f))(x) catch e; e end, c)`` -.. function:: remotecall(f, w::LocalProcess, args...; kwargs...) -> Future - - .. Docstring generated from Julia source - - Call a function ``f`` asynchronously on the given arguments on the specified ``LocalProcess``\ . Returns a ``Future``\ . Keyword arguments, if any, are passed through to ``f``\ . - -.. function:: remotecall(f, w::Worker, args...; kwargs...) -> Future - - .. Docstring generated from Julia source - - Call a function ``f`` asynchronously on the given arguments on the specified ``Worker``\ . Returns a ``Future``\ . Keyword arguments, if any, are passed through to ``f``\ . - .. function:: remotecall(f, id::Integer, args...; kwargs...) -> Future .. Docstring generated from Julia source @@ -345,36 +339,12 @@ General Parallel Computing Support * ``RemoteChannel``\ : Wait for and get the value of a remote reference. Exceptions raised are same as for a ``Future`` . * ``Channel`` : Wait for and get the first available item from the channel. -.. function:: remotecall_wait(f, w::LocalProcess, args...; kwargs...) - - .. Docstring generated from Julia source - - Perform ``wait(remotecall(...))`` in one message on ``LocalProcess`` ``w``\ . Keyword arguments, if any, are passed through to ``f``\ . - -.. function:: remotecall_wait(f, w::Worker, args...; kwargs...) - - .. Docstring generated from Julia source - - Perform a faster ``wait(remotecall(...))`` in one message on ``Worker`` ``w``\ . Keyword arguments, if any, are passed through to ``f``\ . - .. function:: remotecall_wait(f, id::Integer, args...; kwargs...) .. Docstring generated from Julia source Perform a faster ``wait(remotecall(...))`` in one message on the ``Worker`` specified by worker id ``id``\ . Keyword arguments, if any, are passed through to ``f``\ . -.. function:: remotecall_fetch(f, w::LocalProcess, args...; kwargs...) - - .. Docstring generated from Julia source - - Perform a faster ``fetch(remotecall(...))`` in one message. Keyword arguments, if any, are passed through to ``f``\ . Any remote exceptions are captured in a ``RemoteException`` and thrown. - -.. function:: remotecall_fetch(f, w::Worker, args...; kwargs...) - - .. Docstring generated from Julia source - - Perform a faster ``fetch(remotecall(...))`` in one message. Keyword arguments, if any, are passed through to ``f``\ . Any remote exceptions are captured in a ``RemoteException`` and thrown. - .. function:: remotecall_fetch(f, id::Integer, args...; kwargs...) .. Docstring generated from Julia source @@ -411,6 +381,12 @@ General Parallel Computing Support Removes and returns a value from a ``Channel``\ . Blocks till data is available. +.. function:: isready(c::Channel) + + .. Docstring generated from Julia source + + Determine whether a ``Channel`` has a value stored to it. ``isready`` on ``Channel``\ s is non-blocking. + .. function:: isready(rr::RemoteChannel, args...) .. Docstring generated from Julia source @@ -597,7 +573,7 @@ General Parallel Computing Support .. Docstring generated from Julia source - A low-level API which returns the backing ``AbstractChannel`` for an ``id`` returned by ``remoteref_id``\ . The call is valid only on the node where the backing channel exists. + A low-level API which returns the backing ``AbstractChannel`` for an ``id`` returned by :func:`Base.remoteref_id`\ . The call is valid only on the node where the backing channel exists. .. function:: Base.worker_id_from_socket(s) -> pid @@ -609,7 +585,7 @@ General Parallel Computing Support .. Docstring generated from Julia source - Returns the cluster cookie, by default the ``LocalProcess`` cookie. + Returns the cluster cookie. .. function:: Base.cluster_cookie(cookie) -> cookie From 3a582bb768aeb454675d104e39b34203e4c36284 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Sat, 30 Jul 2016 14:19:20 -0700 Subject: [PATCH 0770/1117] [ci skip] Fix some issues from ClusterManagers pr and bad formatting --- base/managers.jl | 4 ++-- base/multi.jl | 13 ++++++------- doc/manual/parallel-computing.rst | 2 +- doc/stdlib/parallel.rst | 26 +++++++++++++++++--------- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/base/managers.jl b/base/managers.jl index 53e0db798df3e..76ee09ef8631f 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -90,7 +90,7 @@ Keyword arguments: + `topology=:all_to_all` : All processes are connected to each other. This is the default. - + `topology=:master_slave` : Only the driver process, i.e. pid 1 connects to the + + `topology=:master_slave` : Only the driver process, i.e. `pid` 1 connects to the workers. The workers do not connect to each other. + `topology=:custom` : The `launch` method of the cluster manager specifes the @@ -104,7 +104,7 @@ Environment variables : If the master process fails to establish a connection with a newly launched worker within 60.0 seconds, the worker treats it as a fatal situation and terminates. This timeout can be controlled via environment variable `JULIA_WORKER_TIMEOUT`. -The value of JULIA_WORKER_TIMEOUT` on the master process specifies the number of seconds a +The value of `JULIA_WORKER_TIMEOUT` on the master process specifies the number of seconds a newly launched worker waits for connection establishment. """ function addprocs(machines::AbstractVector; tunnel=false, sshflags=``, max_parallel=10, kwargs...) diff --git a/base/multi.jl b/base/multi.jl index f8b479de39861..48ae0617c6d51 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -471,8 +471,8 @@ procs() = Int[x.id for x in PGRP.workers] """ procs(pid::Integer) -Returns a list of all process identifiers visible to worker `pid`. -The result depends on the topology of the workgroup. +Returns a list of all process identifiers on the same physical node. +Specifically all workers bound to the same ip-address as `pid` are returned. """ function procs(pid::Integer) if myid() == 1 @@ -578,7 +578,7 @@ end Base.worker_id_from_socket(s) -> pid A low-level API which given a `IO` connection or a `Worker`, -returns the pid of the worker it is connected to. +returns the `pid` of the worker it is connected to. This is useful when writing custom `serialize` methods for a type, which optimizes the data written out depending on the receiving process id. """ @@ -980,10 +980,9 @@ end """ RemoteException(captured) -Contains the `CapturedException` `captured`, which was -generated on a worker and can be passed back to the first -process as a result of `pmap` or other calls to workers -(e.g. `remotecall_fetch`). +Exceptions on remote computations are captured and rethrown locally. A `RemoteException` +wraps the pid of the worker and a captured exception. A `CapturedException` captures the +remote exception and a serializable form of the call stack when the exception was raised. """ RemoteException(captured) = RemoteException(myid(), captured) function showerror(io::IO, re::RemoteException) diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 0317171a1a9c3..ea37e09dc8f7d 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -986,7 +986,7 @@ Keyword argument ``topology`` to ``addprocs`` is used to specify how the workers - ``:all_to_all`` : is the default, where all workers are connected to each other. -- ``:master_slave`` : only the driver process, i.e. `pid` 1 has connections to the workers. +- ``:master_slave`` : only the driver process, i.e. ``pid`` 1 has connections to the workers. - ``:custom`` : the ``launch`` method of the cluster manager specifies the connection topology. Fields ``ident`` and ``connect_idents`` in ``WorkerConfig`` are used to specify the same. diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 58333f2364e0c..4e644d2b80a36 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -164,12 +164,12 @@ General Parallel Computing Support * ``topology``\ : Specifies how the workers connect to each other. Sending a message between unconnected workers results in an error. * ``topology=:all_to_all`` : All processes are connected to each other. This is the default. - * ``topology=:master_slave`` : Only the driver process, i.e. pid 1 connects to the workers. The workers do not connect to each other. + * ``topology=:master_slave`` : Only the driver process, i.e. ``pid`` 1 connects to the workers. The workers do not connect to each other. * ``topology=:custom`` : The ``launch`` method of the cluster manager specifes the connection topology via fields ``ident`` and ``connect_idents`` in ``WorkerConfig``\ . A worker with a cluster manager identity ``ident`` will connect to all workers specified in ``connect_idents``\ . Environment variables : - If the master process fails to establish a connection with a newly launched worker within 60.0 seconds, the worker treats it as a fatal situation and terminates. This timeout can be controlled via environment variable ``JULIA_WORKER_TIMEOUT``\ . The value of JULIA_WORKER_TIMEOUT` on the master process specifies the number of seconds a newly launched worker waits for connection establishment. + If the master process fails to establish a connection with a newly launched worker within 60.0 seconds, the worker treats it as a fatal situation and terminates. This timeout can be controlled via environment variable ``JULIA_WORKER_TIMEOUT``\ . The value of ``JULIA_WORKER_TIMEOUT`` on the master process specifies the number of seconds a newly launched worker waits for connection establishment. .. function:: addprocs(manager::ClusterManager; kwargs...) -> List of process identifiers @@ -203,7 +203,7 @@ General Parallel Computing Support .. Docstring generated from Julia source - Returns a list of all process identifiers visible to worker ``pid``\ . The result depends on the topology of the workgroup. + Returns a list of all process identifiers on the same physical node. Specifically all workers bound to the same ip-address as ``pid`` are returned. .. function:: workers() @@ -281,13 +281,11 @@ General Parallel Computing Support Called by cluster managers using custom transports. It should be called when the custom transport implementation receives the first message from a remote worker. The custom transport must manage a logical connection to the remote worker and provide two ``IO`` objects, one for incoming messages and the other for messages addressed to the remote worker. If ``incoming`` is ``true``\ , the remote peer initiated the connection. Whichever of the pair initiates the connection sends the cluster cookie and its Julia version number to perform the authentication handshake. -.. function:: Future() - .. function:: RemoteException(captured) .. Docstring generated from Julia source - Contains the ``CapturedException`` ``captured``\ , which was generated on a worker and can be passed back to the first process as a result of ``pmap`` or other calls to workers (e.g. ``remotecall_fetch``\ ). + Exceptions on remote computations are captured and rethrown locally. A ``RemoteException`` wraps the pid of the worker and a captured exception. A ``CapturedException`` captures the remote exception and a serializable form of the call stack when the exception was raised. .. function:: Future(pid::Integer=myid()) @@ -563,11 +561,21 @@ General Parallel Computing Support Removes all cached functions from all participating workers. -.. function:: Base.remoteref_id(r::AbstractRemoteRef) -> (whence, id) +.. function:: Base.remoteref_id(r::AbstractRemoteRef) -> RRID .. Docstring generated from Julia source - A low-level API which returns the unique identifying tuple for a remote reference. A reference id is a tuple of two elements - ``pid`` where the reference was created from and a one-up number from that node. + ``Future``\ s and ``RemoteChannel``\ s are identified by fields: + + ``where`` - refers to the node where the underlying object/storage referred to by the reference actually exists. + + ``whence`` - refers to the node the remote reference was created from. Note that this is different from the node where the underlying object referred to actually exists. For example calling ``RemoteChannel(2)`` from the master process would result in a ``where`` value of 2 and a ``whence`` value of 1. + + ``id`` is unique across all references created from the worker specified by ``whence``\ . + + Taken together, ``whence`` and ``id`` uniquely identify a reference across all workers. + + ``Base.remoteref_id`` is a low-level API which returns a ``Base.RRID`` object that wraps ``whence`` and ``id`` values of a remote reference. .. function:: Base.channel_from_id(id) -> c @@ -579,7 +587,7 @@ General Parallel Computing Support .. Docstring generated from Julia source - A low-level API which given a ``IO`` connection or a ``Worker``\ , returns the pid of the worker it is connected to. This is useful when writing custom ``serialize`` methods for a type, which optimizes the data written out depending on the receiving process id. + A low-level API which given a ``IO`` connection or a ``Worker``\ , returns the ``pid`` of the worker it is connected to. This is useful when writing custom ``serialize`` methods for a type, which optimizes the data written out depending on the receiving process id. .. function:: Base.cluster_cookie() -> cookie From 5c6033bb2d8905d785d25e48c15f79db3f588fae Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 1 Aug 2016 00:39:49 -0700 Subject: [PATCH 0771/1117] Remove deprecated AsyncStream from docs and contrib rephrase redundant "setup up" fix mismatched backticks fix typo run genstdlib fix sphinx warning grammar fix add a missing the rephrase "as many workers as cores" double backticks for rst remove a few unnecessary parens when calling at-test comment grammar tweaks --- base/clusterserialize.jl | 2 +- base/managers.jl | 2 +- base/math.jl | 4 ++-- base/multi.jl | 2 +- base/serialize.jl | 2 +- contrib/Julia_Notepad++.xml | 2 +- contrib/julia.lang | 1 - contrib/julia.xml | 1 - doc/devdocs/init.rst | 4 ++-- doc/devdocs/stdio.rst | 2 +- doc/manual/parallel-computing.rst | 2 +- doc/stdlib/parallel.rst | 2 +- src/signal-handling.c | 2 +- src/signals-mach.c | 2 +- test/math.jl | 18 +++++++++--------- test/random.jl | 4 ++-- test/spawn.jl | 3 +-- 17 files changed, 26 insertions(+), 29 deletions(-) diff --git a/base/clusterserialize.jl b/base/clusterserialize.jl index 4f42a862c3ba9..d8bdc25e5151f 100644 --- a/base/clusterserialize.jl +++ b/base/clusterserialize.jl @@ -21,7 +21,7 @@ function deserialize(s::ClusterSerializer, ::Type{TypeName}) if !full_body_sent tn = get(known_object_data, number, nothing)::TypeName if !haskey(object_numbers, tn) - # setup reverse mapping for serialize + # set up reverse mapping for serialize object_numbers[tn] = number end deserialize_cycle(s, tn) diff --git a/base/managers.jl b/base/managers.jl index 76ee09ef8631f..313127598fda1 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -93,7 +93,7 @@ Keyword arguments: + `topology=:master_slave` : Only the driver process, i.e. `pid` 1 connects to the workers. The workers do not connect to each other. - + `topology=:custom` : The `launch` method of the cluster manager specifes the + + `topology=:custom` : The `launch` method of the cluster manager specifies the connection topology via fields `ident` and `connect_idents` in `WorkerConfig`. A worker with a cluster manager identity `ident` will connect to all workers specified in `connect_idents`. diff --git a/base/math.jl b/base/math.jl index 3d062532243bb..69012f7201236 100644 --- a/base/math.jl +++ b/base/math.jl @@ -135,8 +135,8 @@ end Inf64 elseif x <= -1023 # if -1073 < x <= -1023 then Result will be a subnormal number - # Hex literal with padding must be use to work on 32bit machine - reinterpret(Float64, 0x0000_0000_0000_0001 << ((x + 1074)) % UInt) + # Hex literal with padding must be used to work on 32bit machine + reinterpret(Float64, 0x0000_0000_0000_0001 << ((x + 1074)) % UInt) else # We will cast everything to Int64 to avoid errors in case of Int128 # If x is a Int128, and is outside the range of Int64, then it is not -1023<x<=1023 diff --git a/base/multi.jl b/base/multi.jl index 48ae0617c6d51..e299edb35bda3 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -1635,7 +1635,7 @@ const worker_lock = ReentrantLock() Launches worker processes via the specified cluster manager. For example Beowulf clusters are supported via a custom cluster manager implemented in -package `ClusterManagers.jl`. +the package `ClusterManagers.jl`. The number of seconds a newly launched worker waits for connection establishment from the master can be specified via variable `JULIA_WORKER_TIMEOUT` in the worker process's diff --git a/base/serialize.jl b/base/serialize.jl index 39af8a55a7607..55e5f0cdd868a 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -751,7 +751,7 @@ function deserialize_typename(s::AbstractSerializer, number) known_object_data[number] = tn end if !haskey(object_numbers, tn) - # setup up reverse mapping for serialize + # set up reverse mapping for serialize object_numbers[tn] = number end deserialize_cycle(s, tn) diff --git a/contrib/Julia_Notepad++.xml b/contrib/Julia_Notepad++.xml index dca94a236f320..fe0f0a0f27c3f 100644 --- a/contrib/Julia_Notepad++.xml +++ b/contrib/Julia_Notepad++.xml @@ -25,7 +25,7 @@ <Keywords name="Folders in comment, middle"></Keywords> <Keywords name="Folders in comment, close"></Keywords> <Keywords name="Keywords1">true false C_NULL Inf NaN Inf32 NaN32 nothing</Keywords> - <Keywords name="Keywords2">AbstractArray AbstractMatrix AbstractRemoteRef AbstractSparseMatrix AbstractString AbstractVector Any ArgumentError Array Associative AsyncStream BigFloat BigInt BitArray BitMatrix BitVector Bool BunchKaufman Cchar Cdouble Cfloat Char CharString CholeskyDense CholeskyPivotedDense Cint Cintmax_t Clong Clonglong Colon Complex Complex128 Complex64 ComplexPair Cptrdiff_t Cshort Csize_t Cuchar Cuint Cuintmax_t Culong Culonglong Cushort DArray Dict Dims DisconnectException EOFError EachLine EnvHash ErrorException Exception Expr Factorization Filter Float Float32 Float64 Function GSVDDense IO IOBuffer IOStream ImaginaryUnit InsertionSort Int Int128 Int16 Int32 Int64 Int8 IntSet Integer KeyError LDLTTridiagonal LUDense LUTridiagonal LoadError LocalProcess Matrix MergeSort MethodError NTuple Number ObjectIdDict ObjectIdDict OrdinalRange ParseError PipeBuffer ProcessGroup Ptr QRDense QRPivotedDense QuickSort Range Range1 RangeIndex Ranges Rational Real Regex RegexMatch RegexMatchIterator RepString RevString Reverse SVDDense Set Signed SparseMatrixCSC SpawnNullStream Stat StridedArray StridedMatrix StridedVecOrMat StridedVector String SubArray SubDArray SubOrDArray SubString SymTridiagonal Symbol SystemError Task TCPSocket TimSort Tridiagonal Tuple Type TypeError UInt UInt128 UInt16 UInt32 UInt64 UInt8 UVError Union Unsigned VecOrMat Vector VersionNumber Void WeakKeyDict WeakRef Zip</Keywords> + <Keywords name="Keywords2">AbstractArray AbstractMatrix AbstractRemoteRef AbstractSparseMatrix AbstractString AbstractVector Any ArgumentError Array Associative BigFloat BigInt BitArray BitMatrix BitVector Bool BunchKaufman Cchar Cdouble Cfloat Char CharString CholeskyDense CholeskyPivotedDense Cint Cintmax_t Clong Clonglong Colon Complex Complex128 Complex64 ComplexPair Cptrdiff_t Cshort Csize_t Cuchar Cuint Cuintmax_t Culong Culonglong Cushort DArray Dict Dims DisconnectException EOFError EachLine EnvHash ErrorException Exception Expr Factorization Filter Float Float32 Float64 Function GSVDDense IO IOBuffer IOStream ImaginaryUnit InsertionSort Int Int128 Int16 Int32 Int64 Int8 IntSet Integer KeyError LDLTTridiagonal LUDense LUTridiagonal LoadError LocalProcess Matrix MergeSort MethodError NTuple Number ObjectIdDict ObjectIdDict OrdinalRange ParseError PipeBuffer ProcessGroup Ptr QRDense QRPivotedDense QuickSort Range Range1 RangeIndex Ranges Rational Real Regex RegexMatch RegexMatchIterator RepString RevString Reverse SVDDense Set Signed SparseMatrixCSC SpawnNullStream Stat StridedArray StridedMatrix StridedVecOrMat StridedVector String SubArray SubDArray SubOrDArray SubString SymTridiagonal Symbol SystemError Task TCPSocket TimSort Tridiagonal Tuple Type TypeError UInt UInt128 UInt16 UInt32 UInt64 UInt8 UVError Union Unsigned VecOrMat Vector VersionNumber Void WeakKeyDict WeakRef Zip</Keywords> <Keywords name="Keywords3">abstract begin baremodule bitstype break catch ccall const continue do else elseif end export finally for function global if immutable import importall let local macro module quote return try type typealias using while</Keywords> <Keywords name="Keywords4">close enumerate error info open print println read write warn</Keywords> <Keywords name="Keywords5">print println</Keywords> diff --git a/contrib/julia.lang b/contrib/julia.lang index e7fef8da4b51c..15761f75efef3 100644 --- a/contrib/julia.lang +++ b/contrib/julia.lang @@ -255,7 +255,6 @@ <keyword>(Error|Interrupt|Disconnect)?Exception</keyword> <keyword>Algorithm</keyword> <keyword>Associative</keyword> - <keyword>AsyncStream</keyword> <keyword>Box</keyword> <keyword>BunchKaufman</keyword> <keyword>CPUinfo</keyword> diff --git a/contrib/julia.xml b/contrib/julia.xml index 9eae8f34ab56d..578d6e4650ba5 100644 --- a/contrib/julia.xml +++ b/contrib/julia.xml @@ -83,7 +83,6 @@ <item> Any </item> <item> Array </item> <item> Associative </item> - <item> AsyncStream </item> <item> Bidiagonal </item> <item> BitArray </item> <item> BigFloat </item> diff --git a/doc/devdocs/init.rst b/doc/devdocs/init.rst index d6aca3002337b..6603053a0a2c1 100644 --- a/doc/devdocs/init.rst +++ b/doc/devdocs/init.rst @@ -183,7 +183,7 @@ Base.eval `jl_toplevel_eval_in() <https://github.com/JuliaLang/julia/blob/master/src/builtins.c>`_ calls `jl_toplevel_eval_flex() <https://github.com/JuliaLang/julia/blob/master/src/toplevel.c>`_ which calls `eval() in interpreter.c <https://github.com/JuliaLang/julia/blob/master/src/interpreter.c>`_. -The stack dump below shows how the interpreter works its way through various methods of :func:`Base.println` and :func:`Base.print` before arriving at `write{T}(s::AsyncStream, a::Array{T}) <https://github.com/JuliaLang/julia/blob/master/base/stream.jl>`_ which does :code:`ccall(jl_uv_write())`. +The stack dump below shows how the interpreter works its way through various methods of :func:`Base.println` and :func:`Base.print` before arriving at `write{T}(s::IO, a::Array{T}) <https://github.com/JuliaLang/julia/blob/master/base/stream.jl>`_ which does :code:`ccall(jl_uv_write())`. `jl_uv_write() <https://github.com/JuliaLang/julia/blob/master/src/jl_uv.c>`_ calls :c:func:`uv_write` to write "Hello World!" to :c:macro:`JL_STDOUT`. See :ref:`dev-libuv`.:: @@ -195,7 +195,7 @@ calls :c:func:`uv_write` to write "Hello World!" to :c:macro:`JL_STDOUT`. See :r Stack frame Source code Notes ============================ ================= =============================================== jl_uv_write() jl_uv.c called though :func:`Base.ccall` -julia_write_282942 stream.jl function write!{T}(s::AsyncStream, a::Array{T}) +julia_write_282942 stream.jl function write!{T}(s::IO, a::Array{T}) julia_print_284639 ascii.jl print(io::IO, s::String) = (write(io, s);nothing) jlcall_print_284639 jl_apply() julia.h diff --git a/doc/devdocs/stdio.rst b/doc/devdocs/stdio.rst index 0bb6f6c242bbe..cf26a42030e48 100644 --- a/doc/devdocs/stdio.rst +++ b/doc/devdocs/stdio.rst @@ -66,7 +66,7 @@ stream, e.g.: The :func:`Base.read` and :func:`Base.write` methods for these streams use :func:`ccall` to call ``libuv`` wrappers in :code:`src/jl_uv.c`, e.g.:: - stream.jl: function write(s::AsyncStream, p::Ptr, nb::Integer) + stream.jl: function write(s::IO, p::Ptr, nb::Integer) -> ccall(:jl_uv_write, ...) jl_uv.c: -> int jl_uv_write(uv_stream_t *stream, ...) -> uv_write(uvw, stream, buf, ...) diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index ea37e09dc8f7d..ca78dd0f6c4f6 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -860,7 +860,7 @@ Custom cluster managers would typically specify only ``io`` or ``host`` / ``port additional workers. - ``count`` with an integer value ``n`` will launch a total of ``n`` workers. - - ``count`` with a value of ``:auto`` will launch as many workers as cores on that machine. + - ``count`` with a value of ``:auto`` will launch as many workers as the number of cores on that machine. - ``exename`` is the name of the ``julia`` executable including the full path. - ``exeflags`` should be set to the required command line arguments for new workers. diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 4e644d2b80a36..e9fac7fec5b36 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -177,7 +177,7 @@ General Parallel Computing Support Launches worker processes via the specified cluster manager. - For example Beowulf clusters are supported via a custom cluster manager implemented in package ``ClusterManagers.jl``\ . + For example Beowulf clusters are supported via a custom cluster manager implemented in the package ``ClusterManagers.jl``\ . The number of seconds a newly launched worker waits for connection establishment from the master can be specified via variable ``JULIA_WORKER_TIMEOUT`` in the worker process's environment. Relevant only when using TCP/IP as transport. diff --git a/src/signal-handling.c b/src/signal-handling.c index adc91f7a70a03..5838edb6a584d 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -56,7 +56,7 @@ static int jl_check_force_sigint(void) // Force sigint requires pressing `Ctrl-C` repeatedly. // Ignore sigint for a short time after that to avoid rethrowing sigint too // quickly again. (Code that has this issue is inherently racy but this is -// a interactive feature anyway.) +// an interactive feature anyway.) static int jl_ignore_sigint(void) { return jl_disable_sigint_time && jl_disable_sigint_time > uv_hrtime(); diff --git a/src/signals-mach.c b/src/signals-mach.c index dd44d67f527fe..b168f6cd70a11 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -283,7 +283,7 @@ static void jl_try_deliver_sigint(void) kern_return_t ret = thread_suspend(thread); HANDLE_MACH_ERROR("thread_suspend", ret); - // This abort `sleep` and other syscall. + // This aborts `sleep` and other syscall. ret = thread_abort(thread); HANDLE_MACH_ERROR("thread_abort", ret); diff --git a/test/math.jl b/test/math.jl index a46c31702d078..2394bd460e4ea 100644 --- a/test/math.jl +++ b/test/math.jl @@ -192,18 +192,18 @@ end @test exp2(Float16(2.)) ≈ exp2(2.) @test log(e) == 1 -# check exp2(::Integer) matches exp2(::Float)" +# check exp2(::Integer) matches exp2(::Float) for ii in -2048:2048 expected = exp2(float(ii)) - @test(exp2(Int16(ii)) == expected) - @test(exp2(Int32(ii)) == expected) - @test(exp2(Int64(ii)) == expected) - @test(exp2(Int128(ii)) == expected) + @test exp2(Int16(ii)) == expected + @test exp2(Int32(ii)) == expected + @test exp2(Int64(ii)) == expected + @test exp2(Int128(ii)) == expected if ii >= 0 - @test(exp2(UInt16(ii)) == expected) - @test(exp2(UInt32(ii)) == expected) - @test(exp2(UInt64(ii)) == expected) - @test(exp2(UInt128(ii)) == expected) + @test exp2(UInt16(ii)) == expected + @test exp2(UInt32(ii)) == expected + @test exp2(UInt64(ii)) == expected + @test exp2(UInt128(ii)) == expected end end diff --git a/test/random.jl b/test/random.jl index 75a2a871d8a58..1ebffa03c6e29 100644 --- a/test/random.jl +++ b/test/random.jl @@ -10,8 +10,8 @@ srand(0); rand(); x = rand(384) @test -10 <= rand(-10:-5) <= -5 @test -10 <= rand(-10:5) <= 5 @test minimum([rand(Int32(1):Int32(7^7)) for i = 1:100000]) > 0 -@test(typeof(rand(false:true)) === Bool) -@test(typeof(rand(Char)) === Char) +@test typeof(rand(false:true)) === Bool +@test typeof(rand(Char)) === Char @test length(randn(4, 5)) == 20 @test length(bitrand(4, 5)) == 20 diff --git a/test/spawn.jl b/test/spawn.jl index 4bb442b2b08cd..0040630cf2016 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -154,8 +154,7 @@ yield() put!(r,11) yield() -# Test marking of AsyncStream - +# Test marking of IO r = Channel(1) @async begin port, server = listenany(2327) From 7db2fa3e1a5e6de2192b1247e99976f7eab75d9a Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 1 Aug 2016 13:24:48 -0700 Subject: [PATCH 0772/1117] remove call() from special operators table --- doc/manual/functions.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 87414c31fc169..1b6b6883e8f14 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -183,7 +183,6 @@ Expression Calls ``1:n`` :func:`colon` ``A[i]`` :func:`getindex` ``A[i]=x`` :func:`setindex!` -``A(x)`` :func:`call` =================== ================== These functions are included in the ``Base.Operators`` module even From b2955748e1a153f831f12ada23937c153d053de4 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 3 Aug 2016 03:36:50 -0700 Subject: [PATCH 0773/1117] NEWS: move breaking changes to before library improvements --- NEWS.md | 98 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/NEWS.md b/NEWS.md index 520d7cf5f0522..1a855dc87c94f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -102,6 +102,55 @@ Language changes * Parentheses are no longer allowed around iteration specifications, e.g. `for (i = 1:n)` ([#17668]). +Breaking changes +---------------- + +This section lists changes that do not have deprecation warnings. + + * The assignment operations `.+=`, `.*=` and so on now generate calls + to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side + if the latter is an indexing expression, e.g. `a[...]`). This means that they will fail + if the left-hand side is immutable (or does not support `view`), and will otherwise + change the left-hand side in-place ([#17510], [#17546]). + + * Method ambiguities no longer generate warnings when files are loaded, + nor do they dispatch to an arbitrarily-chosen method; instead, a call that + cannot be resolved to a single method results in a `MethodError` at run time, + rather than the previous definition-time warning. ([#6190]) + + * Array comprehensions preserve the dimensions of the input ranges. For example, + `[2x for x in A]` will have the same dimensions as `A` ([#16622]). + + * The result type of an array comprehension depends only on the types of elements + computed, instead of using type inference ([#7258]). If the result is empty, then + type inference is still used to determine the element type. + + * `reshape` is now defined to always share data with the original array. + If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of + the desired shape ([#4211]). + + * `mapslices` now re-uses temporary storage. Recipient functions that expect + input slices to be persistent should copy data to other storage ([#17266]). + All usages of `mapslices` should be carefully audited since this change can cause + silent, incorrect behavior, rather than failing noisily. + + * Local variables and arguments are represented in lowered code as numbered `Slot` + objects instead of as symbols ([#15609]). + + * The information that used to be in the `ast` field of the `LambdaStaticData` type + is now divided among the fields `code`, `slotnames`, `slottypes`, `slotflags`, + `gensymtypes`, `rettype`, `nargs`, and `isva` in the `LambdaInfo` type ([#15609]). + + * `A <: B` is parsed as `Expr(:(<:), :A, :B)` in all cases ([#9503]). + This also applies to the `>:` operator. + + * Simple 2-argument comparisons like `A < B` are parsed as calls instead of using the + `:comparison` expression type ([#15524]). The `:comparison` expression type is still + produced in ASTs when comparisons are chained (e.g. `A < B ≤ C`). + + * `map` on a dictionary now expects a function that expects and returns a `Pair`. + The result is now another dictionary instead of an array ([#16622]). + Library improvements -------------------- @@ -300,55 +349,6 @@ New architectures * [Power](https://github.com/JuliaLang/julia/issues?utf8=%E2%9C%93&q=label%3Apower): [#16455], [#16404] -Breaking changes ----------------- - -This section lists changes that do not have deprecation warnings. - - * The assignment operations `.+=`, `.*=` and so on now generate calls - to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side - if the latter is an indexing expression, e.g. `a[...]`). This means that they will fail - if the left-hand side is immutable (or does not support `view`), and will otherwise - change the left-hand side in-place ([#17510], [#17546]). - - * Method ambiguities no longer generate warnings when files are loaded, - nor do they dispatch to an arbitrarily-chosen method; instead, a call that - cannot be resolved to a single method results in a `MethodError` at run time, - rather than the previous definition-time warning. ([#6190]) - - * Array comprehensions preserve the dimensions of the input ranges. For example, - `[2x for x in A]` will have the same dimensions as `A` ([#16622]). - - * The result type of an array comprehension depends only on the types of elements - computed, instead of using type inference ([#7258]). If the result is empty, then - type inference is still used to determine the element type. - - * `reshape` is now defined to always share data with the original array. - If a reshaped copy is needed, use `copy(reshape(a))` or `copy!` to a new array of - the desired shape ([#4211]). - - * `mapslices` now re-uses temporary storage. Recipient functions that expect - input slices to be persistent should copy data to other storage ([#17266]). - All usages of `mapslices` should be carefully audited since this change can cause - silent, incorrect behavior, rather than failing noisily. - - * Local variables and arguments are represented in lowered code as numbered `Slot` - objects instead of as symbols ([#15609]). - - * The information that used to be in the `ast` field of the `LambdaStaticData` type - is now divided among the fields `code`, `slotnames`, `slottypes`, `slotflags`, - `gensymtypes`, `rettype`, `nargs`, and `isva` in the `LambdaInfo` type ([#15609]). - - * `A <: B` is parsed as `Expr(:(<:), :A, :B)` in all cases ([#9503]). - This also applies to the `>:` operator. - - * Simple 2-argument comparisons like `A < B` are parsed as calls instead of using the - `:comparison` expression type ([#15524]). The `:comparison` expression type is still - produced in ASTs when comparisons are chained (e.g. `A < B ≤ C`). - - * `map` on a dictionary now expects a function that expects and returns a `Pair`. - The result is now another dictionary instead of an array ([#16622]). - Deprecated or removed --------------------- From 10448081c0431598a87e77646807473fe85cb0ae Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 3 Aug 2016 04:03:18 -0700 Subject: [PATCH 0774/1117] NEWS: mention command-line flag deprecations fixes #17740 --- NEWS.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1a855dc87c94f..696c696a0ac1d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -230,7 +230,7 @@ Library improvements * The `open` function now respects `umask` on UNIX when creating files ([#16466], [#16502]). - * A new function `walkdir()` returns an iterator that walks the directory tree of a directory. ([#1765]) + * A new function `walkdir()` returns an iterator that walks the directory tree of a directory. ([#8814], [#13707]) ``` for (root, dirs, files) in walkdir(expanduser("~/.julia/v0.5/Plots/src")) @@ -366,7 +366,7 @@ Deprecated or removed in favor of versions that require the matrix to be in factored form ([#13496]). - * Deprecate `chol(A,Val{:U/:L})` in favor of `chol(A)` ([#13680]). + * `chol(A,Val{:U/:L})` has been deprecated in favor of `chol(A)` ([#13680]). * `issym` is deprecated in favor of `issymmetric` to match similar functions (`ishermitian`, ...) ([#15192]) @@ -384,6 +384,21 @@ Deprecated or removed * `writemime` is deprecated, and output methods specifying a MIME type are now methods of `show` ([#14052]). +Command-line option changes +--------------------------- + + * The `-F` flag to load `~/.juliarc` has been deprecated in favor of + `--startup-file=yes` ([#9482]). + + * The `-f` and `--no-startup` flags to disable loading of `~/.juliarc` have + been deprecated in favor of `--startup-file=no` ([#9482]). + + * The `-P` and `--post-boot` flags for evaluating an expression in "interactive mode" + have been deprecated in favor of `-i -e` ([#16854]). + + * The `--no-history-file` flag to disable loading of `~/.julia_history` has been + deprecated in favor of `--history-file=no` ([#9482]). + Language tooling improvements ----------------------------- @@ -404,7 +419,6 @@ Language tooling improvements [#550]: https://github.com/JuliaLang/julia/issues/550 [#964]: https://github.com/JuliaLang/julia/issues/964 [#1090]: https://github.com/JuliaLang/julia/issues/1090 -[#1765]: https://github.com/JuliaLang/julia/issues/1765 [#2488]: https://github.com/JuliaLang/julia/issues/2488 [#3737]: https://github.com/JuliaLang/julia/issues/3737 [#4163]: https://github.com/JuliaLang/julia/issues/4163 @@ -415,7 +429,9 @@ Language tooling improvements [#7258]: https://github.com/JuliaLang/julia/issues/7258 [#8036]: https://github.com/JuliaLang/julia/issues/8036 [#8599]: https://github.com/JuliaLang/julia/issues/8599 +[#8814]: https://github.com/JuliaLang/julia/issues/8814 [#8846]: https://github.com/JuliaLang/julia/issues/8846 +[#9482]: https://github.com/JuliaLang/julia/issues/9482 [#9503]: https://github.com/JuliaLang/julia/issues/9503 [#9627]: https://github.com/JuliaLang/julia/issues/9627 [#11196]: https://github.com/JuliaLang/julia/issues/11196 @@ -434,6 +450,7 @@ Language tooling improvements [#13612]: https://github.com/JuliaLang/julia/issues/13612 [#13680]: https://github.com/JuliaLang/julia/issues/13680 [#13681]: https://github.com/JuliaLang/julia/issues/13681 +[#13707]: https://github.com/JuliaLang/julia/issues/13707 [#13774]: https://github.com/JuliaLang/julia/issues/13774 [#13780]: https://github.com/JuliaLang/julia/issues/13780 [#13824]: https://github.com/JuliaLang/julia/issues/13824 @@ -485,6 +502,7 @@ Language tooling improvements [#16645]: https://github.com/JuliaLang/julia/issues/16645 [#16663]: https://github.com/JuliaLang/julia/issues/16663 [#16731]: https://github.com/JuliaLang/julia/issues/16731 +[#16854]: https://github.com/JuliaLang/julia/issues/16854 [#16953]: https://github.com/JuliaLang/julia/issues/16953 [#16972]: https://github.com/JuliaLang/julia/issues/16972 [#17037]: https://github.com/JuliaLang/julia/issues/17037 @@ -499,3 +517,4 @@ Language tooling improvements [#17404]: https://github.com/JuliaLang/julia/issues/17404 [#17510]: https://github.com/JuliaLang/julia/issues/17510 [#17546]: https://github.com/JuliaLang/julia/issues/17546 +[#17668]: https://github.com/JuliaLang/julia/issues/17668 From 6ba4922074563b1ea98cfe5e2a7af669d28af1bf Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 3 Aug 2016 04:05:04 -0700 Subject: [PATCH 0775/1117] NEWS: Move scalar index dropping to breaking changes ref https://github.com/JuliaLang/julia/pull/13612#issuecomment-236778544 --- NEWS.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 696c696a0ac1d..1f964f74bc429 100644 --- a/NEWS.md +++ b/NEWS.md @@ -107,6 +107,11 @@ Breaking changes This section lists changes that do not have deprecation warnings. + * All dimensions indexed by scalars are now dropped, whereas previously only + trailing scalar dimensions would be omitted from the result ([#13612]). This + is a very major behavioral change, but should cause obvious failures. To retain + a dimension sliced with a scalar `i` slice with `i:i` instead. + * The assignment operations `.+=`, `.*=` and so on now generate calls to `broadcast!` on the left-hand side (or call to `view(a, ...)` on the left-hand side if the latter is an indexing expression, e.g. `a[...]`). This means that they will fail @@ -192,11 +197,6 @@ Library improvements * Arrays and linear algebra: - * All dimensions indexed by scalars are now dropped, whereas previously only - trailing scalar dimensions would be omitted from the result ([#13612]). This - is a very major behavioral change, but should cause obvious failures. To retain - a dimension sliced with a scalar `i` slice with `i:i` instead. - * Dimensions indexed by multidimensional arrays add dimensions. More generally, the dimensionality of the result is the sum of the dimensionalities of the indices ([#15431]). From 1f5491dc8eff001ab0d41be8be560f22bfa8a090 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 3 Aug 2016 06:10:36 -0700 Subject: [PATCH 0776/1117] NEWS: mention nearly all of the deprecations department of redundancy department [ci skip] --- NEWS.md | 119 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 24 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1f964f74bc429..46ada23e5ccd3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -121,7 +121,7 @@ This section lists changes that do not have deprecation warnings. * Method ambiguities no longer generate warnings when files are loaded, nor do they dispatch to an arbitrarily-chosen method; instead, a call that cannot be resolved to a single method results in a `MethodError` at run time, - rather than the previous definition-time warning. ([#6190]) + rather than the previous definition-time warning ([#6190]). * Array comprehensions preserve the dimensions of the input ranges. For example, `[2x for x in A]` will have the same dimensions as `A` ([#16622]). @@ -170,12 +170,12 @@ Library improvements * The `UTF16String` and `UTF32String` types and corresponding `utf16` and `utf32` converter functions have been removed from the standard library. If you need these types, they have been moved to the - [`LegacyStrings`](https://github.com/JuliaArchive/LegacyStrings.jl) - package. In the future, more robust Unicode string support will be provided - by the [`StringEncodings`](https://github.com/nalimilan/StringEncodings.jl) - package. If you only need these types to call wide string APIs (UTF-16 on - Windows, UTF-32 on UNIX), consider using the new `transcode` function (see - below) or the `Cwstring` type as a `ccall` argument type, which also ensures + [LegacyStrings.jl package](https://github.com/JuliaArchive/LegacyStrings.jl). + In the future, more robust Unicode string support will be provided by the + [StringEncodings.jl package](https://github.com/nalimilan/StringEncodings.jl). + If you only need these types to call wide string APIs (UTF-16 on Windows, + UTF-32 on UNIX), consider using the new `transcode` function (see below) + or the `Cwstring` type as a `ccall` argument type, which also ensures correct NUL termination of string data. * A `transcode(T, src)` function is now exported for converting data @@ -212,10 +212,11 @@ Library improvements * A new `SparseVector` type allows for one-dimensional sparse arrays. Slicing and reshaping sparse matrices now return vectors when appropriate. The `sparsevec` function returns a one-dimensional sparse - vector instead of a one-column sparse matrix. ([#13440]) + vector instead of a one-column sparse matrix. The `SparseMatrix` module + has been renamed to `SparseArrays` ([#13440]). * Rank one update and downdate functions, `lowrankupdate`, `lowrankupdate!`, `lowrankdowndate`, - and `lowrankdowndate!`, for dense Cholesky factorizations ([#14243], [#14424]) + and `lowrankdowndate!`, have been introduced for dense Cholesky factorizations ([#14243], [#14424]). * All `sparse` methods now retain provided numerical zeros as structural nonzeros; to drop numerical zeros, use `dropzeros!` ([#14798], [#15242]). @@ -230,7 +231,7 @@ Library improvements * The `open` function now respects `umask` on UNIX when creating files ([#16466], [#16502]). - * A new function `walkdir()` returns an iterator that walks the directory tree of a directory. ([#8814], [#13707]) + * A new function `walkdir()` returns an iterator that walks the tree of a directory ([#8814], [#13707]). ``` for (root, dirs, files) in walkdir(expanduser("~/.julia/v0.5/Plots/src")) @@ -241,7 +242,7 @@ Library improvements 4 files in /Users/me/.julia/v0.5/Plots/src/deprecated ``` - * A new function `chown()` changes the ownership of files. ([#15007]) + * A new function `chown()` changes the ownership of files ([#15007]). * Display properties can now be passed among output functions (e.g. `show`) using an `IOContext` object ([#13825]). @@ -269,17 +270,21 @@ Library improvements * `cov` and `cor` don't use keyword arguments anymore and are therefore now type stable ([#13465]). + * Histogram functionality has been deprecated in `Base`. Use the + [StatsBase.jl package](https://github.com/JuliaStats/StatsBase.jl) + instead ([#6842], [#16450]). + * Testing: * The `Base.Test` module now has a `@testset` feature to bundle tests together and delay throwing an error until the end ([#13062]). * The new features are mirrored in the - [BaseTestNext](https://github.com/IainNZ/BaseTestNext.jl) - package for users who would like to use the new functionality on Julia v0.4. + [BaseTestNext.jl package](https://github.com/IainNZ/BaseTestNext.jl) + for users who would like to use the new functionality on Julia v0.4. - * The [BaseTestDeprecated](https://github.com/IainNZ/BaseTestDeprecated.jl) - package provides the old-style `handler` functionality, for compatibility + * The [BaseTestDeprecated.jl package](https://github.com/IainNZ/BaseTestDeprecated.jl) + provides the old-style `handler` functionality, for compatibility with code that needs to support both Julia v0.4 and v0.5. * Package management: @@ -292,7 +297,7 @@ Library improvements have been moved to an external [PkgDev] package ([#13387]). * Updating only a subset of the packages is now supported, - e.g. `Pkg.update("Example")` ([#17132]) + e.g. `Pkg.update("Example")` ([#17132]). * Miscellanous: @@ -312,9 +317,10 @@ Library improvements * The `libjulia` library is now properly versioned and installed to the public `<prefix>/lib` directory, instead of the private `<prefix>/lib/julia` directory ([#16362]). - * System reflection is now more consistently exposed from Sys and not Base. - `OS_NAME` has been replaced by `Sys.KERNEL` and always reports the name of the - kernel (as reported by `uname`). The `@windows_only` and `@osx` family of macros + * System reflection is now more consistently exposed from `Sys` and not `Base` + (e.g. constants such as `WORD_SIZE` and `CPU_CORES`). `OS_NAME` has been + replaced by `Sys.KERNEL` and always reports the name of the kernel (as + reported by `uname`). The `@windows_only` and `@osx` family of macros have been replaced with functions such as `is_windows()` and `is_apple()`. There is now also a `@static` macro that will evaluate the condition of an if-statement at compile time, for when a static branch is required ([#16219]). @@ -368,22 +374,65 @@ Deprecated or removed * `chol(A,Val{:U/:L})` has been deprecated in favor of `chol(A)` ([#13680]). + * `rem1(x,y)` is discontinued due to inconsistency for `x==0`. Use `mod1` instead ([#14140]). + + * The `FS` module has been renamed to `Filesystem`. Calling the functions `isreadable`, + `iswritable`, and `isexecutable` on filesystem paths has been deprecated ([#12819]). + + * `RemoteRef` has been deprecated in favor of `RemoteChannel` ([#14458]). + + * `super` has been renamed to `supertype` ([#14335]). + + * `parseip(str)` has been deprecated in favor of `parse(IPAddr, str)` ([#14676]). + + * `readall` has been renamed to `readstring`, and `readbytes` has been renamed to `read` ([#14608], [#14660]). + + * `fieldoffsets(x)` has been deprecated in favor of calling `fieldoffset(x, i)` on each field ([#14777]). + * `issym` is deprecated in favor of `issymmetric` to match similar functions - (`ishermitian`, ...) ([#15192]) + (`ishermitian`, ...) ([#15192]). + + * `scale` is deprecated in favor of either `α*A`, `Diagonal(x)*A`, or `A*Diagonal(x)` ([#15258]). + + * `istext` has been renamed to `istextmime` ([#12872], [#15708]). + + * "Functor" types are no longer necessary and have been deprecated ([#15804]). To maintain + performance on older versions of Julia the [Compat.jl package](https://github.com/JuliaLang/Compat.jl/pull/184) + provides a `@functorize` macro. - * `scale` is deprecated in favor of either `α*A`, `Diagonal(x)*A`, or `A*Diagonal(x)`. ([#15258]) + * `bitunpack(B)` and `bitpack(A)` have been deprecated in favor of + Array(B)` and `BitArray(A)`, respectively ([#16010]). * `xdump` is removed, and `dump` now simply shows the full representation of a value. `dump` should not be overloaded, since it is for examining concrete structure ([#4163]). - * `sub` and `slice` have been deprecated in favor of `view` ([#16972]) + * `sprandbool` has been deprecated in favor of `sprand(Bool, ...)` or + `sprand(rng, Bool, ...)` ([#11688], [#16098]). - * The no-op `transpose` fallback has been deprecated. Consider introducing suitable - `transpose` methods or calling `permutedims(x, [2,1])` ([#13171], [#17075], [#17374]). + * The lowercase `symbol` function has been deprecated in favor of the `Symbol` + constructor ([#16154]). * `writemime` is deprecated, and output methods specifying a MIME type are now methods of `show` ([#14052]). + * BLAS utility functions `blas_set_num_threads`, `blas_vendor`, and `check_blas` + have been moved to the BLAS module as `BLAS.set_num_threads`, `BLAS.vendor`, + and `BLAS.check` ([#10548], [#16600]). + + * `print_escaped` has been renamed to `escape_string`, `print_unescaped` has been + renamed to `unescape_string`, and `print_joined` has been renamed to `join` ([#16603]). + + * `pointer_to_string` has been renamed to `unsafe_wrap(String, ...)`, and + `pointer_to_array` has been renamed to `unsafe_wrap(Array, ...)` ([#16731]). + + * `sub` and `slice` have been deprecated in favor of `view` ([#16972]). + + * Sparse matrix functions `etree`, `ereach`, `csc_permute`, and `symperm` have been moved + to the [SuiteSparse.jl package](https://github.com/JuliaSparse/SuiteSparse.jl) ([#12231], [#17033]). + + * The no-op `transpose` fallback has been deprecated. Consider introducing suitable + `transpose` methods or calling `permutedims(x, [2,1])` ([#13171], [#17075], [#17374]). + Command-line option changes --------------------------- @@ -426,6 +475,7 @@ Language tooling improvements [#4470]: https://github.com/JuliaLang/julia/issues/4470 [#4867]: https://github.com/JuliaLang/julia/issues/4867 [#6190]: https://github.com/JuliaLang/julia/issues/6190 +[#6842]: https://github.com/JuliaLang/julia/issues/6842 [#7258]: https://github.com/JuliaLang/julia/issues/7258 [#8036]: https://github.com/JuliaLang/julia/issues/8036 [#8599]: https://github.com/JuliaLang/julia/issues/8599 @@ -434,8 +484,13 @@ Language tooling improvements [#9482]: https://github.com/JuliaLang/julia/issues/9482 [#9503]: https://github.com/JuliaLang/julia/issues/9503 [#9627]: https://github.com/JuliaLang/julia/issues/9627 +[#10548]: https://github.com/JuliaLang/julia/issues/10548 [#11196]: https://github.com/JuliaLang/julia/issues/11196 [#11242]: https://github.com/JuliaLang/julia/issues/11242 +[#11688]: https://github.com/JuliaLang/julia/issues/11688 +[#12231]: https://github.com/JuliaLang/julia/issues/12231 +[#12819]: https://github.com/JuliaLang/julia/issues/12819 +[#12872]: https://github.com/JuliaLang/julia/issues/12872 [#13062]: https://github.com/JuliaLang/julia/issues/13062 [#13171]: https://github.com/JuliaLang/julia/issues/13171 [#13232]: https://github.com/JuliaLang/julia/issues/13232 @@ -458,14 +513,21 @@ Language tooling improvements [#13897]: https://github.com/JuliaLang/julia/issues/13897 [#14052]: https://github.com/JuliaLang/julia/issues/14052 [#14114]: https://github.com/JuliaLang/julia/issues/14114 +[#14140]: https://github.com/JuliaLang/julia/issues/14140 [#14194]: https://github.com/JuliaLang/julia/issues/14194 [#14243]: https://github.com/JuliaLang/julia/issues/14243 +[#14335]: https://github.com/JuliaLang/julia/issues/14335 [#14413]: https://github.com/JuliaLang/julia/issues/14413 [#14424]: https://github.com/JuliaLang/julia/issues/14424 +[#14458]: https://github.com/JuliaLang/julia/issues/14458 [#14469]: https://github.com/JuliaLang/julia/issues/14469 [#14519]: https://github.com/JuliaLang/julia/issues/14519 +[#14608]: https://github.com/JuliaLang/julia/issues/14608 [#14623]: https://github.com/JuliaLang/julia/issues/14623 +[#14660]: https://github.com/JuliaLang/julia/issues/14660 +[#14676]: https://github.com/JuliaLang/julia/issues/14676 [#14759]: https://github.com/JuliaLang/julia/issues/14759 +[#14777]: https://github.com/JuliaLang/julia/issues/14777 [#14798]: https://github.com/JuliaLang/julia/issues/14798 [#15007]: https://github.com/JuliaLang/julia/issues/15007 [#15032]: https://github.com/JuliaLang/julia/issues/15032 @@ -479,24 +541,32 @@ Language tooling improvements [#15524]: https://github.com/JuliaLang/julia/issues/15524 [#15550]: https://github.com/JuliaLang/julia/issues/15550 [#15609]: https://github.com/JuliaLang/julia/issues/15609 +[#15708]: https://github.com/JuliaLang/julia/issues/15708 [#15731]: https://github.com/JuliaLang/julia/issues/15731 [#15763]: https://github.com/JuliaLang/julia/issues/15763 +[#15804]: https://github.com/JuliaLang/julia/issues/15804 [#15975]: https://github.com/JuliaLang/julia/issues/15975 +[#16010]: https://github.com/JuliaLang/julia/issues/16010 [#16024]: https://github.com/JuliaLang/julia/issues/16024 [#16058]: https://github.com/JuliaLang/julia/issues/16058 [#16071]: https://github.com/JuliaLang/julia/issues/16071 +[#16098]: https://github.com/JuliaLang/julia/issues/16098 [#16107]: https://github.com/JuliaLang/julia/issues/16107 +[#16154]: https://github.com/JuliaLang/julia/issues/16154 [#16219]: https://github.com/JuliaLang/julia/issues/16219 [#16260]: https://github.com/JuliaLang/julia/issues/16260 [#16285]: https://github.com/JuliaLang/julia/issues/16285 [#16362]: https://github.com/JuliaLang/julia/issues/16362 [#16403]: https://github.com/JuliaLang/julia/issues/16403 [#16404]: https://github.com/JuliaLang/julia/issues/16404 +[#16450]: https://github.com/JuliaLang/julia/issues/16450 [#16455]: https://github.com/JuliaLang/julia/issues/16455 [#16466]: https://github.com/JuliaLang/julia/issues/16466 [#16481]: https://github.com/JuliaLang/julia/issues/16481 [#16502]: https://github.com/JuliaLang/julia/issues/16502 [#16510]: https://github.com/JuliaLang/julia/issues/16510 +[#16600]: https://github.com/JuliaLang/julia/issues/16600 +[#16603]: https://github.com/JuliaLang/julia/issues/16603 [#16621]: https://github.com/JuliaLang/julia/issues/16621 [#16622]: https://github.com/JuliaLang/julia/issues/16622 [#16645]: https://github.com/JuliaLang/julia/issues/16645 @@ -505,6 +575,7 @@ Language tooling improvements [#16854]: https://github.com/JuliaLang/julia/issues/16854 [#16953]: https://github.com/JuliaLang/julia/issues/16953 [#16972]: https://github.com/JuliaLang/julia/issues/16972 +[#17033]: https://github.com/JuliaLang/julia/issues/17033 [#17037]: https://github.com/JuliaLang/julia/issues/17037 [#17075]: https://github.com/JuliaLang/julia/issues/17075 [#17132]: https://github.com/JuliaLang/julia/issues/17132 From 9ec52ea98918f59d1ae2b29d527c2088c78e0237 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 3 Aug 2016 08:55:41 -0700 Subject: [PATCH 0777/1117] run genstdlib [ci skip] --- doc/stdlib/arrays.rst | 1 + doc/stdlib/parallel.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 8d6bb1b0ad45e..448dd55c0caa9 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1086,3 +1086,4 @@ dense counterparts. The following functions are specific to sparse arrays. For additional (algorithmic) information, and for versions of these methods that forgo argument checking, see (unexported) parent methods :func:`Base.SparseArrays.unchecked_noalias_permute!` and :func:`Base.SparseArrays.unchecked_aliasing_permute!`\ . See also: :func:`Base.SparseArrays.permute` + diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index e9fac7fec5b36..f7759523d99fb 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -165,7 +165,7 @@ General Parallel Computing Support * ``topology=:all_to_all`` : All processes are connected to each other. This is the default. * ``topology=:master_slave`` : Only the driver process, i.e. ``pid`` 1 connects to the workers. The workers do not connect to each other. - * ``topology=:custom`` : The ``launch`` method of the cluster manager specifes the connection topology via fields ``ident`` and ``connect_idents`` in ``WorkerConfig``\ . A worker with a cluster manager identity ``ident`` will connect to all workers specified in ``connect_idents``\ . + * ``topology=:custom`` : The ``launch`` method of the cluster manager specifies the connection topology via fields ``ident`` and ``connect_idents`` in ``WorkerConfig``\ . A worker with a cluster manager identity ``ident`` will connect to all workers specified in ``connect_idents``\ . Environment variables : From 170d53e254a0016d9634e4a084760d39921ec5f6 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 3 Aug 2016 09:23:09 -0700 Subject: [PATCH 0778/1117] do another test with the commit in the source copy --- test/pkg.jl | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 77c7c152d61c1..7fb6848d270eb 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -141,8 +141,8 @@ temp_pkg_dir() do @test endswith(str, string(Pkg.installed("Example"))) @test isempty(Pkg.dependents("Example")) - # 17364, Pkg.checkout with specific branch - let branch_name = "test-branch", + # 17364 - a, Pkg.checkout with specific local branch + let branch_name = "test-branch-1", branch_commit = "ba3888212e30a7974ac6803a89e64c7098f4865e" # create a branch in Example package @@ -172,6 +172,35 @@ temp_pkg_dir() do end end + # 17364 - b, remote off-tree branch + let branch_name = "test-branch-2", + branch_commit = "ba3888212e30a7974ac6803a89e64c7098f4865e" + + # create a branch in Example package + LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example")) do repo + LibGit2.branch!(repo, branch_name, branch_commit, set_head=true) + end + + # Make changes to local branch + open(Pkg.dir("Example", "README.md"), "w") do f + println(f, "overwritten") + end + + test_commit = LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example")) do repo + LibGit2.add!(repo, "README.md") + test_sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time(), 0), 0) + LibGit2.commit(repo, "testmsg"; author=test_sig, committer=test_sig) + end + Pkg.checkout("Example") + + Pkg.clone(Pkg.dir("Example"), "Example4") + Pkg.checkout("Example4", branch_name) + + LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example4")) do repo + @test LibGit2.head_oid(repo) == test_commit + end + end + # adding a package with unsatisfiable julia version requirements (REPL.jl) errors try Pkg.add("REPL") From 79ea350935f127b85cd3e0ddd501b538f7a72618 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Wed, 3 Aug 2016 09:36:26 -0700 Subject: [PATCH 0779/1117] Add docstring for read with default nb --- base/io.jl | 6 +++++- doc/stdlib/io-network.rst | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/base/io.jl b/base/io.jl index e186abdf9365b..29aa202314678 100644 --- a/base/io.jl +++ b/base/io.jl @@ -342,7 +342,11 @@ function readbytes!(s::IO, b::AbstractArray{UInt8}, nb=length(b)) return nr end -# read up to nb bytes from s, returning a Vector{UInt8} of bytes read. +""" + read(s::IO, nb=typemax(Int)) + +Read at most `nb` bytes from `s`, returning a `Vector{UInt8}` of the bytes read. +""" function read(s::IO, nb=typemax(Int)) # Let readbytes! grow the array progressively by default # instead of taking of risk of over-allocating diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 97e8c7c0c286a..158560b2c9ba3 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -168,6 +168,12 @@ General I/O See ``read`` for a description of the ``all`` option. +.. function:: read(s::IO, nb=typemax(Int)) + + .. Docstring generated from Julia source + + Read at most ``nb`` bytes from ``s``\ , returning a ``Vector{UInt8}`` of the bytes read. + .. function:: read(s::IOStream, nb::Integer; all=true) .. Docstring generated from Julia source From 1a484def83c92c3bcc13a2f4bfd6665e4982a822 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Wed, 3 Aug 2016 11:12:52 -0700 Subject: [PATCH 0780/1117] Moved docs out of helpDB, way more examples Lots of our documentation for some relatively simple iterator methods had no examples. I added a bunch, moved docstrings out of HelpDB, and tried to illustrate some common use cases. --- base/abstractarray.jl | 10 ++ base/array.jl | 95 ++++++++++++++++- base/docs/helpdb/Base.jl | 178 ------------------------------- base/iterator.jl | 124 +++++++++++++++++++++- base/operators.jl | 10 ++ base/range.jl | 18 ++++ base/reduce.jl | 97 +++++++++++++++++ doc/stdlib/collections.rst | 207 ++++++++++++++++++++++++++++++++++++- doc/stdlib/math.rst | 5 + 9 files changed, 562 insertions(+), 182 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index e74181d6428d1..c4e7ba2dbde20 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1338,6 +1338,16 @@ Call function `f` on each element of iterable `c`. For multiple iterable arguments, `f` is called elementwise. `foreach` should be used instead of `map` when the results of `f` are not needed, for example in `foreach(println, array)`. + +```jldoctest +julia> a +1:3:7 + +julia> foreach(x->println(x^2),a) +1 +16 +49 +``` """ foreach(f) = (f(); nothing) foreach(f, itr) = (for x in itr; f(x); end; nothing) diff --git a/base/array.jl b/base/array.jl index e4d98d799b3c8..079bdfd1e5d99 100644 --- a/base/array.jl +++ b/base/array.jl @@ -824,6 +824,17 @@ function findnz{T}(A::AbstractMatrix{T}) return (I, J, NZs) end +""" + findmax(itr) -> (x, index) + +Returns the maximum element and its index. +The collection must not be empty. + +```jldoctest +julia> findmax([8,0.1,-9,pi]) +(8.0,1) +``` +""" function findmax(a) if isempty(a) throw(ArgumentError("collection must be non-empty")) @@ -842,6 +853,17 @@ function findmax(a) return (m, mi) end +""" + findmin(itr) -> (x, index) + +Returns the minimum element and its index. +The collection must not be empty. + +```jldoctest +julia> findmax([8,0.1,-9,pi]) +(-9.0,3) +``` +""" function findmin(a) if isempty(a) throw(ArgumentError("collection must be non-empty")) @@ -860,16 +882,87 @@ function findmin(a) return (m, mi) end +""" + indmax(itr) -> Integer + +Returns the index of the maximum element in a collection. +```jldoctest +julia> indmax([8,0.1,-9,pi]) +1 +``` +""" indmax(a) = findmax(a)[2] + +""" + indmin(itr) -> Integer + +Returns the index of the minimum element in a collection. +```jldoctest +julia> indmin([8,0.1,-9,pi]) +3 +``` +""" indmin(a) = findmin(a)[2] # similar to Matlab's ismember -# returns a vector containing the highest index in b for each value in a that is a member of b +""" + indexin(a, b) + +Returns a vector containing the highest index in `b` for +each value in `a` that is a member of `b` . The output +vector contains 0 wherever `a` is not a member of `b`. + +```jldoctest +julia> a = ['a', 'b', 'c', 'b', 'd', 'a']; + +julia> b = ['a','b','c'] + +julia> indexin(a,b) +6-element Array{Int64,1}: + 1 + 2 + 3 + 2 + 0 + 1 + +julia> indexin(b,a) +3-element Array{Int64,1}: + 6 + 4 + 3 +``` +""" function indexin(a::AbstractArray, b::AbstractArray) bdict = Dict(zip(b, 1:length(b))) [get(bdict, i, 0) for i in a] end +""" + findin(a, b) + +Returns the indices of elements in collection `a` that appear in collection `b`. + +```jldoctest +julia> a = collect(1:3:15) +5-element Array{Int64,1}: + 1 + 4 + 7 + 10 + 13 + +julia> b = collect(2:4:10) +3-element Array{Int64,1}: + 2 + 6 + 10 + +julia> findin(a,b) # 10 is the only common element +1-element Array{Int64,1}: + 4 +``` +""" function findin(a, b) ind = Array{Int,1}(0) bset = Set(b) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 4f6cb0b6dc26c..c66b6fb2ce66a 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -79,13 +79,6 @@ the woken task. """ schedule -""" - step(r) - -Get the step size of a [`Range`](:obj:`Range`) object. -""" -step - """ takebuf_array(b::IOBuffer) @@ -113,13 +106,6 @@ set of characters) is provided, instead remove characters contained in it. """ lstrip -""" - indmin(itr) -> Integer - -Returns the index of the minimum element in a collection. -""" -indmin - """ powermod(x, p, m) @@ -1213,14 +1199,6 @@ greater than 1, and `x` must not be less than 1. """ prevpow -""" - indexin(a, b) - -Returns a vector containing the highest index in `b` for each value in `a` that is a member -of `b` . The output vector contains 0 wherever `a` is not a member of `b`. -""" -indexin - """ permutedims(A, perm) @@ -1376,13 +1354,6 @@ Like `randsubseq`, but the results are stored in `S` (which is resized as needed """ randsubseq! -""" - maximum(itr) - -Returns the largest element in a collection. -""" -maximum(itr) - """ maximum(A, dims) @@ -2473,13 +2444,6 @@ Largest integer less than or equal to `x/y`. """ fld -""" - indmax(itr) -> Integer - -Returns the index of the maximum element in a collection. -""" -indmax - """ writecsv(filename, A) @@ -3009,13 +2973,6 @@ Get the concrete type of `x`. """ typeof -""" - drop(iter, n) - -An iterator that generates all but the first `n` elements of `iter`. -""" -drop - """ acsc(x) @@ -3170,13 +3127,6 @@ Compute the inverse error complementary function of a real `x`, defined by """ erfcinv -""" - minabs(itr) - -Compute the minimum absolute value of a collection of values. -""" -minabs(itr) - """ minabs(A, dims) @@ -3559,16 +3509,6 @@ handle properly. """ OutOfMemoryError -""" - zip(iters...) - -For a set of iterable objects, returns an iterable of tuples, where the `i`th tuple contains -the `i`th component of each input iterable. - -Note that [`zip`](:func:`zip`) is its own inverse: `collect(zip(zip(a...)...)) == collect(a)`. -""" -zip - """ SystemError(prefix::AbstractString, [errno::Int32]) @@ -4647,13 +4587,6 @@ Compute the inverse hyperbolic sine of `x`. """ asinh -""" - count(p, itr) -> Integer - -Count the number of elements in `itr` for which predicate `p` returns `true`. -""" -count - """ atreplinit(f) @@ -4672,20 +4605,6 @@ or vector or set of characters) is provided, instead remove characters contained """ strip -""" - findin(a, b) - -Returns the indices of elements in collection `a` that appear in collection `b`. -""" -findin - -""" - minimum(itr) - -Returns the smallest element in a collection. -""" -minimum(itr) - """ minimum(A, dims) @@ -4741,13 +4660,6 @@ Like redirect_stdout, but for STDIN. Note that the order of the return tuple is """ redirect_stdin -""" - minmax(x, y) - -Return `(min(x,y), max(x,y))`. See also: [`extrema`](:func:`extrema`) that returns `(minimum(x), maximum(x))`. -""" -minmax - """ mktemp([parent=tempdir()]) @@ -4884,13 +4796,6 @@ last argument optionally specifies a size beyond which the buffer may not be gro """ IOBuffer(data=?) -""" - findmax(itr) -> (x, index) - -Returns the maximum element and its index. -""" -findmax(itr) - """ findmax(A, dims) -> (maxval, index) @@ -5502,13 +5407,6 @@ The highest value representable by the given (real) numeric `DataType`. """ typemax -""" - all(itr) -> Bool - -Test whether all elements of a boolean collection are `true`. -""" -all(itr) - """ all(A, dims) @@ -5516,18 +5414,6 @@ Test whether all values along the given dimensions of an array are `true`. """ all(A::AbstractArray, dims) -""" - all(p, itr) -> Bool - -Determine whether predicate `p` returns `true` for all elements of `itr`. - -```jldoctest -julia> all(i->(4<=i<=6), [4,5,6]) -true -``` -""" -all(p, itr) - """ bind(socket::Union{UDPSocket, TCPSocket}, host::IPAddr, port::Integer; ipv6only=false) @@ -6968,13 +6854,6 @@ Compute the trigamma function of `x` (the logarithmic second derivative of `gamm """ trigamma -""" - findmin(itr) -> (x, index) - -Returns the minimum element and its index. -""" -findmin(itr) - """ findmin(A, dims) -> (minval, index) @@ -7472,13 +7351,6 @@ throwing an `ArgumentError` indicating the position of the first non-ASCII byte. """ ascii(s) -""" - maxabs(itr) - -Compute the maximum absolute value of a collection of values. -""" -maxabs(itr) - """ maxabs(A, dims) @@ -8074,13 +7946,6 @@ Get the additive identity element for the type of `x` (`x` can also specify the """ zero -""" - any(itr) -> Bool - -Test whether any elements of a boolean collection are `true`. -""" -any(itr) - """ any(A, dims) @@ -8088,13 +7953,6 @@ Test whether any values along the given dimensions of an array are `true`. """ any(::AbstractArray,dims) -""" - any(p, itr) -> Bool - -Determine whether predicate `p` returns `true` for any elements of `itr`. -""" -any(p,itr) - """ cosc(x) @@ -8230,13 +8088,6 @@ break identities such as `(x-y==0) == (x==y)`. """ set_zero_subnormals -""" - take(iter, n) - -An iterator that generates at most the first `n` elements of `iter`. -""" -take - """ frexp(val) @@ -8827,14 +8678,6 @@ julia> A """ pop!(collection) -""" - filter(function, collection) - -Return a copy of `collection`, removing elements for which `function` is `false`. For -associative collections, the function is passed two arguments (key and value). -""" -filter - """ randperm([rng,] n) @@ -8930,27 +8773,6 @@ Remove a single trailing newline from a string. """ chomp -""" - enumerate(iter) - -An iterator that yields `(i, x)` where `i` is an index starting at 1, and -`x` is the `i`th value from the given iterator. It's useful when you need -not only the values `x` over which you are iterating, but also the index `i` -of the iterations. - -```jldoctest -julia> a = ["a", "b", "c"]; - -julia> for (index, value) in enumerate(a) - println("\$index \$value") - end -1 a -2 b -3 c -``` -""" -enumerate - """ >=(x, y) ≥(x,y) diff --git a/base/iterator.jl b/base/iterator.jl index 56c97b7348618..6c93bcbb6be37 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -17,7 +17,27 @@ _diff_length(a, b, A, B) = max(length(a)-length(b), 0) immutable Enumerate{I} itr::I end -enumerate(itr) = Enumerate(itr) + +""" + enumerate(iter) + +An iterator that yields `(i, x)` where `i` is an index starting at 1, and +`x` is the `i`th value from the given iterator. It's useful when you need +not only the values `x` over which you are iterating, but also the index `i` +of the iterations. + +```jldoctest +julia> a = ["a", "b", "c"]; + +julia> for (index, value) in enumerate(a) + println("\$index \$value") + end +1 a +2 b +3 c +``` +""" +enumerate(iter) = Enumerate(iter) length(e::Enumerate) = length(e.itr) size(e::Enumerate) = size(e.itr) @@ -83,6 +103,37 @@ immutable Zip{I, Z<:AbstractZipIterator} <: AbstractZipIterator a::I z::Z end + +""" + zip(iters...) + +For a set of iterable objects, returns an iterable of tuples, where the `i`th tuple contains +the `i`th component of each input iterable. + +Note that [`zip`](:func:`zip`) is its own inverse: `collect(zip(zip(a...)...)) == collect(a)`. + +```jldoctest +julia> a = 1:5 +1:5 + +julia> b = ["e","d","b","c","a"] +5-element Array{String,1}: + "e" + "d" + "b" + "c" + "a" + +julia> c = zip(a,b) +Base.Zip2{UnitRange{Int64},Array{String,1}}(1:5,String["e","d","b","c","a"]) + +julia> length(c) +5 + +julia> first(c) +(1,"e") +``` +""" zip(a, b, c...) = Zip(a, zip(b, c...)) length(z::Zip) = _min_length(z.a, z.z, iteratorsize(z.a), iteratorsize(z.z)) size(z::Zip) = promote_shape(size(z.a), size(z.z)) @@ -109,6 +160,26 @@ immutable Filter{F,I} flt::F itr::I end + +""" + filter(function, collection) + +Return a copy of `collection`, removing elements for which `function` is `false`. For +associative collections, the function is passed two arguments (key and value). + +```jldocttest +julia> a = 1:10 +1:10 + +julia> filter(isodd, a) +5-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 +``` +""" filter(flt, itr) = Filter(flt, itr) start(f::Filter) = start_filter(f.flt, f.itr) @@ -204,6 +275,32 @@ immutable Take{I} xs::I n::Int end + +""" + take(iter, n) + +An iterator that generates at most the first `n` elements of `iter`. + +```jldoctest +julia> a = 1:2:11 +1:2:11 + +julia> collect(a) +6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + +julia> collect(take(a,3)) +3-element Array{Int64,1}: + 1 + 3 + 5 +``` +""" take(xs, n::Int) = Take(xs, n) eltype{I}(::Type{Take{I}}) = eltype(I) @@ -232,6 +329,31 @@ immutable Drop{I} xs::I n::Int end + +""" + drop(iter, n) + +An iterator that generates all but the first `n` elements of `iter`. + +```jldoctest +julia> a = 1:2:11 +1:2:11 + +julia> collect(a) +6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + +julia> collect(drop(a,4)) +2-element Array{Int64,1}: + 9 + 11 +``` +""" drop(xs, n::Int) = Drop(xs, n) eltype{I}(::Type{Drop{I}}) = eltype(I) diff --git a/base/operators.jl b/base/operators.jl index 1998122216ba1..31faf73b1d77f 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -81,6 +81,16 @@ cmp(x::Integer, y::Integer) = ifelse(isless(x,y), -1, ifelse(isless(y,x), 1, 0)) max(x,y) = ifelse(y < x, x, y) min(x,y) = ifelse(y < x, y, x) +""" + minmax(x, y) + +Return `(min(x,y), max(x,y))`. See also: [`extrema`](:func:`extrema`) that returns `(minimum(x), maximum(x))`. + +```jldoctest +julia> minmax('c','b') +('b','c') +``` +""" minmax(x,y) = y < x ? (y, x) : (x, y) scalarmax(x,y) = max(x,y) diff --git a/base/range.jl b/base/range.jl index 5c60b3078edc1..caca080802553 100644 --- a/base/range.jl +++ b/base/range.jl @@ -332,6 +332,24 @@ isempty(r::AbstractUnitRange) = first(r) > last(r) isempty(r::FloatRange) = length(r) == 0 isempty(r::LinSpace) = length(r) == 0 +""" + step(r) + +Get the step size of a [`Range`](:obj:`Range`) object. +```jldoctest +julia> step(1:10) +1 + +julia> step(1:2:10) +2 + +julia> step(2.5:0.3:10.9) +0.3 + +julia> step(linspace(2.5,10.9,85)) +0.1 +``` +""" step(r::StepRange) = r.step step(r::AbstractUnitRange) = 1 step(r::FloatRange) = r.step/r.divisor diff --git a/base/reduce.jl b/base/reduce.jl index 4d26a67eee985..18ef01b4d63f4 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -286,10 +286,58 @@ end maximum(f::Callable, a) = mapreduce(f, scalarmax, a) minimum(f::Callable, a) = mapreduce(f, scalarmin, a) +""" + maximum(itr) + +Returns the largest element in a collection. + +```jldoctest +julia> maximum(-20.5:10) +9.5 + +julia> maximum([1,2,3]) +3 +``` +""" maximum(a) = mapreduce(identity, scalarmax, a) + +""" + minimum(itr) + +Returns the smallest element in a collection. + +```jldoctest +julia> minimum(-20.5:10) +-20.5 + +julia> minimum([1,2,3]) +1 +``` +""" minimum(a) = mapreduce(identity, scalarmin, a) +""" + maxabs(itr) + +Compute the maximum absolute value of a collection of values. + +```jldoctest +julia> maxabs([-1, 3, 4*im]) +4.0 +``` +""" maxabs(a) = mapreduce(abs, scalarmax, a) + +""" + minabs(itr) + +Compute the minimum absolute value of a collection of values. + +```jldoctest +julia> minabs([-1, 3, 4*im]) +1.0 +``` +""" minabs(a) = mapreduce(abs, scalarmin, a) ## extrema @@ -301,6 +349,14 @@ extrema(x::Real) = (x, x) extrema(itr) -> Tuple Compute both the minimum and maximum element in a single pass, and return them as a 2-tuple. + +```jldoctest +julia> extrema(2:10) +(2,10) + +julia> extrema([9,pi,4.5]) +(3.141592653589793,9.0) +``` """ function extrema(itr) s = start(itr) @@ -362,7 +418,18 @@ end ## all & any +""" + any(itr) -> Bool + +Test whether any elements of a boolean collection are `true`. +""" any(itr) = any(identity, itr) + +""" + all(itr) -> Bool + +Test whether all elements of a boolean collection are `true`. +""" all(itr) = all(identity, itr) nonboolean_error(f, op) = throw(ArgumentError(""" @@ -375,6 +442,16 @@ or_bool_only(a::Bool, b::Bool) = a|b and_bool_only(a, b) = nonboolean_error(:all, :&) and_bool_only(a::Bool, b::Bool) = a&b +""" + any(p, itr) -> Bool + +Determine whether predicate `p` returns `true` for any elements of `itr`. + +```jldoctest +julia> any(i->(4<=i<=6), [3,5,7]) +true +``` +""" any(f::Any, itr) = any(Predicate(f), itr) any(f::Predicate, itr) = mapreduce_sc_impl(f, |, itr) any(f::typeof(identity), itr) = @@ -382,6 +459,16 @@ any(f::typeof(identity), itr) = mapreduce_sc_impl(f, |, itr) : reduce(or_bool_only, itr) +""" + all(p, itr) -> Bool + +Determine whether predicate `p` returns `true` for all elements of `itr`. + +```jldoctest +julia> all(i->(4<=i<=6), [4,5,6]) +true +``` +""" all(f::Any, itr) = all(Predicate(f), itr) all(f::Predicate, itr) = mapreduce_sc_impl(f, &, itr) all(f::typeof(identity), itr) = @@ -408,6 +495,16 @@ end ## countnz & count +""" + count(p, itr) -> Integer + +Count the number of elements in `itr` for which predicate `p` returns `true`. + +```jldoctest +julia> count(i->(4<=i<=6), [2,3,4,5,6]) + 3 +``` +""" function count(pred, itr) n = 0 for x in itr diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 6df9f2c837cdf..e462aecc4ecba 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -55,6 +55,28 @@ type. Note that :func:`zip` is its own inverse: ``collect(zip(zip(a...)...)) == collect(a)``\ . + .. doctest:: + + julia> a = 1:5 + 1:5 + + julia> b = ["e","d","b","c","a"] + 5-element Array{String,1}: + "e" + "d" + "b" + "c" + "a" + + julia> c = zip(a,b) + Base.Zip2{UnitRange{Int64},Array{String,1}}(1:5,String["e","d","b","c","a"]) + + julia> length(c) + 5 + + julia> first(c) + (1,"e") + .. function:: enumerate(iter) .. Docstring generated from Julia source @@ -90,12 +112,51 @@ type. An iterator that generates at most the first ``n`` elements of ``iter``\ . + .. doctest:: + + julia> a = 1:2:11 + 1:2:11 + + julia> collect(a) + 6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + + julia> collect(take(a,3)) + 3-element Array{Int64,1}: + 1 + 3 + 5 + .. function:: drop(iter, n) .. Docstring generated from Julia source An iterator that generates all but the first ``n`` elements of ``iter``\ . + .. doctest:: + + julia> a = 1:2:11 + 1:2:11 + + julia> collect(a) + 6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + + julia> collect(drop(a,4)) + 2-element Array{Int64,1}: + 9 + 11 + .. function:: cycle(iter) .. Docstring generated from Julia source @@ -230,12 +291,53 @@ Iterable Collections Returns a vector containing the highest index in ``b`` for each value in ``a`` that is a member of ``b`` . The output vector contains 0 wherever ``a`` is not a member of ``b``\ . + .. doctest:: + + julia> a = ['a', 'b', 'c', 'b', 'd', 'a']; + + julia> b = ['a','b','c'] + + julia> indexin(a,b) + 6-element Array{Int64,1}: + 1 + 2 + 3 + 2 + 0 + 1 + + julia> indexin(b,a) + 3-element Array{Int64,1}: + 6 + 4 + 3 + .. function:: findin(a, b) .. Docstring generated from Julia source Returns the indices of elements in collection ``a`` that appear in collection ``b``\ . + .. doctest:: + + julia> a = collect(1:3:15) + 5-element Array{Int64,1}: + 1 + 4 + 7 + 10 + 13 + + julia> b = collect(2:4:10) + 3-element Array{Int64,1}: + 2 + 6 + 10 + + julia> findin(a,b) # 10 is the only common element + 1-element Array{Int64,1}: + 4 + .. function:: unique(itr[, dim]) .. Docstring generated from Julia source @@ -308,6 +410,14 @@ Iterable Collections Returns the largest element in a collection. + .. doctest:: + + julia> maximum(-20.5:10) + 9.5 + + julia> maximum([1,2,3]) + 3 + .. function:: maximum(A, dims) .. Docstring generated from Julia source @@ -326,6 +436,14 @@ Iterable Collections Returns the smallest element in a collection. + .. doctest:: + + julia> minimum(-20.5:10) + -20.5 + + julia> minimum([1,2,3]) + 1 + .. function:: minimum(A, dims) .. Docstring generated from Julia source @@ -344,6 +462,14 @@ Iterable Collections Compute both the minimum and maximum element in a single pass, and return them as a 2-tuple. + .. doctest:: + + julia> extrema(2:10) + (2,10) + + julia> extrema([9,pi,4.5]) + (3.141592653589793,9.0) + .. function:: extrema(A,dims) -> Array{Tuple} .. Docstring generated from Julia source @@ -356,17 +482,32 @@ Iterable Collections Returns the index of the maximum element in a collection. + .. doctest:: + + julia> indmax([8,0.1,-9,pi]) + 1 + .. function:: indmin(itr) -> Integer .. Docstring generated from Julia source Returns the index of the minimum element in a collection. + .. doctest:: + + julia> indmin([8,0.1,-9,pi]) + 3 + .. function:: findmax(itr) -> (x, index) .. Docstring generated from Julia source - Returns the maximum element and its index. + Returns the maximum element and its index. The collection must not be empty. + + .. doctest:: + + julia> findmax([8,0.1,-9,pi]) + (8.0,1) .. function:: findmax(A, dims) -> (maxval, index) @@ -378,7 +519,12 @@ Iterable Collections .. Docstring generated from Julia source - Returns the minimum element and its index. + Returns the minimum element and its index. The collection must not be empty. + + .. doctest:: + + julia> findmax([8,0.1,-9,pi]) + (-9.0,3) .. function:: findmin(A, dims) -> (minval, index) @@ -404,6 +550,11 @@ Iterable Collections Compute the maximum absolute value of a collection of values. + .. doctest:: + + julia> maxabs([-1, 3, 4*im]) + 4.0 + .. function:: maxabs(A, dims) .. Docstring generated from Julia source @@ -422,6 +573,11 @@ Iterable Collections Compute the minimum absolute value of a collection of values. + .. doctest:: + + julia> minabs([-1, 3, 4*im]) + 1.0 + .. function:: minabs(A, dims) .. Docstring generated from Julia source @@ -554,12 +710,22 @@ Iterable Collections Count the number of elements in ``itr`` for which predicate ``p`` returns ``true``\ . + .. doctest:: + + julia> count(i->(4<=i<=6), [2,3,4,5,6]) + 3 + .. function:: any(p, itr) -> Bool .. Docstring generated from Julia source Determine whether predicate ``p`` returns ``true`` for any elements of ``itr``\ . + .. doctest:: + + julia> any(i->(4<=i<=6), [3,5,7]) + true + .. function:: all(p, itr) -> Bool .. Docstring generated from Julia source @@ -577,6 +743,16 @@ Iterable Collections Call function ``f`` on each element of iterable ``c``\ . For multiple iterable arguments, ``f`` is called elementwise. ``foreach`` should be used instead of ``map`` when the results of ``f`` are not needed, for example in ``foreach(println, array)``\ . + .. doctest:: + + julia> a + 1:3:7 + + julia> foreach(x->println(x^2),a) + 1 + 16 + 49 + .. function:: map(f, c...) -> collection .. Docstring generated from Julia source @@ -672,6 +848,20 @@ Iterable Collections Get the step size of a :obj:`Range` object. + .. doctest:: + + julia> step(1:10) + 1 + + julia> step(1:2:10) + 2 + + julia> step(2.5:0.3:10.9) + 0.3 + + julia> step(linspace(2.5,10.9,85)) + 0.1 + .. function:: collect(collection) .. Docstring generated from Julia source @@ -699,6 +889,19 @@ Iterable Collections Return a copy of ``collection``\ , removing elements for which ``function`` is ``false``\ . For associative collections, the function is passed two arguments (key and value). + .. code-block:: julia + + julia> a = 1:10 + 1:10 + + julia> filter(isodd, a) + 5-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + .. function:: filter!(function, collection) .. Docstring generated from Julia source diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 6ff3d36e6afce..47620a1b53e14 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1012,6 +1012,11 @@ Mathematical Functions Return ``(min(x,y), max(x,y))``\ . See also: :func:`extrema` that returns ``(minimum(x), maximum(x))``\ . + .. doctest:: + + julia> minmax('c','b') + ('b','c') + .. function:: clamp(x, lo, hi) .. Docstring generated from Julia source From ff272d0828de9de4cb19fa9233e05e873c41bf45 Mon Sep 17 00:00:00 2001 From: pabloferz <pabloferz@yahoo.com.mx> Date: Thu, 4 Aug 2016 01:28:34 +0200 Subject: [PATCH 0781/1117] Restore type parameter for elementwise ops --- base/arraymath.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/arraymath.jl b/base/arraymath.jl index 41cd74e843bb7..f5a0954b8323e 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -53,7 +53,7 @@ function _elementwise(op, ::Type{Any}, A::AbstractArray, B::AbstractArray) promote_shape(A, B) # check size compatibility return broadcast(op, A, B) end -function _elementwise(op, T::Type, A::AbstractArray, B::AbstractArray) +function _elementwise{T}(op, ::Type{T}, A::AbstractArray, B::AbstractArray) F = similar(A, T, promote_shape(A, B)) for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) @inbounds F[iF] = op(A[iA], B[iB]) From 3dcece65723f3f4aad3dcda3db9a3665738423f4 Mon Sep 17 00:00:00 2001 From: Blake Johnson <blakejohnson04@gmail.com> Date: Wed, 3 Aug 2016 21:19:50 -0400 Subject: [PATCH 0782/1117] Add user-extensible bounds checks to NEWS. [ci skip] --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 46ada23e5ccd3..6649743503399 100644 --- a/NEWS.md +++ b/NEWS.md @@ -43,6 +43,10 @@ New language features operator names like `Base.≤` should now use `Base.:≤` (prefixed by `@compat` if you need 0.4 compatibility via the `Compat` package). + * User-extensible bounds check elimination is now possible with the new + `@boundscheck` macro ([#14474]). This macro marks bounds checking code blocks, + which the compiler may remove when encountered inside an `@inbounds` call. + Experimental language features ------------------------------ From 7c0895b7709bf43047e233f2ecc8cb579468eb45 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 3 Aug 2016 21:32:02 -0400 Subject: [PATCH 0783/1117] fix a missing backtick, run news-update doc reference fixed for promote_op changes [ci skip] --- NEWS.md | 3 ++- base/promotion.jl | 2 +- doc/devdocs/julia.rst | 1 - doc/manual/conversion-and-promotion.rst | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS.md b/NEWS.md index 6649743503399..2c185785fe324 100644 --- a/NEWS.md +++ b/NEWS.md @@ -405,7 +405,7 @@ Deprecated or removed provides a `@functorize` macro. * `bitunpack(B)` and `bitpack(A)` have been deprecated in favor of - Array(B)` and `BitArray(A)`, respectively ([#16010]). + `Array(B)` and `BitArray(A)`, respectively ([#16010]). * `xdump` is removed, and `dump` now simply shows the full representation of a value. `dump` should not be overloaded, since it is for examining concrete structure ([#4163]). @@ -525,6 +525,7 @@ Language tooling improvements [#14424]: https://github.com/JuliaLang/julia/issues/14424 [#14458]: https://github.com/JuliaLang/julia/issues/14458 [#14469]: https://github.com/JuliaLang/julia/issues/14469 +[#14474]: https://github.com/JuliaLang/julia/issues/14474 [#14519]: https://github.com/JuliaLang/julia/issues/14519 [#14608]: https://github.com/JuliaLang/julia/issues/14608 [#14623]: https://github.com/JuliaLang/julia/issues/14623 diff --git a/base/promotion.jl b/base/promotion.jl index a3be3c27829d4..449dae5add81a 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -218,7 +218,7 @@ min(x::Real, y::Real) = min(promote(x,y)...) minmax(x::Real, y::Real) = minmax(promote(x, y)...) # "Promotion" that takes a function into account. These are meant to be -# used mainly by broadcast methods, so it is adviced against overriding them +# used mainly by broadcast methods, so it is advised against overriding them if isdefined(Core, :Inference) function _promote_op(op, T::Type) G = Tuple{Generator{Tuple{T},typeof(op)}} diff --git a/doc/devdocs/julia.rst b/doc/devdocs/julia.rst index 141c9f4933b81..9dc165acf5bef 100644 --- a/doc/devdocs/julia.rst +++ b/doc/devdocs/julia.rst @@ -22,7 +22,6 @@ sysimg llvm stdio - promote-op boundscheck locks offset-arrays diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 1ee291d4dddf7..61969179addc5 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -274,8 +274,8 @@ the catch-all method definitions given in *(x::Number, y::Number) = *(promote(x,y)...) /(x::Number, y::Number) = /(promote(x,y)...) -In certain cases, the result type also depends on the operator; how to -handle such scenarios is described :ref:`elsewhere <devdocs-promote-op>`. +In certain cases, the result type also depends on the operator; +such scenarios are handled by the ``promote_op`` function. These method definitions say that in the absence of more specific rules for adding, subtracting, multiplying and dividing pairs of numeric From 56239458ac0fe3174c525295b56081d3cf11b9d9 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 4 Aug 2016 11:46:16 +0800 Subject: [PATCH 0784/1117] Fix `jl_static_show` for `bitstype` The byte gets sign extended when passing to the vararg `jl_printf` and then printed as an unsigned int which might come with unwanted `0xff` prefix... --- src/builtins.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins.c b/src/builtins.c index b64c01be9611c..4698cab3afcbb 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1479,7 +1479,7 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt size_t nb = jl_datatype_size(vt); size_t tlen = jl_datatype_nfields(vt); if (nb > 0 && tlen == 0) { - char *data = (char*)jl_data_ptr(v); + uint8_t *data = (uint8_t*)v; n += jl_printf(out, "0x"); for(int i=nb-1; i >= 0; --i) n += jl_printf(out, "%02" PRIx8, data[i]); From 0be7d825997a73542199f092e05b55eca887ead9 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 4 Aug 2016 00:14:31 -0400 Subject: [PATCH 0785/1117] Fix line numbers in doctests [ci skip] --- base/docs/helpdb/Base.jl | 2 +- doc/manual/complex-and-rational-numbers.rst | 2 +- doc/manual/control-flow.rst | 2 +- doc/manual/mathematical-operations.rst | 2 +- doc/stdlib/collections.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index c66b6fb2ce66a..9599c3a57dbe6 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4915,7 +4915,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:537 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:541 ... ``` """ diff --git a/doc/manual/complex-and-rational-numbers.rst b/doc/manual/complex-and-rational-numbers.rst index 00ab37d26a3b4..aa9cd00783e47 100644 --- a/doc/manual/complex-and-rational-numbers.rst +++ b/doc/manual/complex-and-rational-numbers.rst @@ -159,7 +159,7 @@ versus ``-1 + 0im`` even though ``-1 == -1 + 0im``: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:154 + in sqrt(::Int64) at ./math.jl:169 ... julia> sqrt(-1 + 0im) diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 0e5b8913a3805..0a1180581594b 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -666,7 +666,7 @@ negative real value: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:154 + in sqrt(::Int64) at ./math.jl:169 ... You may define your own exceptions in the following way: diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index c7925dea544e6..36fba25e8c46f 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -427,7 +427,7 @@ The following examples show the different forms. julia> round(Int8,127.6) ERROR: InexactError() - in trunc(::Type{Int8}, ::Float64) at ./float.jl:456 + in trunc(::Type{Int8}, ::Float64) at ./float.jl:458 in round(::Type{Int8}, ::Float64) at ./float.jl:211 ... diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index e462aecc4ecba..4cc7c6eb4be22 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1365,7 +1365,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:537 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:541 ... .. function:: splice!(collection, index, [replacement]) -> item From 73b738e5e158ec29a36a821a645d0d6abeda126e Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 4 Aug 2016 01:23:31 -0400 Subject: [PATCH 0786/1117] VERSION: branch release-0.5, master is now 0.6.0-dev and open for breaking changes (try not to make things too hard to backport just yet) ref #17418 for backporting procedure --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ce3afe3b9964a..fb0b754233e49 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0-rc0 +0.6.0-dev From c9f3d92ea3f5f18aff206461a6572fcf27edf63f Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 4 Aug 2016 08:24:30 +0800 Subject: [PATCH 0787/1117] Deoptimize TLS access * Apparently `ifunc` is not supported by gcc 5 on ARM * Try even harder to workaround LLVM bug when dealing with returntwice function --- src/llvm-ptls.cpp | 45 ++++++++++++++++++++++++++++++++++----------- src/threading.c | 2 +- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 178faf6c3b28e..019975cc3ba05 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -7,6 +7,7 @@ #include "llvm-version.h" #include "support/dtypes.h" +#include <sstream> #include <llvm/Pass.h> #include <llvm/IR/Module.h> @@ -99,23 +100,41 @@ void LowerPTLS::runOnFunction(LLVMContext &ctx, Module &M, Function *F, ptlsStates->addAttribute(AttributeSet::FunctionIndex, Attribute::NoUnwind); } - else if (jl_tls_offset != -1) { #ifdef LLVM37 + else if (jl_tls_offset != -1) { auto T_int8 = Type::getInt8Ty(ctx); auto T_pint8 = PointerType::get(T_int8, 0); - auto T_size = (sizeof(size_t) == 8 ? Type::getInt64Ty(ctx) : - Type::getInt32Ty(ctx)); // Replace the function call with inline assembly if we know // how to generate it. - const char *asm_str = nullptr; +# if defined(_CPU_X86_64_) || defined(_CPU_X86_) + // Workaround LLVM bug by hiding the offset computation + // (and therefore the optimization opportunity) from LLVM. + static const std::string asm_str = [&] () { + std::stringstream stm; # if defined(_CPU_X86_64_) - asm_str = "movq %fs:0, $0"; -# elif defined(_CPU_X86_) - asm_str = "movl %gs:0, $0"; -# elif defined(_CPU_AARCH64_) - asm_str = "mrs $0, tpidr_el0"; + stm << "movq %fs:0, $0;\naddq $$" << jl_tls_offset << ", $0"; +# else + stm << "movl %gs:0, $0;\naddl $$" << jl_tls_offset << ", $0"; # endif - assert(asm_str && "Cannot emit thread pointer for this architecture."); + return stm.str(); + }(); + // The add instruction clobbers flags + auto tp = InlineAsm::get(FunctionType::get(T_pint8, false), + asm_str.c_str(), + "=r,~{dirflag},~{fpsr},~{flags}", false); + Value *tls = CallInst::Create(tp, "ptls_i8", ptlsStates); + tls = new BitCastInst(tls, PointerType::get(T_ppjlvalue, 0), + "ptls", ptlsStates); +# elif defined(_CPU_AARCH64_) + // AArch64 doesn't seem to have this issue. + // (Possibly because there are many more registers and the offset is + // positive and small) + // It's also harder to emit the offset in a generic way on AArch64 + // (need to generate one or two `add` with shift) so let llvm emit + // the add for now. + auto T_size = (sizeof(size_t) == 8 ? Type::getInt64Ty(ctx) : + Type::getInt32Ty(ctx)); + const char *asm_str = "mrs $0, tpidr_el0"; auto offset = ConstantInt::getSigned(T_size, jl_tls_offset); auto tp = InlineAsm::get(FunctionType::get(T_pint8, false), asm_str, "=r", false); @@ -124,10 +143,14 @@ void LowerPTLS::runOnFunction(LLVMContext &ctx, Module &M, Function *F, "ptls_i8", ptlsStates); tls = new BitCastInst(tls, PointerType::get(T_ppjlvalue, 0), "ptls", ptlsStates); +# else + Value *tls = nullptr; + assert(0 && "Cannot emit thread pointer for this architecture."); +# endif ptlsStates->replaceAllUsesWith(tls); ptlsStates->eraseFromParent(); -#endif } +#endif else { ptlsStates->addAttribute(AttributeSet::FunctionIndex, Attribute::ReadNone); diff --git a/src/threading.c b/src/threading.c index b4de746abe526..6b1d373c7f1f6 100644 --- a/src/threading.c +++ b/src/threading.c @@ -147,7 +147,7 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) #if defined(__GLIBC__) && (defined(_CPU_X86_64_) || defined(_CPU_X86_) || \ ((defined(_CPU_AARCH64_) || defined(_CPU_ARM_) || \ defined(_CPU_PPC64_) || defined(_CPU_PPC_)) && \ - __GNUC__ >= 5)) + __GNUC__ >= 6)) // Only enable this on architectures that are tested. // For example, GCC doesn't seem to support the `ifunc` attribute on power yet. # if __GLIBC_PREREQ(2, 12) From 20bc0c44ca6d3d0788bf11d89d0834bd4cf8fd84 Mon Sep 17 00:00:00 2001 From: Helge Eichhorn <git@helgeeichhorn.de> Date: Thu, 4 Aug 2016 13:15:03 +0200 Subject: [PATCH 0788/1117] Add datatype_name function. Fix #17555 --- base/reflection.jl | 7 +++++++ doc/stdlib/base.rst | 6 ++++++ test/reflection.jl | 1 + 3 files changed, 14 insertions(+) diff --git a/base/reflection.jl b/base/reflection.jl index aa18e49cdcfc2..f6facf3ff6123 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -70,6 +70,13 @@ end fieldnames(t::DataType) = Symbol[fieldname(t, n) for n in 1:nfields(t)] fieldnames{T<:Tuple}(t::Type{T}) = Int[n for n in 1:nfields(t)] +""" + Base.datatype_name(t::DataType) -> Symbol + +Get the name of a `DataType` (without its parent module) as a symbol. +""" +datatype_name(t::DataType) = t.name.name + """ Base.datatype_module(t::DataType) -> Module diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 30288f57fbdb1..79f275f7651f5 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -1355,6 +1355,12 @@ Reflection Determine the module containing the definition of a ``DataType``\ . +.. function:: Base.datatype_name(t::DataType) -> Symbol + + .. Docstring generated from Julia source + + Get the name of a ``DataType`` (without its parent module) as a symbol. + .. function:: isconst([m::Module], s::Symbol) -> Bool .. Docstring generated from Julia source diff --git a/test/reflection.jl b/test/reflection.jl index 352ff8db8d9ea..625e61c1cc1ee 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -221,6 +221,7 @@ let @test Base.function_module(foo9475, (Any,)) == TestMod7648.TestModSub9475 @test Base.function_module(foo9475) == TestMod7648.TestModSub9475 @test Base.datatype_module(Foo7648) == TestMod7648 + @test Base.datatype_name(Foo7648) == :Foo7648 @test basename(functionloc(foo7648, (Any,))[1]) == "reflection.jl" @test first(methods(TestMod7648.TestModSub9475.foo7648)) == @which foo7648(5) @test TestMod7648 == @which foo7648 From 435108db15c1ab789dc053d95a33ad3b0607da54 Mon Sep 17 00:00:00 2001 From: jw3126 <jw3126@gmail.com> Date: Thu, 4 Aug 2016 15:51:55 +0200 Subject: [PATCH 0789/1117] Fixed doc/conf.py #17813 Fixed missing UnicodeCharacters doc/conf.py, see #17813 --- doc/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index 4d99c8104f05e..6adb3308fea4b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -201,6 +201,7 @@ \DeclareUnicodeCharacter{025B}{\ensuremath{\varepsilon}} \DeclareUnicodeCharacter{03B3}{\ensuremath{\gamma}} \DeclareUnicodeCharacter{03B8}{\ensuremath{\theta}} ++ \DeclareUnicodeCharacter{03BD}{\ensuremath{\nu}} \DeclareUnicodeCharacter{03C0}{\ensuremath{\pi}} \DeclareUnicodeCharacter{03C6}{\ensuremath{\varphi}} \DeclareUnicodeCharacter{1D34}{\ensuremath{^{\mathrm{H}}}} @@ -213,6 +214,7 @@ \DeclareUnicodeCharacter{2209}{\ensuremath{\notin}} \DeclareUnicodeCharacter{220C}{\ensuremath{\not\ni}} \DeclareUnicodeCharacter{2211}{\ensuremath{\sum}} ++ \DeclareUnicodeCharacter{2213}{\ensuremath{\mp}} \DeclareUnicodeCharacter{221A}{\ensuremath{\sqrt{}}} \DeclareUnicodeCharacter{221B}{\ensuremath{\sqrt[3]{}}} \DeclareUnicodeCharacter{222A}{\ensuremath{\cup}} From b954f3ad25f58ff57fd322861819372925eae238 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 4 Aug 2016 07:28:31 -0700 Subject: [PATCH 0790/1117] Fix #17805, spawn test failure on Windows introduced by #17522 --- test/spawn.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/spawn.jl b/test/spawn.jl index 0040630cf2016..7fa04587e3b28 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -287,7 +287,12 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test_throws ArgumentError write(out, "now closed error") @test isreadable(out) @test !iswritable(out) - is_windows() && Base.process_events(false) # should be enough steps to fully propagate EOF now + if is_windows() + # WINNT kernel does not provide a fast mechanism for async propagation + # of EOF for a blocking stream, so just wait for it to catch up. + # This shouldn't take much more than 32ms. + Base.wait_close(out) + end @test !isopen(out) end wait(ready) # wait for writer task to be ready before using `out` From c2ccbb8831b3e302ec58e62c2ba176c594249fb5 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 4 Aug 2016 07:28:12 -0500 Subject: [PATCH 0791/1117] indices extensions for iterators & collect --- base/array.jl | 4 ++-- base/iterator.jl | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/base/array.jl b/base/array.jl index 079bdfd1e5d99..26f308cd02fc5 100644 --- a/base/array.jl +++ b/base/array.jl @@ -211,7 +211,7 @@ The result has the same shape and number of dimensions as `collection`. collect{T}(::Type{T}, itr) = _collect(T, itr, iteratorsize(itr)) _collect{T}(::Type{T}, itr, isz::HasLength) = copy!(Array{T,1}(Int(length(itr)::Integer)), itr) -_collect{T}(::Type{T}, itr, isz::HasShape) = copy!(Array{T}(convert(Dims,size(itr))), itr) +_collect{T}(::Type{T}, itr, isz::HasShape) = copy!(similar(Array{T}, indices(itr)), itr) function _collect{T}(::Type{T}, itr, isz::SizeUnknown) a = Array{T,1}(0) for x in itr @@ -259,7 +259,7 @@ end _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T _array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = Array{T}(convert(Dims,size(itr))) +_array_for(T, itr, ::HasShape) = similar(Array{T}, indices(itr)) function collect(itr::Generator) isz = iteratorsize(itr.iter) diff --git a/base/iterator.jl b/base/iterator.jl index 6c93bcbb6be37..33852b9a33e5c 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -69,6 +69,7 @@ end zip(a) = Zip1(a) length(z::Zip1) = length(z.a) size(z::Zip1) = size(z.a) +indices(z::Zip1) = indices(z.a) eltype{I}(::Type{Zip1{I}}) = Tuple{eltype(I)} @inline start(z::Zip1) = start(z.a) @inline function next(z::Zip1, st) @@ -87,6 +88,7 @@ end zip(a, b) = Zip2(a, b) length(z::Zip2) = _min_length(z.a, z.b, iteratorsize(z.a), iteratorsize(z.b)) size(z::Zip2) = promote_shape(size(z.a), size(z.b)) +indices(z::Zip2) = promote_shape(indices(z.a), indices(z.b)) eltype{I1,I2}(::Type{Zip2{I1,I2}}) = Tuple{eltype(I1), eltype(I2)} @inline start(z::Zip2) = (start(z.a), start(z.b)) @inline function next(z::Zip2, st) @@ -137,6 +139,7 @@ julia> first(c) zip(a, b, c...) = Zip(a, zip(b, c...)) length(z::Zip) = _min_length(z.a, z.z, iteratorsize(z.a), iteratorsize(z.z)) size(z::Zip) = promote_shape(size(z.a), size(z.z)) +indices(z::Zip) = promote_shape(indices(z.a), indices(z.z)) tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{} function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T}) @_pure_meta @@ -430,8 +433,10 @@ iteratoreltype{O}(::Type{Repeated{O}}) = HasEltype() abstract AbstractProdIterator length(p::AbstractProdIterator) = prod(size(p)) +_length(p::AbstractProdIterator) = prod(map(unsafe_length, indices(p))) size(p::AbstractProdIterator) = _prod_size(p.a, p.b, iteratorsize(p.a), iteratorsize(p.b)) -ndims(p::AbstractProdIterator) = length(size(p)) +indices(p::AbstractProdIterator) = _prod_indices(p.a, p.b, iteratorsize(p.a), iteratorsize(p.b)) +ndims(p::AbstractProdIterator) = length(indices(p)) # generic methods to handle size of Prod* types _prod_size(a, ::HasShape) = size(a) @@ -445,6 +450,17 @@ _prod_size(a, b, ::HasShape, ::HasShape) = (size(a)..., size(b)...) _prod_size(a, b, A, B) = throw(ArgumentError("Cannot construct size for objects of types $(typeof(a)) and $(typeof(b))")) +_prod_indices(a, ::HasShape) = indices(a) +_prod_indices(a, ::HasLength) = (OneTo(length(a)), ) +_prod_indices(a, A) = + throw(ArgumentError("Cannot compute indices for object of type $(typeof(a))")) +_prod_indices(a, b, ::HasLength, ::HasLength) = (OneTo(length(a)), OneTo(length(b))) +_prod_indices(a, b, ::HasLength, ::HasShape) = (OneTo(length(a)), indices(b)...) +_prod_indices(a, b, ::HasShape, ::HasLength) = (indices(a)..., OneTo(length(b))) +_prod_indices(a, b, ::HasShape, ::HasShape) = (indices(a)..., indices(b)...) +_prod_indices(a, b, A, B) = + throw(ArgumentError("Cannot construct indices for objects of types $(typeof(a)) and $(typeof(b))")) + # one iterator immutable Prod1{I} <: AbstractProdIterator a::I @@ -453,6 +469,7 @@ product(a) = Prod1(a) eltype{I}(::Type{Prod1{I}}) = Tuple{eltype(I)} size(p::Prod1) = _prod_size(p.a, iteratorsize(p.a)) +indices(p::Prod1) = _prod_indices(p.a, iteratorsize(p.a)) @inline start(p::Prod1) = start(p.a) @inline function next(p::Prod1, st) From a1bad295a2a769a87f020b23983dd5b513ede574 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 4 Aug 2016 07:30:18 -0500 Subject: [PATCH 0792/1117] Get (c)transpose for vectors working with non-1 indices It was already working for matrices --- base/abstractarray.jl | 7 +++++++ base/arraymath.jl | 4 ++-- test/offsetarray.jl | 17 ++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index c4e7ba2dbde20..82e679aec21a9 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -625,6 +625,13 @@ convert{T,S,N}(::Type{AbstractArray{T }}, A::AbstractArray{S,N}) = convert(Abst convert{T,N}(::Type{Array}, A::AbstractArray{T,N}) = convert(Array{T,N}, A) +""" + of_indices(x, y) + +Represents the array `y` as an array having the same indices type as `x`. +""" +of_indices(x, y) = similar(dims->y, oftype(indices(x), indices(y))) + full(x::AbstractArray) = x map(::Type{Integer}, a::Array) = map!(Integer, similar(a,typeof(Integer(one(eltype(a))))), a) diff --git a/base/arraymath.jl b/base/arraymath.jl index f5a0954b8323e..1f4e636228512 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -272,8 +272,8 @@ function ctranspose(A::AbstractMatrix) end ctranspose{T<:Real}(A::AbstractVecOrMat{T}) = transpose(A) -transpose(x::AbstractVector) = [ transpose(v) for i=1:1, v in x ] -ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=1:1, v in x ] +transpose(x::AbstractVector) = [ transpose(v) for i=of_indices(x, OneTo(1)), v in x ] +ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=of_indices(x, OneTo(1)), v in x ] _cumsum_type{T<:Number}(v::AbstractArray{T}) = typeof(+zero(T)) _cumsum_type(v) = typeof(v[1]+v[1]) diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 2f26454826c7a..d7ebf5a66fd6e 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -101,6 +101,12 @@ using OAs let # Basics +v0 = rand(4) +v = OffsetArray(v0, (-3,)) +@test indices(v) == (-2:1,) +@test_throws ErrorException size(v) +@test_throws ErrorException size(v, 1) + A0 = [1 3; 2 4] A = OffsetArray(A0, (-1,2)) # LinearFast S = OffsetArray(view(A0, 1:2, 1:2), (-1,2)) # LinearSlow @@ -179,6 +185,10 @@ end # show io = IOBuffer() +show(io, v) +str = takebuf_string(io) +show(io, v0) +@test str == takebuf_string(io) show(io, A) str = takebuf_string(io) @test str == "[1 3; 2 4]" @@ -261,7 +271,7 @@ v = view(A0, 1:1, i1) # logical indexing @test A[A .> 2] == [3,4] -# copy! +# copy! and fill! a = OffsetArray{Int}((-3:-1,)) fill!(a, -1) copy!(a, (1,2)) # non-array iterables @@ -316,6 +326,7 @@ copy!(am, b) @test am[1,8] == 2 @test am[1,9] == -1 +# map dest = similar(am) map!(+, dest, am, am) @test dest[1,7] == 2 @@ -326,6 +337,10 @@ am = map(identity, a) @test isa(am, OffsetArray) @test am == a +# other functions +v = OffsetArray(v0, (-3,)) +@test parent(v') == v0' +@test indices(v') === (1:1,-2:1) A = OffsetArray(rand(4,4), (-3,5)) @test maximum(A) == maximum(parent(A)) @test minimum(A) == minimum(parent(A)) From f3c22fe77daf35127ea333e77c443c686a5e0ebf Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 4 Aug 2016 07:30:44 -0500 Subject: [PATCH 0793/1117] Get atsign-test_approx_eq working for non-1 indices --- base/test.jl | 15 ++++++++------- test/offsetarray.jl | 2 ++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/base/test.jl b/base/test.jl index 586f5767906ef..7d23115354bbd 100644 --- a/base/test.jl +++ b/base/test.jl @@ -827,10 +827,11 @@ approx_full(x) = full(x) function test_approx_eq(va, vb, Eps, astr, bstr) va = approx_full(va) vb = approx_full(vb) - if length(va) != length(vb) + la, lb = length(linearindices(va)), length(linearindices(vb)) + if la != lb error("lengths of ", astr, " and ", bstr, " do not match: ", - "\n ", astr, " (length $(length(va))) = ", va, - "\n ", bstr, " (length $(length(vb))) = ", vb) + "\n ", astr, " (length $la) = ", va, + "\n ", bstr, " (length $lb) = ", vb) end diff = real(zero(eltype(va))) for (xa, xb) = zip(va, vb) @@ -856,7 +857,7 @@ array_eps{T}(a::AbstractArray{Complex{T}}) = eps(float(maximum(x->(isfinite(x) ? array_eps(a) = eps(float(maximum(x->(isfinite(x) ? abs(x) : oftype(x,NaN)), a))) test_approx_eq(va, vb, astr, bstr) = - test_approx_eq(va, vb, 1E4*length(va)*max(array_eps(va), array_eps(vb)), astr, bstr) + test_approx_eq(va, vb, 1E4*length(linearindices(va))*max(array_eps(va), array_eps(vb)), astr, bstr) """ @test_approx_eq_eps(a, b, tol) @@ -966,10 +967,10 @@ end # nothing. function test_approx_eq_modphase{S<:Real,T<:Real}( a::StridedVecOrMat{S}, b::StridedVecOrMat{T}, err=nothing) - m, n = size(a) - @test n==size(b, 2) && m==size(b, 1) + @test indices(a,1) == indices(b,1) && indices(a,2) == indices(b,2) + m = length(indices(a,1)) err === nothing && (err=m^3*(eps(S)+eps(T))) - for i=1:n + for i in indices(a,2) v1, v2 = a[:, i], b[:, i] @test_approx_eq_eps min(abs(norm(v1-v2)), abs(norm(v1+v2))) 0.0 err end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index d7ebf5a66fd6e..4289aa523fe17 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -339,9 +339,11 @@ am = map(identity, a) # other functions v = OffsetArray(v0, (-3,)) +@test_approx_eq v v @test parent(v') == v0' @test indices(v') === (1:1,-2:1) A = OffsetArray(rand(4,4), (-3,5)) +@test_approx_eq A A @test maximum(A) == maximum(parent(A)) @test minimum(A) == minimum(parent(A)) @test extrema(A) == extrema(parent(A)) From 99fd261610d461ff5a9af0abe2ced6ac165d65a0 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 4 Aug 2016 07:35:57 -0500 Subject: [PATCH 0794/1117] Generalize findn and findnz for non-1 indices --- base/array.jl | 4 ++-- test/offsetarray.jl | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index 26f308cd02fc5..b843c9043499e 100644 --- a/base/array.jl +++ b/base/array.jl @@ -794,7 +794,7 @@ function findn(A::AbstractMatrix) I = similar(A, Int, nnzA) J = similar(A, Int, nnzA) count = 1 - for j=1:size(A,2), i=1:size(A,1) + for j=indices(A,2), i=indices(A,1) if A[i,j] != 0 I[count] = i J[count] = j @@ -811,7 +811,7 @@ function findnz{T}(A::AbstractMatrix{T}) NZs = Array{T,1}(nnzA) count = 1 if nnzA > 0 - for j=1:size(A,2), i=1:size(A,1) + for j=indices(A,2), i=indices(A,1) Aij = A[i,j] if Aij != 0 I[count] = i diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 4289aa523fe17..5ca538539ec6a 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -369,6 +369,14 @@ pmax, ipmax = findmax(parent(A)) @test amax == pmax @test A[iamax] == amax @test amax == parent(A)[ipmax] +z = OffsetArray([0 0; 2 0; 0 0; 0 0], (-3,-1)) +I,J = findn(z) +@test I == [-1] +@test J == [0] +I,J,N = findnz(z) +@test I == [-1] +@test J == [0] +@test N == [2] v = OffsetArray([1,1e100,1,-1e100], (-3,))*1000 v2 = OffsetArray([1,-1e100,1,1e100], (5,))*1000 From 2e624877ab1e31dc119c7e06498e637819f4509d Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 4 Aug 2016 10:25:29 -0700 Subject: [PATCH 0795/1117] remove patch copy-paste + artifacts from doc/conf.py [ci skip] --- doc/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 6adb3308fea4b..934e10d79bb41 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -201,7 +201,7 @@ \DeclareUnicodeCharacter{025B}{\ensuremath{\varepsilon}} \DeclareUnicodeCharacter{03B3}{\ensuremath{\gamma}} \DeclareUnicodeCharacter{03B8}{\ensuremath{\theta}} -+ \DeclareUnicodeCharacter{03BD}{\ensuremath{\nu}} + \DeclareUnicodeCharacter{03BD}{\ensuremath{\nu}} \DeclareUnicodeCharacter{03C0}{\ensuremath{\pi}} \DeclareUnicodeCharacter{03C6}{\ensuremath{\varphi}} \DeclareUnicodeCharacter{1D34}{\ensuremath{^{\mathrm{H}}}} @@ -214,7 +214,7 @@ \DeclareUnicodeCharacter{2209}{\ensuremath{\notin}} \DeclareUnicodeCharacter{220C}{\ensuremath{\not\ni}} \DeclareUnicodeCharacter{2211}{\ensuremath{\sum}} -+ \DeclareUnicodeCharacter{2213}{\ensuremath{\mp}} + \DeclareUnicodeCharacter{2213}{\ensuremath{\mp}} \DeclareUnicodeCharacter{221A}{\ensuremath{\sqrt{}}} \DeclareUnicodeCharacter{221B}{\ensuremath{\sqrt[3]{}}} \DeclareUnicodeCharacter{222A}{\ensuremath{\cup}} From 2872c6ad89400cb1c9f19449112f7195fed977fe Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Thu, 4 Aug 2016 16:55:55 -0700 Subject: [PATCH 0796/1117] Move array documentation out of HelpDB, add examples (#17795) Lots of examples for things that are probably confusing for new Julia users. Made `reducedim` and `mapreducedim` arguments more similar to the ones for `reduce` and `mapreduce`. --- base/abstractarray.jl | 43 ++++++++ base/arraymath.jl | 141 ++++++++++++++++++++++++ base/docs/helpdb/Base.jl | 95 ---------------- base/multidimensional.jl | 50 ++++++++- base/reducedim.jl | 54 ++++++++++ doc/stdlib/arrays.rst | 226 +++++++++++++++++++++++++++++++++++++-- 6 files changed, 506 insertions(+), 103 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 82e679aec21a9..31e84ab985995 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1365,6 +1365,49 @@ foreach(f, itrs...) = (for z in zip(itrs...); f(z...); end; nothing) ## transform any set of dimensions ## dims specifies which dimensions will be transformed. for example ## dims==1:2 will call f on all slices A[:,:,...] +""" + mapslices(f, A, dims) + +Transform the given dimensions of array `A` using function `f`. `f` is called on each slice +of `A` of the form `A[...,:,...,:,...]`. `dims` is an integer vector specifying where the +colons go in this expression. The results are concatenated along the remaining dimensions. +For example, if `dims` is `[1,2]` and `A` is 4-dimensional, `f` is called on `A[:,:,i,j]` +for all `i` and `j`. + +```jldoctest +julia> a = reshape(collect(1:16),(2,2,2,2)) +2×2×2×2 Array{Int64,4}: +[:, :, 1, 1] = + 1 3 + 2 4 +<BLANKLINE> +[:, :, 2, 1] = + 5 7 + 6 8 +<BLANKLINE> +[:, :, 1, 2] = + 9 11 + 10 12 +<BLANKLINE> +[:, :, 2, 2] = + 13 15 + 14 16 + +julia> mapslices(sum, a, [1,2]) +1×1×2×2 Array{Int64,4}: +[:, :, 1, 1] = + 10 +<BLANKLINE> +[:, :, 2, 1] = + 26 +<BLANKLINE> +[:, :, 1, 2] = + 42 +<BLANKLINE> +[:, :, 2, 2] = + 58 +``` +""" mapslices(f, A::AbstractArray, dims) = mapslices(f, A, [dims...]) function mapslices(f, A::AbstractArray, dims::AbstractVector) if isempty(dims) diff --git a/base/arraymath.jl b/base/arraymath.jl index 1f4e636228512..0de1bd39486f5 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -159,6 +159,23 @@ function flipdim{T}(A::Array{T}, d::Integer) return B end +""" + rotl90(A) + +Rotate matrix `A` left 90 degrees. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rotl90(a) +2×2 Array{Int64,2}: + 2 4 + 1 3 +``` +""" function rotl90(A::AbstractMatrix) ind1, ind2 = indices(A) B = similar(A, (ind2,ind1)) @@ -168,6 +185,24 @@ function rotl90(A::AbstractMatrix) end return B end + +""" + rotr90(A) + +Rotate matrix `A` right 90 degrees. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rotr90(a) +2×2 Array{Int64,2}: + 3 1 + 4 2 +``` +""" function rotr90(A::AbstractMatrix) ind1, ind2 = indices(A) B = similar(A, (ind2,ind1)) @@ -177,6 +212,23 @@ function rotr90(A::AbstractMatrix) end return B end +""" + rot180(A) + +Rotate matrix `A` 180 degrees. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rot180(a) +2×2 Array{Int64,2}: + 4 3 + 2 1 +``` +""" function rot180(A::AbstractMatrix) B = similar(A) ind1, ind2 = indices(A,1), indices(A,2) @@ -186,13 +238,102 @@ function rot180(A::AbstractMatrix) end return B end +""" + rotl90(A, k) + +Rotate matrix `A` left 90 degrees an integer `k` number of times. +If `k` is zero or a multiple of four, this is equivalent to a `copy`. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rotl90(a,1) +2×2 Array{Int64,2}: + 2 4 + 1 3 + +julia> rotl90(a,2) +2×2 Array{Int64,2}: + 4 3 + 2 1 + +julia> rotl90(a,3) +2×2 Array{Int64,2}: + 3 1 + 4 2 + +julia> rotl90(a,4) +2×2 Array{Int64,2}: + 1 2 + 3 4 +``` +""" function rotl90(A::AbstractMatrix, k::Integer) k = mod(k, 4) k == 1 ? rotl90(A) : k == 2 ? rot180(A) : k == 3 ? rotr90(A) : copy(A) end +""" + rotr90(A, k) + +Rotate matrix `A` right 90 degrees an integer `k` number of times. If `k` is zero or a +multiple of four, this is equivalent to a `copy`. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rotr90(a,1) +2×2 Array{Int64,2}: + 3 1 + 4 2 + +julia> rotr90(a,2) +2×2 Array{Int64,2}: + 4 3 + 2 1 + +julia> rotr90(a,3) +2×2 Array{Int64,2}: + 2 4 + 1 3 + +julia> rotr90(a,4) +2×2 Array{Int64,2}: + 1 2 + 3 4 +``` +""" rotr90(A::AbstractMatrix, k::Integer) = rotl90(A,-k) +""" + rot180(A, k) + +Rotate matrix `A` 180 degrees an integer `k` number of times. +If `k` is even, this is equivalent to a `copy`. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rot180(a,1) +2×2 Array{Int64,2}: + 4 3 + 2 1 + +julia> rot180(a,2) +2×2 Array{Int64,2}: + 1 2 + 3 4 +``` +""" rot180(A::AbstractMatrix, k::Integer) = mod(k, 2) == 1 ? rot180(A) : copy(A) ## Transpose ## diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 9599c3a57dbe6..b8ab785fd9778 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -490,19 +490,6 @@ Get a backtrace object for the current program point. """ backtrace -""" - reducedim(f, A, dims[, initial]) - -Reduce 2-argument function `f` along dimensions of `A`. `dims` is a vector specifying the -dimensions to reduce, and `initial` is the initial value to use in the reductions. For `+`, `*`, -`max` and `min` the `initial` argument is optional. - -The associativity of the reduction is implementation-dependent; if you need a particular -associativity, e.g. left-to-right, you should write your own loop. See documentation for -`reduce`. -""" -reducedim - """ -(x) @@ -1105,17 +1092,6 @@ representable. """ ceil -""" - mapslices(f, A, dims) - -Transform the given dimensions of array `A` using function `f`. `f` is called on each slice -of `A` of the form `A[...,:,...,:,...]`. `dims` is an integer vector specifying where the -colons go in this expression. The results are concatenated along the remaining dimensions. -For example, if `dims` is `[1,2]` and `A` is 4-dimensional, `f` is called on `A[:,:,i,j]` -for all `i` and `j`. -""" -mapslices - """ issocket(path) -> Bool @@ -2321,15 +2297,6 @@ Bessel function of the second kind of order 1, ``Y_1(x)``. """ bessely1 -""" - cumprod(A, [dim]) - -Cumulative product along a dimension `dim` (defaults to 1). See also -[`cumprod!`](:func:`cumprod!`) to use a preallocated output array, both for performance and -to control the precision of the output (e.g. to avoid overflow). -""" -cumprod - """ besseljx(nu, x) @@ -3054,21 +3021,6 @@ block to create a new scope with copies of all variables referenced in the expre """ :@async -""" - rotr90(A) - -Rotate matrix `A` right 90 degrees. -""" -rotr90(A) - -""" - rotr90(A, k) - -Rotate matrix `A` right 90 degrees an integer `k` number of times. If `k` is zero or a -multiple of four, this is equivalent to a `copy`. -""" -rotr90(A, k) - """ readdir([dir]) -> Vector{String} @@ -3523,21 +3475,6 @@ Number of ways to choose `k` out of `n` items. """ binomial -""" - rot180(A) - -Rotate matrix `A` 180 degrees. -""" -rot180(A) - -""" - rot180(A, k) - -Rotate matrix `A` 180 degrees an integer `k` number of times. If `k` is even, this is -equivalent to a `copy`. -""" -rot180(A, k) - """ .<=(x, y) .≤(x,y) @@ -3903,21 +3840,6 @@ x == div(x,y)*y + rem(x,y) """ rem -""" - rotl90(A) - -Rotate matrix `A` left 90 degrees. -""" -rotl90(A) - -""" - rotl90(A, k) - -Rotate matrix `A` left 90 degrees an integer `k` number of times. If `k` is zero or a -multiple of four, this is equivalent to a `copy`. -""" -rotl90(A, k) - """ info(msg) @@ -5623,14 +5545,6 @@ handle comparison to other types via promotion rules where possible. """ Base.:(==) -""" - mapreducedim(f, op, A, dims[, initial]) - -Evaluates to the same as `reducedim(op, map(f, A), dims, f(initial))`, but is generally -faster because the intermediate array is avoided. -""" -mapreducedim - """ seekstart(s) @@ -6415,15 +6329,6 @@ the group owning the file """ operm -""" - cumsum(A, [dim]) - -Cumulative sum along a dimension `dim` (defaults to 1). See also [`cumsum!`](:func:`cumsum!`) -to use a preallocated output array, both for performance and to control the precision of the -output (e.g. to avoid overflow). -""" -cumsum - """ rpad(string, n, p) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index f4f4a676d2112..385e9e218f891 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -496,8 +496,56 @@ for (f, fmod, op) = ((:cummin, :_cummin!, :min), (:cummax, :_cummax!, :max)) @eval ($f)(A::AbstractArray) = ($f)(A, 1) end - cumsum(A::AbstractArray, axis::Integer=1) = cumsum!(similar(A, Base._cumsum_type(A)), A, axis) +""" + cumsum(A, dim=1) + +Cumulative sum along a dimension `dim` (defaults to 1). See also [`cumsum!`](:func:`cumsum!`) +to use a preallocated output array, both for performance and to control the precision of the +output (e.g. to avoid overflow). + +```jldoctest +julia> a = [1 2 3; 4 5 6] +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> cumsum(a,1) +2×3 Array{Int64,2}: + 1 2 3 + 5 7 9 + +julia> cumsum(a,2) +2×3 Array{Int64,2}: + 1 3 6 + 4 9 15 +``` +""" +cumsum(A::AbstractArray, axis::Integer=1) = cumsum!(similar(A, Base._cumsum_type(A)), A, axis) cumsum!(B, A::AbstractArray) = cumsum!(B, A, 1) +""" + cumprod(A, dim=1) + +Cumulative product along a dimension `dim` (defaults to 1). See also +[`cumprod!`](:func:`cumprod!`) to use a preallocated output array, both for performance and +to control the precision of the output (e.g. to avoid overflow). + +```jldoctest +julia> a = [1 2 3; 4 5 6] +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> cumprod(a,1) +2×3 Array{Int64,2}: + 1 2 3 + 4 10 18 + +julia> cumprod(a,2) +2×3 Array{Int64,2}: + 1 2 6 + 4 20 120 +``` +""" cumprod(A::AbstractArray, axis::Integer=1) = cumprod!(similar(A), A, axis) cumprod!(B, A) = cumprod!(B, A, 1) diff --git a/base/reducedim.jl b/base/reducedim.jl index f6fa07169a511..047cc9af80e92 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -225,11 +225,65 @@ mapreducedim!(f, op, R::AbstractArray, A::AbstractArray) = reducedim!{RT}(op, R::AbstractArray{RT}, A::AbstractArray) = mapreducedim!(identity, op, R, A, zero(RT)) +""" + mapreducedim(f, op, A, region[, v0]) + +Evaluates to the same as `reducedim(op, map(f, A), region, f(v0))`, but is generally +faster because the intermediate array is avoided. + +```jldoctest +julia> a = reshape(collect(1:16), (4,4)) +4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + +julia> mapreducedim(isodd, *, a, 1) +1×4 Array{Bool,2}: + false false false false + +julia> mapreducedim(isodd, |, a, 1, true) +1×4 Array{Bool,2}: + true true true true +``` +""" mapreducedim(f, op, A::AbstractArray, region, v0) = mapreducedim!(f, op, reducedim_initarray(A, region, v0), A) mapreducedim{T}(f, op, A::AbstractArray{T}, region) = mapreducedim!(f, op, reducedim_init(f, op, A, region), A) +""" + reducedim(f, A, region[, v0]) + +Reduce 2-argument function `f` along dimensions of `A`. `region` is a vector specifying the +dimensions to reduce, and `v0` is the initial value to use in the reductions. For `+`, `*`, +`max` and `min` the `v0` argument is optional. + +The associativity of the reduction is implementation-dependent; if you need a particular +associativity, e.g. left-to-right, you should write your own loop. See documentation for +[`reduce`](:func:`reduce`). + +```jldoctest +julia> a = reshape(collect(1:16), (4,4)) +4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + +julia> reducedim(max, a, 2) +4×1 Array{Int64,2}: + 13 + 14 + 15 + 16 + +julia> reducedim(max, a, 1) +1×4 Array{Int64,2}: + 4 8 12 16 +``` +""" reducedim(op, A::AbstractArray, region, v0) = mapreducedim(identity, op, A, region, v0) reducedim(op, A::AbstractArray, region) = mapreducedim(identity, op, A, region) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 448dd55c0caa9..f72f67f72b10a 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -657,24 +657,58 @@ Indexing, Assignment, and Concatenation Array functions --------------- -.. function:: cumprod(A, [dim]) +.. function:: cumprod(A, dim=1) .. Docstring generated from Julia source Cumulative product along a dimension ``dim`` (defaults to 1). See also :func:`cumprod!` to use a preallocated output array, both for performance and to control the precision of the output (e.g. to avoid overflow). + .. doctest:: + + julia> a = [1 2 3; 4 5 6] + 2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + + julia> cumprod(a,1) + 2×3 Array{Int64,2}: + 1 2 3 + 4 10 18 + + julia> cumprod(a,2) + 2×3 Array{Int64,2}: + 1 2 6 + 4 20 120 + .. function:: cumprod!(B, A, [dim]) .. Docstring generated from Julia source Cumulative product of ``A`` along a dimension, storing the result in ``B``\ . The dimension defaults to 1. -.. function:: cumsum(A, [dim]) +.. function:: cumsum(A, dim=1) .. Docstring generated from Julia source Cumulative sum along a dimension ``dim`` (defaults to 1). See also :func:`cumsum!` to use a preallocated output array, both for performance and to control the precision of the output (e.g. to avoid overflow). + .. doctest:: + + julia> a = [1 2 3; 4 5 6] + 2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + + julia> cumsum(a,1) + 2×3 Array{Int64,2}: + 1 2 3 + 5 7 9 + + julia> cumsum(a,2) + 2×3 Array{Int64,2}: + 1 3 6 + 4 9 15 + .. function:: cumsum!(B, A, [dim]) .. Docstring generated from Julia source @@ -717,49 +751,193 @@ Array functions Rotate matrix ``A`` 180 degrees. + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rot180(a) + 2×2 Array{Int64,2}: + 4 3 + 2 1 + .. function:: rot180(A, k) .. Docstring generated from Julia source Rotate matrix ``A`` 180 degrees an integer ``k`` number of times. If ``k`` is even, this is equivalent to a ``copy``\ . + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rot180(a,1) + 2×2 Array{Int64,2}: + 4 3 + 2 1 + + julia> rot180(a,2) + 2×2 Array{Int64,2}: + 1 2 + 3 4 + .. function:: rotl90(A) .. Docstring generated from Julia source Rotate matrix ``A`` left 90 degrees. + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rotl90(a) + 2×2 Array{Int64,2}: + 2 4 + 1 3 + .. function:: rotl90(A, k) .. Docstring generated from Julia source Rotate matrix ``A`` left 90 degrees an integer ``k`` number of times. If ``k`` is zero or a multiple of four, this is equivalent to a ``copy``\ . + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rotl90(a,1) + 2×2 Array{Int64,2}: + 2 4 + 1 3 + + julia> rotl90(a,2) + 2×2 Array{Int64,2}: + 4 3 + 2 1 + + julia> rotl90(a,3) + 2×2 Array{Int64,2}: + 3 1 + 4 2 + + julia> rotl90(a,4) + 2×2 Array{Int64,2}: + 1 2 + 3 4 + .. function:: rotr90(A) .. Docstring generated from Julia source Rotate matrix ``A`` right 90 degrees. + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rotr90(a) + 2×2 Array{Int64,2}: + 3 1 + 4 2 + .. function:: rotr90(A, k) .. Docstring generated from Julia source Rotate matrix ``A`` right 90 degrees an integer ``k`` number of times. If ``k`` is zero or a multiple of four, this is equivalent to a ``copy``\ . -.. function:: reducedim(f, A, dims[, initial]) + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rotr90(a,1) + 2×2 Array{Int64,2}: + 3 1 + 4 2 + + julia> rotr90(a,2) + 2×2 Array{Int64,2}: + 4 3 + 2 1 + + julia> rotr90(a,3) + 2×2 Array{Int64,2}: + 2 4 + 1 3 + + julia> rotr90(a,4) + 2×2 Array{Int64,2}: + 1 2 + 3 4 + +.. function:: reducedim(f, A, region[, v0]) .. Docstring generated from Julia source - Reduce 2-argument function ``f`` along dimensions of ``A``\ . ``dims`` is a vector specifying the dimensions to reduce, and ``initial`` is the initial value to use in the reductions. For ``+``\ , ``*``\ , ``max`` and ``min`` the ``initial`` argument is optional. + Reduce 2-argument function ``f`` along dimensions of ``A``\ . ``region`` is a vector specifying the dimensions to reduce, and ``v0`` is the initial value to use in the reductions. For ``+``\ , ``*``\ , ``max`` and ``min`` the ``v0`` argument is optional. + + The associativity of the reduction is implementation-dependent; if you need a particular associativity, e.g. left-to-right, you should write your own loop. See documentation for :func:`reduce`\ . + + .. doctest:: - The associativity of the reduction is implementation-dependent; if you need a particular associativity, e.g. left-to-right, you should write your own loop. See documentation for ``reduce``\ . + julia> a = reshape(collect(1:16), (4,4)) + 4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + + julia> reducedim(max, a, 2) + 4×1 Array{Int64,2}: + 13 + 14 + 15 + 16 + + julia> reducedim(max, a, 1) + 1×4 Array{Int64,2}: + 4 8 12 16 -.. function:: mapreducedim(f, op, A, dims[, initial]) +.. function:: mapreducedim(f, op, A, region[, v0]) .. Docstring generated from Julia source - Evaluates to the same as ``reducedim(op, map(f, A), dims, f(initial))``\ , but is generally faster because the intermediate array is avoided. + Evaluates to the same as ``reducedim(op, map(f, A), region, f(v0))``\ , but is generally faster because the intermediate array is avoided. + + .. doctest:: + + julia> a = reshape(collect(1:16), (4,4)) + 4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + + julia> mapreducedim(isodd, *, a, 1) + 1×4 Array{Bool,2}: + false false false false + + julia> mapreducedim(isodd, |, a, 1, true) + 1×4 Array{Bool,2}: + true true true true .. function:: mapslices(f, A, dims) @@ -767,6 +945,40 @@ Array functions Transform the given dimensions of array ``A`` using function ``f``\ . ``f`` is called on each slice of ``A`` of the form ``A[...,:,...,:,...]``\ . ``dims`` is an integer vector specifying where the colons go in this expression. The results are concatenated along the remaining dimensions. For example, if ``dims`` is ``[1,2]`` and ``A`` is 4-dimensional, ``f`` is called on ``A[:,:,i,j]`` for all ``i`` and ``j``\ . + .. doctest:: + + julia> a = reshape(collect(1:16),(2,2,2,2)) + 2×2×2×2 Array{Int64,4}: + [:, :, 1, 1] = + 1 3 + 2 4 + <BLANKLINE> + [:, :, 2, 1] = + 5 7 + 6 8 + <BLANKLINE> + [:, :, 1, 2] = + 9 11 + 10 12 + <BLANKLINE> + [:, :, 2, 2] = + 13 15 + 14 16 + + julia> mapslices(sum, a, [1,2]) + 1×1×2×2 Array{Int64,4}: + [:, :, 1, 1] = + 10 + <BLANKLINE> + [:, :, 2, 1] = + 26 + <BLANKLINE> + [:, :, 1, 2] = + 42 + <BLANKLINE> + [:, :, 2, 2] = + 58 + .. function:: sum_kbn(A) .. Docstring generated from Julia source From 2ad30ddc0cc3ec9848c74bde1edb7fc3c286f33a Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 4 Aug 2016 14:42:30 -0400 Subject: [PATCH 0797/1117] fix deserializer jl_recache_type to walk type parameter list completely previously, it only walked through types with `uid = -1`, but in fact we needed to walk through any uncached type, including those with `uid = 0` and those gotten from jl_typeof fix #17809 --- src/dump.c | 53 +++++++++++++++++++++++++++++-------------------- test/compile.jl | 19 ++++++++++++++++++ 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/src/dump.c b/src/dump.c index d0e3f88f5c8a3..7157d89bb5a36 100644 --- a/src/dump.c +++ b/src/dump.c @@ -2222,35 +2222,46 @@ static jl_datatype_t *jl_recache_type(jl_datatype_t *dt, size_t start, jl_value_ { if (v == NULL) v = dt->instance; // the instance before unique'ing - jl_datatype_t *t; // the type after unique'ing - if (dt->uid == -1) { - jl_svec_t *tt = dt->parameters; - size_t l = jl_svec_len(tt); - if (l == 0) { // jl_cache_type doesn't work if length(parameters) == 0 - dt->uid = jl_assign_type_uid(); - t = dt; - } - else { - // recache all type parameters, then type type itself - size_t i; - for (i = 0; i < l; i++) { - jl_datatype_t *p = (jl_datatype_t*)jl_svecref(tt, i); - if (jl_is_datatype(p) && p->uid == -1) { + jl_svec_t *tt = dt->parameters; + if (dt->uid == 0 || dt->uid == -1) { + // recache all type parameters + size_t i, l = jl_svec_len(tt); + for (i = 0; i < l; i++) { + jl_datatype_t *p = (jl_datatype_t*)jl_svecref(tt, i); + if (jl_is_datatype(p)) { + if (p->uid == -1 || p->uid == 0) { jl_datatype_t *cachep = jl_recache_type(p, start, NULL); - if (p != cachep) + if (p != cachep) { + assert(jl_types_equal((jl_value_t*)p, (jl_value_t*)cachep)); jl_svecset(tt, i, cachep); + } } + } + else { jl_datatype_t *tp = (jl_datatype_t*)jl_typeof(p); - if (jl_is_datatype_singleton(tp)) { - if (tp->uid == -1) { - tp = jl_recache_type(tp, start, NULL); - } - if ((jl_value_t*)p != tp->instance) - jl_svecset(tt, i, tp->instance); + assert(tp->uid != 0); + if (tp->uid == -1) { + tp = jl_recache_type(tp, start, NULL); } + if (tp->instance && (jl_value_t*)p != tp->instance) + jl_svecset(tt, i, tp->instance); } + } + } + + jl_datatype_t *t; // the type after unique'ing + if (dt->uid == 0) { + return dt; + } + else if (dt->uid == -1) { + if (jl_svec_len(tt) == 0) { // jl_cache_type doesn't work if length(parameters) == 0 + dt->uid = jl_assign_type_uid(); + t = dt; + } + else { dt->uid = 0; t = (jl_datatype_t*)jl_cache_type_(dt); + assert(jl_types_equal((jl_value_t*)t, (jl_value_t*)dt)); } } else { diff --git a/test/compile.jl b/test/compile.jl index ce768211a96cc..09dc51390edc6 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -31,6 +31,17 @@ try include_dependency("bar.jl") end + # test for creation of some reasonably complicated type + immutable MyType{T} end + const t17809s = Any[ + Tuple{ + Type{Ptr{MyType{i}}}, + Array{Ptr{MyType{MyType{:sym}()}}(0), 0}, + Val{Complex{Int}(1, 2)}, + Val{3}, + Val{nothing}} + for i = 0:25] + # test that types and methods get reconnected correctly # issue 16529 (adding a method to a type with no instances) (::Task)(::UInt8, ::UInt16, ::UInt32) = 2 @@ -110,6 +121,14 @@ try @test Vector{Foo.NominalValue{Int32, Int64}}() == 3 @test Vector{Foo.NominalValue{UInt, UInt}}() == 4 @test Vector{Foo.NominalValue{Int, Int}}() == 5 + @test all(i -> Foo.t17809s[i + 1] === + Tuple{ + Type{Ptr{Foo.MyType{i}}}, + Array{Ptr{Foo.MyType{Foo.MyType{:sym}()}}(0), 0}, + Val{Complex{Int}(1, 2)}, + Val{3}, + Val{nothing}}, + 0:25) end Baz_file = joinpath(dir, "Baz.jl") From a1e1c0ae2e2378786daeffb220b3b85191a77075 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 4 Aug 2016 17:52:35 -0400 Subject: [PATCH 0798/1117] make Libuv types RAII with better error handling and bump libuv version for bugfixes --- base/process.jl | 9 +- base/socket.jl | 115 +++++++-------- base/stream.jl | 131 ++++++++---------- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/libuv.version | 2 +- test/spawn.jl | 17 +++ 9 files changed, 137 insertions(+), 141 deletions(-) create mode 100644 deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 create mode 100644 deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 delete mode 100644 deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 delete mode 100644 deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 diff --git a/base/process.jl b/base/process.jl index 88d51cb4f1634..22c39e4527f64 100644 --- a/base/process.jl +++ b/base/process.jl @@ -403,22 +403,19 @@ end function setup_stdio(stdio::PipeEndpoint, readable::Bool) closeafter = false - if stdio.handle == C_NULL - io = Libc.malloc(_sizeof_uv_named_pipe) + if stdio.status == StatusUninit if readable link_pipe(io, false, stdio, true) else link_pipe(stdio, true, io, false) end closeafter = true - else - io = stdio.handle end - return (io, closeafter) + return (stdio.handle, closeafter) end function setup_stdio(stdio::Pipe, readable::Bool) - if stdio.in.handle == C_NULL && stdio.out.handle == C_NULL + if stdio.in.status == StatusUninit && stdio.out.status == StatusUninit link_pipe(stdio) end io = readable ? stdio.out : stdio.in diff --git a/base/socket.jl b/base/socket.jl index 8365f219fe711..d037173ea62b5 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -255,33 +255,30 @@ type TCPSocket <: LibuvStream lock::ReentrantLock throttle::Int - TCPSocket(handle) = new( - handle, - StatusUninit, - true, - PipeBuffer(), - false, Condition(), - false, Condition(), - false, Condition(), - nothing, - ReentrantLock(), - DEFAULT_READ_BUFFER_SZ - ) + function TCPSocket(handle::Ptr{Void}, status) + tcp = new( + handle, + status, + true, + PipeBuffer(), + false, Condition(), + false, Condition(), + false, Condition(), + nothing, + ReentrantLock(), + DEFAULT_READ_BUFFER_SZ) + associate_julia_struct(tcp.handle, tcp) + finalizer(tcp, uvfinalize) + return tcp + end end function TCPSocket() - this = TCPSocket(Libc.malloc(_sizeof_uv_tcp)) - associate_julia_struct(this.handle,this) - finalizer(this,uvfinalize) - err = ccall(:uv_tcp_init,Cint,(Ptr{Void},Ptr{Void}), - eventloop(),this.handle) - if err != 0 - #TODO: this codepath is not currently tested - Libc.free(this.handle) - this.handle = C_NULL - throw(UVError("failed to create tcp socket",err)) - end - this.status = StatusInit - return this + tcp = TCPSocket(Libc.malloc(_sizeof_uv_tcp), StatusUninit) + err = ccall(:uv_tcp_init, Cint, (Ptr{Void}, Ptr{Void}), + eventloop(), tcp.handle) + uv_error("failed to create tcp socket", err) + tcp.status = StatusInit + return tcp end type TCPServer <: LibuvServer @@ -292,27 +289,24 @@ type TCPServer <: LibuvServer closecb::Callback closenotify::Condition - TCPServer(handle) = new( - handle, - StatusUninit, - false, Condition(), - false, Condition() - ) + function TCPServer(handle::Ptr{Void}, status) + tcp = new( + handle, + status, + false, Condition(), + false, Condition()) + associate_julia_struct(tcp.handle, tcp) + finalizer(tcp, uvfinalize) + return tcp + end end function TCPServer() - this = TCPServer(Libc.malloc(_sizeof_uv_tcp)) - associate_julia_struct(this.handle, this) - finalizer(this,uvfinalize) - err = ccall(:uv_tcp_init,Cint,(Ptr{Void},Ptr{Void}), - eventloop(),this.handle) - if err != 0 - #TODO: this codepath is not currently tested - Libc.free(this.handle) - this.handle = C_NULL - throw(UVError("failed to create tcp server",err)) - end - this.status = StatusInit - return this + tcp = TCPServer(Libc.malloc(_sizeof_uv_tcp), StatusUninit) + err = ccall(:uv_tcp_init, Cint, (Ptr{Void}, Ptr{Void}), + eventloop(), tcp.handle) + uv_error("failed to create tcp server", err) + tcp.status = StatusInit + return tcp end isreadable(io::TCPSocket) = isopen(io) || nb_available(io) > 0 @@ -344,26 +338,23 @@ type UDPSocket <: LibuvStream sendnotify::Condition closenotify::Condition - UDPSocket(handle::Ptr) = new( - handle, - StatusUninit, - Condition(), - Condition(), - Condition() - ) + function UDPSocket(handle::Ptr{Void}, status) + udp = new( + handle, + status, + Condition(), + Condition(), + Condition()) + associate_julia_struct(udp.handle, udp) + finalizer(udp, uvfinalize) + return udp + end end function UDPSocket() - this = UDPSocket(Libc.malloc(_sizeof_uv_udp)) - associate_julia_struct(this.handle, this) - err = ccall(:uv_udp_init,Cint,(Ptr{Void},Ptr{Void}), - eventloop(),this.handle) - finalizer(this, uvfinalize) - if err != 0 - #TODO: this codepath is not currently tested - Libc.free(this.handle) - this.handle = C_NULL - throw(UVError("failed to create udp socket",err)) - end + this = UDPSocket(Libc.malloc(_sizeof_uv_udp), StatusUninit) + err = ccall(:uv_udp_init, Cint, (Ptr{Void}, Ptr{Void}), + eventloop(), this.handle) + uv_error("failed to create udp socket", err) this.status = StatusInit return this end diff --git a/base/stream.jl b/base/stream.jl index 733bba8d450ce..2c7ad60244ccd 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -113,16 +113,21 @@ type PipeEndpoint <: LibuvStream lock::ReentrantLock throttle::Int - PipeEndpoint(handle::Ptr{Void} = C_NULL) = new( - handle, - StatusUninit, - PipeBuffer(), - true, - false,Condition(), - false,Condition(), - false,Condition(), - nothing, ReentrantLock(), - DEFAULT_READ_BUFFER_SZ) + PipeEndpoint() = PipeEndpoint(Libc.malloc(_sizeof_uv_named_pipe), StatusUninit) + function PipeEndpoint(handle::Ptr{Void}, status) + p = new(handle, + status, + PipeBuffer(), + true, + false,Condition(), + false,Condition(), + false,Condition(), + nothing, ReentrantLock(), + DEFAULT_READ_BUFFER_SZ) + associate_julia_struct(handle, p) + finalizer(p, uvfinalize) + return p + end end type PipeServer <: LibuvServer @@ -132,26 +137,22 @@ type PipeServer <: LibuvServer connectnotify::Condition closecb::Callback closenotify::Condition - PipeServer(handle) = new( - handle, - StatusUninit, - false,Condition(), - false,Condition()) + function PipeServer(handle::Ptr{Void}, status) + p = new(handle, + status, + false,Condition(), + false,Condition()) + associate_julia_struct(p.handle, p) + finalizer(p, uvfinalize) + return p + end end typealias LibuvPipe Union{PipeEndpoint, PipeServer} function PipeServer() - handle = Libc.malloc(_sizeof_uv_named_pipe) - try - ret = PipeServer(handle) - associate_julia_struct(ret.handle,ret) - finalizer(ret,uvfinalize) - return init_pipe!(ret;readable=true) - catch - Libc.free(handle) - rethrow() - end + p = PipeServer(Libc.malloc(_sizeof_uv_named_pipe), StatusUninit) + return init_pipe!(p; readable=true) end type TTY <: LibuvStream @@ -167,16 +168,19 @@ type TTY <: LibuvStream lock::ReentrantLock throttle::Int @static if is_windows(); ispty::Bool; end - function TTY(handle) + TTY() = TTY(Libc.malloc(_sizeof_uv_tty), StatusUninit) + function TTY(handle::Ptr{Void}, status) tty = new( handle, - StatusUninit, + status, true, PipeBuffer(), false,Condition(), false,Condition(), nothing, ReentrantLock(), DEFAULT_READ_BUFFER_SZ) + associate_julia_struct(handle, tty) + finalizer(tty, uvfinalize) @static if is_windows() tty.ispty = ccall(:jl_ispty, Cint, (Ptr{Void},), handle) != 0 end @@ -185,16 +189,15 @@ type TTY <: LibuvStream end function TTY(fd::RawFD; readable::Bool = false) - handle = Libc.malloc(_sizeof_uv_tty) - ret = TTY(handle) - associate_julia_struct(handle,ret) - finalizer(ret,uvfinalize) + tty = TTY() # This needs to go after associate_julia_struct so that there # is no garbage in the ->data field - uv_error("TTY",ccall(:uv_tty_init,Int32,(Ptr{Void},Ptr{Void},Int32,Int32),eventloop(),handle,fd.fd,readable)) - ret.status = StatusOpen - ret.line_buffered = false - return ret + err = ccall(:uv_tty_init, Int32, (Ptr{Void}, Ptr{Void}, Int32, Int32), + eventloop(), tty.handle, fd.fd, readable) + uv_error("TTY", err) + tty.status = StatusOpen + tty.line_buffered = false + return tty end show(io::IO,stream::LibuvServer) = print(io, typeof(stream), "(", uv_status_string(stream), ")") @@ -230,18 +233,15 @@ function init_stdio(handle::Ptr{Void}) # return File(RawFD(ccall(:jl_uv_file_handle,Int32,(Ptr{Void},),handle))) else if t == UV_TTY - ret = TTY(handle) + ret = TTY(handle, StatusOpen) elseif t == UV_TCP - ret = TCPSocket(handle) + ret = TCPSocket(handle, StatusOpen) elseif t == UV_NAMED_PIPE - ret = PipeEndpoint(handle) + ret = PipeEndpoint(handle, StatusOpen) else throw(ArgumentError("invalid stdio type: $t")) end - ret.status = StatusOpen ret.line_buffered = false - associate_julia_struct(ret.handle, ret) - finalizer(ret, uvfinalize) return ret end end @@ -322,7 +322,9 @@ function wait_close(x::Union{LibuvStream, LibuvServer}) end function close(stream::Union{LibuvStream, LibuvServer}) - if isopen(stream) + if stream.status == StatusInit + ccall(:jl_forceclose_uv, Void, (Ptr{Void},), stream.handle) + elseif isopen(stream) if stream.status != StatusClosing ccall(:jl_close_uv, Void, (Ptr{Void},), stream.handle) stream.status = StatusClosing @@ -337,11 +339,13 @@ end function uvfinalize(uv::Union{LibuvStream, LibuvServer}) if uv.handle != C_NULL disassociate_julia_struct(uv.handle) # not going to call the usual close hooks - if uv.status != StatusUninit && uv.status != StatusInit + if uv.status != StatusUninit close(uv) - uv.handle = C_NULL - uv.status = StatusClosed + else + Libc.free(uv.handle) end + uv.status = StatusClosed + uv.handle = C_NULL end nothing end @@ -571,24 +575,23 @@ function init_pipe!(pipe::LibuvPipe; if pipe.status != StatusUninit error("pipe is already initialized") end - if pipe.handle == C_NULL - malloc_julia_pipe!(pipe) - end - uv_error("init_pipe",ccall(:jl_init_pipe, Cint, + err = ccall(:jl_init_pipe, Cint, (Ptr{Void}, Int32, Int32, Int32), - pipe.handle, writable, readable, julia_only)) + pipe.handle, writable, readable, julia_only) + uv_error( + if readable && writable + "init_pipe(ipc)" + elseif readable + "init_pipe(read)" + elseif writable + "init_pipe(write)" + else + "init_pipe(none)" + end, err) pipe.status = StatusInit return pipe end -function malloc_julia_pipe!(x::LibuvPipe) - assert(x.handle == C_NULL) - x.handle = Libc.malloc(_sizeof_uv_named_pipe) - associate_julia_struct(x.handle, x) - finalizer(x, uvfinalize) - nothing -end - function _link_pipe(read_end::Ptr{Void}, write_end::Ptr{Void}) uv_error("pipe_link", ccall(:uv_pipe_link, Int32, (Ptr{Void}, Ptr{Void}), read_end, write_end)) @@ -620,9 +623,6 @@ end function link_pipe(read_end::PipeEndpoint, readable_julia_only::Bool, write_end::Ptr{Void}, writable_julia_only::Bool) - if read_end.handle == C_NULL - malloc_julia_pipe!(read_end) - end init_pipe!(read_end; readable = true, writable = false, julia_only = readable_julia_only) uv_error("init_pipe", @@ -634,9 +634,6 @@ end function link_pipe(read_end::Ptr{Void}, readable_julia_only::Bool, write_end::PipeEndpoint, writable_julia_only::Bool) - if write_end.handle == C_NULL - malloc_julia_pipe!(write_end) - end uv_error("init_pipe", ccall(:jl_init_pipe, Cint, (Ptr{Void},Int32,Int32,Int32), read_end, 0, 1, readable_julia_only)) init_pipe!(write_end; @@ -648,12 +645,6 @@ end function link_pipe(read_end::PipeEndpoint, readable_julia_only::Bool, write_end::PipeEndpoint, writable_julia_only::Bool) - if write_end.handle == C_NULL - malloc_julia_pipe!(write_end) - end - if read_end.handle == C_NULL - malloc_julia_pipe!(read_end) - end init_pipe!(read_end; readable = true, writable = false, julia_only = readable_julia_only) init_pipe!(write_end; diff --git a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 new file mode 100644 index 0000000000000..e42a20ffd5a35 --- /dev/null +++ b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 @@ -0,0 +1 @@ +cc07a8ef026fa42eeaddf7b6e074096e diff --git a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 new file mode 100644 index 0000000000000..bec2d8d44203f --- /dev/null +++ b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 @@ -0,0 +1 @@ +e128cec9548ff2f52a78743203d3f06dceef3cf7cbeb3a7a5f7453b132576833f82688eed52cf74c035f57828deac079cd845ad47c8cd6197a8e0bebf3586fc2 diff --git a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 deleted file mode 100644 index 793f153e26212..0000000000000 --- a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -c6a019d79d20eabc39619a04961c9a3b diff --git a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 deleted file mode 100644 index f3f1ec9dca483..0000000000000 --- a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -478ab473244b01bef344892a75e09fef50da8fb1a7212e0257c53f3223de4fde5f6bd449eef34bc1f025481c7d9f854002acb6eb203b447a50a34bae4ad9dee4 diff --git a/deps/libuv.version b/deps/libuv.version index 51f79989c5fc3..478ebca66589a 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv1.9.0 -LIBUV_SHA1=cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68 +LIBUV_SHA1=28743d6091531340cfe316de2b2d385fe1778ff5 diff --git a/test/spawn.jl b/test/spawn.jl index 7fa04587e3b28..b4d7f7ee1990e 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -392,3 +392,20 @@ end @test_throws ArgumentError reduce(&, Base.AbstractCmd[]) @test_throws ArgumentError reduce(&, Base.Cmd[]) @test reduce(&, [`$echo abc`, `$echo def`, `$echo hij`]) == `$echo abc` & `$echo def` & `$echo hij` + +# test for proper handling of FD exhaustion +let ps = Pipe[] + try + for i = 1:100_000 + p = Pipe() + Base.link_pipe(p) + push!(ps, p) + end + @test false + catch ex + for p in ps + close(p) + end + @test (ex::Base.UVError).code == Base.UV_EMFILE + end +end From 78c4bf9056f65be89216d9eeb0705b9b95d07e46 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 25 Jul 2016 15:30:29 -0400 Subject: [PATCH 0799/1117] allow redefinition of egal parametric types --- src/interpreter.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/interpreter.c b/src/interpreter.c index 67da253f7e6e0..857807f0c4d1f 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -94,20 +94,35 @@ extern int jl_boot_file_loaded; extern int inside_typedef; // this is a heuristic for allowing "redefining" a type to something identical +static int equiv_svec_dt(jl_svec_t *sa, jl_svec_t *sb) +{ + size_t i, l = jl_svec_len(sa); + if (l != jl_svec_len(sb)) return 0; + for (i = 0; i < l; i++) { + jl_value_t *a = jl_svecref(sa, i); + jl_value_t *b = jl_svecref(sb, i); + if (jl_typeof(a) != jl_typeof(b)) + return 0; + if (jl_is_typevar(a) && ((jl_tvar_t*)a)->name != ((jl_tvar_t*)b)->name) + return 0; + if (!jl_subtype(a, b, 0) || !jl_subtype(b, a, 0)) + return 0; + } + return 1; +} static int equiv_type(jl_datatype_t *dta, jl_datatype_t *dtb) { return (jl_typeof(dta) == jl_typeof(dtb) && - // TODO: can't yet handle parametric types due to how constructors work - dta->parameters == jl_emptysvec && dta->name->name == dtb->name->name && - jl_egal((jl_value_t*)dta->types, (jl_value_t*)dtb->types) && dta->abstract == dtb->abstract && dta->mutabl == dtb->mutabl && dta->size == dtb->size && dta->ninitialized == dtb->ninitialized && - jl_egal((jl_value_t*)dta->super, (jl_value_t*)dtb->super) && - jl_egal((jl_value_t*)dta->name->names, (jl_value_t*)dtb->name->names) && - jl_egal((jl_value_t*)dta->parameters, (jl_value_t*)dtb->parameters)); + equiv_svec_dt(dta->parameters, dtb->parameters) && + equiv_svec_dt(dta->types, dtb->types) && + jl_subtype((jl_value_t*)dta->super, (jl_value_t*)dtb->super, 0) && + jl_subtype((jl_value_t*)dtb->super, (jl_value_t*)dta->super, 0) && + jl_egal((jl_value_t*)dta->name->names, (jl_value_t*)dtb->name->names)); } static void check_can_assign_type(jl_binding_t *b) @@ -297,7 +312,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) jl_rethrow(); } b->value = temp; - if (temp==NULL || !equiv_type(dt, (jl_datatype_t*)temp)) { + if (temp == NULL || !equiv_type(dt, (jl_datatype_t*)temp)) { jl_checked_assignment(b, (jl_value_t*)dt); } JL_GC_POP(); @@ -339,7 +354,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) jl_rethrow(); } b->value = temp; - if (temp==NULL || !equiv_type(dt, (jl_datatype_t*)temp)) { + if (temp == NULL || !equiv_type(dt, (jl_datatype_t*)temp)) { jl_checked_assignment(b, (jl_value_t*)dt); } JL_GC_POP(); @@ -405,12 +420,9 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) } b->value = temp; - if (temp==NULL || !equiv_type(dt, (jl_datatype_t*)temp)) { + if (temp == NULL || !equiv_type(dt, (jl_datatype_t*)temp)) { jl_checked_assignment(b, (jl_value_t*)dt); } - else { - // TODO: remove all old ctors and set temp->name->ctor_factory = dt->name->ctor_factory - } JL_GC_POP(); return (jl_value_t*)jl_nothing; From 98d27ba10723948e44d6fb4ecf9c88567daed058 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 4 May 2016 18:02:37 -0400 Subject: [PATCH 0800/1117] improve efficiency of Dict and accuracy of age (nee dirty) flag ref comment in #15923 --- base/dict.jl | 106 ++++++++++++++++++++------------------------------- 1 file changed, 42 insertions(+), 64 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 00e56434d2745..ae7635bf1cdb6 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -331,13 +331,13 @@ type Dict{K,V} <: Associative{K,V} vals::Array{V,1} ndel::Int count::Int - dirty::Bool + age::UInt idxfloor::Int # an index <= the indexes of all used slots maxprobe::Int function Dict() n = 16 - new(zeros(UInt8,n), Array{K}(n), Array{V}(n), 0, 0, false, 1, 0) + new(zeros(UInt8,n), Array{K,1}(n), Array{V,1}(n), 0, 0, 0, 1, 0) end function Dict(kv) h = Dict{K,V}() @@ -360,7 +360,7 @@ type Dict{K,V} <: Associative{K,V} rehash!(d) end @assert d.ndel == 0 - new(copy(d.slots), copy(d.keys), copy(d.vals), 0, d.count, d.dirty, d.idxfloor, + new(copy(d.slots), copy(d.keys), copy(d.vals), 0, d.count, d.age, d.idxfloor, d.maxprobe) end end @@ -427,7 +427,7 @@ function convert{K,V}(::Type{Dict{K,V}},d::Associative) end convert{K,V}(::Type{Dict{K,V}},d::Dict{K,V}) = d -hashindex(key, sz) = ((hash(key)%Int) & (sz-1)) + 1 +hashindex(key, sz) = (((hash(key)%Int) & (sz-1)) + 1)::Int isslotempty(h::Dict, i::Int) = h.slots[i] == 0x0 isslotfilled(h::Dict, i::Int) = h.slots[i] == 0x1 @@ -439,7 +439,7 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys)) oldv = h.vals sz = length(olds) newsz = _tablesz(newsz) - h.dirty = true + h.age += 1 h.idxfloor = 1 if h.count == 0 resize!(h.slots, newsz) @@ -451,9 +451,9 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys)) end slots = zeros(UInt8,newsz) - keys = Array{K}(newsz) - vals = Array{V}(newsz) - count0 = h.count + keys = Array{K,1}(newsz) + vals = Array{V,1}(newsz) + age0 = h.age count = 0 maxprobe = h.maxprobe @@ -472,8 +472,8 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys)) vals[index] = v count += 1 - if h.count != count0 - # if items are removed by finalizers, retry + if h.age != age0 + # if `h` is changed by a finalizer, retry return rehash!(h, newsz) end end @@ -485,6 +485,7 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys)) h.count = count h.ndel = 0 h.maxprobe = maxprobe + @assert h.age == age0 return h end @@ -511,7 +512,7 @@ function empty!{K,V}(h::Dict{K,V}) resize!(h.vals, sz) h.ndel = 0 h.count = 0 - h.dirty = true + h.age += 1 h.idxfloor = 1 return h end @@ -528,7 +529,7 @@ function ht_keyindex{K,V}(h::Dict{K,V}, key) if isslotempty(h,index) break end - if !isslotmissing(h,index) && isequal(key,keys[index]) + if !isslotmissing(h,index) && (key === keys[index] || isequal(key,keys[index])) return index end @@ -543,6 +544,7 @@ end # and the key would be inserted at pos # This version is for use by setindex! and get! function ht_keyindex2{K,V}(h::Dict{K,V}, key) + age0 = h.age sz = length(h.keys) iter = 0 maxprobe = h.maxprobe @@ -552,7 +554,9 @@ function ht_keyindex2{K,V}(h::Dict{K,V}, key) while true if isslotempty(h,index) - avail < 0 && return avail + if avail < 0 + return avail + end return -index end @@ -562,7 +566,7 @@ function ht_keyindex2{K,V}(h::Dict{K,V}, key) # in case "key" already exists in a later collided slot. avail = -index end - elseif isequal(key, keys[index]) + elseif key === keys[index] || isequal(key, keys[index]) return index end @@ -594,7 +598,7 @@ function _setindex!(h::Dict, v, key, index) h.keys[index] = key h.vals[index] = v h.count += 1 - h.dirty = true + h.age += 1 if index < h.idxfloor h.idxfloor = index end @@ -620,6 +624,7 @@ function setindex!{K,V}(h::Dict{K,V}, v0, key::K) index = ht_keyindex2(h, key) if index > 0 + h.age += 1 h.keys[index] = key h.vals[index] = v else @@ -629,30 +634,13 @@ function setindex!{K,V}(h::Dict{K,V}, v0, key::K) return h end -function get!{K,V}(h::Dict{K,V}, key0, default) - key = convert(K,key0) - if !isequal(key,key0) - throw(ArgumentError("$key0 is not a valid key for type $K")) - end - get!(h, key, default) -end - -function get!{K, V}(h::Dict{K,V}, key::K, default) - index = ht_keyindex2(h, key) - - index > 0 && return h.vals[index] - - v = convert(V, default) - _setindex!(h, v, key, -index) - return v -end - +get!{K,V}(h::Dict{K,V}, key0, default) = get!(()->default, h, key0) function get!{K,V}(default::Callable, h::Dict{K,V}, key0) - key = convert(K,key0) - if !isequal(key,key0) + key = convert(K, key0) + if !isequal(key, key0) throw(ArgumentError("$key0 is not a valid key for type $K")) end - get!(default, h, key) + return get!(default, h, key) end function get!{K,V}(default::Callable, h::Dict{K,V}, key::K) @@ -660,12 +648,13 @@ function get!{K,V}(default::Callable, h::Dict{K,V}, key::K) index > 0 && return h.vals[index] - h.dirty = false - v = convert(V, default()) - if h.dirty + age0 = h.age + v = convert(V, default()) + if h.age != age0 index = ht_keyindex2(h, key) end if index > 0 + h.age += 1 h.keys[index] = key h.vals[index] = v else @@ -674,41 +663,28 @@ function get!{K,V}(default::Callable, h::Dict{K,V}, key::K) return v end -# NOTE: this macro is specific to Dict, not Associative, and should +# NOTE: this macro is trivial, and should # therefore not be exported as-is: it's for internal use only. macro get!(h, key0, default) - quote - K, V = keytype($(esc(h))), valtype($(esc(h))) - key = convert(K, $(esc(key0))) - if !isequal(key, $(esc(key0))) - throw(ArgumentError(string($(esc(key0)), " is not a valid key for type ", K))) - end - idx = ht_keyindex2($(esc(h)), key) - if idx < 0 - idx = -idx - v = convert(V, $(esc(default))) - _setindex!($(esc(h)), v, key, idx) - else - @inbounds v = $(esc(h)).vals[idx] - end - v + return quote + get!(()->$(esc(default)), $(esc(h)), $(esc(key0))) end end function getindex{K,V}(h::Dict{K,V}, key) index = ht_keyindex(h, key) - return (index<0) ? throw(KeyError(key)) : h.vals[index]::V + return (index < 0) ? throw(KeyError(key)) : h.vals[index]::V end function get{K,V}(h::Dict{K,V}, key, default) index = ht_keyindex(h, key) - return (index<0) ? default : h.vals[index]::V + return (index < 0) ? default : h.vals[index]::V end function get{K,V}(default::Callable, h::Dict{K,V}, key) index = ht_keyindex(h, key) - return (index<0) ? default() : h.vals[index]::V + return (index < 0) ? default() : h.vals[index]::V end haskey(h::Dict, key) = (ht_keyindex(h, key) >= 0) @@ -727,12 +703,12 @@ end function pop!(h::Dict, key) index = ht_keyindex(h, key) - index > 0 ? _pop!(h, index) : throw(KeyError(key)) + return index > 0 ? _pop!(h, index) : throw(KeyError(key)) end function pop!(h::Dict, key, default) index = ht_keyindex(h, key) - index > 0 ? _pop!(h, index) : default + return index > 0 ? _pop!(h, index) : default end function _delete!(h::Dict, index) @@ -741,14 +717,16 @@ function _delete!(h::Dict, index) ccall(:jl_arrayunset, Void, (Any, UInt), h.vals, index-1) h.ndel += 1 h.count -= 1 - h.dirty = true - h + h.age += 1 + return h end function delete!(h::Dict, key) index = ht_keyindex(h, key) - if index > 0; _delete!(h, index); end - h + if index > 0 + _delete!(h, index) + end + return h end function skip_deleted(h::Dict, i) From 03abe01bd0066a21d1e0c236446d19d3f9877d20 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 25 Jul 2016 17:26:52 -0400 Subject: [PATCH 0801/1117] split weakkeydict into a separate file --- base/dict.jl | 63 +---------------------------------------- base/sysimg.jl | 1 + base/weakkeydict.jl | 69 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 62 deletions(-) create mode 100644 base/weakkeydict.jl diff --git a/base/dict.jl b/base/dict.jl index ae7635bf1cdb6..e0f947cd23a1a 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -751,70 +751,9 @@ length(t::Dict) = t.count next{T<:Dict}(v::KeyIterator{T}, i) = (v.dict.keys[i], skip_deleted(v.dict,i+1)) next{T<:Dict}(v::ValueIterator{T}, i) = (v.dict.vals[i], skip_deleted(v.dict,i+1)) -# weak key dictionaries - -type WeakKeyDict{K,V} <: Associative{K,V} - ht::Dict{Any,V} - deleter::Function - - WeakKeyDict() = new(Dict{Any,V}(), identity) -end -WeakKeyDict() = WeakKeyDict{Any,Any}() - -function weak_key_delete!(t::Dict, k) - # when a weak key is finalized, remove from dictionary if it is still there - wk = getkey(t, k, secret_table_token) - if !is(wk,secret_table_token) && is(wk.value, k) - delete!(t, k) - end -end - -function setindex!{K}(wkh::WeakKeyDict{K}, v, key) - t = wkh.ht - k = convert(K, key) - if is(wkh.deleter, identity) - wkh.deleter = x->weak_key_delete!(t, x) - end - t[WeakRef(k)] = v - # TODO: it might be better to avoid the finalizer, allow - # wiped WeakRefs to remain in the table, and delete them as - # they are discovered by getindex and setindex!. - finalizer(k, wkh.deleter) - return t -end - - -function getkey{K}(wkh::WeakKeyDict{K}, kk, default) - k = getkey(wkh.ht, kk, secret_table_token) - if is(k, secret_table_token) - return default - end - return k.value::K -end - -get{K}(wkh::WeakKeyDict{K}, key, default) = get(wkh.ht, key, default) -get{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get(default, wkh.ht, key) -get!{K}(wkh::WeakKeyDict{K}, key, default) = get!(wkh.ht, key, default) -get!{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get!(default, wkh.ht, key) -pop!{K}(wkh::WeakKeyDict{K}, key) = pop!(wkh.ht, key) -pop!{K}(wkh::WeakKeyDict{K}, key, default) = pop!(wkh.ht, key, default) -delete!{K}(wkh::WeakKeyDict{K}, key) = delete!(wkh.ht, key) -empty!(wkh::WeakKeyDict) = (empty!(wkh.ht); wkh) -haskey{K}(wkh::WeakKeyDict{K}, key) = haskey(wkh.ht, key) -getindex{K}(wkh::WeakKeyDict{K}, key) = getindex(wkh.ht, key) -isempty(wkh::WeakKeyDict) = isempty(wkh.ht) - -start(t::WeakKeyDict) = start(t.ht) -done(t::WeakKeyDict, i) = done(t.ht, i) -function next{K,V}(t::WeakKeyDict{K,V}, i) - kv, i = next(t.ht, i) - (Pair{K,V}(kv[1].value::K,kv[2]), i) -end -length(t::WeakKeyDict) = length(t.ht) - # For these Associative types, it is safe to implement filter! # by deleting keys during iteration. -function filter!(f, d::Union{ObjectIdDict,Dict,WeakKeyDict}) +function filter!(f, d::Union{ObjectIdDict,Dict}) for (k,v) in d if !f(k,v) delete!(d,k) diff --git a/base/sysimg.jl b/base/sysimg.jl index f2cf7fb13d96d..9461866639070 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -115,6 +115,7 @@ include("reshapedarray.jl") include("bitarray.jl") include("intset.jl") include("dict.jl") +include("weakkeydict.jl") include("set.jl") include("iterator.jl") diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl new file mode 100644 index 0000000000000..a1f5a0ded821e --- /dev/null +++ b/base/weakkeydict.jl @@ -0,0 +1,69 @@ +# weak key dictionaries + +type WeakKeyDict{K,V} <: Associative{K,V} + ht::Dict{Any,V} + deleter::Function + + WeakKeyDict() = new(Dict{Any,V}(), identity) +end +WeakKeyDict() = WeakKeyDict{Any,Any}() + +function weak_key_delete!(t::Dict, k) + # when a weak key is finalized, remove from dictionary if it is still there + wk = getkey(t, k, secret_table_token) + if !is(wk,secret_table_token) && is(wk.value, k) + delete!(t, k) + end +end + +function setindex!{K}(wkh::WeakKeyDict{K}, v, key) + t = wkh.ht + k = convert(K, key) + if is(wkh.deleter, identity) + wkh.deleter = x->weak_key_delete!(t, x) + end + t[WeakRef(k)] = v + # TODO: it might be better to avoid the finalizer, allow + # wiped WeakRefs to remain in the table, and delete them as + # they are discovered by getindex and setindex!. + finalizer(k, wkh.deleter) + return t +end + + +function getkey{K}(wkh::WeakKeyDict{K}, kk, default) + k = getkey(wkh.ht, kk, secret_table_token) + if is(k, secret_table_token) + return default + end + return k.value::K +end + +get{K}(wkh::WeakKeyDict{K}, key, default) = get(wkh.ht, key, default) +get{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get(default, wkh.ht, key) +get!{K}(wkh::WeakKeyDict{K}, key, default) = get!(wkh.ht, key, default) +get!{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get!(default, wkh.ht, key) +pop!{K}(wkh::WeakKeyDict{K}, key) = pop!(wkh.ht, key) +pop!{K}(wkh::WeakKeyDict{K}, key, default) = pop!(wkh.ht, key, default) +delete!{K}(wkh::WeakKeyDict{K}, key) = delete!(wkh.ht, key) +empty!(wkh::WeakKeyDict) = (empty!(wkh.ht); wkh) +haskey{K}(wkh::WeakKeyDict{K}, key) = haskey(wkh.ht, key) +getindex{K}(wkh::WeakKeyDict{K}, key) = getindex(wkh.ht, key) +isempty(wkh::WeakKeyDict) = isempty(wkh.ht) + +start(t::WeakKeyDict) = start(t.ht) +done(t::WeakKeyDict, i) = done(t.ht, i) +function next{K,V}(t::WeakKeyDict{K,V}, i) + kv, i = next(t.ht, i) + (Pair{K,V}(kv[1].value::K,kv[2]), i) +end +length(t::WeakKeyDict) = length(t.ht) + +function filter!(f, d::WeakKeyDict) + for (k, v) in d + if !f(k, v) + delete!(d, k) + end + end + return d +end From c0483212dab742ee22ad610961cdb6496342b8a7 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 25 Jul 2016 17:32:15 -0400 Subject: [PATCH 0802/1117] move Thread support much earlier in bootstrapping --- base/sysimg.jl | 16 ++++++++-------- base/threads.jl | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/base/sysimg.jl b/base/sysimg.jl index 9461866639070..8428a4a5166ec 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -154,6 +154,12 @@ include("show.jl") include("base64.jl") importall .Base64 +# nullable types +include("nullable.jl") + +# version +include("version.jl") + # system & environment include("libc.jl") using .Libc: getpid, gethostname, time @@ -161,14 +167,12 @@ include("libdl.jl") using .Libdl: DL_LOAD_PATH include("env.jl") -# nullable types -include("nullable.jl") - # Scheduling include("libuv.jl") include("event.jl") include("task.jl") include("lock.jl") +include("threads.jl") # I/O include("stream.jl") @@ -218,9 +222,6 @@ include("collections.jl") include("sort.jl") importall .Sort -# version -include("version.jl") - function deepcopy_internal end # BigInts and BigFloats @@ -343,8 +344,7 @@ import .Dates: Date, DateTime, now include("sparse/sparse.jl") importall .SparseArrays -# threads -include("threads.jl") +# worker threads include("threadcall.jl") # deprecated functions diff --git a/base/threads.jl b/base/threads.jl index 4550e41dd0ba7..ef9f6957fa3b7 100644 --- a/base/threads.jl +++ b/base/threads.jl @@ -7,4 +7,3 @@ include("atomics.jl") include("locks.jl") end - From e9bb7fe306c4ad591e3cc390a83fcade6b0fb5ef Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 25 Jul 2016 19:41:50 -0400 Subject: [PATCH 0803/1117] make WeakKeyDict finalizer usage gc-safe also use this `client_refs.lock` to protect other data-structures from being interrupted by finalizers, in the multi.jl logic we may want to start indicating which mutable data-structures are safe to call from finalizers, since generally that isn't possible to make a finalizer API gc-safe, that code should observe the standard thread-safe restrictions (there's no guarantee of which thread it'll run on), plus, if the data-structures uses locks for synchronization, use the `islocked` pattern (demonstrated herein) in the `finalizer` to re-schedule the finalizer when the mutable data-structure is not available for mutation. this ensures that the lock cannot be acquired recursively, and furthermore, this pattern will continue to work if finalizers get moved to their own separate thread. close #14445 fix #16550 reverts workaround #14456 (shouldn't break #14295, due to new locks) should fix #16091 (with #17619) --- base/dict.jl | 10 --- base/lock.jl | 20 ++++++ base/multi.jl | 155 ++++++++++++++++++++++++-------------------- base/precompile.jl | 1 - base/serialize.jl | 11 +++- base/sysimg.jl | 2 +- base/weakkeydict.jl | 88 ++++++++++++++----------- test/misc.jl | 17 ++++- 8 files changed, 180 insertions(+), 124 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index e0f947cd23a1a..8990bb198c1a2 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -309,16 +309,6 @@ get!(o::ObjectIdDict, key, default) = (o[key] = get(o, key, default)) abstract AbstractSerializer -# Serializer type needed as soon as ObjectIdDict is available -type SerializationState{I<:IO} <: AbstractSerializer - io::I - counter::Int - table::ObjectIdDict - SerializationState(io::I) = new(io, 0, ObjectIdDict()) -end - -SerializationState(io::IO) = SerializationState{typeof(io)}(io) - # dict # These can be changed, to trade off better performance for space diff --git a/base/lock.jl b/base/lock.jl index ea42b8e454dd2..e0bd86dfeff81 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -95,6 +95,26 @@ function unlock(rl::ReentrantLock) return end +function lock(f, l) + lock(l) + try + return f() + finally + unlock(l) + end +end + +function trylock(f, l) + if trylock(l) + try + return f() + finally + unlock(l) + end + end + return false +end + """ Semaphore(sem_size) diff --git a/base/multi.jl b/base/multi.jl index e299edb35bda3..3049e5732fd2e 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -637,27 +637,38 @@ function deregister_worker(pg, pid) # delete this worker from our remote reference client sets ids = [] tonotify = [] - for (id,rv) in pg.refs - if in(pid,rv.clientset) - push!(ids, id) + lock(client_refs) do + for (id,rv) in pg.refs + if in(pid,rv.clientset) + push!(ids, id) + end + if rv.waitingfor == pid + push!(tonotify, (id,rv)) + end end - if rv.waitingfor == pid - push!(tonotify, (id,rv)) + for id in ids + del_client(pg, id, pid) end - end - for id in ids - del_client(pg, id, pid) - end - # throw exception to tasks waiting for this pid - for (id,rv) in tonotify - notify_error(rv.c, ProcessExitedException()) - delete!(pg.refs, id) + # throw exception to tasks waiting for this pid + for (id,rv) in tonotify + notify_error(rv.c, ProcessExitedException()) + delete!(pg.refs, id) + end end end ## remote refs ## -const client_refs = WeakKeyDict() + +""" + client_refs + +Tracks whether a particular AbstractRemoteRef +(identified by its RRID) exists on this worker. + +The client_refs lock is also used to synchronize access to `.refs` and associated clientset state +""" +const client_refs = WeakKeyDict{Any, Void}() # used as a WeakKeySet abstract AbstractRemoteRef @@ -680,34 +691,26 @@ type RemoteChannel{T<:AbstractChannel} <: AbstractRemoteRef end function test_existing_ref(r::AbstractRemoteRef) - found = getkey(client_refs, r, false) - if found !== false - if client_refs[r] == true - @assert r.where > 0 - if isa(r, Future) && isnull(found.v) && !isnull(r.v) - # we have recd the value from another source, probably a deserialized ref, send a del_client message - send_del_client(r) - found.v = r.v - end - return found - else - # just delete the entry. - delete!(client_refs, found) - end + found = getkey(client_refs, r, nothing) + if found !== nothing + @assert r.where > 0 + if isa(r, Future) && isnull(found.v) && !isnull(r.v) + # we have recd the value from another source, probably a deserialized ref, send a del_client message + send_del_client(r) + found.v = r.v + end + return found::typeof(r) end - client_refs[r] = true + client_refs[r] = nothing finalizer(r, finalize_ref) return r end function finalize_ref(r::AbstractRemoteRef) - if r.where > 0 # Handle the case of the finalizer having being called manually - if haskey(client_refs, r) - # NOTE: Change below line to deleting the entry once issue https://github.com/JuliaLang/julia/issues/14445 - # is fixed. - client_refs[r] = false - end + if r.where > 0 # Handle the case of the finalizer having been called manually + islocked(client_refs) && return finalizer(r, finalize_ref) # delay finalizer for later, when it's not already locked + delete!(client_refs, r) if isa(r, RemoteChannel) send_del_client(r) else @@ -717,7 +720,7 @@ function finalize_ref(r::AbstractRemoteRef) end r.where = 0 end - return r + nothing end Future(w::LocalProcess) = Future(w.id) @@ -791,23 +794,27 @@ A low-level API which returns the backing `AbstractChannel` for an `id` returned The call is valid only on the node where the backing channel exists. """ function channel_from_id(id) - rv = get(PGRP.refs, id, false) + rv = lock(client_refs) do + return get(PGRP.refs, id, false) + end if rv === false throw(ErrorException("Local instance of remote reference not found")) end - rv.c + return rv.c end lookup_ref(rrid::RRID, f=def_rv_channel) = lookup_ref(PGRP, rrid, f) function lookup_ref(pg, rrid, f) - rv = get(pg.refs, rrid, false) - if rv === false - # first we've heard of this ref - rv = RemoteValue(f()) - pg.refs[rrid] = rv - push!(rv.clientset, rrid.whence) - end - rv + return lock(client_refs) do + rv = get(pg.refs, rrid, false) + if rv === false + # first we've heard of this ref + rv = RemoteValue(f()) + pg.refs[rrid] = rv + push!(rv.clientset, rrid.whence) + end + return rv + end::RemoteValue end """ @@ -827,7 +834,7 @@ function isready(rr::Future) !isnull(rr.v) && return true rid = remoteref_id(rr) - if rr.where == myid() + return if rr.where == myid() isready(lookup_ref(rid).c) else remotecall_fetch(rid->isready(lookup_ref(rid).c), rr.where, rid) @@ -844,7 +851,7 @@ it can be safely used on a `Future` since they are assigned only once. """ function isready(rr::RemoteChannel, args...) rid = remoteref_id(rr) - if rr.where == myid() + return if rr.where == myid() isready(lookup_ref(rid).c, args...) else remotecall_fetch(rid->isready(lookup_ref(rid).c, args...), rr.where, rid) @@ -855,11 +862,7 @@ del_client(rr::AbstractRemoteRef) = del_client(remoteref_id(rr), myid()) del_client(id, client) = del_client(PGRP, id, client) function del_client(pg, id, client) -# As a workaround to issue https://github.com/JuliaLang/julia/issues/14445 -# the dict/set updates are executed asynchronously so that they do -# not occur in the midst of a gc. The `@async` prefix must be removed once -# 14445 is fixed. - @async begin + lock(client_refs) do rv = get(pg.refs, id, false) if rv !== false delete!(rv.clientset, client) @@ -898,8 +901,10 @@ function send_del_client(rr) end function add_client(id, client) - rv = lookup_ref(id) - push!(rv.clientset, client) + lock(client_refs) do + rv = lookup_ref(id) + push!(rv.clientset, client) + end nothing end @@ -999,7 +1004,7 @@ function run_work_thunk(thunk, print_error) result = RemoteException(ce) print_error && showerror(STDERR, ce) end - result + return result end function run_work_thunk(rv::RemoteValue, thunk) put!(rv, run_work_thunk(thunk, false)) @@ -1007,11 +1012,13 @@ function run_work_thunk(rv::RemoteValue, thunk) end function schedule_call(rid, thunk) - rv = RemoteValue(def_rv_channel()) - (PGRP::ProcessGroup).refs[rid] = rv - push!(rv.clientset, rid.whence) - schedule(@task(run_work_thunk(rv,thunk))) - rv + return lock(client_refs) do + rv = RemoteValue(def_rv_channel()) + (PGRP::ProcessGroup).refs[rid] = rv + push!(rv.clientset, rid.whence) + @schedule run_work_thunk(rv, thunk) + return rv + end end # make a thunk to call f on args in a way that simulates what would happen if @@ -1026,13 +1033,13 @@ end function remotecall(f, w::LocalProcess, args...; kwargs...) rr = Future(w) schedule_call(remoteref_id(rr), local_remotecall_thunk(f, args, kwargs)) - rr + return rr end function remotecall(f, w::Worker, args...; kwargs...) rr = Future(w) send_msg(w, MsgHeader(remoteref_id(rr)), CallMsg{:call}(f, args, kwargs)) - rr + return rr end """ @@ -1046,7 +1053,7 @@ remotecall(f, id::Integer, args...; kwargs...) = remotecall(f, worker_from_id(id function remotecall_fetch(f, w::LocalProcess, args...; kwargs...) v=run_work_thunk(local_remotecall_thunk(f,args, kwargs), false) - isa(v, RemoteException) ? throw(v) : v + return isa(v, RemoteException) ? throw(v) : v end function remotecall_fetch(f, w::Worker, args...; kwargs...) @@ -1057,8 +1064,10 @@ function remotecall_fetch(f, w::Worker, args...; kwargs...) rv.waitingfor = w.id send_msg(w, MsgHeader(RRID(0,0), oid), CallMsg{:call_fetch}(f, args, kwargs)) v = take!(rv) - delete!(PGRP.refs, oid) - isa(v, RemoteException) ? throw(v) : v + lock(client_refs) do + delete!(PGRP.refs, oid) + end + return isa(v, RemoteException) ? throw(v) : v end """ @@ -1080,9 +1089,11 @@ function remotecall_wait(f, w::Worker, args...; kwargs...) rr = Future(w) send_msg(w, MsgHeader(remoteref_id(rr), prid), CallWaitMsg(f, args, kwargs)) v = fetch(rv.c) - delete!(PGRP.refs, prid) + lock(client_refs) do + delete!(PGRP.refs, prid) + end isa(v, RemoteException) && throw(v) - rr + return rr end """ @@ -1834,9 +1845,11 @@ function create_worker(manager, wconfig) @schedule manage(w.manager, w.id, w.config, :register) wait(rr_ntfy_join) - delete!(PGRP.refs, ntfy_oid) + lock(client_refs) do + delete!(PGRP.refs, ntfy_oid) + end - w.id + return w.id end @@ -1859,7 +1872,7 @@ function launch_additional(np::Integer, cmd::Cmd) additional_io_objs[port] = io end - addresses + return addresses end function redirect_output_from_additional_worker(pid, port) diff --git a/base/precompile.jl b/base/precompile.jl index 9c8519886a1de..e189959ff2759 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -444,7 +444,6 @@ precompile(Base.get, (Base.Dict{Any, Any}, Tuple{Int64, Int64}, Bool)) precompile(Base.LineEdit.refresh_multi_line, (Array{Any, 1}, Base.Terminals.TerminalBuffer, Base.Terminals.TTYTerminal, Base.IOBuffer, Base.LineEdit.InputAreaState, Base.LineEdit.PromptState)) precompile(Base.schedule, (Array{Any, 1}, Task, Void)) precompile(Base.LineEdit.match_input, (Function, Base.LineEdit.MIState, Base.Terminals.TTYTerminal, Array{Char, 1}, Base.Dict{Char, Any})) -precompile(Base.weak_key_delete!, (Base.Dict{Any, Any}, Base.RemoteChannel)) precompile(==, (Base.RemoteChannel, WeakRef)) precompile(==, (Base.RemoteChannel, Base.RemoteChannel)) precompile(Base.send_del_client, (Base.RemoteChannel,)) diff --git a/base/serialize.jl b/base/serialize.jl index 55e5f0cdd868a..b50fe29937fc5 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -6,7 +6,16 @@ import Base: GMP, Bottom, unsafe_convert, uncompressed_ast, datatype_pointerfree import Core: svec using Base: ViewIndex, index_lengths -export serialize, deserialize +export serialize, deserialize, SerializationState + +type SerializationState{I<:IO} <: AbstractSerializer + io::I + counter::Int + table::ObjectIdDict + SerializationState(io::I) = new(io, 0, ObjectIdDict()) +end + +SerializationState(io::IO) = SerializationState{typeof(io)}(io) ## serializing values ## diff --git a/base/sysimg.jl b/base/sysimg.jl index 8428a4a5166ec..7005898b07050 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -115,7 +115,6 @@ include("reshapedarray.jl") include("bitarray.jl") include("intset.jl") include("dict.jl") -include("weakkeydict.jl") include("set.jl") include("iterator.jl") @@ -173,6 +172,7 @@ include("event.jl") include("task.jl") include("lock.jl") include("threads.jl") +include("weakkeydict.jl") # I/O include("stream.jl") diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl index a1f5a0ded821e..6485c83af38e4 100644 --- a/base/weakkeydict.jl +++ b/base/weakkeydict.jl @@ -2,62 +2,74 @@ type WeakKeyDict{K,V} <: Associative{K,V} ht::Dict{Any,V} - deleter::Function + lock::Threads.RecursiveSpinLock + finalizer::Function - WeakKeyDict() = new(Dict{Any,V}(), identity) + function WeakKeyDict() + t = new(Dict{Any,V}(), Threads.RecursiveSpinLock(), identity) + t.finalizer = function(k) + # when a weak key is finalized, remove from dictionary if it is still there + islocked(t) && return finalizer(k, t.finalizer) + delete!(t, k) + end + return t + end end WeakKeyDict() = WeakKeyDict{Any,Any}() -function weak_key_delete!(t::Dict, k) - # when a weak key is finalized, remove from dictionary if it is still there - wk = getkey(t, k, secret_table_token) - if !is(wk,secret_table_token) && is(wk.value, k) - delete!(t, k) - end -end +islocked(wkh::WeakKeyDict) = islocked(wkh.lock) +lock(f, wkh::WeakKeyDict) = lock(f, wkh.lock) +trylock(f, wkh::WeakKeyDict) = trylock(f, wkh.lock) function setindex!{K}(wkh::WeakKeyDict{K}, v, key) - t = wkh.ht k = convert(K, key) - if is(wkh.deleter, identity) - wkh.deleter = x->weak_key_delete!(t, x) + finalizer(k, wkh.finalizer) + lock(wkh) do + wkh.ht[WeakRef(k)] = v end - t[WeakRef(k)] = v - # TODO: it might be better to avoid the finalizer, allow - # wiped WeakRefs to remain in the table, and delete them as - # they are discovered by getindex and setindex!. - finalizer(k, wkh.deleter) - return t + return wkh end - function getkey{K}(wkh::WeakKeyDict{K}, kk, default) - k = getkey(wkh.ht, kk, secret_table_token) - if is(k, secret_table_token) - return default + return lock(wkh) do + k = getkey(wkh.ht, kk, secret_table_token) + is(k, secret_table_token) && return default + return k.value::K end - return k.value::K end -get{K}(wkh::WeakKeyDict{K}, key, default) = get(wkh.ht, key, default) -get{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get(default, wkh.ht, key) -get!{K}(wkh::WeakKeyDict{K}, key, default) = get!(wkh.ht, key, default) -get!{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get!(default, wkh.ht, key) -pop!{K}(wkh::WeakKeyDict{K}, key) = pop!(wkh.ht, key) -pop!{K}(wkh::WeakKeyDict{K}, key, default) = pop!(wkh.ht, key, default) -delete!{K}(wkh::WeakKeyDict{K}, key) = delete!(wkh.ht, key) -empty!(wkh::WeakKeyDict) = (empty!(wkh.ht); wkh) -haskey{K}(wkh::WeakKeyDict{K}, key) = haskey(wkh.ht, key) -getindex{K}(wkh::WeakKeyDict{K}, key) = getindex(wkh.ht, key) +get{K}(wkh::WeakKeyDict{K}, key, default) = lock(() -> get(wkh.ht, key, default), wkh) +get{K}(default::Callable, wkh::WeakKeyDict{K}, key) = lock(() -> get(default, wkh.ht, key), wkh) +get!{K}(wkh::WeakKeyDict{K}, key, default) = lock(() -> get!(wkh.ht, key, default), wkh) +get!{K}(default::Callable, wkh::WeakKeyDict{K}, key) = lock(() -> get!(default, wkh.ht, key), wkh) +pop!{K}(wkh::WeakKeyDict{K}, key) = lock(() -> pop!(wkh.ht, key), wkh) +pop!{K}(wkh::WeakKeyDict{K}, key, default) = lock(() -> pop!(wkh.ht, key, default), wkh) +delete!{K}(wkh::WeakKeyDict{K}, key) = lock(() -> delete!(wkh.ht, key), wkh) +empty!(wkh::WeakKeyDict) = (lock(() -> empty!(wkh.ht)); wkh) +haskey{K}(wkh::WeakKeyDict{K}, key) = lock(() -> haskey(wkh.ht, key), wkh) +getindex{K}(wkh::WeakKeyDict{K}, key) = lock(() -> getindex(wkh.ht, key), wkh) isempty(wkh::WeakKeyDict) = isempty(wkh.ht) +length(t::WeakKeyDict) = length(t.ht) -start(t::WeakKeyDict) = start(t.ht) -done(t::WeakKeyDict, i) = done(t.ht, i) +function start{K,V}(t::WeakKeyDict{K,V}) + gc_token = Ref{Bool}(false) # no keys will be deleted via finalizers until this token is gc'd + finalizer(gc_token, function(r) + if r[] + r[] = false + unlock(t.lock) + end + end) + s = lock(t.lock) + gc_token[] = true + return (start(t.ht), gc_token) +end +done(t::WeakKeyDict, i) = done(t.ht, i[1]) function next{K,V}(t::WeakKeyDict{K,V}, i) - kv, i = next(t.ht, i) - (Pair{K,V}(kv[1].value::K,kv[2]), i) + gc_token = i[2] + wkv, i = next(t.ht, i[1]) + kv = Pair{K,V}(wkv[1].value::K, wkv[2]) + return (kv, (i, gc_token)) end -length(t::WeakKeyDict) = length(t.ht) function filter!(f, d::WeakKeyDict) for (k, v) in d diff --git a/test/misc.jl b/test/misc.jl index 633576319e108..d10b46aafa732 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -138,8 +138,21 @@ end # lock / unlock let l = ReentrantLock() lock(l) - @test trylock(l) - unlock(l) + success = Ref(false) + @test trylock(l) do + @test lock(l) do + success[] = true + return :foo + end === :foo + return :bar + end === :bar + @test success[] + t = @async begin + @test trylock(l) do + @test false + end === false + end + wait(t) unlock(l) @test_throws ErrorException unlock(l) end From cb3c2bb6cf39ce2a4423b67c8b781ee7d0c6e640 Mon Sep 17 00:00:00 2001 From: WooKyoung Noh <wookay.noh@gmail.com> Date: Fri, 5 Aug 2016 20:08:19 +0900 Subject: [PATCH 0804/1117] Add tests for date parsing (#10817) --- test/dates/io.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/dates/io.jl b/test/dates/io.jl index 11aea94137a1b..bdcade050c16f 100644 --- a/test/dates/io.jl +++ b/test/dates/io.jl @@ -359,3 +359,8 @@ let @test DateTime(ds, format) == dt @test DateTime(ds, escaped_format) == dt end + +# Issue 10817 +@test Dates.Date("Apr 01 2014", "uuu dd yyyy") == Dates.Date(2014,4,1) +@test_throws ArgumentError Dates.Date("Apr 01 xx 2014", "uuu dd zz yyyy") +@test_throws ArgumentError Dates.Date("Apr 01 xx 2014", "uuu dd yyyy") From afc95ab9c82299b4f4557228c3870c1116c56b2e Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Thu, 4 Aug 2016 23:35:28 +0200 Subject: [PATCH 0805/1117] Fix `@threadcall` argument passing. Fixes #17819. Also document not being allowed to call back into Julia. --- base/threadcall.jl | 3 +++ test/ccall.jl | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/base/threadcall.jl b/base/threadcall.jl index c9da92c5a49ea..da877d9707109 100644 --- a/base/threadcall.jl +++ b/base/threadcall.jl @@ -23,6 +23,8 @@ function without causing the main `julia` thread to become blocked. Concurrency is limited by size of the libuv thread pool, which defaults to 4 threads but can be increased by setting the `UV_THREADPOOL_SIZE` environment variable and restarting the `julia` process. + +Note that the called function should never call back into Julia. """ macro threadcall(f, rettype, argtypes, argvals...) # check for usage errors @@ -73,6 +75,7 @@ function do_threadcall(wrapper::Function, rettype::Type, argtypes::Vector, argva y = cconvert(T, x) push!(roots, y) unsafe_store!(convert(Ptr{T}, ptr), unsafe_convert(T, y)) + ptr += sizeof(T) end # create return buffer diff --git a/test/ccall.jl b/test/ccall.jl index 7f0792a87fdd5..e5cf4709d6d12 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -589,6 +589,10 @@ threadcall_test_func(x) = @test threadcall_test_func(3) == 1 @test threadcall_test_func(259) == 1 +f17819(a,b) = Cint(a+b) +cf17819 = cfunction(f17819, Cint, (Cint,Cint)) +@test @threadcall(cf17819, Cint, (Cint, Cint), 1, 2) == 3 + let n=3 tids = Culong[] @sync for i in 1:10^n From ba7460c496eb5a118885d540482fb69d5525b8cb Mon Sep 17 00:00:00 2001 From: Valentin Churavy <v.churavy@gmail.com> Date: Wed, 6 Jul 2016 23:10:27 +0900 Subject: [PATCH 0806/1117] Operations between Float16 and Integer now return Float16 Previously we would promote to Float32 leading to subtle type instabilities when Float16 was used for computations instead of a pure data-storage format. By defining `promote_type(Float16, Integer) = Float16` this incurs an overhead by first converting the `Integer` argument to `Float16` and then `Float32` for mathematical operations, that are performed in Float32. --- NEWS.md | 2 ++ base/float.jl | 2 +- base/float16.jl | 1 + base/statistics.jl | 1 - test/float16.jl | 8 ++++---- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index 2c185785fe324..f30ae3b4b50e9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -160,6 +160,8 @@ This section lists changes that do not have deprecation warnings. * `map` on a dictionary now expects a function that expects and returns a `Pair`. The result is now another dictionary instead of an array ([#16622]). + * Operations between `Float16` and `Integers` now return `Float16` instead of `Float32`. ([#17261]) + Library improvements -------------------- diff --git a/base/float.jl b/base/float.jl index 01767dd3d4123..dae5ad233f39f 100644 --- a/base/float.jl +++ b/base/float.jl @@ -45,7 +45,7 @@ const NaN = NaN64 ## conversions to floating-point ## convert(::Type{Float16}, x::Integer) = convert(Float16, convert(Float32,x)) for t in (Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128) - @eval promote_rule(::Type{Float16}, ::Type{$t}) = Float32 + @eval promote_rule(::Type{Float16}, ::Type{$t}) = Float16 end promote_rule(::Type{Float16}, ::Type{Bool}) = Float16 diff --git a/base/float16.jl b/base/float16.jl index bfd72a8f66f67..22c686a5e3648 100644 --- a/base/float16.jl +++ b/base/float16.jl @@ -135,6 +135,7 @@ abs(x::Float16) = reinterpret(Float16, reinterpret(UInt16,x) & 0x7fff) for op in (:+,:-,:*,:/,:\,:^) @eval ($op)(a::Float16, b::Float16) = Float16(($op)(Float32(a), Float32(b))) end + function fma(a::Float16, b::Float16, c::Float16) Float16(fma(Float32(a), Float32(b), Float32(c))) end diff --git a/base/statistics.jl b/base/statistics.jl index d85569ec52526..5f4ebeb3ba2ef 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -451,7 +451,6 @@ Compute the middle of a scalar value, which is equivalent to `x` itself, but of # Specialized functions for real types allow for improved performance middle(x::Union{Bool,Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128}) = Float64(x) middle(x::AbstractFloat) = x -middle(x::Float16) = Float32(x) middle(x::Real) = (x + zero(x)) / 1 """ diff --git a/test/float16.jl b/test/float16.jl index 8ada5d87f1780..5ea6fa198699e 100644 --- a/test/float16.jl +++ b/test/float16.jl @@ -63,10 +63,10 @@ g = Float16(1.) @test f^g === Float16(2f0) @test f^-g === Float16(0.5f0) -@test f + 2 === Float32(4f0) -@test f - 2 === Float32(0f0) -@test f*2 === Float32(4f0) -@test f/2 === Float32(1f0) +@test f + 2 === Float16(4f0) +@test f - 2 === Float16(0f0) +@test f*2 === Float16(4f0) +@test f/2 === Float16(1f0) @test f + 2. === 4. @test f - 2. === 0. @test f*2. === 4. From a240267df87878367262553f35289991a08c6e62 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Fri, 5 Aug 2016 08:27:20 -0700 Subject: [PATCH 0807/1117] Added documentation for doctests (#17825) The manual entry for doctests was way out of date! Updated with an example of what to do. --- doc/manual/documentation.rst | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index f03f60a8b4ba2..a6d481f340a58 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -99,14 +99,29 @@ documentation: ... """ -5. Group examples under an ``# Examples`` section and use ````julia`` blocks instead of +5. Group examples under an ``# Examples`` section and use `````julia`` blocks instead of standard text. Examples should consist of verbatim copies of the Julia REPL, including the ``julia>`` - prompt (see example above). This will be used in the future to allow running examples - automatically and checking that their actual output is consistent with that presented - in the documentation (a feature called *doctests*). This way, the code will be tested and - examples won't get out of date without notice. + prompt (see example above). This is used to allow running examples automatically and + checking that their actual output is consistent with that presented in the + documentation (a feature called *doctests*). This way, the code will be tested and + examples won't get out of date without notice. An example:: + + """ + Some nice documentation here. + + ```jldoctest + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + ``` + """ + + You can then run ``make -C doc doctest`` to run all the doctests, which will ensure + that your example works. Note that whitespace in your doctest is significant! The + doctest will fail if you misalign the output of pretty-printing an array, for example. 6. Use backticks to identify code and equations. From 2523cab8d89f147578818ed1721301245807fece Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Fri, 5 Aug 2016 08:28:09 -0700 Subject: [PATCH 0808/1117] Move math docs out of HelpDB, more examples, fix typos (#17791) * Move math docs out of HelpDB, more examples, fix typos Found a series of typos in `cov` and friends. Added more notes about `NaN` and Julia. Made the function signatures reflect what's actually in the code. More examples for quite a few functions. * Move quadgk docs out, update formatting * Moved special functions out of HelpDB, insert some links * Updated docs for some array ops as well * Updated in response to feedback Removed calls to `rand` in doctests. Made examples better. Cleaned up function signatures. --- base/abstractarray.jl | 113 ++++++- base/abstractarraymath.jl | 89 ++++++ base/array.jl | 5 +- base/docs/helpdb/Base.jl | 335 -------------------- base/iterator.jl | 12 +- base/math.jl | 16 + base/quadgk.jl | 55 ++++ base/rational.jl | 22 +- base/reduce.jl | 2 +- base/special/bessel.jl | 104 +++++- base/statistics.jl | 115 ++++++- doc/manual/arrays.rst | 2 +- doc/manual/complex-and-rational-numbers.rst | 2 +- doc/manual/control-flow.rst | 2 +- doc/manual/strings.rst | 2 +- doc/stdlib/arrays.rst | 136 +++++++- doc/stdlib/collections.rst | 21 +- doc/stdlib/math.rst | 116 +++++-- doc/stdlib/pkg.rst | 2 +- 19 files changed, 751 insertions(+), 400 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 31e84ab985995..65303a33d7b08 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1024,7 +1024,80 @@ function cat_t(catdims, typeC::Type, X...) return C end +""" + vcat(A...) + +Concatenate along dimension 1. + +```jldoctest +julia> a = [1 2 3 4 5] +1×5 Array{Int64,2}: + 1 2 3 4 5 + +julia> b = [6 7 8 9 10; 11 12 13 14 15] +2×5 Array{Int64,2}: + 6 7 8 9 10 + 11 12 13 14 15 + +julia> vcat(a,b) +3×5 Array{Int64,2}: + 1 2 3 4 5 + 6 7 8 9 10 + 11 12 13 14 15 + +julia> c = ([1 2 3], [4 5 6]) +( +[1 2 3], +<BLANKLINE> +[4 5 6]) + +julia> vcat(c...) +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 +``` +""" vcat(X...) = cat(1, X...) +""" + hcat(A...) + +Concatenate along dimension 2. + +```jldoctest +julia> a = [1; 2; 3; 4; 5] +5-element Array{Int64,1}: + 1 + 2 + 3 + 4 + 5 + +julia> b = [6 7; 8 9; 10 11; 12 13; 14 15] +5×2 Array{Int64,2}: + 6 7 + 8 9 + 10 11 + 12 13 + 14 15 + +julia> hcat(a,b) +5×3 Array{Int64,2}: + 1 6 7 + 2 8 9 + 3 10 11 + 4 12 13 + 5 14 15 + +julia> c = ([1; 2; 3], [4; 5; 6]) +([1,2,3],[4,5,6]) + +julia> hcat(c...) +3×2 Array{Int64,2}: + 1 4 + 2 5 + 3 6 +``` +""" hcat(X...) = cat(2, X...) typed_vcat(T::Type, X...) = cat_t(1, T, X...) @@ -1061,6 +1134,43 @@ function hvcat(nbc::Integer, as...) hvcat(ntuple(i->nbc, nbr), as...) end +""" + hvcat(rows::Tuple{Vararg{Int}}, values...) + +Horizontal and vertical concatenation in one call. This function is called for block matrix +syntax. The first argument specifies the number of arguments to concatenate in each block +row. + +```jldoctest +julia> a, b, c, d, e, f = 1, 2, 3, 4, 5, 6 +(1,2,3,4,5,6) + +julia> [a b c; d e f] +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> hvcat((3,3), a,b,c,d,e,f) +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> [a b;c d; e f] +3×2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 + +julia> hvcat((2,2,2), a,b,c,d,e,f) +3×2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 +``` + +If the first argument is a single integer `n`, then all block rows are assumed to have `n` +block columns. +""" hvcat(rows::Tuple{Vararg{Int}}, xs::AbstractMatrix...) = typed_hvcat(promote_eltype(xs...), rows, xs...) hvcat{T}(rows::Tuple{Vararg{Int}}, xs::AbstractMatrix{T}...) = typed_hvcat(T, rows, xs...) @@ -1347,8 +1457,7 @@ For multiple iterable arguments, `f` is called elementwise. needed, for example in `foreach(println, array)`. ```jldoctest -julia> a -1:3:7 +julia> a = 1:3:7; julia> foreach(x->println(x^2),a) 1 diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 61b95287155ad..7f95b1f1651ea 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -11,6 +11,27 @@ transpose(a::AbstractArray) = error("transpose not implemented for $(typeof(a)). ## Constructors ## +""" + vec(a::AbstractArray) -> Vector + +Reshape array `a` as a one-dimensional column vector. + +```jldoctest +julia> a = [1 2 3; 4 5 6] +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> vec(a) +6-element Array{Int64,1}: + 1 + 4 + 2 + 5 + 3 + 6 +``` +""" vec(a::AbstractArray) = reshape(a,_length(a)) vec(a::AbstractVector) = a @@ -18,6 +39,27 @@ _sub(::Tuple{}, ::Tuple{}) = () _sub(t::Tuple, ::Tuple{}) = t _sub(t::Tuple, s::Tuple) = _sub(tail(t), tail(s)) +""" + squeeze(A, dims) + +Remove the dimensions specified by `dims` from array `A`. +Elements of `dims` must be unique and within the range `1:ndims(A)`. +`size(A,i)` must equal 1 for all `i` in `dims`. + +```jldoctest +julia> a = reshape(collect(1:4),(2,2,1,1)) +2×2×1×1 Array{Int64,4}: +[:, :, 1, 1] = + 1 3 + 2 4 + +julia> squeeze(a,3) +2×2×1 Array{Int64,3}: +[:, :, 1] = + 1 3 + 2 4 +``` +""" function squeeze(A::AbstractArray, dims::Dims) for i in 1:length(dims) 1 <= dims[i] <= ndims(A) || throw(ArgumentError("squeezed dims must be in range 1:ndims(A)")) @@ -71,6 +113,23 @@ function flipdim(A::AbstractVector, d::Integer) reverse(A) end +""" + flipdim(A, d) + +Reverse `A` in dimension `d`. + +```jldoctest +julia> b = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> flipdim(b,2) +2×2 Array{Int64,2}: + 2 1 + 4 3 +``` +""" function flipdim(A::AbstractArray, d::Integer) nd = ndims(A) 1 ≤ d ≤ nd || throw(ArgumentError("dimension $d is not 1 ≤ $d ≤ $nd")) @@ -100,6 +159,36 @@ function flipdim(A::AbstractArray, d::Integer) end circshift(a::AbstractArray, shiftamt::Real) = circshift(a, [Integer(shiftamt)]) + +""" + circshift(A, shifts) + +Circularly shift the data in an array. The second argument is a vector giving the amount to +shift in each dimension. + +```jldoctest +julia> b = reshape(collect(1:16), (4,4)) +4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + +julia> circshift(b, [0,2]) +4×4 Array{Int64,2}: + 9 13 1 5 + 10 14 2 6 + 11 15 3 7 + 12 16 4 8 + +julia> circshift(b, [-1,0]) +4×4 Array{Int64,2}: + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + 1 5 9 13 +``` +""" function circshift{T,N}(a::AbstractArray{T,N}, shiftamts) I = () for i=1:N diff --git a/base/array.jl b/base/array.jl index b843c9043499e..783b93e0e5956 100644 --- a/base/array.jl +++ b/base/array.jl @@ -667,6 +667,7 @@ function hcat{T}(V::Vector{T}...) end return [ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ] end + function vcat{T}(arrays::Vector{T}...) n = 0 for a in arrays @@ -860,7 +861,7 @@ Returns the minimum element and its index. The collection must not be empty. ```jldoctest -julia> findmax([8,0.1,-9,pi]) +julia> findmin([8,0.1,-9,pi]) (-9.0,3) ``` """ @@ -915,7 +916,7 @@ vector contains 0 wherever `a` is not a member of `b`. ```jldoctest julia> a = ['a', 'b', 'c', 'b', 'd', 'a']; -julia> b = ['a','b','c'] +julia> b = ['a','b','c']; julia> indexin(a,b) 6-element Array{Int64,1}: diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index b8ab785fd9778..9a08432feec8e 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -146,13 +146,6 @@ Print (using [`print`](:func:`print`)) `x` followed by a newline. """ println -""" - besselj(nu, x) - -Bessel function of the first kind of order `nu`, ``J_\\nu(x)``. -""" -besselj - """ //(num, den) @@ -211,13 +204,6 @@ Multiply elements of `A` over the singleton dimensions of `r`, and write results """ prod! -""" - airybi(x) - -Airy function ``\\operatorname{Bi}(x)``. -""" -airybi - """ gensym([tag]) @@ -356,13 +342,6 @@ cannot be used with empty collections (see `reduce(op, itr)`). """ foldl(op, itr) -""" - airybiprime(x) - -Airy function derivative ``\\operatorname{Bi}'(x)``. -""" -airybiprime - """ Ac_rdiv_B(A, B) @@ -757,13 +736,6 @@ Get the local machine's host name. """ gethostname -""" - hankelh1x(nu, x) - -Scaled Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x) e^{-x i}``. -""" -hankelh1x - """ replace(string, pat, r[, n]) @@ -913,13 +885,6 @@ This would create a 25-by-30000 `BitArray`, linked to the file associated with s """ Mmap.mmap(io, ::BitArray, dims = ?, offset = ?) -""" - airyprime(x) - -Airy function derivative ``\\operatorname{Ai}'(x)``. -""" -airyprime - """ bessely0(x) @@ -950,13 +915,6 @@ Decodes the base64-encoded `string` and returns a `Vector{UInt8}` of the decoded """ base64decode -""" - besselkx(nu, x) - -Scaled modified Bessel function of the second kind of order `nu`, ``K_\\nu(x) e^x``. -""" -besselkx - """ oct(n, [pad]) @@ -1213,13 +1171,6 @@ Wrap an expression in a `Task` and add it to the local machine's scheduler queue """ :@schedule -""" - bessely(nu, x) - -Bessel function of the second kind of order `nu`, ``Y_\\nu(x)``. -""" -bessely - """ gradient(F, [h]) @@ -1389,15 +1340,6 @@ Connect to the named pipe / UNIX domain socket at `path`. """ connect(path) -""" - mean(v[, region]) - -Compute the mean of whole array `v`, or optionally along the dimensions in `region`. Note: -Julia does not ignore `NaN` values in the computation. For applications requiring the -handling of missing data, the `DataArray` package is recommended. -""" -mean - """ split(string, [chars]; limit=0, keep=true) @@ -1440,13 +1382,6 @@ be passed, to be returned from the last `produce` call in the producer. """ consume -""" - hankelh2x(nu, x) - -Scaled Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x) e^{x i}``. -""" -hankelh2x - """ ndigits(n, b = 10) @@ -1594,14 +1529,6 @@ Dict{String,Float64} with 3 entries: """ merge -""" - circshift(A,shifts) - -Circularly shift the data in an array. The second argument is a vector giving the amount to -shift in each dimension. -""" -circshift - """ yield() @@ -1720,13 +1647,6 @@ Construct a tuple of the given objects. """ tuple -""" - besseli(nu, x) - -Modified Bessel function of the first kind of order `nu`, ``I_\\nu(x)``. -""" -besseli - """ eachmatch(r::Regex, s::AbstractString[, overlap::Bool=false]) @@ -1848,14 +1768,6 @@ Bitwise and. """ & -""" - besselyx(nu, x) - -Scaled Bessel function of the second kind of order `nu`, -``Y_\\nu(x) e^{- | \\operatorname{Im}(x) |}``. -""" -besselyx - """ eigmax(A) @@ -1910,13 +1822,6 @@ and/or speed. """ logdet -""" - hcat(A...) - -Concatenate along dimension 2. -""" -hcat - """ select(v, k, [by=<transform>,] [lt=<comparison>,] [rev=false]) @@ -2130,14 +2035,6 @@ Element-wise greater-than-or-equals comparison operator. """ Base.:(.>=) -""" - stdm(v, m) - -Compute the sample standard deviation of a vector `v` with known mean `m`. Note: Julia does -not ignore `NaN` values in the computation. -""" -stdm - """ mv(src::AbstractString,dst::AbstractString; remove_destination::Bool=false) @@ -2217,13 +2114,6 @@ In-place version of [`reverse`](:func:`reverse`). """ reverse! -""" - flipdim(A, d) - -Reverse `A` in dimension `d`. -""" -flipdim - """ num(x) @@ -2297,13 +2187,6 @@ Bessel function of the second kind of order 1, ``Y_1(x)``. """ bessely1 -""" - besseljx(nu, x) - -Scaled Bessel function of the first kind of order `nu`, ``J_\\nu(x) e^{- | \\operatorname{Im}(x) |}``. -""" -besseljx - """ print(x) @@ -2530,45 +2413,6 @@ Extract a named field from a `value` of composite type. The syntax `a.b` calls """ getfield -""" - hvcat(rows::Tuple{Vararg{Int}}, values...) - -Horizontal and vertical concatenation in one call. This function is called for block matrix -syntax. The first argument specifies the number of arguments to concatenate in each block -row. - -```jldoctest -julia> a, b, c, d, e, f = 1, 2, 3, 4, 5, 6 -(1,2,3,4,5,6) - -julia> [a b c; d e f] -2×3 Array{Int64,2}: - 1 2 3 - 4 5 6 - -julia> hvcat((3,3), a,b,c,d,e,f) -2×3 Array{Int64,2}: - 1 2 3 - 4 5 6 - -julia> [a b;c d; e f] -3×2 Array{Int64,2}: - 1 2 - 3 4 - 5 6 - -julia> hvcat((2,2,2), a,b,c,d,e,f) -3×2 Array{Int64,2}: - 1 2 - 3 4 - 5 6 -``` - -If the first argument is a single integer `n`, then all block rows are assumed to have `n` -block columns. -""" -hvcat - """ besselj1(x) @@ -2700,13 +2544,6 @@ Create all directories in the given `path`, with permissions `mode`. `mode` defa """ mkpath -""" - besselix(nu, x) - -Scaled modified Bessel function of the first kind of order `nu`, ``I_\\nu(x) e^{- | \\operatorname{Re}(x) |}``. -""" -besselix - """ union(s1,s2...) ∪(s1,s2...) @@ -3183,17 +3020,6 @@ Compute ``\\sin(\\pi x) / (\\pi x)`` if ``x \\neq 0``, and ``1`` if ``x = 0``. """ sinc -""" - median(v[, region]) - -Compute the median of whole array `v`, or optionally along the dimensions in `region`. For -even number of elements no exact median element exists, so the result is equivalent to -calculating mean of two median elements. `NaN` is returned if the data contains any `NaN` -values. For applications requiring the handling of missing data, the `DataArrays` package is -recommended. -""" -median - """ cglobal((symbol, library) [, type=Void]) @@ -3212,14 +3038,6 @@ itself). For matrices, returns an identity matrix of the appropriate size and ty """ one -""" - rationalize([Type=Int,] x; tol=eps(x)) - -Approximate floating point number `x` as a Rational number with components of the given -integer type. The result will differ from `x` by no more than `tol`. -""" -rationalize - """ splice!(collection, index, [replacement]) -> item @@ -3437,13 +3255,6 @@ Test whether `n` is a power of two. """ ispow2 -""" - vcat(A...) - -Concatenate along dimension 1. -""" -vcat - """ isgraph(c::Union{Char,AbstractString}) -> Bool @@ -3805,14 +3616,6 @@ array, but with the specified element type. """ reinterpret -""" - squeeze(A, dims) - -Remove the dimensions specified by `dims` from array `A`. Elements of `dims` must be unique -and within the range `1:ndims(A)`. -""" -squeeze - """ ~(x) @@ -3820,13 +3623,6 @@ Bitwise not. """ ~ -""" - hankelh1(nu, x) - -Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x)``. -""" -hankelh1 - """ rem(x, y) %(x, y) @@ -4141,18 +3937,6 @@ Delete the mapping for the given key in a collection, and return the collection. """ delete! -""" - std(v[, region]) - -Compute the sample standard deviation of a vector or array `v`, optionally along dimensions -in `region`. The algorithm returns an estimator of the generative distribution's standard -deviation under the assumption that each entry of `v` is an IID drawn from that generative -distribution. This computation is equivalent to calculating `sqrt(sum((v - mean(v)).^2) / -(length(v) - 1))`. Note: Julia does not ignore `NaN` values in the computation. For -applications requiring the handling of missing data, the `DataArray` package is recommended. -""" -std - """ chr2ind(string, i) @@ -4275,13 +4059,6 @@ Get a module's enclosing `Module`. `Main` is its own parent, as is `LastMain` af """ module_parent -""" - airyaiprime(x) - -Airy function derivative ``\\operatorname{Ai}'(x)``. -""" -airyaiprime - """ prepend!(collection, items) -> collection @@ -5043,13 +4820,6 @@ called in last in first out (LIFO) order and run before object finalizers. """ atexit -""" - besselk(nu, x) - -Modified Bessel function of the second kind of order `nu`, ``K_\\nu(x)``. -""" -besselk - """ readchomp(x) @@ -5653,13 +5423,6 @@ general. """ setdiff -""" - airyai(x) - -Airy function ``\\operatorname{Ai}(x)``. -""" -airyai - """ error(message::AbstractString) @@ -5982,14 +5745,6 @@ Compute the secant of `x`, where `x` is in degrees. """ secd -""" - varm(v, m) - -Compute the sample variance of a vector `v` with known mean `m`. Note: Julia does not ignore -`NaN` values in the computation. -""" -varm - """ OverflowError() @@ -6113,13 +5868,6 @@ julia> leading_zeros(Int32(1)) """ leading_zeros -""" - hankelh2(nu, x) - -Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x)``. -""" -hankelh2 - """ lexcmp(x, y) @@ -6781,13 +6529,6 @@ characters, tests whether the last character of `string` belongs to that set. """ endswith -""" - airy(k,x) - -The `k`th derivative of the Airy function ``\\operatorname{Ai}(x)``. -""" -airy - """ !(x) @@ -7192,15 +6933,6 @@ Get the backtrace of the current exception, for use within `catch` blocks. """ catch_backtrace -""" - airyx(k,x) - -scaled `k`th derivative of the Airy function, return ``\\operatorname{Ai}(x) e^{\\frac{2}{3} x \\sqrt{x}}`` -for `k == 0 || k == 1`, and ``\\operatorname{Ai}(x) e^{- \\left| \\operatorname{Re} \\left( \\frac{2}{3} x \\sqrt{x} \\right) \\right|}`` -for `k == 2 || k == 3`. -""" -airyx - """ get_zero_subnormals() -> Bool @@ -7553,13 +7285,6 @@ available in the `.error` field. """ InitError -""" - vec(Array) -> Vector - -Vectorize an array using column-major convention. -""" -vec - """ copy!(dest, src) @@ -8159,58 +7884,6 @@ Compute ``x \\times 2^n``. """ ldexp -""" - quadgk(f, a,b,c...; reltol=sqrt(eps), abstol=0, maxevals=10^7, order=7, norm=vecnorm) - -Numerically integrate the function `f(x)` from `a` to `b`, and optionally over additional -intervals `b` to `c` and so on. Keyword options include a relative error tolerance `reltol` -(defaults to `sqrt(eps)` in the precision of the endpoints), an absolute error tolerance -`abstol` (defaults to 0), a maximum number of function evaluations `maxevals` (defaults to -`10^7`), and the `order` of the integration rule (defaults to 7). - -Returns a pair `(I,E)` of the estimated integral `I` and an estimated upper bound on the -absolute error `E`. If `maxevals` is not exceeded then `E <= max(abstol, reltol*norm(I))` -will hold. (Note that it is useful to specify a positive `abstol` in cases where `norm(I)` -may be zero.) - -The endpoints `a` etcetera can also be complex (in which case the integral is performed over -straight-line segments in the complex plane). If the endpoints are `BigFloat`, then the -integration will be performed in `BigFloat` precision as well (note: it is advisable to -increase the integration `order` in rough proportion to the precision, for smooth -integrands). More generally, the precision is set by the precision of the integration -endpoints (promoted to floating-point types). - -The integrand `f(x)` can return any numeric scalar, vector, or matrix type, or in fact any -type supporting `+`, `-`, multiplication by real values, and a `norm` (i.e., any normed -vector space). Alternatively, a different norm can be specified by passing a `norm`-like -function as the `norm` keyword argument (which defaults to `vecnorm`). - -\[Only one-dimensional integrals are provided by this function. For multi-dimensional -integration (cubature), there are many different algorithms (often much better than simple -nested 1d integrals) and the optimal choice tends to be very problem-dependent. See the -Julia external-package listing for available algorithms for multidimensional integration or -other specialized tasks (such as integrals of highly oscillatory or singular functions).\] - -The algorithm is an adaptive Gauss-Kronrod integration technique: the integral in each -interval is estimated using a Kronrod rule (`2*order+1` points) and the error is estimated -using an embedded Gauss rule (`order` points). The interval with the largest error is then -subdivided into two intervals and the process is repeated until the desired error tolerance -is achieved. - -These quadrature rules work best for smooth functions within each interval, so if your -function has a known discontinuity or other singularity, it is best to subdivide your -interval to put the singularity at an endpoint. For example, if `f` has a discontinuity at -`x=0.7` and you want to integrate from 0 to 1, you should use `quadgk(f, 0,0.7,1)` to -subdivide the interval at the point of discontinuity. The integrand is never evaluated -exactly at the endpoints of the intervals, so it is possible to integrate functions that -diverge at the endpoints as long as the singularity is integrable (for example, a `log(x)` -or `1/sqrt(x)` singularity). - -For real-valued endpoints, the starting and/or ending points may be infinite. (A coordinate -transformation is performed internally to map the infinite interval to a finite one.) -""" -quadgk - """ islower(c::Union{Char,AbstractString}) -> Bool @@ -8268,14 +7941,6 @@ Returns `true` if the value of the sign of `x` is negative, otherwise `false`. """ signbit -""" - clamp(x, lo, hi) - -Return `x` if `lo <= x <= hi`. If `x < lo`, return `lo`. If `x > hi`, return `hi`. Arguments -are promoted to a common type. Operates elementwise over `x` if it is an array. -""" -clamp - """ cscd(x) diff --git a/base/iterator.jl b/base/iterator.jl index 33852b9a33e5c..3d74fd5da57db 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -295,13 +295,13 @@ julia> collect(a) 5 7 9 - 11 + 11 julia> collect(take(a,3)) 3-element Array{Int64,1}: - 1 - 3 - 5 + 1 + 3 + 5 ``` """ take(xs, n::Int) = Take(xs, n) @@ -349,12 +349,12 @@ julia> collect(a) 5 7 9 - 11 + 11 julia> collect(drop(a,4)) 2-element Array{Int64,1}: 9 - 11 + 11 ``` """ drop(xs, n::Int) = Drop(xs, n) diff --git a/base/math.jl b/base/math.jl index 69012f7201236..6c96f5ab67319 100644 --- a/base/math.jl +++ b/base/math.jl @@ -34,6 +34,20 @@ import Core.Intrinsics: sqrt_llvm, box, unbox, powi_llvm # non-type specific math functions +""" + clamp(x, lo, hi) + +Return `x` if `lo <= x <= hi`. If `x < lo`, return `lo`. If `x > hi`, return `hi`. Arguments +are promoted to a common type. Operates elementwise over `x` if `x` is an array. + +```jldoctest +julia> clamp([pi, 1.0, big(10.)], 2., 9.) +3-element Array{BigFloat,1}: + 3.141592653589793238462643383279502884197169399375105820974944592307816406286198 + 2.000000000000000000000000000000000000000000000000000000000000000000000000000000 + 9.000000000000000000000000000000000000000000000000000000000000000000000000000000 +``` +""" clamp{X,L,H}(x::X, lo::L, hi::H) = ifelse(x > hi, convert(promote_type(X,L,H), hi), ifelse(x < lo, @@ -43,6 +57,7 @@ clamp{X,L,H}(x::X, lo::L, hi::H) = clamp{T}(x::AbstractArray{T,1}, lo, hi) = [clamp(xx, lo, hi) for xx in x] clamp{T}(x::AbstractArray{T,2}, lo, hi) = [clamp(x[i,j], lo, hi) for i in indices(x,1), j in indices(x,2)] + clamp{T}(x::AbstractArray{T}, lo, hi) = reshape([clamp(xx, lo, hi) for xx in x], size(x)) @@ -50,6 +65,7 @@ clamp{T}(x::AbstractArray{T}, lo, hi) = clamp!(array::AbstractArray, lo, hi) Restrict values in `array` to the specified range, in-place. +See also [`clamp`](:func:`clamp`). """ function clamp!{T}(x::AbstractArray{T}, lo, hi) @inbounds for i in eachindex(x) diff --git a/base/quadgk.jl b/base/quadgk.jl index dbd99d518493c..d80e1a4ae516e 100644 --- a/base/quadgk.jl +++ b/base/quadgk.jl @@ -164,6 +164,61 @@ function quadgk{T<:AbstractFloat}(f, a::Complex{T}, do_quadgk(f, [a, b, c...], order, T, abstol, reltol, maxevals, norm) end +""" + quadgk(f, a,b,c...; reltol=sqrt(eps), abstol=0, maxevals=10^7, order=7, norm=vecnorm) + +Numerically integrate the function `f(x)` from `a` to `b`, and optionally over additional +intervals `b` to `c` and so on. Keyword options include a relative error tolerance `reltol` +(defaults to `sqrt(eps)` in the precision of the endpoints), an absolute error tolerance +`abstol` (defaults to 0), a maximum number of function evaluations `maxevals` (defaults to +`10^7`), and the `order` of the integration rule (defaults to 7). + +Returns a pair `(I,E)` of the estimated integral `I` and an estimated upper bound on the +absolute error `E`. If `maxevals` is not exceeded then `E <= max(abstol, reltol*norm(I))` +will hold. (Note that it is useful to specify a positive `abstol` in cases where `norm(I)` +may be zero.) + +The endpoints `a` et cetera can also be complex (in which case the integral is performed over +straight-line segments in the complex plane). If the endpoints are `BigFloat`, then the +integration will be performed in `BigFloat` precision as well. + +!!! note + It is advisable to increase the integration `order` in rough proportion to the + precision, for smooth integrands. + +More generally, the precision is set by the precision of the integration +endpoints (promoted to floating-point types). + +The integrand `f(x)` can return any numeric scalar, vector, or matrix type, or in fact any +type supporting `+`, `-`, multiplication by real values, and a `norm` (i.e., any normed +vector space). Alternatively, a different norm can be specified by passing a `norm`-like +function as the `norm` keyword argument (which defaults to `vecnorm`). + +!!! note + Only one-dimensional integrals are provided by this function. For multi-dimensional + integration (cubature), there are many different algorithms (often much better than simple + nested 1d integrals) and the optimal choice tends to be very problem-dependent. See the + Julia external-package listing for available algorithms for multidimensional integration or + other specialized tasks (such as integrals of highly oscillatory or singular functions). + +The algorithm is an adaptive Gauss-Kronrod integration technique: the integral in each +interval is estimated using a Kronrod rule (`2*order+1` points) and the error is estimated +using an embedded Gauss rule (`order` points). The interval with the largest error is then +subdivided into two intervals and the process is repeated until the desired error tolerance +is achieved. + +These quadrature rules work best for smooth functions within each interval, so if your +function has a known discontinuity or other singularity, it is best to subdivide your +interval to put the singularity at an endpoint. For example, if `f` has a discontinuity at +`x=0.7` and you want to integrate from 0 to 1, you should use `quadgk(f, 0,0.7,1)` to +subdivide the interval at the point of discontinuity. The integrand is never evaluated +exactly at the endpoints of the intervals, so it is possible to integrate functions that +diverge at the endpoints as long as the singularity is integrable (for example, a `log(x)` +or `1/sqrt(x)` singularity). + +For real-valued endpoints, the starting and/or ending points may be infinite. (A coordinate +transformation is performed internally to map the infinite interval to a finite one.) +""" # generic version: determine precision from a combination of # all the integration-segment endpoints function quadgk(f, a, b, c...; kws...) diff --git a/base/rational.jl b/base/rational.jl index 438e8e1724f4e..067b8b570deac 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -90,8 +90,28 @@ promote_rule{T<:Integer,S<:AbstractFloat}(::Type{Rational{T}}, ::Type{S}) = prom widen{T}(::Type{Rational{T}}) = Rational{widen(T)} +""" + rationalize([T<:Integer=Int,] x; tol::Real=eps(x)) + +Approximate floating point number `x` as a `Rational` number with components +of the given integer type. The result will differ from `x` by no more than `tol`. +If `T` is not provided, it defaults to `Int`. + +```jldoctest +julia> rationalize(5.6) +28//5 + +julia> a = rationalize(BigInt, 10.3) +103//10 + +julia> typeof(num(a)) +BigInt +``` +""" function rationalize{T<:Integer}(::Type{T}, x::AbstractFloat; tol::Real=eps(x)) - tol < 0 && throw(ArgumentError("negative tolerance")) + if tol < 0 + throw(ArgumentError("negative tolerance $tol")) + end isnan(x) && return zero(T)//zero(T) isinf(x) && return (x < 0 ? -one(T) : one(T))//zero(T) diff --git a/base/reduce.jl b/base/reduce.jl index 18ef01b4d63f4..fe8b7e3320da4 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -502,7 +502,7 @@ Count the number of elements in `itr` for which predicate `p` returns `true`. ```jldoctest julia> count(i->(4<=i<=6), [2,3,4,5,6]) - 3 +3 ``` """ function count(pred, itr) diff --git a/base/special/bessel.jl b/base/special/bessel.jl index 286770488759e..3151306de7c6a 100644 --- a/base/special/bessel.jl +++ b/base/special/bessel.jl @@ -40,6 +40,11 @@ let end end +""" + airy(k,x) + +The `k`th derivative of the Airy function ``\\operatorname{Ai}(x)``. +""" function airy(k::Integer, z::Complex128) id = Int32(k==1 || k==3) if k == 0 || k == 1 @@ -51,14 +56,43 @@ function airy(k::Integer, z::Complex128) end end +""" + airyprime(x) + +Airy function derivative ``\\operatorname{Ai}'(x)``. +""" airyprime(z) = airy(1,z) @vectorize_1arg Number airyprime + +""" + airyai(x) + +Airy function ``\\operatorname{Ai}(x)``. +""" airyai(z) = airy(0,z) @vectorize_1arg Number airyai + +""" + airyaiprime(x) + +Airy function derivative ``\\operatorname{Ai}'(x)``. +""" airyaiprime(z) = airy(1,z) @vectorize_1arg Number airyaiprime + +""" + airybi(x) + +Airy function ``\\operatorname{Bi}(x)``. +""" airybi(z) = airy(2,z) @vectorize_1arg Number airybi + +""" + airybiprime(x) + +Airy function derivative ``\\operatorname{Bi}'(x)``. +""" airybiprime(z) = airy(3,z) @vectorize_1arg Number airybiprime @@ -87,6 +121,14 @@ for afn in (:airy,:airyx) @vectorize_2arg Number $afn end end +""" + airyx(k,x) + +scaled `k`th derivative of the Airy function, return ``\\operatorname{Ai}(x) e^{\\frac{2}{3} x \\sqrt{x}}`` +for `k == 0 || k == 1`, and ``\\operatorname{Ai}(x) e^{- \\left| \\operatorname{Re} \\left( \\frac{2}{3} x \\sqrt{x} \\right) \\right|}`` +for `k == 2 || k == 3`. +""" +function airyx(k,x) end ## Bessel functions @@ -194,7 +236,8 @@ end besselh(nu, [k=1,] x) Bessel function of the third kind of order `nu` (the Hankel function). `k` is either 1 or 2, -selecting `hankelh1` or `hankelh2`, respectively. `k` defaults to 1 if it is omitted. +selecting [`hankelh1`](:func:`hankelh1`) or [`hankelh2`](:func:`hankelh2`), respectively. +`k` defaults to 1 if it is omitted. (See also [`besselhx`](:func:`besselhx`) for an exponentially scaled variant.) """ function besselh end @@ -299,7 +342,11 @@ function besselyx(nu::Float64, z::Complex128) end end +""" + besseli(nu, x) +Modified Bessel function of the first kind of order `nu`, ``I_\\nu(x)``. +""" function besseli(nu::Real, x::AbstractFloat) if x < 0 && !isinteger(nu) throw(DomainError()) @@ -307,6 +354,11 @@ function besseli(nu::Real, x::AbstractFloat) real(besseli(float(nu), complex(x))) end +""" + besselix(nu, x) + +Scaled modified Bessel function of the first kind of order `nu`, ``I_\\nu(x) e^{- | \\operatorname{Re}(x) |}``. +""" function besselix(nu::Real, x::AbstractFloat) if x < 0 && !isinteger(nu) throw(DomainError()) @@ -314,6 +366,11 @@ function besselix(nu::Real, x::AbstractFloat) real(besselix(float(nu), complex(x))) end +""" + besselj(nu, x) + +Bessel function of the first kind of order `nu`, ``J_\\nu(x)``. +""" function besselj(nu::Real, x::AbstractFloat) if isinteger(nu) if typemin(Cint) <= nu <= typemax(Cint) @@ -325,6 +382,11 @@ function besselj(nu::Real, x::AbstractFloat) real(besselj(float(nu), complex(x))) end +""" + besseljx(nu, x) + +Scaled Bessel function of the first kind of order `nu`, ``J_\\nu(x) e^{- | \\operatorname{Im}(x) |}``. +""" function besseljx(nu::Real, x::AbstractFloat) if x < 0 && !isinteger(nu) throw(DomainError()) @@ -332,6 +394,11 @@ function besseljx(nu::Real, x::AbstractFloat) real(besseljx(float(nu), complex(x))) end +""" + besselk(nu, x) + +Modified Bessel function of the second kind of order `nu`, ``K_\\nu(x)``. +""" function besselk(nu::Real, x::AbstractFloat) if x < 0 throw(DomainError()) @@ -341,6 +408,11 @@ function besselk(nu::Real, x::AbstractFloat) real(besselk(float(nu), complex(x))) end +""" + besselkx(nu, x) + +Scaled modified Bessel function of the second kind of order `nu`, ``K_\\nu(x) e^x``. +""" function besselkx(nu::Real, x::AbstractFloat) if x < 0 throw(DomainError()) @@ -350,6 +422,11 @@ function besselkx(nu::Real, x::AbstractFloat) real(besselkx(float(nu), complex(x))) end +""" + bessely(nu, x) + +Bessel function of the second kind of order `nu`, ``Y_\\nu(x)``. +""" function bessely(nu::Real, x::AbstractFloat) if x < 0 throw(DomainError()) @@ -359,6 +436,12 @@ function bessely(nu::Real, x::AbstractFloat) real(bessely(float(nu), complex(x))) end +""" + besselyx(nu, x) + +Scaled Bessel function of the second kind of order `nu`, +``Y_\\nu(x) e^{- | \\operatorname{Im}(x) |}``. +""" function besselyx(nu::Real, x::AbstractFloat) if x < 0 throw(DomainError()) @@ -398,15 +481,34 @@ for bfn in (:besselh, :besselhx) end end +""" + hankelh1(nu, x) +Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x)``. +""" hankelh1(nu, z) = besselh(nu, 1, z) @vectorize_2arg Number hankelh1 +""" + hankelh2(nu, x) + +Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x)``. +""" hankelh2(nu, z) = besselh(nu, 2, z) @vectorize_2arg Number hankelh2 +""" + hankelh1x(nu, x) + +Scaled Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x) e^{-x i}``. +""" hankelh1x(nu, z) = besselhx(nu, 1, z) @vectorize_2arg Number hankelh1x +""" + hankelh2x(nu, x) + +Scaled Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x) e^{x i}``. +""" hankelh2x(nu, z) = besselhx(nu, 2, z) @vectorize_2arg Number hankelh2x diff --git a/base/statistics.jl b/base/statistics.jl index 5f4ebeb3ba2ef..37ef2afe459c5 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -37,6 +37,15 @@ momenttype{T}(::Type{T}) = typeof((zero(T) + zero(T)) / 2) momenttype(::Type{Float32}) = Float32 momenttype{T<:Union{Float64,Int32,Int64,UInt32,UInt64}}(::Type{T}) = Float64 +""" + mean(v[, region]) + +Compute the mean of whole array `v`, or optionally along the dimensions in `region`. + +!!! note + Julia does not ignore `NaN` values in the computation. For applications requiring the + handling of missing data, the `DataArrays.jl` package is recommended. +""" mean{T}(A::AbstractArray{T}, region) = mean!(reducedim_initarray(A, region, 0, momenttype(T)), A) @@ -147,6 +156,19 @@ function varm!{S}(R::AbstractArray{S}, A::AbstractArray, m::AbstractArray; corre return R end +""" + varm(v, m[, region]; corrected::Bool=true) + +Compute the sample variance of a collection `v` with known mean(s) `m`, +optionally over `region`. `m` may contain means for each dimension of +`v`. If `corrected` is `true`, then the sum is scaled with `n-1`, +whereas the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. + +!!! note + Julia does not ignore `NaN` values in the computation. For + applications requiring the handling of missing data, the + `DataArrays.jl` package is recommended. +""" varm{T}(A::AbstractArray{T}, m::AbstractArray, region; corrected::Bool=true) = varm!(reducedim_initarray(A, region, 0, real(momenttype(T))), A, m; corrected=corrected) @@ -198,12 +220,41 @@ stdm(A::AbstractArray, m::Number; corrected::Bool=true) = std(A::AbstractArray; corrected::Bool=true, mean=nothing) = sqrt(var(A; corrected=corrected, mean=mean)) +""" + std(v[, region]; corrected::Bool=true, mean=nothing) + +Compute the sample standard deviation of a vector or array `v`, optionally along dimensions +in `region`. The algorithm returns an estimator of the generative distribution's standard +deviation under the assumption that each entry of `v` is an IID drawn from that generative +distribution. This computation is equivalent to calculating `sqrt(sum((v - mean(v)).^2) / +(length(v) - 1))`. A pre-computed `mean` may be provided. If `corrected` is `true`, +then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is +`false` where `n = length(x)`. + +!!! note + Julia does not ignore `NaN` values in the computation. For + applications requiring the handling of missing data, the + `DataArrays.jl` package is recommended. +""" std(A::AbstractArray, region; corrected::Bool=true, mean=nothing) = sqrt!(var(A, region; corrected=corrected, mean=mean)) std(iterable; corrected::Bool=true, mean=nothing) = sqrt(var(iterable, corrected=corrected, mean=mean)) +""" + stdm(v, m::Number; corrected::Bool=true) + +Compute the sample standard deviation of a vector `v` +with known mean `m`. If `corrected` is `true`, +then the sum is scaled with `n-1`, whereas the sum is +scaled with `n` if `corrected` is `false` where `n = length(x)`. + +!!! note + Julia does not ignore `NaN` values in the computation. For + applications requiring the handling of missing data, the + `DataArrays.jl` package is recommended. +""" stdm(iterable, m::Number; corrected::Bool=true) = std(iterable, corrected=corrected, mean=m) @@ -266,7 +317,7 @@ covm(x::AbstractVecOrMat, xmean, y::AbstractVecOrMat, ymean, vardim::Int=1, corr cov(x[, corrected=true]) Compute the variance of the vector `x`. If `corrected` is `true` (the default) then the sum -is scaled with `n-1` wheares the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. +is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. """ cov(x::AbstractVector, corrected::Bool) = covm(x, Base.mean(x), corrected) # This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged @@ -276,7 +327,7 @@ cov{T<:AbstractVector}(x::T) = covm(x, Base.mean(x), true) cov(X[, vardim=1, corrected=true]) Compute the covariance matrix of the matrix `X` along the dimension `vardim`. If `corrected` -is `true` (the default) then the sum is scaled with `n-1` wheares the sum is scaled with `n` +is `true` (the default) then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = size(X, vardim)`. """ cov(X::AbstractMatrix, vardim::Int, corrected::Bool=true) = @@ -288,7 +339,7 @@ cov{T<:AbstractMatrix}(X::T) = cov(X, 1, true) cov(x, y[, corrected=true]) Compute the covariance between the vectors `x` and `y`. If `corrected` is `true` (the default) -then the sum is scaled with `n-1` wheares the sum is scaled with `n` if `corrected` is `false` +then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = length(x) = length(y)`. """ cov(x::AbstractVector, y::AbstractVector, corrected::Bool) = @@ -301,7 +352,7 @@ cov{T<:AbstractVector,S<:AbstractVector}(x::T, y::S) = cov(X, Y[, vardim=1, corrected=true]) Compute the covariance between the vectors or matrices `X` and `Y` along the dimension -`vardim`. If `corrected` is `true` (the default) then the sum is scaled with `n-1` wheares +`vardim`. If `corrected` is `true` (the default) then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = size(X, vardim) = size(Y, vardim)`. """ cov(X::AbstractVecOrMat, Y::AbstractVecOrMat, vardim::Int, corrected::Bool=true) = @@ -456,21 +507,41 @@ middle(x::Real) = (x + zero(x)) / 1 """ middle(x, y) -Compute the middle of two reals `x` and `y`, which is equivalent in both value and type to computing their mean (`(x + y) / 2`). +Compute the middle of two reals `x` and `y`, which is +equivalent in both value and type to computing their mean (`(x + y) / 2`). """ middle(x::Real, y::Real) = x/2 + y/2 """ middle(range) -Compute the middle of a range, which consists in computing the mean of its extrema. Since a range is sorted, the mean is performed with the first and last element. +Compute the middle of a range, which consists of computing the mean of its extrema. +Since a range is sorted, the mean is performed with the first and last element. + +```jldoctest +julia> middle(1:10) +5.5 +``` """ middle(a::Range) = middle(a[1], a[end]) """ - middle(array) - -Compute the middle of an array, which consists in finding its extrema and then computing their mean. + middle(a) + +Compute the middle of an array `a`, which consists of finding its +extrema and then computing their mean. + +```jldoctest +julia> a = [1,2,3.6,10.9] +4-element Array{Float64,1}: + 1.0 + 2.0 + 3.6 + 10.9 + +julia> middle(a) +5.95 +``` """ middle(a::AbstractArray) = ((v1, v2) = extrema(a); middle(v1, v2)) @@ -492,6 +563,18 @@ end median!{T}(v::AbstractArray{T}) = median!(vec(v)) median{T}(v::AbstractArray{T}) = median!(copy!(Array{T,1}(length(v)), v)) +""" + median(v[, region]) + +Compute the median of an entire array `v`, or, optionally, +along the dimensions in `region`. For an even number of +elements no exact median element exists, so the result is +equivalent to calculating mean of two median elements. + +!!! note + Julia does not ignore `NaN` values in the computation. For applications requiring the + handling of missing data, the `DataArrays.jl` package is recommended. +""" median{T}(v::AbstractArray{T}, region) = mapslices(median!, v, region) # for now, use the R/S definition of quantile; may want variants later @@ -511,12 +594,19 @@ Quantiles are computed via linear interpolation between the points `((k-1)/(n-1) for `k = 1:n` where `n = length(v)`. This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. +!!! note + Julia does not ignore `NaN` values in the computation. For applications requiring the + handling of missing data, the `DataArrays.jl` package is recommended. `quantile!` will + throw an `ArgumentError` in the presence of `NaN` values in the data array. + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 """ function quantile!(q::AbstractArray, v::AbstractVector, p::AbstractArray; sorted::Bool=false) - size(p) == size(q) || throw(DimensionMismatch()) + if size(p) != size(q) + throw(DimensionMismatch("size of p, $(size(p)), must equal size of q, $(size(q))")) + end isempty(v) && throw(ArgumentError("empty data vector")) @@ -592,6 +682,11 @@ Quantiles are computed via linear interpolation between the points `((k-1)/(n-1) for `k = 1:n` where `n = length(v)`. This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. +!!! note + Julia does not ignore `NaN` values in the computation. For applications requiring the + handling of missing data, the `DataArrays.jl` package is recommended. `quantile` will + throw an `ArgumentError` in the presence of `NaN` values in the data array. + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 """ diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index e04ae53de2ae0..5644e2d047bf6 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -517,7 +517,7 @@ the name of the function to vectorize. Here is a simple example: julia> methods(square) # 2 methods for generic function "square": - square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:540 + square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:550 square(x) at none:1 julia> square([1 2 4; 5 6 7]) diff --git a/doc/manual/complex-and-rational-numbers.rst b/doc/manual/complex-and-rational-numbers.rst index aa9cd00783e47..ae30e44e838eb 100644 --- a/doc/manual/complex-and-rational-numbers.rst +++ b/doc/manual/complex-and-rational-numbers.rst @@ -159,7 +159,7 @@ versus ``-1 + 0im`` even though ``-1 == -1 + 0im``: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:169 + in sqrt(::Int64) at ./math.jl:185 ... julia> sqrt(-1 + 0im) diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 0a1180581594b..4168cb0118ce0 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -666,7 +666,7 @@ negative real value: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:169 + in sqrt(::Int64) at ./math.jl:185 ... You may define your own exceptions in the following way: diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index a1838fd57e903..e8e3c0813e843 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -553,7 +553,7 @@ contained in a string: julia> contains("Xylophon", 'o') ERROR: MethodError: no method matching contains(::String, ::Char) Closest candidates are: - contains(!Matched::Function, ::Any, !Matched::Any) at reduce.jl:402 + contains(!Matched::Function, ::Any, !Matched::Any) at reduce.jl:489 contains(::AbstractString, !Matched::AbstractString) at strings/search.jl:310 ... diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index f72f67f72b10a..f45ff006269a5 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -437,12 +437,75 @@ Indexing, Assignment, and Concatenation Concatenate along dimension 1. + .. doctest:: + + julia> a = [1 2 3 4 5] + 1×5 Array{Int64,2}: + 1 2 3 4 5 + + julia> b = [6 7 8 9 10; 11 12 13 14 15] + 2×5 Array{Int64,2}: + 6 7 8 9 10 + 11 12 13 14 15 + + julia> vcat(a,b) + 3×5 Array{Int64,2}: + 1 2 3 4 5 + 6 7 8 9 10 + 11 12 13 14 15 + + julia> c = ([1 2 3], [4 5 6]) + ( + [1 2 3], + <BLANKLINE> + [4 5 6]) + + julia> vcat(c...) + 2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + .. function:: hcat(A...) .. Docstring generated from Julia source Concatenate along dimension 2. + .. doctest:: + + julia> a = [1; 2; 3; 4; 5] + 5-element Array{Int64,1}: + 1 + 2 + 3 + 4 + 5 + + julia> b = [6 7; 8 9; 10 11; 12 13; 14 15] + 5×2 Array{Int64,2}: + 6 7 + 8 9 + 10 11 + 12 13 + 14 15 + + julia> hcat(a,b) + 5×3 Array{Int64,2}: + 1 6 7 + 2 8 9 + 3 10 11 + 4 12 13 + 5 14 15 + + julia> c = ([1; 2; 3], [4; 5; 6]) + ([1,2,3],[4,5,6]) + + julia> hcat(c...) + 3×2 Array{Int64,2}: + 1 4 + 2 5 + 3 6 + .. function:: hvcat(rows::Tuple{Vararg{Int}}, values...) .. Docstring generated from Julia source @@ -484,12 +547,47 @@ Indexing, Assignment, and Concatenation Reverse ``A`` in dimension ``d``\ . -.. function:: circshift(A,shifts) + .. doctest:: + + julia> b = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> flipdim(b,2) + 2×2 Array{Int64,2}: + 2 1 + 4 3 + +.. function:: circshift(A, shifts) .. Docstring generated from Julia source Circularly shift the data in an array. The second argument is a vector giving the amount to shift in each dimension. + .. doctest:: + + julia> b = reshape(collect(1:16), (4,4)) + 4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + + julia> circshift(b, [0,2]) + 4×4 Array{Int64,2}: + 9 13 1 5 + 10 14 2 6 + 11 15 3 7 + 12 16 4 8 + + julia> circshift(b, [-1,0]) + 4×4 Array{Int64,2}: + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + 1 5 9 13 + .. function:: find(A) .. Docstring generated from Julia source @@ -608,13 +706,43 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Remove the dimensions specified by ``dims`` from array ``A``\ . Elements of ``dims`` must be unique and within the range ``1:ndims(A)``\ . + Remove the dimensions specified by ``dims`` from array ``A``\ . Elements of ``dims`` must be unique and within the range ``1:ndims(A)``\ . ``size(A,i)`` must equal 1 for all ``i`` in ``dims``\ . -.. function:: vec(Array) -> Vector + .. doctest:: + + julia> a = reshape(collect(1:4),(2,2,1,1)) + 2×2×1×1 Array{Int64,4}: + [:, :, 1, 1] = + 1 3 + 2 4 + + julia> squeeze(a,3) + 2×2×1 Array{Int64,3}: + [:, :, 1] = + 1 3 + 2 4 + +.. function:: vec(a::AbstractArray) -> Vector .. Docstring generated from Julia source - Vectorize an array using column-major convention. + Reshape array ``a`` as a one-dimensional column vector. + + .. doctest:: + + julia> a = [1 2 3; 4 5 6] + 2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + + julia> vec(a) + 6-element Array{Int64,1}: + 1 + 4 + 2 + 5 + 3 + 6 .. function:: promote_shape(s1, s2) diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 4cc7c6eb4be22..e37157208b22a 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -124,13 +124,13 @@ type. 5 7 9 - 11 + 11 julia> collect(take(a,3)) 3-element Array{Int64,1}: - 1 - 3 - 5 + 1 + 3 + 5 .. function:: drop(iter, n) @@ -150,12 +150,12 @@ type. 5 7 9 - 11 + 11 julia> collect(drop(a,4)) 2-element Array{Int64,1}: 9 - 11 + 11 .. function:: cycle(iter) @@ -295,7 +295,7 @@ Iterable Collections julia> a = ['a', 'b', 'c', 'b', 'd', 'a']; - julia> b = ['a','b','c'] + julia> b = ['a','b','c']; julia> indexin(a,b) 6-element Array{Int64,1}: @@ -523,7 +523,7 @@ Iterable Collections .. doctest:: - julia> findmax([8,0.1,-9,pi]) + julia> findmin([8,0.1,-9,pi]) (-9.0,3) .. function:: findmin(A, dims) -> (minval, index) @@ -713,7 +713,7 @@ Iterable Collections .. doctest:: julia> count(i->(4<=i<=6), [2,3,4,5,6]) - 3 + 3 .. function:: any(p, itr) -> Bool @@ -745,8 +745,7 @@ Iterable Collections .. doctest:: - julia> a - 1:3:7 + julia> a = 1:3:7; julia> foreach(x->println(x^2),a) 1 diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 47620a1b53e14..7c9a7c653e808 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -201,11 +201,22 @@ Mathematical Operators Divide two integers or rational numbers, giving a ``Rational`` result. -.. function:: rationalize([Type=Int,] x; tol=eps(x)) +.. function:: rationalize([T<:Integer=Int,] x; tol::Real=eps(x)) .. Docstring generated from Julia source - Approximate floating point number ``x`` as a Rational number with components of the given integer type. The result will differ from ``x`` by no more than ``tol``\ . + Approximate floating point number ``x`` as a ``Rational`` number with components of the given integer type. The result will differ from ``x`` by no more than ``tol``\ . If ``T`` is not provided, it defaults to ``Int``\ . + + .. doctest:: + + julia> rationalize(5.6) + 28//5 + + julia> a = rationalize(BigInt, 10.3) + 103//10 + + julia> typeof(num(a)) + BigInt .. function:: num(x) @@ -1021,13 +1032,21 @@ Mathematical Functions .. Docstring generated from Julia source - Return ``x`` if ``lo <= x <= hi``\ . If ``x < lo``\ , return ``lo``\ . If ``x > hi``\ , return ``hi``\ . Arguments are promoted to a common type. Operates elementwise over ``x`` if it is an array. + Return ``x`` if ``lo <= x <= hi``\ . If ``x < lo``\ , return ``lo``\ . If ``x > hi``\ , return ``hi``\ . Arguments are promoted to a common type. Operates elementwise over ``x`` if ``x`` is an array. + + .. doctest:: + + julia> clamp([pi, 1.0, big(10.)], 2., 9.) + 3-element Array{BigFloat,1}: + 3.141592653589793238462643383279502884197169399375105820974944592307816406286198 + 2.000000000000000000000000000000000000000000000000000000000000000000000000000000 + 9.000000000000000000000000000000000000000000000000000000000000000000000000000000 .. function:: clamp!(array::AbstractArray, lo, hi) .. Docstring generated from Julia source - Restrict values in ``array`` to the specified range, in-place. + Restrict values in ``array`` to the specified range, in-place. See also :func:`clamp`\ . .. function:: abs(x) @@ -1495,7 +1514,7 @@ Mathematical Functions .. Docstring generated from Julia source - Bessel function of the third kind of order ``nu`` (the Hankel function). ``k`` is either 1 or 2, selecting ``hankelh1`` or ``hankelh2``\ , respectively. ``k`` defaults to 1 if it is omitted. (See also :func:`besselhx` for an exponentially scaled variant.) + Bessel function of the third kind of order ``nu`` (the Hankel function). ``k`` is either 1 or 2, selecting :func:`hankelh1` or :func:`hankelh2`\ , respectively. ``k`` defaults to 1 if it is omitted. (See also :func:`besselhx` for an exponentially scaled variant.) .. function:: besselhx(nu, [k=1,] z) @@ -1584,7 +1603,11 @@ Statistics .. Docstring generated from Julia source - Compute the mean of whole array ``v``\ , or optionally along the dimensions in ``region``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. + Compute the mean of whole array ``v``\ , or optionally along the dimensions in ``region``\ . + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: mean(f::Function, v) @@ -1598,17 +1621,25 @@ Statistics Compute the mean of ``v`` over the singleton dimensions of ``r``\ , and write results to ``r``\ . -.. function:: std(v[, region]) +.. function:: std(v[, region]; corrected::Bool=true, mean=nothing) .. Docstring generated from Julia source - Compute the sample standard deviation of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm returns an estimator of the generative distribution's standard deviation under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sqrt(sum((v - mean(v)).^2) / (length(v) - 1))``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. + Compute the sample standard deviation of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm returns an estimator of the generative distribution's standard deviation under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sqrt(sum((v - mean(v)).^2) / (length(v) - 1))``\ . A pre-computed ``mean`` may be provided. If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . -.. function:: stdm(v, m) + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + + +.. function:: stdm(v, m::Number; corrected::Bool=true) .. Docstring generated from Julia source - Compute the sample standard deviation of a vector ``v`` with known mean ``m``\ . Note: Julia does not ignore ``NaN`` values in the computation. + Compute the sample standard deviation of a vector ``v`` with known mean ``m``\ . If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: var(v[, region]) @@ -1616,11 +1647,15 @@ Statistics Compute the sample variance of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sumabs2(v - mean(v)) / (length(v) - 1)``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. -.. function:: varm(v, m) +.. function:: varm(v, m[, region]; corrected::Bool=true) .. Docstring generated from Julia source - Compute the sample variance of a vector ``v`` with known mean ``m``\ . Note: Julia does not ignore ``NaN`` values in the computation. + Compute the sample variance of a collection ``v`` with known mean(s) ``m``\ , optionally over ``region``\ . ``m`` may contain means for each dimension of ``v``\ . If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: middle(x) @@ -1638,19 +1673,40 @@ Statistics .. Docstring generated from Julia source - Compute the middle of a range, which consists in computing the mean of its extrema. Since a range is sorted, the mean is performed with the first and last element. + Compute the middle of a range, which consists of computing the mean of its extrema. Since a range is sorted, the mean is performed with the first and last element. + + .. doctest:: + + julia> middle(1:10) + 5.5 -.. function:: middle(array) +.. function:: middle(a) .. Docstring generated from Julia source - Compute the middle of an array, which consists in finding its extrema and then computing their mean. + Compute the middle of an array ``a``\ , which consists of finding its extrema and then computing their mean. + + .. doctest:: + + julia> a = [1,2,3.6,10.9] + 4-element Array{Float64,1}: + 1.0 + 2.0 + 3.6 + 10.9 + + julia> middle(a) + 5.95 .. function:: median(v[, region]) .. Docstring generated from Julia source - Compute the median of whole array ``v``\ , or optionally along the dimensions in ``region``\ . For even number of elements no exact median element exists, so the result is equivalent to calculating mean of two median elements. ``NaN`` is returned if the data contains any ``NaN`` values. For applications requiring the handling of missing data, the ``DataArrays`` package is recommended. + Compute the median of an entire array ``v``\ , or, optionally, along the dimensions in ``region``\ . For an even number of elements no exact median element exists, so the result is equivalent to calculating mean of two median elements. + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: median!(v) @@ -1674,6 +1730,10 @@ Statistics Quantiles are computed via linear interpolation between the points ``((k-1)/(n-1), v[k])``\ , for ``k = 1:n`` where ``n = length(v)``\ . This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. ``quantile`` will throw an ``ArgumentError`` in the presence of ``NaN`` values in the data array. + + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 .. function:: quantile!([q, ] v, p; sorted=false) @@ -1686,31 +1746,35 @@ Statistics Quantiles are computed via linear interpolation between the points ``((k-1)/(n-1), v[k])``\ , for ``k = 1:n`` where ``n = length(v)``\ . This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. ``quantile!`` will throw an ``ArgumentError`` in the presence of ``NaN`` values in the data array. + + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 .. function:: cov(x[, corrected=true]) .. Docstring generated from Julia source - Compute the variance of the vector ``x``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . + Compute the variance of the vector ``x``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . .. function:: cov(X[, vardim=1, corrected=true]) .. Docstring generated from Julia source - Compute the covariance matrix of the matrix ``X`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim)``\ . + Compute the covariance matrix of the matrix ``X`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim)``\ . .. function:: cov(x, y[, corrected=true]) .. Docstring generated from Julia source - Compute the covariance between the vectors ``x`` and ``y``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x) = length(y)``\ . + Compute the covariance between the vectors ``x`` and ``y``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x) = length(y)``\ . .. function:: cov(X, Y[, vardim=1, corrected=true]) .. Docstring generated from Julia source - Compute the covariance between the vectors or matrices ``X`` and ``Y`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim) = size(Y, vardim)``\ . + Compute the covariance between the vectors or matrices ``X`` and ``Y`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim) = size(Y, vardim)``\ . .. function:: cor(x) @@ -2047,11 +2111,19 @@ some built-in integration support in Julia. Returns a pair ``(I,E)`` of the estimated integral ``I`` and an estimated upper bound on the absolute error ``E``\ . If ``maxevals`` is not exceeded then ``E <= max(abstol, reltol*norm(I))`` will hold. (Note that it is useful to specify a positive ``abstol`` in cases where ``norm(I)`` may be zero.) - The endpoints ``a`` etcetera can also be complex (in which case the integral is performed over straight-line segments in the complex plane). If the endpoints are ``BigFloat``\ , then the integration will be performed in ``BigFloat`` precision as well (note: it is advisable to increase the integration ``order`` in rough proportion to the precision, for smooth integrands). More generally, the precision is set by the precision of the integration endpoints (promoted to floating-point types). + The endpoints ``a`` et cetera can also be complex (in which case the integral is performed over straight-line segments in the complex plane). If the endpoints are ``BigFloat``\ , then the integration will be performed in ``BigFloat`` precision as well. + + .. note:: + It is advisable to increase the integration ``order`` in rough proportion to the precision, for smooth integrands. + + + More generally, the precision is set by the precision of the integration endpoints (promoted to floating-point types). The integrand ``f(x)`` can return any numeric scalar, vector, or matrix type, or in fact any type supporting ``+``\ , ``-``\ , multiplication by real values, and a ``norm`` (i.e., any normed vector space). Alternatively, a different norm can be specified by passing a ``norm``\ -like function as the ``norm`` keyword argument (which defaults to ``vecnorm``\ ). - [Only one-dimensional integrals are provided by this function. For multi-dimensional integration (cubature), there are many different algorithms (often much better than simple nested 1d integrals) and the optimal choice tends to be very problem-dependent. See the Julia external-package listing for available algorithms for multidimensional integration or other specialized tasks (such as integrals of highly oscillatory or singular functions).] + .. note:: + Only one-dimensional integrals are provided by this function. For multi-dimensional integration (cubature), there are many different algorithms (often much better than simple nested 1d integrals) and the optimal choice tends to be very problem-dependent. See the Julia external-package listing for available algorithms for multidimensional integration or other specialized tasks (such as integrals of highly oscillatory or singular functions). + The algorithm is an adaptive Gauss-Kronrod integration technique: the integral in each interval is estimated using a Kronrod rule (``2*order+1`` points) and the error is estimated using an embedded Gauss rule (``order`` points). The interval with the largest error is then subdivided into two intervals and the process is repeated until the desired error tolerance is achieved. diff --git a/doc/stdlib/pkg.rst b/doc/stdlib/pkg.rst index 33d35f6ef11c3..f0d455b9e0829 100644 --- a/doc/stdlib/pkg.rst +++ b/doc/stdlib/pkg.rst @@ -13,7 +13,7 @@ Functions for package development (e.g. ``tag``, ``publish``, etc.) have been mo .. Docstring generated from Julia source - Returns the absolute path of the package directory. This defaults to ``joinpath(homedir(),".julia","v$(VERSION.major).$(VERSION.minor)")`` on all platforms (i.e. ``~/.julia/v0.5`` in UNIX shell syntax). If the ``JULIA_PKGDIR`` environment variable is set, then that path is used in the returned value as ``joinpath(ENV["JULIA_PKGDIR"],"v$(VERSION.major).$(VERSION.minor)")``\ . If ``JULIA_PKGDIR`` is a relative path, it is interpreted relative to whatever the current working directory is. + Returns the absolute path of the package directory. This defaults to ``joinpath(homedir(),".julia","v$(VERSION.major).$(VERSION.minor)")`` on all platforms (i.e. ``~/.julia/v0.6`` in UNIX shell syntax). If the ``JULIA_PKGDIR`` environment variable is set, then that path is used in the returned value as ``joinpath(ENV["JULIA_PKGDIR"],"v$(VERSION.major).$(VERSION.minor)")``\ . If ``JULIA_PKGDIR`` is a relative path, it is interpreted relative to whatever the current working directory is. .. function:: dir(names...) -> AbstractString From 2426803750ee3adfe0f247d722a75c67ab43b91a Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Wed, 3 Aug 2016 16:32:33 -0700 Subject: [PATCH 0809/1117] More HelpDB pruning for Bit/SparseArray, more examples Yet MORE examples from the REPL. Fixed and added links, changd some wording. --- base/bitarray.jl | 134 +++++++++++++++++++++++++++ base/docs/helpdb/Base.jl | 49 ---------- base/sparse/sparsematrix.jl | 81 +++++++++++++++-- doc/stdlib/arrays.rst | 177 +++++++++++++++++++++++++++++++++--- 4 files changed, 368 insertions(+), 73 deletions(-) diff --git a/base/bitarray.jl b/base/bitarray.jl index 81ba017866ee9..8ea68191bb8a0 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1005,6 +1005,25 @@ function (~)(B::BitArray) return C end +""" + flipbits!(B::BitArray{N}) -> BitArray{N} + +Performs a bitwise not operation on `B`. See [`~`](:ref:`~ operator <~>`). + +```jldoctest +julia> A = bitrand(3,5) +3×5 BitArray{2}: + true true true true true + false false false true false + true false false false true + +julia> flipbits!(A) +3×5 BitArray{2}: + false false false false false + true true true false true + false true true true false +``` +""" function flipbits!(B::BitArray) Bc = B.chunks @inbounds if !isempty(Bc) @@ -1416,6 +1435,39 @@ end (>>)(B::BitVector, i::Int) = B >>> i +""" + rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector + +Performs a left rotation operation on `src` and put the result into `dest`. + +```jldoctest +julia> a = bitrand(5) +5-element BitArray{1}: + false + false + true + true + true + +julia> b = falses(5); + +julia> rol!(b,a,2) +5-element BitArray{1}: + true + true + true + false + false + +julia> rol!(b,a,3) +5-element BitArray{1}: + true + true + false + false + true +``` +""" function rol!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) n = length(dest) @@ -1428,14 +1480,67 @@ function rol!(dest::BitVector, src::BitVector, i::Integer) return dest end +""" + rol!(B::BitVector, i::Integer) -> BitVector + +Performs a left rotation operation on `B`. + +```jldoctest +julia> a = bitrand(5) +5-element BitArray{1}: + true + false + false + true + true + +julia> rol!(a,4) +5-element BitArray{1}: + true + true + false + false + true +""" function rol!(B::BitVector, i::Integer) return rol!(B, B, i) end +""" + rol(B::BitVector, i::Integer) -> BitVector + +Performs a left rotation operation without modifying `B`. +See also [`rol!`](:func:`rol!`). +""" function rol(B::BitVector, i::Integer) return rol!(similar(B), B, i) end +""" + ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation on `src` and put the result into `dest`. + +```jldoctest +julia> a = bitrand(5) +5-element BitArray{1}: + false + false + true + true + true + +julia> b = falses(5); + +julia> ror!(b,a,2) +5-element BitArray{1}: + true + true + false + false + true +``` +""" function ror!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) n = length(dest) @@ -1448,10 +1553,39 @@ function ror!(dest::BitVector, src::BitVector, i::Integer) return dest end +""" + ror!(B::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation on `B`. + +```jldoctest +julia> a = bitrand(5) +5-element BitArray{1}: + false + true + false + true + false + +julia> ror!(a,3) +5-element BitArray{1}: + false + true + false + false + true +``` +""" function ror!(B::BitVector, i::Integer) return ror!(B, B, i) end +""" + ror(B::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation without modifying `B`. +See also [`ror!`](:func:`ror!`). +""" function ror(B::BitVector, i::Integer) return ror!(similar(B), B, i) end diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 9a08432feec8e..68fb612f5bac0 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -400,20 +400,6 @@ Element-wise multiplication operator. """ Base.:(.*) -""" - ror!(dest::BitArray{1}, src::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a right rotation operation on `src` and put the result into `dest`. -""" -ror!(dest::BitArray{1}, src::BitArray{1}, i::Integer) - -""" - ror!(B::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a right rotation operation on `B`. -""" -ror!(B::BitArray{1}, i::Integer) - """ range(start, [step], length) @@ -805,13 +791,6 @@ Returns `true` if `path` has the sticky bit set, `false` otherwise. """ issticky -""" - rol(B::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a left rotation operation. -""" -rol - """ Mmap.mmap(io::Union{IOStream,AbstractString,Mmap.AnonymousMmap}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) Mmap.mmap(type::Type{Array{T,N}}, dims) @@ -4330,13 +4309,6 @@ Returns `string` with the first character converted to lowercase. """ lcfirst -""" - flipbits!(B::BitArray{N}) -> BitArray{N} - -Performs a bitwise not operation on `B`. See [`~`](:ref:`~ operator <~>`). -""" -flipbits! - """ readlink(path) -> AbstractString @@ -7465,20 +7437,6 @@ dims)` will return an array filled with the result of evaluating `Foo()` once. """ fill -""" - rol!(dest::BitArray{1}, src::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a left rotation operation on `src` and put the result into `dest`. -""" -rol!(::BitArray,::BitArray,::Integer) - -""" - rol!(B::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a left rotation operation on `B`. -""" -rol!(::BitArray,::Integer) - """ issubset(a, b) ⊆(a,b) -> Bool @@ -8280,13 +8238,6 @@ Optional argument `msg` is a descriptive error string. """ AssertionError -""" - ror(B::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a right rotation operation. -""" -ror - """ Ac_ldiv_Bc(A, B) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 821e04d4d18da..fa25fa2e64ac4 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -40,8 +40,8 @@ countnz(S::SparseMatrixCSC) = countnz(S.nzval) Return a vector of the structural nonzero values in sparse array `A`. This includes zeros that are explicitly stored in the sparse array. The returned vector points directly to the internal nonzero storage of `A`, and any -modifications to the returned vector will mutate `A` as well. See `rowvals(A)` -and `nzrange(A, col)`. +modifications to the returned vector will mutate `A` as well. See +[`rowvals`](:func:`rowvals`) and [`nzrange`](:func:`nzrange`). """ nonzeros(S::SparseMatrixCSC) = S.nzval @@ -51,7 +51,7 @@ nonzeros(S::SparseMatrixCSC) = S.nzval Return a vector of the row indices of `A`. Any modifications to the returned vector will mutate `A` as well. Providing access to how the row indices are stored internally can be useful in conjunction with iterating over structural -nonzero values. See also `nonzeros(A)` and `nzrange(A, col)`. +nonzero values. See also [`nonzeros`](:func:`nonzeros`) and [`nzrange`](:func:`nzrange`). """ rowvals(S::SparseMatrixCSC) = S.rowval @@ -59,8 +59,8 @@ rowvals(S::SparseMatrixCSC) = S.rowval nzrange(A::SparseMatrixCSC, col) Return the range of indices to the structural nonzero values of a sparse matrix -column. In conjunction with `nonzeros(A)` and `rowvals(A)`, this allows for -convenient iterating over a sparse matrix : +column. In conjunction with [`nonzeros`](:func:`nonzeros`) and +[`rowvals`](:func:`rowvals`), this allows for convenient iterating over a sparse matrix : A = sparse(I,J,V) rows = rowvals(A) @@ -374,7 +374,7 @@ are set to `maximum(I)` and `maximum(J)` respectively. If the `combine` function supplied, `combine` defaults to `+` unless the elements of `V` are Booleans in which case `combine` defaults to `|`. All elements of `I` must satisfy `1 <= I[k] <= m`, and all elements of `J` must satisfy `1 <= J[k] <= n`. Numerical zeros in (`I`, `J`, `V`) are -retained as structural nonzeros; to drop numerical zeros, use `dropzeros!`. +retained as structural nonzeros; to drop numerical zeros, use [`dropzeros!`](:func:`dropzeros!`). For additional documentation and an expert driver, see `Base.SparseArrays.sparse!`. """ @@ -1044,7 +1044,7 @@ droptol!(A::SparseMatrixCSC, tol, trim::Bool = true) = Removes stored numerical zeros from `A`, optionally trimming resulting excess space from `A.rowval` and `A.nzval` when `trim` is `true`. -For an out-of-place version, see [`dropzeros`](:func:`Base.SparseArrays.dropzeros`). For +For an out-of-place version, see [`dropzeros`](:func:`dropzeros`). For algorithmic information, see [`Base.SparseArrays.fkeep!`](:func:`Base.SparseArrays.fkeep!`). """ dropzeros!(A::SparseMatrixCSC, trim::Bool = true) = fkeep!(A, (i, j, x) -> x != 0, trim) @@ -1054,7 +1054,7 @@ dropzeros!(A::SparseMatrixCSC, trim::Bool = true) = fkeep!(A, (i, j, x) -> x != Generates a copy of `A` and removes stored numerical zeros from that copy, optionally trimming excess space from the result's `rowval` and `nzval` arrays when `trim` is `true`. -For an in-place version and algorithmic information, see [`dropzeros!`](:func:`Base.SparseArrays.dropzeros!`). +For an in-place version and algorithmic information, see [`dropzeros!`](:func:`dropzeros!`). """ dropzeros(A::SparseMatrixCSC, trim::Bool = true) = dropzeros!(copy(A), trim) @@ -1212,6 +1212,32 @@ sprandn(m::Integer, n::Integer, density::AbstractFloat) = sprandn(GLOBAL_RNG,m,n Create a sparse array with the same structure as that of `S`, but with every nonzero element having the value `1.0`. + +```jldoctest +julia> A = sprand(5,6,0.2) +5×6 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 0.639431 + [5, 1] = 0.881209 + [3, 2] = 0.355834 + [4, 2] = 0.904768 + [2, 3] = 0.760943 + [3, 5] = 0.525942 + [4, 5] = 0.936283 + [5, 5] = 0.432364 + +julia> spones(A) +5×6 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 1.0 + [5, 1] = 1.0 + [3, 2] = 1.0 + [4, 2] = 1.0 + [2, 3] = 1.0 + [3, 5] = 1.0 + [4, 5] = 1.0 + [5, 5] = 1.0 +``` + +Note the difference from [`speye`](:func:`speye`). """ spones{T}(S::SparseMatrixCSC{T}) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), ones(T, S.colptr[end]-1)) @@ -1240,7 +1266,31 @@ speye(m::Integer, n::Integer) = speye(Float64, m, n) """ speye(S) -Create a sparse identity matrix with the same structure as that of `S`. +Create a sparse identity matrix with the same structure as that of `S`. + +```jldoctest +julia> A = sprand(5,6,0.2) +5×6 sparse matrix with 9 Float64 nonzero entries: + [1, 1] = 0.102874 + [2, 1] = 0.780098 + [1, 2] = 0.610378 + [1, 3] = 0.422308 + [3, 3] = 0.546398 + [4, 3] = 0.43053 + [5, 3] = 0.909283 + [2, 4] = 0.391321 + [5, 6] = 0.97785 + +julia> speye(A) +5×6 sparse matrix with 5 Float64 nonzero entries: + [1, 1] = 1.0 + [2, 2] = 1.0 + [3, 3] = 1.0 + [4, 4] = 1.0 + [5, 5] = 1.0 +``` + +Note the difference from [`spones`](:func:`spones`). """ speye{T}(S::SparseMatrixCSC{T}) = speye(T, size(S, 1), size(S, 2)) eye(S::SparseMatrixCSC) = speye(S) @@ -3461,6 +3511,19 @@ Construct a sparse diagonal matrix. `B` is a tuple of vectors containing the dia one diagonal, `B` can be a vector (instead of a tuple) and `d` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, `m` and `n` specify the size of the resulting sparse matrix. + +```jldoctest +julia> spdiagm((rand(4), rand(4)), (-1, 1)) +5×5 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 0.962245 + [1, 2] = 0.919341 + [3, 2] = 0.59239 + [2, 3] = 0.628924 + [4, 3] = 0.694011 + [3, 4] = 0.0660923 + [5, 4] = 0.494409 + [4, 5] = 0.54209 +``` """ function spdiagm(B, d, m::Integer, n::Integer) (I,J,V) = spdiagm_internal(B, d) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index f45ff006269a5..91ed246b883c5 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1198,41 +1198,125 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively Performs a bitwise not operation on ``B``\ . See :ref:`~ operator <~>`\ . -.. function:: rol!(dest::BitArray{1}, src::BitArray{1}, i::Integer) -> BitArray{1} + .. doctest:: + + julia> A = bitrand(3,5) + 3×5 BitArray{2}: + true true true true true + false false false true false + true false false false true + + julia> flipbits!(A) + 3×5 BitArray{2}: + false false false false false + true true true false true + false true true true false + +.. function:: rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source Performs a left rotation operation on ``src`` and put the result into ``dest``\ . -.. function:: rol!(B::BitArray{1}, i::Integer) -> BitArray{1} + .. doctest:: + + julia> a = bitrand(5) + 5-element BitArray{1}: + false + false + true + true + true + + julia> b = falses(5); + + julia> rol!(b,a,2) + 5-element BitArray{1}: + true + true + true + false + false + + julia> rol!(b,a,3) + 5-element BitArray{1}: + true + true + false + false + true + +.. function:: rol!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source Performs a left rotation operation on ``B``\ . -.. function:: rol(B::BitArray{1}, i::Integer) -> BitArray{1} + ```jldoctest julia> a = bitrand(5) 5-element BitArray{1}: true false false true true + + julia> rol!(a,4) 5-element BitArray{1}: true true false false true + +.. function:: rol(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation. + Performs a left rotation operation without modifying ``B``\ . See also :func:`rol!`\ . -.. function:: ror!(dest::BitArray{1}, src::BitArray{1}, i::Integer) -> BitArray{1} +.. function:: ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source Performs a right rotation operation on ``src`` and put the result into ``dest``\ . -.. function:: ror!(B::BitArray{1}, i::Integer) -> BitArray{1} + .. doctest:: + + julia> a = bitrand(5) + 5-element BitArray{1}: + false + false + true + true + true + + julia> b = falses(5); + + julia> ror!(b,a,2) + 5-element BitArray{1}: + true + true + false + false + true + +.. function:: ror!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source Performs a right rotation operation on ``B``\ . -.. function:: ror(B::BitArray{1}, i::Integer) -> BitArray{1} + .. doctest:: + + julia> a = bitrand(5) + 5-element BitArray{1}: + false + true + false + true + false + + julia> ror!(a,3) + 5-element BitArray{1}: + false + true + false + false + true + +.. function:: ror(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a right rotation operation. + Performs a right rotation operation without modifying ``B``\ . See also :func:`ror!`\ . .. _stdlib-sparse: @@ -1246,7 +1330,7 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Create a sparse matrix ``S`` of dimensions ``m x n`` such that ``S[I[k], J[k]] = V[k]``\ . The ``combine`` function is used to combine duplicates. If ``m`` and ``n`` are not specified, they are set to ``maximum(I)`` and ``maximum(J)`` respectively. If the ``combine`` function is not supplied, ``combine`` defaults to ``+`` unless the elements of ``V`` are Booleans in which case ``combine`` defaults to ``|``\ . All elements of ``I`` must satisfy ``1 <= I[k] <= m``\ , and all elements of ``J`` must satisfy ``1 <= J[k] <= n``\ . Numerical zeros in (``I``\ , ``J``\ , ``V``\ ) are retained as structural nonzeros; to drop numerical zeros, use ``dropzeros!``\ . + Create a sparse matrix ``S`` of dimensions ``m x n`` such that ``S[I[k], J[k]] = V[k]``\ . The ``combine`` function is used to combine duplicates. If ``m`` and ``n`` are not specified, they are set to ``maximum(I)`` and ``maximum(J)`` respectively. If the ``combine`` function is not supplied, ``combine`` defaults to ``+`` unless the elements of ``V`` are Booleans in which case ``combine`` defaults to ``|``\ . All elements of ``I`` must satisfy ``1 <= I[k] <= m``\ , and all elements of ``J`` must satisfy ``1 <= J[k] <= n``\ . Numerical zeros in (``I``\ , ``J``\ , ``V``\ ) are retained as structural nonzeros; to drop numerical zeros, use :func:`dropzeros!`\ . For additional documentation and an expert driver, see ``Base.SparseArrays.sparse!``\ . @@ -1304,6 +1388,32 @@ dense counterparts. The following functions are specific to sparse arrays. Create a sparse array with the same structure as that of ``S``\ , but with every nonzero element having the value ``1.0``\ . + .. doctest:: + + julia> A = sprand(5,6,0.2) + 5×6 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 0.639431 + [5, 1] = 0.881209 + [3, 2] = 0.355834 + [4, 2] = 0.904768 + [2, 3] = 0.760943 + [3, 5] = 0.525942 + [4, 5] = 0.936283 + [5, 5] = 0.432364 + + julia> spones(A) + 5×6 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 1.0 + [5, 1] = 1.0 + [3, 2] = 1.0 + [4, 2] = 1.0 + [2, 3] = 1.0 + [3, 5] = 1.0 + [4, 5] = 1.0 + [5, 5] = 1.0 + + Note the difference from :func:`speye`\ . + .. function:: speye([type,]m[,n]) .. Docstring generated from Julia source @@ -1314,7 +1424,31 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Create a sparse identity matrix with the same structure as that of ``S``\ . + Create a sparse identity matrix with the same structure as that of ``S``\ . + + .. doctest:: + + julia> A = sprand(5,6,0.2) + 5×6 sparse matrix with 9 Float64 nonzero entries: + [1, 1] = 0.102874 + [2, 1] = 0.780098 + [1, 2] = 0.610378 + [1, 3] = 0.422308 + [3, 3] = 0.546398 + [4, 3] = 0.43053 + [5, 3] = 0.909283 + [2, 4] = 0.391321 + [5, 6] = 0.97785 + + julia> speye(A) + 5×6 sparse matrix with 5 Float64 nonzero entries: + [1, 1] = 1.0 + [2, 2] = 1.0 + [3, 3] = 1.0 + [4, 4] = 1.0 + [5, 5] = 1.0 + + Note the difference from :func:`spones`\ . .. function:: spdiagm(B, d[, m, n]) @@ -1322,6 +1456,19 @@ dense counterparts. The following functions are specific to sparse arrays. Construct a sparse diagonal matrix. ``B`` is a tuple of vectors containing the diagonals and ``d`` is a tuple containing the positions of the diagonals. In the case the input contains only one diagonal, ``B`` can be a vector (instead of a tuple) and ``d`` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, ``m`` and ``n`` specify the size of the resulting sparse matrix. + .. doctest:: + + julia> spdiagm((rand(4), rand(4)), (-1, 1)) + 5×5 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 0.962245 + [1, 2] = 0.919341 + [3, 2] = 0.59239 + [2, 3] = 0.628924 + [4, 3] = 0.694011 + [3, 4] = 0.0660923 + [5, 4] = 0.494409 + [4, 5] = 0.54209 + .. function:: sprand([rng],[type],m,[n],p::AbstractFloat,[rfn]) .. Docstring generated from Julia source @@ -1338,19 +1485,19 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Return a vector of the structural nonzero values in sparse array ``A``\ . This includes zeros that are explicitly stored in the sparse array. The returned vector points directly to the internal nonzero storage of ``A``\ , and any modifications to the returned vector will mutate ``A`` as well. See ``rowvals(A)`` and ``nzrange(A, col)``\ . + Return a vector of the structural nonzero values in sparse array ``A``\ . This includes zeros that are explicitly stored in the sparse array. The returned vector points directly to the internal nonzero storage of ``A``\ , and any modifications to the returned vector will mutate ``A`` as well. See :func:`rowvals` and :func:`nzrange`\ . .. function:: rowvals(A::SparseMatrixCSC) .. Docstring generated from Julia source - Return a vector of the row indices of ``A``\ . Any modifications to the returned vector will mutate ``A`` as well. Providing access to how the row indices are stored internally can be useful in conjunction with iterating over structural nonzero values. See also ``nonzeros(A)`` and ``nzrange(A, col)``\ . + Return a vector of the row indices of ``A``\ . Any modifications to the returned vector will mutate ``A`` as well. Providing access to how the row indices are stored internally can be useful in conjunction with iterating over structural nonzero values. See also :func:`nonzeros` and :func:`nzrange`\ . .. function:: nzrange(A::SparseMatrixCSC, col) .. Docstring generated from Julia source - Return the range of indices to the structural nonzero values of a sparse matrix column. In conjunction with ``nonzeros(A)`` and ``rowvals(A)``\ , this allows for convenient iterating over a sparse matrix : + Return the range of indices to the structural nonzero values of a sparse matrix column. In conjunction with :func:`nonzeros` and :func:`rowvals`\ , this allows for convenient iterating over a sparse matrix : .. code-block:: julia @@ -1372,7 +1519,7 @@ dense counterparts. The following functions are specific to sparse arrays. Removes stored numerical zeros from ``A``\ , optionally trimming resulting excess space from ``A.rowval`` and ``A.nzval`` when ``trim`` is ``true``\ . - For an out-of-place version, see :func:`Base.SparseArrays.dropzeros`\ . For algorithmic information, see :func:`Base.SparseArrays.fkeep!`\ . + For an out-of-place version, see :func:`dropzeros`\ . For algorithmic information, see :func:`Base.SparseArrays.fkeep!`\ . .. function:: dropzeros(A::SparseMatrixCSC, trim::Bool = true) @@ -1380,7 +1527,7 @@ dense counterparts. The following functions are specific to sparse arrays. Generates a copy of ``A`` and removes stored numerical zeros from that copy, optionally trimming excess space from the result's ``rowval`` and ``nzval`` arrays when ``trim`` is ``true``\ . - For an in-place version and algorithmic information, see :func:`Base.SparseArrays.dropzeros!`\ . + For an in-place version and algorithmic information, see :func:`dropzeros!`\ . .. function:: dropzeros!(x::SparseVector, trim::Bool = true) From 82099277a7ec9b51beb5da23fbc5116d832ac196 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Thu, 4 Aug 2016 09:27:23 -0700 Subject: [PATCH 0810/1117] Added examples for ones, eye. Fixed up BitArray docs. Removed calls to `rand` in the doctests. --- base/array.jl | 63 ++++++++++++ base/bitarray.jl | 135 ++++++++++++------------ base/docs/helpdb/Base.jl | 35 ------- base/sparse/sparsematrix.jl | 72 ++++++------- doc/stdlib/arrays.rst | 198 +++++++++++++++++++----------------- 5 files changed, 262 insertions(+), 241 deletions(-) diff --git a/base/array.jl b/base/array.jl index 783b93e0e5956..3a4ac020af589 100644 --- a/base/array.jl +++ b/base/array.jl @@ -171,7 +171,42 @@ for (fname, felt) in ((:zeros,:zero), (:ones,:one)) ($fname){T}(A::AbstractArray{T}) = fill!(similar(A), ($felt)(T)) end end +""" + ones([T::Type=Float64,] dims) + +Create an array of all ones of specified type. The type defaults to `Float64` if not specified. +""" +function ones(dims::Dims) end + +""" + ones(A) + +Create an array of all ones with the same element type and shape as `A`. + +```jldoctest +julia> A = [1 2 3; 4 5 6; 7 8 9] +3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + +julia> ones(A) +3×3 Array{Int64,2}: + 1 1 1 + 1 1 1 + 1 1 1 +``` + +Note the difference from [`eye`](:func:`eye`). +""" +function ones(A) end + +""" + eye([T::Type=Float64,] m::Integer, n::Integer) +`m`-by-`n` identity matrix. +The default element type is `Float64`. +""" function eye(T::Type, m::Integer, n::Integer) a = zeros(T,m,n) for i = 1:min(m,n) @@ -181,7 +216,35 @@ function eye(T::Type, m::Integer, n::Integer) end eye(m::Integer, n::Integer) = eye(Float64, m, n) eye(T::Type, n::Integer) = eye(T, n, n) +""" + eye([T::Type=Float64,] n::Integer) + +`n`-by-`n` identity matrix. +The default element type is `Float64`. +""" eye(n::Integer) = eye(Float64, n) + +""" + eye(A) + +Constructs an identity matrix of the same dimensions and type as `A`. + +```jldoctest +julia> A = [1 2 3; 4 5 6; 7 8 9] +3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + +julia> eye(A) +3×3 Array{Int64,2}: + 1 0 0 + 0 1 0 + 0 0 1 +``` + +Note the difference from [`ones`](:func:`ones`). +""" eye{T}(x::AbstractMatrix{T}) = eye(T, size(x, 1), size(x, 2)) function one{T}(x::AbstractMatrix{T}) diff --git a/base/bitarray.jl b/base/bitarray.jl index 8ea68191bb8a0..c134ae9f8c3f3 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1438,35 +1438,8 @@ end """ rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector -Performs a left rotation operation on `src` and put the result into `dest`. - -```jldoctest -julia> a = bitrand(5) -5-element BitArray{1}: - false - false - true - true - true - -julia> b = falses(5); - -julia> rol!(b,a,2) -5-element BitArray{1}: - true - true - true - false - false - -julia> rol!(b,a,3) -5-element BitArray{1}: - true - true - false - false - true -``` +Performs a left rotation operation on `src` and puts the result into `dest`. +`i` controls how far to rotate each bit. """ function rol!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) @@ -1483,46 +1456,38 @@ end """ rol!(B::BitVector, i::Integer) -> BitVector -Performs a left rotation operation on `B`. +Performs a left rotation operation in-place on `B`. +`i` controls how far to rotate each bit. +""" +function rol!(B::BitVector, i::Integer) + return rol!(B, B, i) +end + +""" + rol(B::BitVector, i::Integer) -> BitVector + +Performs a left rotation operation, returning a new `BitVector`. +`i` controls how far to rotate each bit. +See also [`rol!`](:func:`rol!`). ```jldoctest -julia> a = bitrand(5) +julia> A = BitArray([true, true, false, false, true]) 5-element BitArray{1}: true + true false false true - true -julia> rol!(a,4) +julia> rol(A,1) 5-element BitArray{1}: true - true false false true -""" -function rol!(B::BitVector, i::Integer) - return rol!(B, B, i) -end - -""" - rol(B::BitVector, i::Integer) -> BitVector - -Performs a left rotation operation without modifying `B`. -See also [`rol!`](:func:`rol!`). -""" -function rol(B::BitVector, i::Integer) - return rol!(similar(B), B, i) -end - -""" - ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector - -Performs a right rotation operation on `src` and put the result into `dest`. + true -```jldoctest -julia> a = bitrand(5) +julia> rol(A,2) 5-element BitArray{1}: false false @@ -1530,9 +1495,7 @@ julia> a = bitrand(5) true true -julia> b = falses(5); - -julia> ror!(b,a,2) +julia> rol(A,5) 5-element BitArray{1}: true true @@ -1541,6 +1504,16 @@ julia> ror!(b,a,2) true ``` """ +function rol(B::BitVector, i::Integer) + return rol!(similar(B), B, i) +end + +""" + ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation on `src` and puts the result into `dest`. +`i` controls how far to rotate each bit. +""" function ror!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) n = length(dest) @@ -1556,36 +1529,54 @@ end """ ror!(B::BitVector, i::Integer) -> BitVector -Performs a right rotation operation on `B`. +Performs a right rotation operation in-place on `B`. +`i` controls how far to rotate each bit. +""" +function ror!(B::BitVector, i::Integer) + return ror!(B, B, i) +end + +""" + ror(B::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation on `B`, returning a new `BitVector`. +`i` controls how far to rotate each bit. +See also [`ror!`](:func:`ror!`). ```jldoctest -julia> a = bitrand(5) +julia> A = BitArray([true, true, false, false, true]) 5-element BitArray{1}: - false true + true + false false true + +julia> ror(A,1) +5-element BitArray{1}: + true + true + true + false false -julia> ror!(a,3) +julia> ror(A,2) 5-element BitArray{1}: false true + true + true + false + +julia> ror(A,5) +5-element BitArray{1}: + true + true false false true ``` """ -function ror!(B::BitVector, i::Integer) - return ror!(B, B, i) -end - -""" - ror(B::BitVector, i::Integer) -> BitVector - -Performs a right rotation operation without modifying `B`. -See also [`ror!`](:func:`ror!`). -""" function ror(B::BitVector, i::Integer) return ror!(similar(B), B, i) end diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 68fb612f5bac0..a6be74723d897 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1225,20 +1225,6 @@ of the problem that is solved on each processor. """ peakflops -""" - ones(type, dims) - -Create an array of all ones of specified type. The type defaults to `Float64` if not specified. -""" -ones(t,dims) - -""" - ones(A) - -Create an array of all ones with the same element type and shape as `A`. -""" -ones(A) - """ ind2chr(string, i) @@ -4068,27 +4054,6 @@ Euler integral of the first kind ``\\operatorname{B}(x,y) = \\Gamma(x)\\Gamma(y) """ beta -""" - eye(n) - -`n`-by-`n` identity matrix. -""" -eye(n::Int) - -""" - eye(m, n) - -`m`-by-`n` identity matrix. -""" -eye(m, n) - -""" - eye(A) - -Constructs an identity matrix of the same dimensions and type as `A`. -""" -eye(A) - """ diagind(M[, k]) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index fa25fa2e64ac4..b943dc5dee93d 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1214,27 +1214,19 @@ Create a sparse array with the same structure as that of `S`, but with every non element having the value `1.0`. ```jldoctest -julia> A = sprand(5,6,0.2) -5×6 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 0.639431 - [5, 1] = 0.881209 - [3, 2] = 0.355834 - [4, 2] = 0.904768 - [2, 3] = 0.760943 - [3, 5] = 0.525942 - [4, 5] = 0.936283 - [5, 5] = 0.432364 +julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) +4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> spones(A) -5×6 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 1.0 - [5, 1] = 1.0 - [3, 2] = 1.0 - [4, 2] = 1.0 - [2, 3] = 1.0 - [3, 5] = 1.0 - [4, 5] = 1.0 - [5, 5] = 1.0 +4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 1.0 + [1, 2] = 1.0 + [3, 3] = 1.0 + [2, 4] = 1.0 ``` Note the difference from [`speye`](:func:`speye`). @@ -1266,28 +1258,22 @@ speye(m::Integer, n::Integer) = speye(Float64, m, n) """ speye(S) -Create a sparse identity matrix with the same structure as that of `S`. +Create a sparse identity matrix with the same size as that of `S`. ```jldoctest -julia> A = sprand(5,6,0.2) -5×6 sparse matrix with 9 Float64 nonzero entries: - [1, 1] = 0.102874 - [2, 1] = 0.780098 - [1, 2] = 0.610378 - [1, 3] = 0.422308 - [3, 3] = 0.546398 - [4, 3] = 0.43053 - [5, 3] = 0.909283 - [2, 4] = 0.391321 - [5, 6] = 0.97785 +julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) +4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> speye(A) -5×6 sparse matrix with 5 Float64 nonzero entries: +4×4 sparse matrix with 4 Float64 nonzero entries: [1, 1] = 1.0 [2, 2] = 1.0 [3, 3] = 1.0 [4, 4] = 1.0 - [5, 5] = 1.0 ``` Note the difference from [`spones`](:func:`spones`). @@ -3513,16 +3499,16 @@ one diagonal, `B` can be a vector (instead of a tuple) and `d` can be the diagon of the resulting sparse matrix. ```jldoctest -julia> spdiagm((rand(4), rand(4)), (-1, 1)) -5×5 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 0.962245 - [1, 2] = 0.919341 - [3, 2] = 0.59239 - [2, 3] = 0.628924 - [4, 3] = 0.694011 - [3, 4] = 0.0660923 - [5, 4] = 0.494409 - [4, 5] = 0.54209 +julia> spdiagm(([1,2,3,4],[4,3,2,1]),(-1,1)) +5×5 sparse matrix with 8 Int64 nonzero entries: + [2, 1] = 1 + [1, 2] = 4 + [3, 2] = 2 + [2, 3] = 3 + [4, 3] = 3 + [3, 4] = 2 + [5, 4] = 4 + [4, 5] = 1 ``` """ function spdiagm(B, d, m::Integer, n::Integer) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 91ed246b883c5..b51ab8bae7118 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -187,7 +187,7 @@ Constructors Create an array of all zeros with the same element type and shape as ``A``\ . -.. function:: ones(type, dims) +.. function:: ones([T::Type=Float64,] dims) .. Docstring generated from Julia source @@ -199,6 +199,22 @@ Constructors Create an array of all ones with the same element type and shape as ``A``\ . + .. doctest:: + + julia> A = [1 2 3; 4 5 6; 7 8 9] + 3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + + julia> ones(A) + 3×3 Array{Int64,2}: + 1 1 1 + 1 1 1 + 1 1 1 + + Note the difference from :func:`eye`\ . + .. function:: trues(dims) .. Docstring generated from Julia source @@ -309,17 +325,17 @@ Constructors Change the type-interpretation of a block of memory. For example, ``reinterpret(Float32, UInt32(7))`` interprets the 4 bytes corresponding to ``UInt32(7)`` as a ``Float32``\ . For arrays, this constructs an array with the same binary data as the given array, but with the specified element type. -.. function:: eye(n) +.. function:: eye([T::Type=Float64,] n::Integer) .. Docstring generated from Julia source - ``n``\ -by-``n`` identity matrix. + ``n``\ -by-``n`` identity matrix. The default element type is ``Float64``\ . -.. function:: eye(m, n) +.. function:: eye([T::Type=Float64,] m::Integer, n::Integer) .. Docstring generated from Julia source - ``m``\ -by-``n`` identity matrix. + ``m``\ -by-``n`` identity matrix. The default element type is ``Float64``\ . .. function:: eye(A) @@ -327,6 +343,22 @@ Constructors Constructs an identity matrix of the same dimensions and type as ``A``\ . + .. doctest:: + + julia> A = [1 2 3; 4 5 6; 7 8 9] + 3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + + julia> eye(A) + 3×3 Array{Int64,2}: + 1 0 0 + 0 1 0 + 0 0 1 + + Note the difference from :func:`ones`\ . + .. function:: linspace(start, stop, n=50) .. Docstring generated from Julia source @@ -1216,29 +1248,47 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively .. Docstring generated from Julia source - Performs a left rotation operation on ``src`` and put the result into ``dest``\ . + Performs a left rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate each bit. + +.. function:: rol!(B::BitVector, i::Integer) -> BitVector + + .. Docstring generated from Julia source + + Performs a left rotation operation in-place on ``B``\ . ``i`` controls how far to rotate each bit. + +.. function:: rol(B::BitVector, i::Integer) -> BitVector + + .. Docstring generated from Julia source + + Performs a left rotation operation, returning a new ``BitVector``\ . ``i`` controls how far to rotate each bit. See also :func:`rol!`\ . .. doctest:: - julia> a = bitrand(5) + julia> A = BitArray([true, true, false, false, true]) 5-element BitArray{1}: - false - false true true + false + false true - julia> b = falses(5); - - julia> rol!(b,a,2) + julia> rol(A,1) 5-element BitArray{1}: true + false + false true true + + julia> rol(A,2) + 5-element BitArray{1}: false false + true + true + true - julia> rol!(b,a,3) + julia> rol(A,5) 5-element BitArray{1}: true true @@ -1246,78 +1296,58 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively false true -.. function:: rol!(B::BitVector, i::Integer) -> BitVector +.. function:: ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation on ``B``\ . - - ```jldoctest julia> a = bitrand(5) 5-element BitArray{1}: true false false true true + Performs a right rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate each bit. - julia> rol!(a,4) 5-element BitArray{1}: true true false false true - -.. function:: rol(B::BitVector, i::Integer) -> BitVector +.. function:: ror!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation without modifying ``B``\ . See also :func:`rol!`\ . + Performs a right rotation operation in-place on ``B``\ . ``i`` controls how far to rotate each bit. -.. function:: ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector +.. function:: ror(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a right rotation operation on ``src`` and put the result into ``dest``\ . + Performs a right rotation operation on ``B``\ , returning a new ``BitVector``\ . ``i`` controls how far to rotate each bit. See also :func:`ror!`\ . .. doctest:: - julia> a = bitrand(5) + julia> A = BitArray([true, true, false, false, true]) 5-element BitArray{1}: - false - false true true + false + false true - julia> b = falses(5); - - julia> ror!(b,a,2) + julia> ror(A,1) 5-element BitArray{1}: true true + true false false - true -.. function:: ror!(B::BitVector, i::Integer) -> BitVector - - .. Docstring generated from Julia source - - Performs a right rotation operation on ``B``\ . - - .. doctest:: - - julia> a = bitrand(5) + julia> ror(A,2) 5-element BitArray{1}: false true - false + true true false - julia> ror!(a,3) + julia> ror(A,5) 5-element BitArray{1}: - false + true true false false true -.. function:: ror(B::BitVector, i::Integer) -> BitVector - - .. Docstring generated from Julia source - - Performs a right rotation operation without modifying ``B``\ . See also :func:`ror!`\ . - .. _stdlib-sparse: Sparse Vectors and Matrices @@ -1390,27 +1420,19 @@ dense counterparts. The following functions are specific to sparse arrays. .. doctest:: - julia> A = sprand(5,6,0.2) - 5×6 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 0.639431 - [5, 1] = 0.881209 - [3, 2] = 0.355834 - [4, 2] = 0.904768 - [2, 3] = 0.760943 - [3, 5] = 0.525942 - [4, 5] = 0.936283 - [5, 5] = 0.432364 + julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) + 4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> spones(A) - 5×6 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 1.0 - [5, 1] = 1.0 - [3, 2] = 1.0 - [4, 2] = 1.0 - [2, 3] = 1.0 - [3, 5] = 1.0 - [4, 5] = 1.0 - [5, 5] = 1.0 + 4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 1.0 + [1, 2] = 1.0 + [3, 3] = 1.0 + [2, 4] = 1.0 Note the difference from :func:`speye`\ . @@ -1424,29 +1446,23 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Create a sparse identity matrix with the same structure as that of ``S``\ . + Create a sparse identity matrix with the same size as that of ``S``\ . .. doctest:: - julia> A = sprand(5,6,0.2) - 5×6 sparse matrix with 9 Float64 nonzero entries: - [1, 1] = 0.102874 - [2, 1] = 0.780098 - [1, 2] = 0.610378 - [1, 3] = 0.422308 - [3, 3] = 0.546398 - [4, 3] = 0.43053 - [5, 3] = 0.909283 - [2, 4] = 0.391321 - [5, 6] = 0.97785 + julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) + 4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> speye(A) - 5×6 sparse matrix with 5 Float64 nonzero entries: + 4×4 sparse matrix with 4 Float64 nonzero entries: [1, 1] = 1.0 [2, 2] = 1.0 [3, 3] = 1.0 [4, 4] = 1.0 - [5, 5] = 1.0 Note the difference from :func:`spones`\ . @@ -1458,16 +1474,16 @@ dense counterparts. The following functions are specific to sparse arrays. .. doctest:: - julia> spdiagm((rand(4), rand(4)), (-1, 1)) - 5×5 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 0.962245 - [1, 2] = 0.919341 - [3, 2] = 0.59239 - [2, 3] = 0.628924 - [4, 3] = 0.694011 - [3, 4] = 0.0660923 - [5, 4] = 0.494409 - [4, 5] = 0.54209 + julia> spdiagm(([1,2,3,4],[4,3,2,1]),(-1,1)) + 5×5 sparse matrix with 8 Int64 nonzero entries: + [2, 1] = 1 + [1, 2] = 4 + [3, 2] = 2 + [2, 3] = 3 + [4, 3] = 3 + [3, 4] = 2 + [5, 4] = 4 + [4, 5] = 1 .. function:: sprand([rng],[type],m,[n],p::AbstractFloat,[rfn]) From 8b71132dc4fb6c825c8095bc05929f1b2ff5da04 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Thu, 4 Aug 2016 16:44:34 -0700 Subject: [PATCH 0811/1117] Doctests passed, gave up on ones --- base/array.jl | 29 ------------- base/bitarray.jl | 16 ++++---- base/docs/helpdb/Base.jl | 28 +++++++++++++ base/sparse/sparsematrix.jl | 48 +++++++++++----------- doc/stdlib/arrays.rst | 82 +++++++++++++++---------------------- 5 files changed, 91 insertions(+), 112 deletions(-) diff --git a/base/array.jl b/base/array.jl index 3a4ac020af589..8c753b3e3924d 100644 --- a/base/array.jl +++ b/base/array.jl @@ -171,35 +171,6 @@ for (fname, felt) in ((:zeros,:zero), (:ones,:one)) ($fname){T}(A::AbstractArray{T}) = fill!(similar(A), ($felt)(T)) end end -""" - ones([T::Type=Float64,] dims) - -Create an array of all ones of specified type. The type defaults to `Float64` if not specified. -""" -function ones(dims::Dims) end - -""" - ones(A) - -Create an array of all ones with the same element type and shape as `A`. - -```jldoctest -julia> A = [1 2 3; 4 5 6; 7 8 9] -3×3 Array{Int64,2}: - 1 2 3 - 4 5 6 - 7 8 9 - -julia> ones(A) -3×3 Array{Int64,2}: - 1 1 1 - 1 1 1 - 1 1 1 -``` - -Note the difference from [`eye`](:func:`eye`). -""" -function ones(A) end """ eye([T::Type=Float64,] m::Integer, n::Integer) diff --git a/base/bitarray.jl b/base/bitarray.jl index c134ae9f8c3f3..b883b94aadc2c 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1011,17 +1011,15 @@ end Performs a bitwise not operation on `B`. See [`~`](:ref:`~ operator <~>`). ```jldoctest -julia> A = bitrand(3,5) -3×5 BitArray{2}: - true true true true true - false false false true false - true false false false true +julia> A = trues(2,2) +2×2 BitArray{2}: + true true + true true julia> flipbits!(A) -3×5 BitArray{2}: - false false false false false - true true true false true - false true true true false +2×2 BitArray{2}: + false false + false false ``` """ function flipbits!(B::BitArray) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index a6be74723d897..253d25d800c5f 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1225,6 +1225,20 @@ of the problem that is solved on each processor. """ peakflops +""" + ones(type, dims) + +Create an array of all ones of specified type. The type defaults to `Float64` if not specified. +""" +ones(t,dims) + +""" + ones(A) + +Create an array of all ones with the same element type and shape as `A`. +""" +ones(A) + """ ind2chr(string, i) @@ -4054,6 +4068,20 @@ Euler integral of the first kind ``\\operatorname{B}(x,y) = \\Gamma(x)\\Gamma(y) """ beta +""" + eye(n) + +`n`-by-`n` identity matrix. +""" +eye(n::Int) + +""" + eye(m, n) + +`m`-by-`n` identity matrix. +""" +eye(m, n) + """ diagind(M[, k]) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index b943dc5dee93d..0e3f2bf68c174 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1216,17 +1216,17 @@ element having the value `1.0`. ```jldoctest julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 2.0 - [1, 2] = 5.0 - [3, 3] = 3.0 - [2, 4] = 4.0 + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> spones(A) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 1.0 - [1, 2] = 1.0 - [3, 3] = 1.0 - [2, 4] = 1.0 + [4, 1] = 1.0 + [1, 2] = 1.0 + [3, 3] = 1.0 + [2, 4] = 1.0 ``` Note the difference from [`speye`](:func:`speye`). @@ -1263,17 +1263,17 @@ Create a sparse identity matrix with the same size as that of `S`. ```jldoctest julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 2.0 - [1, 2] = 5.0 - [3, 3] = 3.0 - [2, 4] = 4.0 + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> speye(A) 4×4 sparse matrix with 4 Float64 nonzero entries: - [1, 1] = 1.0 - [2, 2] = 1.0 - [3, 3] = 1.0 - [4, 4] = 1.0 + [1, 1] = 1.0 + [2, 2] = 1.0 + [3, 3] = 1.0 + [4, 4] = 1.0 ``` Note the difference from [`spones`](:func:`spones`). @@ -3501,14 +3501,14 @@ of the resulting sparse matrix. ```jldoctest julia> spdiagm(([1,2,3,4],[4,3,2,1]),(-1,1)) 5×5 sparse matrix with 8 Int64 nonzero entries: - [2, 1] = 1 - [1, 2] = 4 - [3, 2] = 2 - [2, 3] = 3 - [4, 3] = 3 - [3, 4] = 2 - [5, 4] = 4 - [4, 5] = 1 + [2, 1] = 1 + [1, 2] = 4 + [3, 2] = 2 + [2, 3] = 3 + [4, 3] = 3 + [3, 4] = 2 + [5, 4] = 4 + [4, 5] = 1 ``` """ function spdiagm(B, d, m::Integer, n::Integer) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index b51ab8bae7118..78cdb2e7e44ee 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -187,7 +187,7 @@ Constructors Create an array of all zeros with the same element type and shape as ``A``\ . -.. function:: ones([T::Type=Float64,] dims) +.. function:: ones(type, dims) .. Docstring generated from Julia source @@ -199,22 +199,6 @@ Constructors Create an array of all ones with the same element type and shape as ``A``\ . - .. doctest:: - - julia> A = [1 2 3; 4 5 6; 7 8 9] - 3×3 Array{Int64,2}: - 1 2 3 - 4 5 6 - 7 8 9 - - julia> ones(A) - 3×3 Array{Int64,2}: - 1 1 1 - 1 1 1 - 1 1 1 - - Note the difference from :func:`eye`\ . - .. function:: trues(dims) .. Docstring generated from Julia source @@ -1232,17 +1216,15 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively .. doctest:: - julia> A = bitrand(3,5) - 3×5 BitArray{2}: - true true true true true - false false false true false - true false false false true + julia> A = trues(2,2) + 2×2 BitArray{2}: + true true + true true julia> flipbits!(A) - 3×5 BitArray{2}: - false false false false false - true true true false true - false true true true false + 2×2 BitArray{2}: + false false + false false .. function:: rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector @@ -1422,17 +1404,17 @@ dense counterparts. The following functions are specific to sparse arrays. julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 2.0 - [1, 2] = 5.0 - [3, 3] = 3.0 - [2, 4] = 4.0 + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> spones(A) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 1.0 - [1, 2] = 1.0 - [3, 3] = 1.0 - [2, 4] = 1.0 + [4, 1] = 1.0 + [1, 2] = 1.0 + [3, 3] = 1.0 + [2, 4] = 1.0 Note the difference from :func:`speye`\ . @@ -1452,17 +1434,17 @@ dense counterparts. The following functions are specific to sparse arrays. julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 2.0 - [1, 2] = 5.0 - [3, 3] = 3.0 - [2, 4] = 4.0 + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> speye(A) 4×4 sparse matrix with 4 Float64 nonzero entries: - [1, 1] = 1.0 - [2, 2] = 1.0 - [3, 3] = 1.0 - [4, 4] = 1.0 + [1, 1] = 1.0 + [2, 2] = 1.0 + [3, 3] = 1.0 + [4, 4] = 1.0 Note the difference from :func:`spones`\ . @@ -1476,14 +1458,14 @@ dense counterparts. The following functions are specific to sparse arrays. julia> spdiagm(([1,2,3,4],[4,3,2,1]),(-1,1)) 5×5 sparse matrix with 8 Int64 nonzero entries: - [2, 1] = 1 - [1, 2] = 4 - [3, 2] = 2 - [2, 3] = 3 - [4, 3] = 3 - [3, 4] = 2 - [5, 4] = 4 - [4, 5] = 1 + [2, 1] = 1 + [1, 2] = 4 + [3, 2] = 2 + [2, 3] = 3 + [4, 3] = 3 + [3, 4] = 2 + [5, 4] = 4 + [4, 5] = 1 .. function:: sprand([rng],[type],m,[n],p::AbstractFloat,[rfn]) From 02f9ba10f2dd1f361093996294c348d8a5c0f53e Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Fri, 5 Aug 2016 08:46:02 -0700 Subject: [PATCH 0812/1117] Fixed line in helpdb for doctest --- base/docs/helpdb/Base.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 253d25d800c5f..916db9fc34049 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4579,7 +4579,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:541 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:575 ... ``` """ From 6c73070145fabce2398d156018964c51b53adc0c Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Fri, 5 Aug 2016 10:08:22 -0700 Subject: [PATCH 0813/1117] Move docs out of helpDB, make them less verbose. --- base/docs/helpdb/Base.jl | 144 --------------------------------------- base/special/gamma.jl | 40 +++++++++++ base/special/trig.jl | 36 ++++++++-- 3 files changed, 71 insertions(+), 149 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 916db9fc34049..7b45975e24423 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -22,13 +22,6 @@ tab-delimited text to `f` by either `writedlm(f, [x y])` or by `writedlm(f, zip( """ writedlm -""" - digamma(x) - -Compute the digamma function of `x` (the logarithmic derivative of `gamma(x)`) -""" -digamma - """ fill!(A, x) @@ -509,13 +502,6 @@ A string giving the literal bit representation of a number. """ bits -""" - invdigamma(x) - -Compute the inverse digamma function of `x`. -""" -invdigamma - """ getindex(type[, elements...]) @@ -595,13 +581,6 @@ used as the final delimiter instead of `delim`. """ join(io, items, delim, last) -""" - lfact(x) - -Compute the logarithmic factorial of `x` -""" -lfact - """ deconv(b,a) @@ -986,15 +965,6 @@ Test whether a matrix is lower triangular. """ istril -""" - lgamma(x) - -Compute the logarithm of the absolute value of [`gamma`](:func:`gamma`) for -[`Real`](:obj:`Real`) `x`, while for [`Complex`](:obj:`Complex`) `x` it computes the -logarithm of `gamma(x)`. -""" -lgamma - """ bin(n, [pad]) @@ -1424,13 +1394,6 @@ Close an I/O stream. Performs a `flush` first. """ close(stream::IO) -""" - cospi(x) - -Compute ``\\cos(\\pi x)`` more accurately than `cos(pi*x)`, especially for large `x`. -""" -cospi - """ parentindexes(A) @@ -2399,13 +2362,6 @@ Bessel function of the first kind of order 1, ``J_1(x)``. """ besselj1 -""" - sinpi(x) - -Compute ``\\sin(\\pi x)`` more accurately than `sin(pi*x)`, especially for large `x`. -""" -sinpi - """ select!(v, k, [by=<transform>,] [lt=<comparison>,] [rev=false]) @@ -2851,13 +2807,6 @@ Seek a stream to the given position. """ seek -""" - acosd(x) - -Compute the inverse cosine of `x`, where the output is in degrees. -""" -acosd - """ triu(M) @@ -2992,13 +2941,6 @@ Equivalent to `stat(file).size`. """ filesize -""" - sinc(x) - -Compute ``\\sin(\\pi x) / (\\pi x)`` if ``x \\neq 0``, and ``1`` if ``x = 0``. -""" -sinc - """ cglobal((symbol, library) [, type=Void]) @@ -3629,13 +3571,6 @@ Returns the smallest eigenvalue of `A`. """ eigmin -""" - acscd(x) - -Compute the inverse cosecant of `x`, where the output is in degrees. -""" -acscd - """ ltoh(x) @@ -4114,13 +4049,6 @@ is `-1` the corresponding ID will not change. Only integer `owner`s and `group`s """ chown -""" - gamma(x) - -Compute the gamma function of `x`. -""" -gamma - """ sin(x) @@ -4820,13 +4748,6 @@ For more information, see [^issue8859], [^B96], [^S84], [^KY88]. """ pinv -""" - asecd(x) - -Compute the inverse secant of `x`, where the output is in degrees. -""" -asecd - """ readbytes!(stream::IO, b::AbstractVector{UInt8}, nb=length(b); all=true) @@ -4853,13 +4774,6 @@ descriptive error string. """ ArgumentError -""" - atand(x) - -Compute the inverse tangent of `x`, where the output is in degrees. -""" -atand - """ KeyError(key) @@ -5027,13 +4941,6 @@ Test whether a matrix is Hermitian. """ ishermitian -""" - sind(x) - -Compute sine of `x`, where `x` is in degrees. -""" -sind - """ min(x, y, ...) @@ -5345,14 +5252,6 @@ two strings. For example """ join(strings, delim, last) -""" - polygamma(m, x) - -Compute the polygamma function of order `m` of argument `x` (the `(m+1)th` derivative of the -logarithm of `gamma(x)`) -""" -polygamma - """ isless(x, y) @@ -6020,13 +5919,6 @@ Returns `string` with all characters converted to uppercase. """ uppercase -""" - cosd(x) - -Compute cosine of `x`, where `x` is in degrees. -""" -cosd - """ cycle(iter) @@ -6465,13 +6357,6 @@ lengths of dimensions you asked for. """ size -""" - trigamma(x) - -Compute the trigamma function of `x` (the logarithmic second derivative of `gamma(x)`). -""" -trigamma - """ findmin(A, dims) -> (minval, index) @@ -7153,13 +7038,6 @@ but throws an error for unordered arguments. """ cmp -""" - tand(x) - -Compute tangent of `x`, where `x` is in degrees. -""" -tand - """ issorted(v, [by=<transform>,] [lt=<comparison>,] [rev=false]) @@ -7534,14 +7412,6 @@ Test whether any values along the given dimensions of an array are `true`. """ any(::AbstractArray,dims) -""" - cosc(x) - -Compute ``\\cos(\\pi x) / x - \\sin(\\pi x) / (\\pi x^2)`` if ``x \\neq 0``, and ``0`` if -``x = 0``. This is the derivative of `sinc(x)`. -""" -cosc - """ getkey(collection, key, default) @@ -7563,13 +7433,6 @@ For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ Bᴴ``. """ Ac_mul_Bc -""" - acotd(x) - -Compute the inverse cotangent of `x`, where the output is in degrees. -""" -acotd - """ zeros(type, dims) @@ -8257,13 +8120,6 @@ julia> f(apple) """ :@enum -""" - asind(x) - -Compute the inverse sine of `x`, where the output is in degrees. -""" -asind - """ widemul(x, y) diff --git a/base/special/gamma.jl b/base/special/gamma.jl index e1bbe630f1e33..083ad14c30fdb 100644 --- a/base/special/gamma.jl +++ b/base/special/gamma.jl @@ -2,6 +2,12 @@ gamma(x::Float64) = nan_dom_err(ccall((:tgamma,libm), Float64, (Float64,), x), x) gamma(x::Float32) = nan_dom_err(ccall((:tgammaf,libm), Float32, (Float32,), x), x) + +""" + gamma(x) + +Compute the gamma function of `x`. +""" gamma(x::Real) = gamma(float(x)) @vectorize_1arg Number gamma @@ -19,6 +25,11 @@ lgamma_r(x::Real) = lgamma_r(float(x)) lgamma_r(x::Number) = lgamma(x), 1 # lgamma does not take abs for non-real x "`lgamma_r(x)`: return L,s such that `gamma(x) = s * exp(L)`" lgamma_r +""" + lfact(x) + +Compute the logarithmic factorial of `x` +""" lfact(x::Real) = (x<=1 ? zero(float(x)) : lgamma(x+one(x))) @vectorize_1arg Number lfact @@ -47,6 +58,13 @@ function clgamma_lanczos(z) return log(zz) - temp end +""" + lgamma(x) + +Compute the logarithm of the absolute value of [`gamma`](:func:`gamma`) for +[`Real`](:obj:`Real`) `x`, while for [`Complex`](:obj:`Complex`) `x` it computes the +logarithm of `gamma(x)`. +""" function lgamma(z::Complex) if real(z) <= 0.5 a = clgamma_lanczos(1-z) @@ -69,6 +87,11 @@ gamma(z::Complex) = exp(lgamma(z)) # const A002445 = [1,6,30,42,30,66,2730,6,510,798,330,138,2730,6,870,14322,510,6,1919190,6,13530] # const bernoulli = A000367 .// A002445 # even-index Bernoulli numbers +""" + digamma(x) + +Compute the digamma function of `x` (the logarithmic derivative of `gamma(x)`) +""" function digamma(z::Union{Float64,Complex{Float64}}) # Based on eq. (12), without looking at the accompanying source # code, of: K. S. Kölbig, "Programs for computing the logarithm of @@ -98,6 +121,11 @@ function digamma(z::Union{Float64,Complex{Float64}}) ψ -= t * @evalpoly(t,0.08333333333333333,-0.008333333333333333,0.003968253968253968,-0.004166666666666667,0.007575757575757576,-0.021092796092796094,0.08333333333333333,-0.4432598039215686) end +""" + trigamma(x) + +Compute the trigamma function of `x` (the logarithmic second derivative of `gamma(x)`). +""" function trigamma(z::Union{Float64,Complex{Float64}}) # via the derivative of the Kölbig digamma formulation x = real(z) @@ -360,6 +388,12 @@ function zeta(s::Union{Int,Float64,Complex{Float64}}, return ζ end +""" + polygamma(m, x) + +Compute the polygamma function of order `m` of argument `x` (the `(m+1)th` derivative of the +logarithm of `gamma(x)`) +""" function polygamma(m::Integer, z::Union{Float64,Complex{Float64}}) m == 0 && return digamma(z) m == 1 && return trigamma(z) @@ -445,6 +479,12 @@ function invdigamma(y::Float64) return x_new end invdigamma(x::Float32) = Float32(invdigamma(Float64(x))) + +""" + invdigamma(x) + +Compute the inverse digamma function of `x`. +""" invdigamma(x::Real) = invdigamma(Float64(x)) @vectorize_1arg Real invdigamma diff --git a/base/special/trig.jl b/base/special/trig.jl index ab1ac1b4efafe..087f2ad0338e6 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -98,7 +98,11 @@ mulpi_ext(x::Float32) = DoubleFloat32(pi*Float64(x)) mulpi_ext(x::Rational) = mulpi_ext(float(x)) mulpi_ext(x::Real) = pi*x # Fallback +""" + sinpi(x) +Compute ``\\sin(\\pi x)`` more accurately than `sin(pi*x)`, especially for large `x`. +""" function sinpi{T<:AbstractFloat}(x::T) if !isfinite(x) isnan(x) && return x @@ -157,6 +161,11 @@ function sinpi{T<:Real}(x::T) end end +""" + cospi(x) + +Compute ``\\cos(\\pi x)`` more accurately than `cos(pi*x)`, especially for large `x`. +""" function cospi{T<:AbstractFloat}(x::T) if !isfinite(x) isnan(x) && return x @@ -278,13 +287,23 @@ end @vectorize_1arg Number sinpi @vectorize_1arg Number cospi +""" + sinc(x) +Compute ``\\sin(\\pi x) / (\\pi x)`` if ``x \\neq 0``, and ``1`` if ``x = 0``. +""" sinc(x::Number) = x==0 ? one(x) : oftype(x,sinpi(x)/(pi*x)) sinc(x::Integer) = x==0 ? one(x) : zero(x) sinc{T<:Integer}(x::Complex{T}) = sinc(float(x)) sinc(x::Real) = x==0 ? one(x) : isinf(x) ? zero(x) : sinpi(x)/(pi*x) @vectorize_1arg Number sinc +""" + cosc(x) + +Compute ``\\cos(\\pi x) / x - \\sin(\\pi x) / (\\pi x^2)`` if ``x \\neq 0``, and ``0`` if +``x = 0``. This is the derivative of `sinc(x)`. +""" cosc(x::Number) = x==0 ? zero(x) : oftype(x,(cospi(x)-sinpi(x)/(pi*x))/x) cosc(x::Integer) = cosc(float(x)) cosc{T<:Integer}(x::Complex{T}) = cosc(float(x)) @@ -389,16 +408,23 @@ end tand(x::Real) = sind(x) / cosd(x) @vectorize_1arg Real tand -for (fd, f) in ((:sind, :sin), (:cosd, :cos), (:tand, :tan)) +for (fd, f, fn) in ((:sind, :sin, "sine"), (:cosd, :cos, "cosine"), (:tand, :tan, "tangent")) + name = string(fd) @eval begin - ($fd)(z) = ($f)(deg2rad(z)) + @doc """ + $($name)(x) + Compute $($fn) of `x`, where `x` is in degrees. """ ($fd)(z) = ($f)(deg2rad(z)) end end -for (fd, f) in ((:asind, :asin), (:acosd, :acos), (:atand, :atan), - (:asecd, :asec), (:acscd, :acsc), (:acotd, :acot)) +for (fd, f, fn) in ((:asind, :asin, "sine"), (:acosd, :acos, "cosine"), (:atand, :atan, "tangent"), + (:asecd, :asec, "secant"), (:acscd, :acsc, "cosecant"), (:acotd, :acot, "cotangent")) + name = string(fd) @eval begin - ($fd)(y) = rad2deg(($f)(y)) + @doc """ + $($name)(x) + + Compute the inverse $($fn) of `x`, where the output is in degrees. """ ($fd)(y) = rad2deg(($f)(y)) @vectorize_1arg Real $fd end end From 29d11a36f53de19a2fc93eb1ff99fa6035fcd52f Mon Sep 17 00:00:00 2001 From: Valentin Churavy <v.churavy@gmail.com> Date: Sat, 6 Aug 2016 02:51:54 +0900 Subject: [PATCH 0814/1117] enable two additional Float16 tests --- test/numbers.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/numbers.jl b/test/numbers.jl index a0ccc138e776f..8d7c680a0a709 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -120,11 +120,9 @@ end let eps = 1//BigInt(2)^7, one_eps = 1+eps, eps16 = Float16(Float32(eps)), one_eps16 = Float16(Float32(one_eps)) @test eps16 == Float16(Float32(eps)) - # Currently broken in Julia -- enable when "rationalize" is fixed; - # see <https://github.com/JuliaLang/julia/issues/9897> - # @test rationalize(BigInt, eps16, tol=0) == eps + @test rationalize(BigInt, eps16, tol=0) == eps @test one_eps16 == Float16(Float32(one_eps)) - # @test rationalize(BigInt, one_eps16, tol=0) == one_eps + @test rationalize(BigInt, one_eps16, tol=0) == one_eps @test one_eps16 * one_eps16 - 1 != Float16(Float32(one_eps * one_eps - 1)) @test (fma(one_eps16, one_eps16, -1) == Float16(Float32(one_eps * one_eps - 1))) From e64c174d8bd5fcff023752f4e596c3ce04e2bb1c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 5 Aug 2016 13:57:05 -0400 Subject: [PATCH 0815/1117] sharper inference of `===` with singletons, e.g. `x===nothing` --- base/inference.jl | 3 +++ test/inference.jl | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/base/inference.jl b/base/inference.jl index 90ae3d82356d9..aa97ef4f710f5 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -286,6 +286,9 @@ add_tfunc(is, 2, 2, return Const(x.parameters[1]===y.parameters[1]) elseif typeintersect(widenconst(x), widenconst(y)) === Bottom return Const(false) + elseif (isa(x,Const) && y === typeof(x.val) && isdefined(y,:instance)) || + (isa(y,Const) && x === typeof(y.val) && isdefined(x,:instance)) + return Const(true) else return Bool end diff --git a/test/inference.jl b/test/inference.jl index 5776354b91e6e..dc8677df4d71e 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -269,3 +269,8 @@ function f17572{A}(::Type{Val{A}}) end # test that inference doesn't error @test isa(code_typed(f17572, (Type{Val{0}},)), Array) + +# === with singleton constants +let f(x) = (x===nothing) ? 1 : 1.0 + @test Base.return_types(f, (Void,)) == Any[Int] +end From c7599507462b7223139f807daedba351f21eb352 Mon Sep 17 00:00:00 2001 From: Valentin Churavy <v.churavy@gmail.com> Date: Sat, 6 Aug 2016 03:51:45 +0900 Subject: [PATCH 0816/1117] Start v0.6 release notes [ci skip] --- NEWS.md | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index f30ae3b4b50e9..80094d7659355 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,28 @@ +Julia v0.6.0 Release Notes +========================== + +New language features +--------------------- + +Language changes +---------------- + +Breaking changes +---------------- + +This section lists changes that do not have deprecation warnings. + + * Operations between `Float16` and `Integers` now return `Float16` instead of `Float32`. ([#17261]) + +Library improvements +-------------------- + +Compiler/Runtime improvements +----------------------------- + +Deprecated or removed +--------------------- + Julia v0.5.0 Release Notes ========================== @@ -160,8 +185,6 @@ This section lists changes that do not have deprecation warnings. * `map` on a dictionary now expects a function that expects and returns a `Pair`. The result is now another dictionary instead of an array ([#16622]). - * Operations between `Float16` and `Integers` now return `Float16` instead of `Float32`. ([#17261]) - Library improvements -------------------- From e632525e15bf665c10832a88719bb2e95bbd5a7b Mon Sep 17 00:00:00 2001 From: Kadar Qian <kadar.qian@gmail.com> Date: Fri, 5 Aug 2016 16:00:28 -0400 Subject: [PATCH 0817/1117] Clarifies location of Sublime Text editing mode It is now in a separate repo and not in `contrib/` --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 87aa47027db59..6baaf4c2343a1 100644 --- a/README.md +++ b/README.md @@ -418,9 +418,10 @@ The following distributions include julia, but the versions may be out of date d Currently, Julia editing mode support is available for a number of editors. While Julia modes for -[Emacs](https://github.com/JuliaLang/julia-emacs) and +[Emacs](https://github.com/JuliaLang/julia-emacs), +[Sublime Text](https://github.com/JuliaEditorSupport/Julia-sublime), and [Vim](https://github.com/JuliaLang/julia-vim) have their own repos, -others such as Textmate, Sublime Text, Notepad++, and Kate, are in +others such as Textmate, Notepad++, and Kate, are in `contrib/`. Two major IDEs are supported for Julia: [Juno](http://junolab.org/), From 3a237bccdfaf0e3fb31f407216f17383b7e9949a Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 5 Aug 2016 14:03:16 -0700 Subject: [PATCH 0818/1117] Only download busybox-w32 when USE_GPL_LIBS is 1 some of the spawn and cmdlineargs tests may fail in a no-GPL Windows build if executed outside of the build environment (all of which are GPL themselves...), we could either skip them or try to get them to work via calling powershell or some other non-GPL coreutils replacement (toybox? uutils? sbase + now-LPGL cygwin?) --- Makefile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7a5b562896b47..e978824d0ff96 100644 --- a/Makefile +++ b/Makefile @@ -468,7 +468,11 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll busybox.exe libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) + cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) +ifeq ($(USE_GPL_LIBS), 1) + [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ + cp busybox.exe $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) +endif cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. @@ -624,12 +628,15 @@ endif cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920_extra.7z && \ $(JLDOWNLOAD) https://unsis.googlecode.com/files/nsis-2.46.5-Unicode-setup.exe && \ - $(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-483-g31277ab.exe && \ chmod a+x 7z.exe && \ chmod a+x 7z.dll && \ $(call spawn,./7z.exe) x -y -onsis nsis-2.46.5-Unicode-setup.exe && \ - chmod a+x ./nsis/makensis.exe && \ + chmod a+x ./nsis/makensis.exe +ifeq ($(USE_GPL_LIBS), 1) + cd $(JULIAHOME)/dist-extras && \ + $(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-483-g31277ab.exe && \ chmod a+x busybox.exe +endif # various statistics about the build that may interest the user ifeq ($(USE_SYSTEM_LLVM), 1) From 0d6b4955ed490b69b481a249c446276afb2b8fbf Mon Sep 17 00:00:00 2001 From: Spencer Russell <spencer.f.russell@gmail.com> Date: Fri, 5 Aug 2016 18:14:42 -0400 Subject: [PATCH 0819/1117] AST doc - comparisons now use `call` unless chained --- doc/devdocs/ast.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/devdocs/ast.rst b/doc/devdocs/ast.rst index 7cfe51bb4ac18..72712803a5fcc 100644 --- a/doc/devdocs/ast.rst +++ b/doc/devdocs/ast.rst @@ -267,7 +267,7 @@ Input AST ``a:b`` ``(: a b)`` ``a:b:c`` ``(: a b c)`` ``a,b`` ``(tuple a b)`` -``a==b`` ``(comparison a == b)`` +``a==b`` ``(call == a b)`` ``1<i<=n`` ``(comparison 1 < i <= n)`` ``a.b`` ``(. a (quote b))`` ``a.(b)`` ``(. a b)`` From 0900a8c3891d25bb53511b1461b83a85b5590264 Mon Sep 17 00:00:00 2001 From: Tracy Wadleigh <tracy.wadleigh@gmail.com> Date: Fri, 5 Aug 2016 15:27:49 -0700 Subject: [PATCH 0820/1117] Recognize atom with a file extension. The `edit` function on windows fails if the editor command is `atom`. It succeeds if it is instead `atom.cmd`, but then Julia fails to recognize the `atom` editor. This fixes that. --- base/interactiveutil.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 0a757ee10b619..7ae43796fedf0 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -40,7 +40,7 @@ function edit(path::AbstractString, line::Integer=0) background = false elseif name == "textmate" || name == "mate" || name == "kate" cmd = line != 0 ? `$command $path -l $line` : `$command $path` - elseif startswith(name, "subl") || name == "atom" + elseif startswith(name, "subl") || startswith(name, "atom") cmd = line != 0 ? `$command $path:$line` : `$command $path` elseif is_windows() && (name == "start" || name == "open") cmd = `cmd /c start /b $path` From dc4da1da312e41caef4db73b48be9d6ce36177f4 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Sat, 6 Aug 2016 00:20:57 -0700 Subject: [PATCH 0821/1117] [ci skip] fix docstring wording --- base/bitarray.jl | 12 ++++++------ base/sparse/sparsematrix.jl | 2 +- doc/stdlib/arrays.rst | 14 +++++++------- doc/stdlib/collections.rst | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/base/bitarray.jl b/base/bitarray.jl index b883b94aadc2c..3e67d3c86e911 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1437,7 +1437,7 @@ end rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector Performs a left rotation operation on `src` and puts the result into `dest`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. """ function rol!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) @@ -1455,7 +1455,7 @@ end rol!(B::BitVector, i::Integer) -> BitVector Performs a left rotation operation in-place on `B`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. """ function rol!(B::BitVector, i::Integer) return rol!(B, B, i) @@ -1465,7 +1465,7 @@ end rol(B::BitVector, i::Integer) -> BitVector Performs a left rotation operation, returning a new `BitVector`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. See also [`rol!`](:func:`rol!`). ```jldoctest @@ -1510,7 +1510,7 @@ end ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector Performs a right rotation operation on `src` and puts the result into `dest`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. """ function ror!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) @@ -1528,7 +1528,7 @@ end ror!(B::BitVector, i::Integer) -> BitVector Performs a right rotation operation in-place on `B`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. """ function ror!(B::BitVector, i::Integer) return ror!(B, B, i) @@ -1538,7 +1538,7 @@ end ror(B::BitVector, i::Integer) -> BitVector Performs a right rotation operation on `B`, returning a new `BitVector`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. See also [`ror!`](:func:`ror!`). ```jldoctest diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 0e3f2bf68c174..b50e3bef99781 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1258,7 +1258,7 @@ speye(m::Integer, n::Integer) = speye(Float64, m, n) """ speye(S) -Create a sparse identity matrix with the same size as that of `S`. +Create a sparse identity matrix with the same size as `S`. ```jldoctest julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 78cdb2e7e44ee..a475fd69b31ad 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1230,19 +1230,19 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively .. Docstring generated from Julia source - Performs a left rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate each bit. + Performs a left rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate the bits. .. function:: rol!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation in-place on ``B``\ . ``i`` controls how far to rotate each bit. + Performs a left rotation operation in-place on ``B``\ . ``i`` controls how far to rotate the bits. .. function:: rol(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation, returning a new ``BitVector``\ . ``i`` controls how far to rotate each bit. See also :func:`rol!`\ . + Performs a left rotation operation, returning a new ``BitVector``\ . ``i`` controls how far to rotate the bits. See also :func:`rol!`\ . .. doctest:: @@ -1282,19 +1282,19 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively .. Docstring generated from Julia source - Performs a right rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate each bit. + Performs a right rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate the bits. .. function:: ror!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a right rotation operation in-place on ``B``\ . ``i`` controls how far to rotate each bit. + Performs a right rotation operation in-place on ``B``\ . ``i`` controls how far to rotate the bits. .. function:: ror(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a right rotation operation on ``B``\ , returning a new ``BitVector``\ . ``i`` controls how far to rotate each bit. See also :func:`ror!`\ . + Performs a right rotation operation on ``B``\ , returning a new ``BitVector``\ . ``i`` controls how far to rotate the bits. See also :func:`ror!`\ . .. doctest:: @@ -1428,7 +1428,7 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Create a sparse identity matrix with the same size as that of ``S``\ . + Create a sparse identity matrix with the same size as ``S``\ . .. doctest:: diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index e37157208b22a..7d3fba881b555 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1364,7 +1364,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:541 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:575 ... .. function:: splice!(collection, index, [replacement]) -> item From f2535c91de9b3bc71c74c32a66ca21799fed2cfc Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Wed, 3 Aug 2016 19:05:10 +0530 Subject: [PATCH 0822/1117] Fix pcre clean target. --- deps/pcre.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/pcre.mk b/deps/pcre.mk index c66197fc7f50c..5ae612452e763 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -36,7 +36,7 @@ $(PCRE_OBJ_TARGET): $(PCRE_SRC_TARGET) touch -c $@ clean-pcre: - -$(MAKE) -C pcre2-$(PCRE_VER) clean + -$(MAKE) -C $(BUILDDIR)/pcre2-$(PCRE_VER) clean -rm -f $(build_shlibdir)/libpcre* distclean-pcre: -rm -rf $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 $(SRCDIR)/srccache/pcre2-$(PCRE_VER) $(BUILDDIR)/pcre2-$(PCRE_VER) From 341a212e330824878aa5cd9efffbb52768b04ecb Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Sat, 6 Aug 2016 14:32:52 +0530 Subject: [PATCH 0823/1117] Fix mbedtls clean target to delete all installed files. --- deps/mbedtls.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 79c8513a44579..9ea8181408a7d 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -71,7 +71,7 @@ endif clean-mbedtls: -$(MAKE) -C $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) clean - -rm -f $(MBEDTLS_OBJ_TARGET) + -rm -f $(MBEDTLS_OBJ_TARGET) $(build_shlibdir)/libmbed* distclean-mbedtls: -rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz \ From 123177afd440d95957cf5867faea100f144bdf39 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Wed, 3 Aug 2016 19:07:09 +0530 Subject: [PATCH 0824/1117] Build curl to provide proxy support for Pkg. The curl build depends on mbedtls and libssh2 Patch for mbedtls to allow curl to build with mbedtls support Add curl to LICENSE.md, README.md and all the other various locations --- LICENSE.md | 1 + Make.inc | 1 + Makefile | 3 ++ README.md | 1 + deps/Makefile | 9 ++++- deps/Versions.make | 1 + deps/checksums/curl-7.50.1.tar.bz2/md5 | 1 + deps/checksums/curl-7.50.1.tar.bz2/sha512 | 1 + deps/curl.mk | 46 +++++++++++++++++++++++ deps/mbedtls.mk | 6 ++- deps/patches/mbedtls-ssl.h.patch | 12 ++++++ 11 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/curl-7.50.1.tar.bz2/md5 create mode 100644 deps/checksums/curl-7.50.1.tar.bz2/sha512 create mode 100644 deps/curl.mk create mode 100644 deps/patches/mbedtls-ssl.h.patch diff --git a/LICENSE.md b/LICENSE.md index fd312fcee06f3..ab11841e04e27 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -57,6 +57,7 @@ their own licenses: - [FFTW](http://fftw.org/doc/License-and-Copyright.html) [GPL2+] - [GMP](http://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+] - [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception] +- [CURL](https://curl.haxx.se/docs/copyright.html) [MIT/X derivative] - [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3] - [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0] - [MPFR](http://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+] diff --git a/Make.inc b/Make.inc index 8bd586dd92cb8..568a94cb22cf7 100644 --- a/Make.inc +++ b/Make.inc @@ -38,6 +38,7 @@ USE_SYSTEM_LIBUV:=0 USE_SYSTEM_UTF8PROC:=0 USE_SYSTEM_MBEDTLS:=0 USE_SYSTEM_LIBSSH2:=0 +USE_SYSTEM_CURL:=0 USE_SYSTEM_LIBGIT2:=0 USE_SYSTEM_PATCHELF:=0 diff --git a/Makefile b/Makefile index e978824d0ff96..7f0ef928f902f 100644 --- a/Makefile +++ b/Makefile @@ -277,6 +277,9 @@ endif ifeq ($(USE_SYSTEM_LIBSSH2),0) JL_PRIVATE_LIBS += ssh2 endif +ifeq ($(USE_SYSTEM_CURL),0) +JL_PRIVATE_LIBS += curl +endif ifeq ($(USE_SYSTEM_LIBGIT2),0) JL_PRIVATE_LIBS += git2 endif diff --git a/README.md b/README.md index 6baaf4c2343a1..f75dbeafa867d 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,7 @@ Julia uses the following external libraries, which are automatically downloaded - **[GMP]** (>= 5.0) — GNU multiple precision arithmetic library, needed for `BigInt` support. - **[MPFR]** (>= 3.0) — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support. - **[libgit2]** (>= 0.23) — Git linkable library, used by Julia's package manager +- **[curl]** (>= 7.50) — libcurl provides download and proxy support for Julia's package manager - **[libssh2]** (>= 1.7) — library for SSH transport, used by libgit2 for packages with SSH remotes - **[mbedtls]** (>= 2.2) — library used for cryptography and transport layer security, used by libssh2 - **[utf8proc]** (>= 2.0) — a library for processing UTF-8 encoded Unicode strings diff --git a/deps/Makefile b/deps/Makefile index 08c3f96fc503e..9d6dd32375950 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -17,7 +17,7 @@ include $(SRCDIR)/llvm-ver.make # additionally all targets should be listed in the getall target for easier off-line compilation # if you are adding a new target, it can help to copy an similar, existing target # -# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv +# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv curl # custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind # entirely custom: virtualenv # CMake libs: libgit2 libssh2 mbedtls @@ -171,6 +171,10 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0) STAGE2_DEPS += libssh2 endif +ifeq ($(USE_SYSTEM_CURL), 0) +STAGE2_DEPS += curl +endif + ifeq ($(USE_SYSTEM_LIBGIT2), 0) STAGE3_DEPS += libgit2 endif @@ -222,7 +226,7 @@ install: $(addprefix install-, $(DEP_LIBS)) cleanall: $(addprefix clean-, $(DEP_LIBS)) distcleanall: $(addprefix distclean-, $(DEP_LIBS)) rm -rf $(build_prefix) -getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-libgit2 +getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-curl get-libgit2 ## PATHS ## # sort is used to remove potential duplicates @@ -271,6 +275,7 @@ include $(SRCDIR)/mpfr.mk include $(SRCDIR)/patchelf.mk include $(SRCDIR)/mbedtls.mk include $(SRCDIR)/libssh2.mk +include $(SRCDIR)/curl.mk include $(SRCDIR)/libgit2.mk include $(SRCDIR)/virtualenv.mk diff --git a/deps/Versions.make b/deps/Versions.make index 4d6c5fc751b79..cdf2c1eb84de9 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -13,3 +13,4 @@ MPFR_VER = 3.1.4 PATCHELF_VER = 0.9 VIRTUALENV_VER = 15.0.0 MBEDTLS_VER = 2.3.0 +CURL_VER = 7.50.1 diff --git a/deps/checksums/curl-7.50.1.tar.bz2/md5 b/deps/checksums/curl-7.50.1.tar.bz2/md5 new file mode 100644 index 0000000000000..111b575728535 --- /dev/null +++ b/deps/checksums/curl-7.50.1.tar.bz2/md5 @@ -0,0 +1 @@ +015f6a0217ca6f2c5442ca406476920b diff --git a/deps/checksums/curl-7.50.1.tar.bz2/sha512 b/deps/checksums/curl-7.50.1.tar.bz2/sha512 new file mode 100644 index 0000000000000..95a02a7febe1c --- /dev/null +++ b/deps/checksums/curl-7.50.1.tar.bz2/sha512 @@ -0,0 +1 @@ +94acd91fcf8ff2605e1ba2d086c2c366257b61eaf516b9ea44e574e315feb5b30f6e47d89051f259e026ef5dd9edde5f7b15a6af9ee6a38f641da354e1e677b1 diff --git a/deps/curl.mk b/deps/curl.mk new file mode 100644 index 0000000000000..12295cde6ad08 --- /dev/null +++ b/deps/curl.mk @@ -0,0 +1,46 @@ +## CURL ## + +CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/.libs/libcurl.$(SHLIB_EXT) +CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT) + +ifneq ($(OS),WINNT) +CURL_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) +endif + +$(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache + $(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2 +$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET) + $(JLCHECKSUM) $< + cd $(dir $<) && $(TAR) jxf $(notdir $<) + touch -c $@ +$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --with-mbedtls=$(build_shlibdir)/.. CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + touch -c $@ +$(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status + $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) + touch -c $@ +$(BUILDDIR)/curl-$(CURL_VER)/checked: $(CURL_SRC_TARGET) +ifeq ($(OS),$(BUILD_OS)) +ifneq ($(OS),WINNT) + $(MAKE) -C $(dir $@) check -j1 +endif +endif + echo 1 > $@ +$(CURL_OBJ_TARGET): $(CURL_SRC_TARGET) + $(call make-install,curl-$(CURL_VER),$(LIBTOOL_CCLD)) + $(INSTALL_NAME_CMD)libcurl.$(SHLIB_EXT) $@ + touch -c $@ + +clean-curl: + -$(MAKE) -C $(BUILDDIR)/curl-$(CURL_VER) clean + -rm -f $(build_shlibdir)/libcurl* +distclean-curl: + -rm -rf $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(SRCDIR)/srccache/curl-$(CURL_VER) $(BUILDDIR)/curl-$(CURL_VER) + +get-curl: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 +configure-curl: $(BUILDDIR)/curl-$(CURL_VER)/config.status +compile-curl: $(CURL_SRC_TARGET) +check-curl: $(BUILDDIR)/curl-$(CURL_VER)/checked +install-curl: $(CURL_OBJ_TARGET) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 9ea8181408a7d..ce0396a80f895 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -39,7 +39,11 @@ $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied: | $(SRCDIR)/srcc cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls-config.patch echo 1 > $@ -$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied +$(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied: | $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(MBEDTLS_SRC)/include/mbedtls && patch -p0 -f < $(SRCDIR)/patches/mbedtls-ssl.h.patch + echo 1 > $@ + +$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(MBEDTLS_OPTS) diff --git a/deps/patches/mbedtls-ssl.h.patch b/deps/patches/mbedtls-ssl.h.patch new file mode 100644 index 0000000000000..11785c5d88c60 --- /dev/null +++ b/deps/patches/mbedtls-ssl.h.patch @@ -0,0 +1,12 @@ +--- ssl.h.old 2016-06-28 18:12:06.000000000 +0530 ++++ ssl.h 2016-08-03 18:51:34.000000000 +0530 +@@ -54,7 +54,8 @@ + #endif + + #if defined(MBEDTLS_HAVE_TIME) +-#include <time.h> ++//#include <time.h> ++#include "platform.h" + #endif + + /* From 16a7d6a392d22e9fed1978df487c7681cfd0ed06 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Wed, 3 Aug 2016 19:38:25 +0530 Subject: [PATCH 0825/1117] Have libgit2 use our curl. Enforce curl as a dependency for the libgit2 build. Fix the libgit2 clean target Build curl only on linux and osx. Add pkg-config to build dependencies in README.md. Required to detect curl in libgit2 build. --- README.md | 2 ++ contrib/windows/msys_build.sh | 1 + deps/Makefile | 8 +++++--- deps/libgit2.mk | 9 +++++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f75dbeafa867d..97601eda402ef 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,7 @@ Building Julia requires that the following software be installed: - **[patch]** — for modifying source code. - **[cmake]** — needed to build `libgit2`. - **[openssl]** — needed for HTTPS support in `libgit2` on Linux, install via `apt-get install libssl-dev` or `yum install openssl-devel`. +- **[pkg-config]** - needed to build libgit2 correctly, especially for proxy support Julia uses the following external libraries, which are automatically downloaded (or in a few cases, included in the Julia source repository) and then compiled from source the first time you run `make`: @@ -326,6 +327,7 @@ For a longer overview of Julia's dependencies, see these [slides](https://github [openssl]: https://www.openssl.org [libssh2]: https://www.libssh2.org [mbedtls]: https://tls.mbed.org/ +[pkg-config]: https://www.freedesktop.org/wiki/Software/pkg-config/ <a name="System-Provided-Libraries"> ### System Provided Libraries diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 41258a21f414d..034127a9d7536 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -176,6 +176,7 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user echo 'override STAGE1_DEPS = libuv' >> Make.user echo 'override STAGE2_DEPS = utf8proc' >> Make.user echo 'override STAGE3_DEPS = ' >> Make.user +echo 'override STAGE4_DEPS = ' >> Make.user if [ -n "$USEMSVC" ]; then # Openlibm doesn't build well with MSVC right now diff --git a/deps/Makefile b/deps/Makefile index 9d6dd32375950..37b4defd90884 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -171,12 +171,14 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0) STAGE2_DEPS += libssh2 endif +ifneq ($(OS), WINNT) ifeq ($(USE_SYSTEM_CURL), 0) -STAGE2_DEPS += curl +STAGE3_DEPS += curl +endif endif ifeq ($(USE_SYSTEM_LIBGIT2), 0) -STAGE3_DEPS += libgit2 +STAGE4_DEPS += libgit2 endif ifeq ($(USE_SYSTEM_MPFR), 0) @@ -215,7 +217,7 @@ endif ## Common build target prefixes -DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) +DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) $(STAGE4_DEPS) default: install | $(build_prefix) get: $(addprefix get-, $(DEP_LIBS)) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index b23daf7317a1b..224d30f9304ec 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -19,6 +19,8 @@ else LIBGIT2_OPTS += -DBUILD_CLAR=OFF -DDLLTOOL=`which $(CROSS_COMPILE)dlltool` LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY endif +else +LIBGIT2_OPTS += -DCURL_INCLUDE_DIRS=$(build_includedir) -DCURL_LIBRARIES="-L$(build_shlibdir) -lcurl" endif ifeq ($(OS),Linux) @@ -40,6 +42,9 @@ $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied: | $(SR ifeq ($(OS),Linux) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied endif +ifneq ($(OS),WINNT) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(CURL_OBJ_TARGET) +endif $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied $(LIBSSH2_OBJ_TARGET) mkdir -p $(dir $@) cd $(dir $@) && \ @@ -69,14 +74,14 @@ ifeq ($(OS),Linux) for LIB in libssl libcrypto; do \ LIB_PATH=$$(echo "$$LIBGIT_LIBS" | grep "$$LIB"); \ echo "LIB_PATH for $$LIB: $$LIB_PATH"; \ - [ ! -z "$$LIB_PATH" ] && cp -v "$$LIB_PATH" $(build_shlibdir); \ + [ ! -z "$$LIB_PATH" ] && cp -v -f "$$LIB_PATH" $(build_shlibdir); \ done endif touch -c $@ clean-libgit2: -$(MAKE) -C $(BUILDDIR)/$(LIBGIT2_SRC_DIR) clean - -rm -f $(LIBGIT2_OBJ_TARGET) + -rm -f $(build_shlibdir)/libgit2* $(build_shlibdir)/libssl* $(build_shlibdir)/libcrypto* get-libgit2: $(LIBGIT2_SRC_FILE) configure-libgit2: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile From 06112675c37549dc003469885f09afbe27604f49 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Sat, 6 Aug 2016 20:32:42 +0530 Subject: [PATCH 0826/1117] Bump openlibm. --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/openlibm.version | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 create mode 100644 deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 delete mode 100644 deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/md5 delete mode 100644 deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/sha512 diff --git a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 new file mode 100644 index 0000000000000..8e7d4cc7c4029 --- /dev/null +++ b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 @@ -0,0 +1 @@ +3b852052db9052a3668122449ce1f82c diff --git a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 new file mode 100644 index 0000000000000..a840095574b1a --- /dev/null +++ b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 @@ -0,0 +1 @@ +7125800186428a8aefea3030ff1d35f4169c29cb3ae30c2bd7965bd5e66f0958b8b33fcd2e5a30b012ecececd09ce3ce0195b5a8fc9066efd31e124cc32ec2c0 diff --git a/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/md5 b/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/md5 deleted file mode 100644 index 5b6d306e94650..0000000000000 --- a/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -7d15bdbd8ba248ab8ff6a120c7806cf0 diff --git a/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/sha512 b/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/sha512 deleted file mode 100644 index f5322da6ded37..0000000000000 --- a/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -9977fcc79e54c86ec8373f73f98f95dc7c4b20ca28884998fbb7f0aa3759ff904ee326cfcbbaf0d000de9161e8d1511e71440b259a95517742b803ef8be32bdc diff --git a/deps/openlibm.version b/deps/openlibm.version index 4e15529fd3583..ceb185d9bc9de 100644 --- a/deps/openlibm.version +++ b/deps/openlibm.version @@ -1,2 +1,2 @@ -OPENLIBM_BRANCH=v0.5.1 -OPENLIBM_SHA1=e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5 +OPENLIBM_BRANCH=v0.5.2 +OPENLIBM_SHA1=0fa599cce845be67ed1729d9753f67e366c37b96 From 08ac1467c390ba16221de3eabe3368bf560e7bfb Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sat, 6 Aug 2016 12:54:57 -0400 Subject: [PATCH 0827/1117] only run Pipe exhaustion test on unix windows doesn't stop you from allocating millions of pipes, until Julia runs out of memory --- test/spawn.jl | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/spawn.jl b/test/spawn.jl index b4d7f7ee1990e..cfff6b6b6425a 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -394,18 +394,20 @@ end @test reduce(&, [`$echo abc`, `$echo def`, `$echo hij`]) == `$echo abc` & `$echo def` & `$echo hij` # test for proper handling of FD exhaustion -let ps = Pipe[] - try - for i = 1:100_000 - p = Pipe() - Base.link_pipe(p) - push!(ps, p) - end - @test false - catch ex - for p in ps - close(p) +if is_unix() + let ps = Pipe[] + try + for i = 1:100_000 + p = Pipe() + Base.link_pipe(p) + push!(ps, p) + end + @test false + catch ex + for p in ps + close(p) + end + @test (ex::Base.UVError).code == Base.UV_EMFILE end - @test (ex::Base.UVError).code == Base.UV_EMFILE end end From 2621cd9bbc60a8c93d7acfcd844aa0049fc6a973 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 28 Jul 2016 12:13:56 -0700 Subject: [PATCH 0828/1117] Make concatenations involving combinations of special matrices with special matrices, sparse matrices, or dense matrices/vectors yield sparse arrays. --- base/sparse/sparsematrix.jl | 14 +++++++++----- test/linalg/special.jl | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index b50e3bef99781..280ffbf5cd179 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -3265,19 +3265,23 @@ function hcat(X::SparseMatrixCSC...) end -# Sparse/dense concatenation +# Sparse/special/dense concatenation -function hcat(Xin::Union{Vector, Matrix, SparseMatrixCSC}...) +# TODO: A similar definition also exists in base/linalg/bidiag.jl. These definitions should +# be consolidated in a more appropriate location, for example base/linalg/special.jl. +SpecialArrays = Union{Diagonal, Bidiagonal, Tridiagonal, SymTridiagonal} + +function hcat(Xin::Union{Vector, Matrix, SparseMatrixCSC, SpecialArrays}...) X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] hcat(X...) end -function vcat(Xin::Union{Vector, Matrix, SparseMatrixCSC}...) +function vcat(Xin::Union{Vector, Matrix, SparseMatrixCSC, SpecialArrays}...) X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] vcat(X...) end -function hvcat(rows::Tuple{Vararg{Int}}, X::Union{Vector, Matrix, SparseMatrixCSC}...) +function hvcat(rows::Tuple{Vararg{Int}}, X::Union{Vector, Matrix, SparseMatrixCSC, SpecialArrays}...) nbr = length(rows) # number of block rows tmp_rows = Array{SparseMatrixCSC}(nbr) @@ -3289,7 +3293,7 @@ function hvcat(rows::Tuple{Vararg{Int}}, X::Union{Vector, Matrix, SparseMatrixCS vcat(tmp_rows...) end -function cat(catdims, Xin::Union{Vector, Matrix, SparseMatrixCSC}...) +function cat(catdims, Xin::Union{Vector, Matrix, SparseMatrixCSC, SpecialArrays}...) X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] T = promote_eltype(Xin...) Base.cat_t(catdims, T, X...) diff --git a/test/linalg/special.jl b/test/linalg/special.jl index 89ddde5ee4b00..05105acd149a8 100644 --- a/test/linalg/special.jl +++ b/test/linalg/special.jl @@ -128,3 +128,41 @@ for typ in [UpperTriangular,LowerTriangular,Base.LinAlg.UnitUpperTriangular,Base @test Base.LinAlg.A_mul_Bc(atri,qrb[:Q]) ≈ full(atri) * qrb[:Q]' @test Base.LinAlg.A_mul_Bc!(copy(atri),qrb[:Q]) ≈ full(atri) * qrb[:Q]' end + +# Test that concatenations of combinations of special and other matrix types yield sparse arrays +let + N = 4 + # Test concatenating pairwise combinations of special matrices + diagmat = Diagonal(ones(N)) + bidiagmat = Bidiagonal(ones(N), ones(N-1), true) + tridiagmat = Tridiagonal(ones(N-1), ones(N), ones(N-1)) + symtridiagmat = SymTridiagonal(ones(N), ones(N-1)) + specialmats = (diagmat, bidiagmat, tridiagmat, symtridiagmat) + for specialmata in specialmats, specialmatb in specialmats + @test issparse(hcat(specialmata, specialmatb)) + @test issparse(vcat(specialmata, specialmatb)) + @test issparse(hvcat((1,1), specialmata, specialmatb)) + @test issparse(cat((1,2), specialmata, specialmatb)) + end + # Test concatenating pairwise combinations of special matrices with sparse matrices, + # dense matrices, or dense vectors + densevec = ones(N) + densemat = diagm(ones(N)) + spmat = spdiagm(ones(N)) + for specialmat in specialmats + # --> Tests applicable only to pairs of matrices + for othermat in (spmat, densemat) + @test issparse(vcat(specialmat, othermat)) + @test issparse(vcat(othermat, specialmat)) + end + # --> Tests applicable also to pairs including vectors + for specialmat in specialmats, othermatorvec in (spmat, densemat, densevec) + @test issparse(hcat(specialmat, othermatorvec)) + @test issparse(hcat(othermatorvec, specialmat)) + @test issparse(hvcat((2,), specialmat, othermatorvec)) + @test issparse(hvcat((2,), othermatorvec, specialmat)) + @test issparse(cat((1,2), specialmat, othermatorvec)) + @test issparse(cat((1,2), othermatorvec, specialmat)) + end + end +end From 8225b2b8fb3e7189fdc937bfb420b793eb148d7a Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Mon, 1 Aug 2016 11:19:13 -0700 Subject: [PATCH 0829/1117] Make concatenations involving combinations of sparse vectors with various matrix and vector types consistently yield sparse arrays. --- base/sparse/sparsematrix.jl | 35 -------------------- base/sparse/sparsevector.jl | 58 +++++++++++++++++++++++++++++++--- test/sparsedir/sparsevector.jl | 35 ++++++++++++++++++++ 3 files changed, 89 insertions(+), 39 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 280ffbf5cd179..d8d3fa47d0f42 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -3264,41 +3264,6 @@ function hcat(X::SparseMatrixCSC...) SparseMatrixCSC(m, n, colptr, rowval, nzval) end - -# Sparse/special/dense concatenation - -# TODO: A similar definition also exists in base/linalg/bidiag.jl. These definitions should -# be consolidated in a more appropriate location, for example base/linalg/special.jl. -SpecialArrays = Union{Diagonal, Bidiagonal, Tridiagonal, SymTridiagonal} - -function hcat(Xin::Union{Vector, Matrix, SparseMatrixCSC, SpecialArrays}...) - X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] - hcat(X...) -end - -function vcat(Xin::Union{Vector, Matrix, SparseMatrixCSC, SpecialArrays}...) - X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] - vcat(X...) -end - -function hvcat(rows::Tuple{Vararg{Int}}, X::Union{Vector, Matrix, SparseMatrixCSC, SpecialArrays}...) - nbr = length(rows) # number of block rows - - tmp_rows = Array{SparseMatrixCSC}(nbr) - k = 0 - @inbounds for i = 1 : nbr - tmp_rows[i] = hcat(X[(1 : rows[i]) + k]...) - k += rows[i] - end - vcat(tmp_rows...) -end - -function cat(catdims, Xin::Union{Vector, Matrix, SparseMatrixCSC, SpecialArrays}...) - X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] - T = promote_eltype(Xin...) - Base.cat_t(catdims, T, X...) -end - """ blkdiag(A...) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index db2a603232e33..cd002a2574bdb 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -720,7 +720,13 @@ complex(x::AbstractSparseVector) = ### Concatenation -function hcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) +# Without the first of these methods, horizontal concatenations of SparseVectors fall +# back to the horizontal concatenation method that ensures that combinations of +# sparse/special/dense matrix/vector types concatenate to SparseMatrixCSCs, instead +# of _absspvec_hcat below. The <:Integer qualifications are necessary for correct dispatch. +hcat{Tv,Ti<:Integer}(X::SparseVector{Tv,Ti}...) = _absspvec_hcat(X...) +hcat{Tv,Ti<:Integer}(X::AbstractSparseVector{Tv,Ti}...) = _absspvec_hcat(X...) +function _absspvec_hcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) # check sizes n = length(X) m = length(X[1]) @@ -749,7 +755,13 @@ function hcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) SparseMatrixCSC{Tv,Ti}(m, n, colptr, nzrow, nzval) end -function vcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) +# Without the first of these methods, vertical concatenations of SparseVectors fall +# back to the vertical concatenation method that ensures that combinations of +# sparse/special/dense matrix/vector types concatenate to SparseMatrixCSCs, instead +# of _absspvec_vcat below. The <:Integer qualifications are necessary for correct dispatch. +vcat{Tv,Ti<:Integer}(X::SparseVector{Tv,Ti}...) = _absspvec_vcat(X...) +vcat{Tv,Ti<:Integer}(X::AbstractSparseVector{Tv,Ti}...) = _absspvec_vcat(X...) +function _absspvec_vcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) # check sizes n = length(X) tnnz = 0 @@ -777,11 +789,49 @@ function vcat{Tv,Ti}(X::AbstractSparseVector{Tv,Ti}...) SparseVector(len, rnzind, rnzval) end -hcat(Xin::Union{AbstractSparseVector, SparseMatrixCSC}...) = hcat(map(SparseMatrixCSC, Xin)...) -vcat(Xin::Union{AbstractSparseVector, SparseMatrixCSC}...) = vcat(map(SparseMatrixCSC, Xin)...) hcat(Xin::Union{Vector, AbstractSparseVector}...) = hcat(map(sparse, Xin)...) vcat(Xin::Union{Vector, AbstractSparseVector}...) = vcat(map(sparse, Xin)...) + +### Sparse/special/dense vector/matrix concatenation + +# TODO: These methods should be moved to a more appropriate location, particularly some +# future equivalent of base/linalg/special.jl dedicated to interactions between a broader +# set of matrix types. + +# TODO: A similar definition also exists in base/linalg/bidiag.jl. These definitions should +# be consolidated in a more appropriate location, for example base/linalg/special.jl. +SpecialArrays = Union{Diagonal, Bidiagonal, Tridiagonal, SymTridiagonal} + +function hcat(Xin::Union{Vector, Matrix, SparseVector, SparseMatrixCSC, SpecialArrays}...) + X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] + hcat(X...) +end + +function vcat(Xin::Union{Vector, Matrix, SparseVector, SparseMatrixCSC, SpecialArrays}...) + X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] + vcat(X...) +end + +function hvcat(rows::Tuple{Vararg{Int}}, X::Union{Vector, Matrix, SparseVector, SparseMatrixCSC, SpecialArrays}...) + nbr = length(rows) # number of block rows + + tmp_rows = Array{SparseMatrixCSC}(nbr) + k = 0 + @inbounds for i = 1 : nbr + tmp_rows[i] = hcat(X[(1 : rows[i]) + k]...) + k += rows[i] + end + vcat(tmp_rows...) +end + +function cat(catdims, Xin::Union{Vector, Matrix, SparseVector, SparseMatrixCSC, SpecialArrays}...) + X = SparseMatrixCSC[issparse(x) ? x : sparse(x) for x in Xin] + T = promote_eltype(Xin...) + Base.cat_t(catdims, T, X...) +end + + ### math functions ### Unary Map diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index 62a7649cc1e4d..b9079c95807be 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -404,6 +404,41 @@ let m = 80, n = 100 @test full(V) == Vr end +# Test that concatenations of combinations of sparse vectors with various other +# matrix/vector types yield sparse arrays +let + N = 4 + spvec = spzeros(N) + spmat = spzeros(N, 1) + densevec = ones(N) + densemat = ones(N, 1) + diagmat = Diagonal(ones(4)) + # Test that concatenations of pairwise combinations of sparse vectors with dense + # vectors/matrices, sparse matrices, or special matrices yield sparse arrays + for othervecormat in (densevec, densemat, spmat) + @test issparse(vcat(spvec, othervecormat)) + @test issparse(vcat(othervecormat, spvec)) + end + for othervecormat in (densevec, densemat, spmat, diagmat) + @test issparse(hcat(spvec, othervecormat)) + @test issparse(hcat(othervecormat, spvec)) + @test issparse(hvcat((2,), spvec, othervecormat)) + @test issparse(hvcat((2,), othervecormat, spvec)) + @test issparse(cat((1,2), spvec, othervecormat)) + @test issparse(cat((1,2), othervecormat, spvec)) + end + # The preceding tests should cover multi-way combinations of those types, but for good + # measure test a few multi-way combinations involving those types + @test issparse(vcat(spvec, densevec, spmat, densemat)) + @test issparse(vcat(densevec, spvec, densemat, spmat)) + @test issparse(hcat(spvec, densemat, spmat, densevec, diagmat)) + @test issparse(hcat(densemat, spmat, spvec, densevec, diagmat)) + @test issparse(hvcat((5,), diagmat, densevec, spvec, densemat, spmat)) + @test issparse(hvcat((5,), spvec, densemat, diagmat, densevec, spmat)) + @test issparse(cat((1,2), densemat, diagmat, spmat, densevec, spvec)) + @test issparse(cat((1,2), spvec, diagmat, densevec, spmat, densemat)) +end + ## sparsemat: combinations with sparse matrix From d0b3a412f91d855cf1e6798e4cd078e41c7de647 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Fri, 5 Aug 2016 10:36:34 -0700 Subject: [PATCH 0830/1117] Move datafmt docs out, update them a bit --- base/datafmt.jl | 82 ++++++++++++++++++++++++++++++++++ base/docs/helpdb/Base.jl | 93 ++------------------------------------- doc/stdlib/io-network.rst | 4 +- 3 files changed, 87 insertions(+), 92 deletions(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index cc4bd51dcc445..7016d09a3d6d3 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -30,14 +30,79 @@ function countlines(io::IO, eol::Char='\n') nl end +""" + readdlm(source, T::Type; options...) + +The columns are assumed to be separated by one or more whitespaces. The end of line +delimiter is taken as `\\n`. +""" readdlm(input, T::Type; opts...) = readdlm(input, invalid_dlm(Char), T, '\n'; opts...) + +""" + readdlm(source, delim::Char, T::Type; options...) + +The end of line delimiter is taken as `\\n`. +""" readdlm(input, dlm::Char, T::Type; opts...) = readdlm(input, dlm, T, '\n'; opts...) +""" + readdlm(source; options...) + +The columns are assumed to be separated by one or more whitespaces. The end of line +delimiter is taken as `\\n`. If all data is numeric, the result will be a numeric array. If +some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings +is returned. +""" readdlm(input; opts...) = readdlm(input, invalid_dlm(Char), '\n'; opts...) + +""" + readdlm(source, delim::Char; options...) + +The end of line delimiter is taken as `\\n`. If all data is numeric, the result will be a +numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of +numbers and strings is returned. +""" readdlm(input, dlm::Char; opts...) = readdlm(input, dlm, '\n'; opts...) +""" + readdlm(source, delim::Char, eol::Char; options...) + +If all data is numeric, the result will be a numeric array. If some elements cannot be +parsed as numbers, a heterogeneous array of numbers and strings is returned. +""" readdlm(input, dlm::Char, eol::Char; opts...) = readdlm_auto(input, dlm, Float64, eol, true; opts...) + +""" + readdlm(source, delim::Char, T::Type, eol::Char; header=false, skipstart=0, skipblanks=true, use_mmap, quotes=true, dims, comments=true, comment_char='#') + +Read a matrix from the source where each line (separated by `eol`) gives one row, with +elements separated by the given delimiter. The source can be a text file, stream or byte +array. Memory mapped files can be used by passing the byte array representation of the +mapped segment as source. + +If `T` is a numeric type, the result is an array of that type, with any non-numeric elements +as `NaN` for floating-point types, or zero. Other useful values of `T` include +`String`, `AbstractString`, and `Any`. + +If `header` is `true`, the first row of data will be read as header and the tuple +`(data_cells, header_cells)` is returned instead of only `data_cells`. + +Specifying `skipstart` will ignore the corresponding number of initial lines from the input. + +If `skipblanks` is `true`, blank lines in the input will be ignored. + +If `use_mmap` is `true`, the file specified by `source` is memory mapped for potential +speedups. Default is `true` except on Windows. On Windows, you may want to specify `true` if +the file is large, and is only read once and not written to. + +If `quotes` is `true`, columns enclosed within double-quote (\") characters are allowed to +contain new lines and column delimiters. Double-quote characters within a quoted field must +be escaped with another double-quote. Specifying `dims` as a tuple of the expected rows and +columns (including header, if any) may speed up reading of large files. If `comments` is +`true`, lines beginning with `comment_char` and text following `comment_char` in any line +are ignored. +""" readdlm(input, dlm::Char, T::Type, eol::Char; opts...) = readdlm_auto(input, dlm, T, eol, false; opts...) @@ -624,7 +689,24 @@ function writedlm(fname::AbstractString, a, dlm; opts...) end end +""" + writedlm(f, A, dl='\\t'; opts) + +Write `A` (a vector, matrix or an iterable collection of iterable rows) as text to `f` +(either a filename string or an `IO` stream) using the given delimiter `delim` (which +defaults to tab, but can be any printable Julia object, typically a `Char` or +`AbstractString`). + +For example, two vectors `x` and `y` of the same length can be written as two columns of +tab-delimited text to `f` by either `writedlm(f, [x y])` or by `writedlm(f, zip(x, y))`. +""" writedlm(io, a; opts...) = writedlm(io, a, '\t'; opts...) + +""" + writecsv(filename, A; opts) + +Equivalent to `writedlm` with `delim` set to comma. +""" writecsv(io, a; opts...) = writedlm(io, a, ','; opts...) show(io::IO, ::MIME"text/csv", a) = writedlm(io, a, ',') diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 7b45975e24423..67812fa2c1f2d 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -10,17 +10,11 @@ Raises a `SystemError` for `errno` with the descriptive string `sysfunc` if `ift systemerror """ - writedlm(f, A, delim='\\t') + digamma(x) -Write `A` (a vector, matrix or an iterable collection of iterable rows) as text to `f` -(either a filename string or an `IO` stream) using the given delimiter `delim` (which -defaults to tab, but can be any printable Julia object, typically a `Char` or -`AbstractString`). - -For example, two vectors `x` and `y` of the same length can be written as two columns of -tab-delimited text to `f` by either `writedlm(f, [x y])` or by `writedlm(f, zip(x, y))`. +Compute the digamma function of `x` (the logarithmic derivative of `gamma(x)`) """ -writedlm +digamma """ fill!(A, x) @@ -2236,13 +2230,6 @@ Largest integer less than or equal to `x/y`. """ fld -""" - writecsv(filename, A) - -Equivalent to `writedlm` with `delim` set to comma. -""" -writecsv - """ withenv(f::Function, kv::Pair...) @@ -2860,80 +2847,6 @@ second variant. """ popdisplay -""" - readdlm(source, delim::Char, T::Type, eol::Char; header=false, skipstart=0, skipblanks=true, use_mmap, quotes=true, dims, comments=true, comment_char='#') - -Read a matrix from the source where each line (separated by `eol`) gives one row, with -elements separated by the given delimiter. The source can be a text file, stream or byte -array. Memory mapped files can be used by passing the byte array representation of the -mapped segment as source. - -If `T` is a numeric type, the result is an array of that type, with any non-numeric elements -as `NaN` for floating-point types, or zero. Other useful values of `T` include -`String`, `AbstractString`, and `Any`. - -If `header` is `true`, the first row of data will be read as header and the tuple -`(data_cells, header_cells)` is returned instead of only `data_cells`. - -Specifying `skipstart` will ignore the corresponding number of initial lines from the input. - -If `skipblanks` is `true`, blank lines in the input will be ignored. - -If `use_mmap` is `true`, the file specified by `source` is memory mapped for potential -speedups. Default is `true` except on Windows. On Windows, you may want to specify `true` if -the file is large, and is only read once and not written to. - -If `quotes` is `true`, columns enclosed within double-quote (\") characters are allowed to -contain new lines and column delimiters. Double-quote characters within a quoted field must -be escaped with another double-quote. Specifying `dims` as a tuple of the expected rows and -columns (including header, if any) may speed up reading of large files. If `comments` is -`true`, lines beginning with `comment_char` and text following `comment_char` in any line -are ignored. -""" -readdlm(source, delim, T, eol) - -""" - readdlm(source, delim::Char, eol::Char; options...) - -If all data is numeric, the result will be a numeric array. If some elements cannot be -parsed as numbers, a heterogeneous array of numbers and strings is returned. -""" -readdlm(source, delim::Char, eol::Char) - -""" - readdlm(source, delim::Char, T::Type; options...) - -The end of line delimiter is taken as `\\n`. -""" -readdlm(source, delim::Char, T::Type) - -""" - readdlm(source, delim::Char; options...) - -The end of line delimiter is taken as `\\n`. If all data is numeric, the result will be a -numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of -numbers and strings is returned. -""" -readdlm(source, delim::Char) - -""" - readdlm(source, T::Type; options...) - -The columns are assumed to be separated by one or more whitespaces. The end of line -delimiter is taken as `\\n`. -""" -readdlm(source, T::Type) - -""" - readdlm(source; options...) - -The columns are assumed to be separated by one or more whitespaces. The end of line -delimiter is taken as `\\n`. If all data is numeric, the result will be a numeric array. If -some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings -is returned. -""" -readdlm(source) - """ filesize(path...) diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 158560b2c9ba3..6f4737b133498 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -624,7 +624,7 @@ Text I/O The columns are assumed to be separated by one or more whitespaces. The end of line delimiter is taken as ``\n``\ . If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. -.. function:: writedlm(f, A, delim='\\t') +.. function:: writedlm(f, A, dl='\\t'; opts) .. Docstring generated from Julia source @@ -638,7 +638,7 @@ Text I/O Equivalent to ``readdlm`` with ``delim`` set to comma. -.. function:: writecsv(filename, A) +.. function:: writecsv(filename, A; opts) .. Docstring generated from Julia source From 15021904f1d07c546013493b860a7acebc36fa5e Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 6 Aug 2016 19:32:28 -0400 Subject: [PATCH 0831/1117] Fix compilation on LLVM 4.0 Renaming of `orc::JITSymbol` and `RuntimeDyld::SymbolInfo`. --- src/jitlayers.cpp | 27 +++++++++++++++++---------- src/jitlayers.h | 14 ++++++++++++-- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index aa653c333507b..50fb1683a68f1 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -376,7 +376,7 @@ void JuliaOJIT::DebugObjectRegistrar::operator()(ObjectLinkingLayerBase::ObjSetH auto NameOrError = Symbol.getName(); assert(NameOrError); auto Name = NameOrError.get(); - orc::JITSymbol Sym = JIT.CompileLayer.findSymbolIn(H, Name, true); + auto Sym = JIT.CompileLayer.findSymbolIn(H, Name, true); assert(Sym); // note: calling getAddress here eagerly finalizes H // as an alternative, we could store the JITSymbol instead @@ -507,18 +507,25 @@ void JuliaOJIT::addModule(std::unique_ptr<Module> M) // TODO: consider moving the FunctionMover resolver here // Step 0: ObjectLinkingLayer has checked whether it is in the current module // Step 1: See if it's something known to the ExecutionEngine - if (auto Sym = findSymbol(Name, true)) - return RuntimeDyld::SymbolInfo(Sym.getAddress(), - Sym.getFlags()); + if (auto Sym = findSymbol(Name, true)) { +#ifdef LLVM40 + // `findSymbol` already eagerly resolved the address + // return it directly. + return Sym; +#else + return RuntimeDyld::SymbolInfo(Sym.getAddress(), + Sym.getFlags()); +#endif + } // Step 2: Search the program symbols if (uint64_t addr = SectionMemoryManager::getSymbolAddressInProcess(Name)) - return RuntimeDyld::SymbolInfo(addr, JITSymbolFlags::Exported); + return JL_SymbolInfo(addr, JITSymbolFlags::Exported); #ifdef _OS_LINUX_ if (uint64_t addr = resolve_atomic(Name.c_str())) - return RuntimeDyld::SymbolInfo(addr, JITSymbolFlags::Exported); + return JL_SymbolInfo(addr, JITSymbolFlags::Exported); #endif // Return failure code - return RuntimeDyld::SymbolInfo(nullptr); + return JL_SymbolInfo(nullptr); }, [](const std::string &S) { return nullptr; } ); @@ -536,7 +543,7 @@ void JuliaOJIT::removeModule(ModuleHandleT H) CompileLayer.removeModuleSet(H); } -orc::JITSymbol JuliaOJIT::findSymbol(const std::string &Name, bool ExportedSymbolsOnly) +JL_JITSymbol JuliaOJIT::findSymbol(const std::string &Name, bool ExportedSymbolsOnly) { void *Addr = nullptr; if (ExportedSymbolsOnly) { @@ -546,10 +553,10 @@ orc::JITSymbol JuliaOJIT::findSymbol(const std::string &Name, bool ExportedSymbo // Step 2: Search all previously emitted symbols if (Addr == nullptr) Addr = LocalSymbolTable[Name]; - return orc::JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); + return JL_JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); } -orc::JITSymbol JuliaOJIT::findUnmangledSymbol(const std::string Name) +JL_JITSymbol JuliaOJIT::findUnmangledSymbol(const std::string Name) { return findSymbol(getMangledName(Name), true); } diff --git a/src/jitlayers.h b/src/jitlayers.h index 6ed1c43782fcf..a87138f46c10a 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -35,7 +35,17 @@ extern PassManager *jl_globalPM; #ifdef LLVM35 #include <llvm/Target/TargetMachine.h> +#endif + +#ifdef LLVM40 +typedef JITSymbol JL_JITSymbol; +// The type that is similar to SymbolInfo on LLVM 4.0 is actually +// `JITEvaluatedSymbol`. However, we only use this type when a JITSymbol +// is expected. +typedef JITSymbol JL_SymbolInfo; #else +typedef orc::JITSymbol JL_JITSymbol; +typedef RuntimeDyld::SymbolInfo JL_SymbolInfo; #endif extern "C" { @@ -184,8 +194,8 @@ class JuliaOJIT { void *getPointerToGlobalIfAvailable(const GlobalValue *GV); void addModule(std::unique_ptr<Module> M); void removeModule(ModuleHandleT H); - orc::JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly); - orc::JITSymbol findUnmangledSymbol(const std::string Name); + JL_JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly); + JL_JITSymbol findUnmangledSymbol(const std::string Name); uint64_t getGlobalValueAddress(const std::string &Name); uint64_t getFunctionAddress(const std::string &Name); Function *FindFunctionNamed(const std::string &Name); From 7816f01ae6cc543f88d21c7a9b8240e9ba941cef Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Sat, 6 Aug 2016 19:14:21 -0700 Subject: [PATCH 0832/1117] Update some function sigs for tasks and events, clear out HelpDB (#17846) --- base/docs/helpdb/Base.jl | 123 --------------------------------------- base/event.jl | 47 +++++++++++++++ base/task.jl | 53 ++++++++++++++++- doc/stdlib/parallel.rst | 24 ++++---- 4 files changed, 111 insertions(+), 136 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 67812fa2c1f2d..82604311cd1cc 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -54,18 +54,6 @@ Subtype operator, equivalent to `issubtype(T1,T2)`. """ Base.:(<:) -""" - schedule(t::Task, [val]; error=false) - -Add a task to the scheduler's queue. This causes the task to run constantly when the system -is otherwise idle, unless the task performs a blocking operation such as `wait`. - -If a second argument is provided, it will be passed to the task (via the return value of -`yieldto`) when it runs again. If `error` is `true`, the value is raised as an exception in -the woken task. -""" -schedule - """ takebuf_array(b::IOBuffer) @@ -258,29 +246,6 @@ Returns `true` if `path` is a regular file, `false` otherwise. """ isfile -""" - task_local_storage(symbol) - -Look up the value of a symbol in the current task's task-local storage. -""" -task_local_storage(symbol) - -""" - task_local_storage(symbol, value) - -Assign a value to a symbol in the current task's task-local storage. -""" -task_local_storage(symbol, value) - -""" - task_local_storage(body, symbol, value) - -Call the function `body` with a modified task-local storage, in which `value` is assigned to -`symbol`; the previous value of `symbol`, or lack thereof, is restored afterwards. Useful -for emulating dynamic scoping. -""" -task_local_storage(body, symbol, value) - """ diff(A, [dim]) @@ -1107,13 +1072,6 @@ Convert all arguments to their common promotion type (if any), and return them a """ promote -""" - @schedule - -Wrap an expression in a `Task` and add it to the local machine's scheduler queue. -""" -:@schedule - """ gradient(F, [h]) @@ -1465,15 +1423,6 @@ Dict{String,Float64} with 3 entries: """ merge -""" - yield() - -Switch to the scheduler to allow another scheduled task to run. A task that calls this -function is still runnable, and will be restarted immediately if there are no other runnable -tasks. -""" -yield - """ transpose!(dest,src) @@ -2215,14 +2164,6 @@ Compute the LU factorization of `A`, such that `A[p,:] = L*U`. """ lu -""" - @task - -Wrap an expression in a `Task` without executing it, and return the `Task`. This only -creates a task, and does not run it. -""" -:@task - """ fld(x, y) @@ -2770,16 +2711,6 @@ Compute the inverse error function of a real `x`, defined by ``\\operatorname{er """ erfinv -""" - @async - -Like `@schedule`, `@async` wraps an expression in a `Task` and adds it to the local -machine's scheduler queue. Additionally it adds the task to the set of items that the -nearest enclosing `@sync` waits for. `@async` also wraps the expression in a `let x=x, y=y, ...` -block to create a new scope with copies of all variables referenced in the expression. -""" -:@async - """ readdir([dir]) -> Vector{String} @@ -4041,13 +3972,6 @@ Like `selectperm`, but accepts a preallocated index vector `ix`. If `initialized """ selectperm! -""" - istaskdone(task) -> Bool - -Tell whether a task has exited. -""" -istaskdone - """ .>(x, y) @@ -4187,15 +4111,6 @@ Determine whether a stream is read-only. """ isreadonly -""" - notify(condition, val=nothing; all=true, error=false) - -Wake up tasks waiting for a condition, passing them `val`. If `all` is `true` (the default), -all waiting tasks are woken, otherwise only one is. If `error` is `true`, the passed value -is raised as an exception in the woken tasks. -""" -notify - """ view(A, inds...) @@ -4868,15 +4783,6 @@ Type conversion cannot be done exactly. """ InexactError -""" - @sync - -Wait until all dynamically-enclosed uses of `@async`, `@spawn`, `@spawnat` and `@parallel` -are complete. All exceptions thrown by enclosed async operations are collected and thrown as -a `CompositeException`. -""" -:@sync - """ typemax(T) @@ -6723,18 +6629,6 @@ base64-encoded string. """ base64encode -""" - Condition() - -Create an edge-triggered event source that tasks can wait for. Tasks that call `wait` on a -`Condition` are suspended and queued. Tasks are woken up when `notify` is later called on -the `Condition`. Edge triggering means that only tasks waiting at the time `notify` is -called can be woken up. For level-triggered notifications, you must keep extra state to keep -track of whether a notification has happened. The `Channel` type does this, and so can be -used for level-triggered events. -""" -Condition - """ filt!(out, b, a, x, [si]) @@ -7912,16 +7806,6 @@ Bitwise or. """ Base.:(|) -""" - yieldto(task, arg = nothing) - -Switch to the given task. The first time a task is switched to, the task's function is -called with no arguments. On subsequent switches, `arg` is returned from the task's last -call to `yieldto`. This is a low-level call that only switches tasks, not considering states -or scheduling in any way. Its use is discouraged. -""" -yieldto - """ readandwrite(command) @@ -8072,13 +7956,6 @@ Compute the Dawson function (scaled imaginary error function) of `x`, defined by """ dawson -""" - current_task() - -Get the currently running `Task`. -""" -current_task - """ randjump(r::MersenneTwister, jumps, [jumppoly]) -> Vector{MersenneTwister} diff --git a/base/event.jl b/base/event.jl index e91849b40b11f..67beaa308a867 100644 --- a/base/event.jl +++ b/base/event.jl @@ -2,6 +2,16 @@ ## condition variables +""" + Condition() + +Create an edge-triggered event source that tasks can wait for. Tasks that call `wait` on a +`Condition` are suspended and queued. Tasks are woken up when `notify` is later called on +the `Condition`. Edge triggering means that only tasks waiting at the time `notify` is +called can be woken up. For level-triggered notifications, you must keep extra state to keep +track of whether a notification has happened. The [`Channel`](:class:`Channel`) type does +this, and so can be used for level-triggered events. +""" type Condition waitq::Vector{Any} @@ -21,6 +31,13 @@ function wait(c::Condition) end end +""" + notify(condition, val=nothing; all=true, error=false) + +Wake up tasks waiting for a condition, passing them `val`. If `all` is `true` (the default), +all waiting tasks are woken, otherwise only one is. If `error` is `true`, the passed value +is raised as an exception in the woken tasks. +""" notify(c::Condition, arg::ANY=nothing; all=true, error=false) = notify(c, arg, all, error) function notify(c::Condition, arg, all, error) if all @@ -42,6 +59,11 @@ notify1_error(c::Condition, err) = notify(c, err, error=true, all=false) # schedule an expression to run asynchronously, with minimal ceremony +""" + @schedule + +Wrap an expression in a `Task` and add it to the local machine's scheduler queue. +""" macro schedule(expr) expr = :(()->($expr)) :(enq_work(Task($(esc(expr))))) @@ -61,6 +83,16 @@ end schedule(t::Task) = enq_work(t) +""" + schedule(t::Task, [val]; error=false) + +Add a task to the scheduler's queue. This causes the task to run constantly when the system +is otherwise idle, unless the task performs a blocking operation such as `wait`. + +If a second argument `val` is provided, it will be passed to the task (via the return value of +`yieldto`) when it runs again. If `error` is `true`, the value is raised as an exception in +the woken task. +""" function schedule(t::Task, arg; error=false) # schedule a task to be (re)started with the given value or exception if error @@ -84,8 +116,23 @@ function schedule_and_wait(t::Task, v=nothing) return wait() end +""" + yield() + +Switch to the scheduler to allow another scheduled task to run. A task that calls this +function is still runnable, and will be restarted immediately if there are no other runnable +tasks. +""" yield() = (enq_work(current_task()); wait()) +""" + yieldto(task, arg = nothing) + +Switch to the given task. The first time a task is switched to, the task's function is +called with no arguments. On subsequent switches, `arg` is returned from the task's last +call to `yieldto`. This is a low-level call that only switches tasks, not considering states +or scheduling in any way. Its use is discouraged. +""" yieldto(t::Task, x::ANY = nothing) = ccall(:jl_switchto, Any, (Any, Any), t, x) # yield to a task, throwing an exception in it diff --git a/base/task.jl b/base/task.jl index 148b77861cc58..cc06115a23554 100644 --- a/base/task.jl +++ b/base/task.jl @@ -50,17 +50,34 @@ function show(io::IO, t::Task) print(io, "Task ($(t.state)) @0x$(hex(convert(UInt, pointer_from_objref(t)), Sys.WORD_SIZE>>2))") end +""" + @task + +Wrap an expression in a [`Task`](:class:`Task`) without executing it, and return the [`Task`](:class:`Task`). This only +creates a task, and does not run it. +""" macro task(ex) :(Task(()->$(esc(ex)))) end +""" + current_task() + +Get the currently running [`Task`](:class:`Task`). +""" current_task() = ccall(:jl_get_current_task, Ref{Task}, ()) + +""" + istaskdone(task) -> Bool + +Determine whether a task has exited. +""" istaskdone(t::Task) = ((t.state == :done) | (t.state == :failed)) """ istaskstarted(task) -> Bool -Tell whether a task has started executing. +Determine whether a task has started executing. """ istaskstarted(t::Task) = ccall(:jl_is_task_started, Cint, (Any,), t) != 0 @@ -71,9 +88,28 @@ function get_task_tls(t::Task) end (t.storage)::ObjectIdDict end + +""" + task_local_storage(key) + +Look up the value of a key in the current task's task-local storage. +""" task_local_storage(key) = task_local_storage()[key] + +""" + task_local_storage(key, value) + +Assign a value to a key in the current task's task-local storage. +""" task_local_storage(key, val) = (task_local_storage()[key] = val) +""" + task_local_storage(body, key, value) + +Call the function `body` with a modified task-local storage, in which `value` is assigned to +`key`; the previous value of `key`, or lack thereof, is restored afterwards. Useful +for emulating dynamic scoping. +""" function task_local_storage(body::Function, key, val) tls = task_local_storage() hadkey = haskey(tls,key) @@ -277,6 +313,13 @@ function sync_end() nothing end +""" + @sync + +Wait until all dynamically-enclosed uses of `@async`, `@spawn`, `@spawnat` and `@parallel` +are complete. All exceptions thrown by enclosed async operations are collected and thrown as +a `CompositeException`. +""" macro sync(block) quote sync_begin() @@ -305,6 +348,14 @@ function async_run_thunk(thunk) t end +""" + @async + +Like `@schedule`, `@async` wraps an expression in a `Task` and adds it to the local +machine's scheduler queue. Additionally it adds the task to the set of items that the +nearest enclosing `@sync` waits for. `@async` also wraps the expression in a `let x=x, y=y, ...` +block to create a new scope with copies of all variables referenced in the expression. +""" macro async(expr) expr = localize_vars(esc(:(()->($expr))), false) :(async_run_thunk($expr)) diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index f7759523d99fb..9aeaeb22d096f 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -23,19 +23,19 @@ Tasks .. Docstring generated from Julia source - Get the currently running ``Task``\ . + Get the currently running :class:`Task`\ . .. function:: istaskdone(task) -> Bool .. Docstring generated from Julia source - Tell whether a task has exited. + Determine whether a task has exited. .. function:: istaskstarted(task) -> Bool .. Docstring generated from Julia source - Tell whether a task has started executing. + Determine whether a task has started executing. .. function:: consume(task, values...) @@ -55,29 +55,29 @@ Tasks Switch to the scheduler to allow another scheduled task to run. A task that calls this function is still runnable, and will be restarted immediately if there are no other runnable tasks. -.. function:: task_local_storage(symbol) +.. function:: task_local_storage(key) .. Docstring generated from Julia source - Look up the value of a symbol in the current task's task-local storage. + Look up the value of a key in the current task's task-local storage. -.. function:: task_local_storage(symbol, value) +.. function:: task_local_storage(key, value) .. Docstring generated from Julia source - Assign a value to a symbol in the current task's task-local storage. + Assign a value to a key in the current task's task-local storage. -.. function:: task_local_storage(body, symbol, value) +.. function:: task_local_storage(body, key, value) .. Docstring generated from Julia source - Call the function ``body`` with a modified task-local storage, in which ``value`` is assigned to ``symbol``\ ; the previous value of ``symbol``\ , or lack thereof, is restored afterwards. Useful for emulating dynamic scoping. + Call the function ``body`` with a modified task-local storage, in which ``value`` is assigned to ``key``\ ; the previous value of ``key``\ , or lack thereof, is restored afterwards. Useful for emulating dynamic scoping. .. function:: Condition() .. Docstring generated from Julia source - Create an edge-triggered event source that tasks can wait for. Tasks that call ``wait`` on a ``Condition`` are suspended and queued. Tasks are woken up when ``notify`` is later called on the ``Condition``\ . Edge triggering means that only tasks waiting at the time ``notify`` is called can be woken up. For level-triggered notifications, you must keep extra state to keep track of whether a notification has happened. The ``Channel`` type does this, and so can be used for level-triggered events. + Create an edge-triggered event source that tasks can wait for. Tasks that call ``wait`` on a ``Condition`` are suspended and queued. Tasks are woken up when ``notify`` is later called on the ``Condition``\ . Edge triggering means that only tasks waiting at the time ``notify`` is called can be woken up. For level-triggered notifications, you must keep extra state to keep track of whether a notification has happened. The :class:`Channel` type does this, and so can be used for level-triggered events. .. function:: notify(condition, val=nothing; all=true, error=false) @@ -91,7 +91,7 @@ Tasks Add a task to the scheduler's queue. This causes the task to run constantly when the system is otherwise idle, unless the task performs a blocking operation such as ``wait``\ . - If a second argument is provided, it will be passed to the task (via the return value of ``yieldto``\ ) when it runs again. If ``error`` is ``true``\ , the value is raised as an exception in the woken task. + If a second argument ``val`` is provided, it will be passed to the task (via the return value of ``yieldto``\ ) when it runs again. If ``error`` is ``true``\ , the value is raised as an exception in the woken task. .. function:: @schedule @@ -103,7 +103,7 @@ Tasks .. Docstring generated from Julia source - Wrap an expression in a ``Task`` without executing it, and return the ``Task``\ . This only creates a task, and does not run it. + Wrap an expression in a :class:`Task` without executing it, and return the :class:`Task`\ . This only creates a task, and does not run it. .. function:: sleep(seconds) From 09ab79c83597148f2d6ecbd75f2e1f24af8582bc Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Sat, 6 Aug 2016 10:16:39 -0700 Subject: [PATCH 0833/1117] More trig docstring cleanout --- base/docs/helpdb/Base.jl | 42 ---------------------------------------- base/special/trig.jl | 9 ++++++--- 2 files changed, 6 insertions(+), 45 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 82604311cd1cc..463fac30e923e 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -976,13 +976,6 @@ seeding. """ srand -""" - acot(x) - -Compute the inverse cotangent of `x`, where the output is in radians. -""" -acot - """ oftype(x, y) @@ -2640,13 +2633,6 @@ Get the concrete type of `x`. """ typeof -""" - acsc(x) - -Compute the inverse cosecant of `x`, where the output is in radians -""" -acsc - """ log(x) @@ -3059,13 +3045,6 @@ Element-wise less-than-or-equals comparison operator. """ Base.:(.<=) -""" - asec(x) - -Compute the inverse secant of `x`, where the output is in radians. -""" -asec - """ rank(M) @@ -4925,13 +4904,6 @@ recurses infinitely. """ StackOverflowError -""" - acsch(x) - -Compute the inverse hyperbolic cosecant of `x`. -""" -acsch - """ process_running(p::Process) @@ -5689,13 +5661,6 @@ Read a UDP packet from the specified socket, and return the bytes received. This """ recv -""" - acoth(x) - -Compute the inverse hyperbolic cotangent of `x`. -""" -acoth - """ det(M) @@ -6038,13 +6003,6 @@ the integrity and correctness of data read from `stream`. """ deserialize -""" - asech(x) - -Compute the inverse hyperbolic secant of `x`. -""" -asech - """ ismarked(s) diff --git a/base/special/trig.jl b/base/special/trig.jl index 087f2ad0338e6..fa4c33717f784 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -319,10 +319,13 @@ for (finv, f) in ((:sec, :cos), (:csc, :sin), (:cot, :tan), end end -for (fa, fainv) in ((:asec, :acos), (:acsc, :asin), (:acot, :atan), - (:asech, :acosh), (:acsch, :asinh), (:acoth, :atanh)) +for (fa, fainv, fn) in ((:asec, :acos, "secant"), (:acsc, :asin, "cosecant"), (:acot, :atan, "cotangent"), + (:asech, :acosh, "hyperbolic secant"), (:acsch, :asinh, "hyperbolic cosecant"), (:acoth, :atanh, "hyperbolic cotangent")) + name = string(fa) @eval begin - ($fa){T<:Number}(y::T) = ($fainv)(one(T) / y) + @doc """ + $($name)(x) + Compute the $($fn) of `x`, where the output is in radians. """ ($fa){T<:Number}(y::T) = ($fainv)(one(T) / y) ($fa){T<:Number}(y::AbstractArray{T}) = ($fainv)(one(T) ./ y) end end From a5812737a898253e96200cfc65141fede1826ac8 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Sat, 6 Aug 2016 19:26:25 -0700 Subject: [PATCH 0834/1117] Fix a failing doctest --- doc/manual/integers-and-floating-point-numbers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index 9b37bff26eb08..0e79f2feb4952 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -317,7 +317,7 @@ only as a storage format. In calculations they'll be converted to ``Float32``: 2 julia> 2*Float16(4.) - 8.0f0 + Float16(8.0) The underscore ``_`` can be used as digit separator: From 79b52d8a5d7c6fde87ae8723ec16545848a32ae1 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sun, 7 Aug 2016 01:19:58 -0400 Subject: [PATCH 0835/1117] move the typecache and inference locks to protect more of their global state --- base/inference.jl | 6 ++++-- doc/devdocs/locks.rst | 6 ------ src/jltypes.c | 23 +++++++++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index aa97ef4f710f5..4022da0726a60 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1487,6 +1487,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr linfo = specialize_method(method, atypes, sparams, cached) end + ccall(:jl_typeinf_begin, Void, ()) # XXX: the following logic is likely subtly broken if code.code was nothing, # although it seems unlikely something bad (infinite recursion) will happen as a result if linfo.inInference @@ -1529,6 +1530,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr end end typeinf_loop(frame) + ccall(:jl_typeinf_end, Void, ()) return (frame.linfo, widenconst(frame.bestguess), frame.inferred) end @@ -1576,8 +1578,10 @@ function typeinf_ext(linfo::LambdaInfo) else # toplevel lambda - infer directly linfo.inInference = true + ccall(:jl_typeinf_begin, Void, ()) frame = InferenceState(linfo, true, inlining_enabled(), true) typeinf_loop(frame) + ccall(:jl_typeinf_end, Void, ()) @assert frame.inferred # TODO: deal with this better return linfo end @@ -1591,7 +1595,6 @@ function typeinf_loop(frame) frame.inworkq || typeinf_frame(frame) return end - ccall(:jl_typeinf_begin, Void, ()) try in_typeinf_loop = true # the core type-inference algorithm @@ -1638,7 +1641,6 @@ function typeinf_loop(frame) println(ex) ccall(:jlbacktrace, Void, ()) end - ccall(:jl_typeinf_end, Void, ()) nothing end diff --git a/doc/devdocs/locks.rst b/doc/devdocs/locks.rst index 191312d811323..ea37560bd2eaf 100644 --- a/doc/devdocs/locks.rst +++ b/doc/devdocs/locks.rst @@ -118,12 +118,6 @@ The following locks are broken: fix: create it -* typecache - - this only protects cache lookup and insertion but it doesn't make the lookup-and-construct atomic - - fix: lock for ``apply_type`` / global (level 2?) - Shared Global Data Structures ----------------------------- diff --git a/src/jltypes.c b/src/jltypes.c index d4977719145b9..74f54e98eb768 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1997,10 +1997,8 @@ static jl_value_t *lookup_type(jl_typename_t *tn, jl_value_t **key, size_t n) { JL_TIMING(TYPE_CACHE_LOOKUP); int ord = is_typekey_ordered(key, n); - JL_LOCK(&typecache_lock); // Might GC ssize_t idx = lookup_type_idx(tn, key, n, ord); jl_value_t *t = (idx < 0) ? NULL : jl_svecref(ord ? tn->cache : tn->linearcache, idx); - JL_UNLOCK(&typecache_lock); // Might GC return t; } @@ -2071,14 +2069,12 @@ jl_value_t *jl_cache_type_(jl_datatype_t *type) if (is_cacheable(type)) { JL_TIMING(TYPE_CACHE_INSERT); int ord = is_typekey_ordered(jl_svec_data(type->parameters), jl_svec_len(type->parameters)); - JL_LOCK(&typecache_lock); // Might GC ssize_t idx = lookup_type_idx(type->name, jl_svec_data(type->parameters), jl_svec_len(type->parameters), ord); if (idx >= 0) type = (jl_datatype_t*)jl_svecref(ord ? type->name->cache : type->name->linearcache, idx); else cache_insert_type((jl_value_t*)type, ~idx, ord); - JL_UNLOCK(&typecache_lock); // Might GC } return (jl_value_t*)type; } @@ -2163,13 +2159,18 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i int istuple = (tn == jl_tuple_typename); // check type cache if (cacheable) { + JL_LOCK(&typecache_lock); // Might GC jl_value_t *lkup = (jl_value_t*)lookup_type(tn, iparams, ntp); - if (lkup != NULL) + if (lkup != NULL) { + JL_UNLOCK(&typecache_lock); // Might GC return lkup; + } } jl_value_t *stack_lkup = lookup_type_stack(stack, dt, ntp, iparams); - if (stack_lkup) + if (stack_lkup) { + if (cacheable) JL_UNLOCK(&typecache_lock); // Might GC return stack_lkup; + } if (istuple && ntp > 0 && jl_is_vararg_type(iparams[ntp - 1])) { // normalize Tuple{..., Vararg{Int, 3}} to Tuple{..., Int, Int, Int} @@ -2180,6 +2181,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i jl_errorf("apply_type: Vararg length N is negative: %zd", nt); va = jl_tparam0(va); if (nt == 0 || !jl_has_typevars(va)) { + if (cacheable) JL_UNLOCK(&typecache_lock); // Might GC if (ntp == 1) return jl_tupletype_fill(nt, va); size_t i, l; @@ -2206,10 +2208,13 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i jl_type_error_rt("apply_type", "Vararg count", (jl_value_t*)jl_long_type, iparams[1]); } } - if (tc != (jl_value_t*)dt) + if (tc != (jl_value_t*)dt) { + if (cacheable) JL_UNLOCK(&typecache_lock); // Might GC return (jl_value_t*)jl_apply_type_(tc, iparams, ntp); + } } else if (ntp == 0 && jl_emptytuple != NULL) { + if (cacheable) JL_UNLOCK(&typecache_lock); // Might GC return jl_typeof(jl_emptytuple); } @@ -2299,8 +2304,10 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i else ndt->ninitialized = dt->ninitialized; - if (cacheable) + if (cacheable) { jl_cache_type_(ndt); + JL_UNLOCK(&typecache_lock); // Might GC + } JL_GC_POP(); return (jl_value_t*)ndt; From d891d57023ca6dccd2bc659f066f5279a701efb0 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer+github@college.harvard.edu> Date: Sun, 7 Aug 2016 06:45:29 +0000 Subject: [PATCH 0836/1117] Test LibGit2 SSH authentication (#17651) --- base/libgit2/callbacks.jl | 17 ++-- base/libgit2/error.jl | 1 + test/TestHelpers.jl | 27 ++++++ test/choosetests.jl | 2 +- test/libgit2.jl | 177 ++++++++++++++++++++++++++++++++++++++ test/repl.jl | 51 ++++------- 6 files changed, 231 insertions(+), 44 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 81aa56c56d78a..0143125c3c2f7 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -64,9 +64,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, else keydefpath = creds.prvkey # check if credentials were already used keydefpath === nothing && (keydefpath = "") - if !isempty(keydefpath) && !isusedcreds - keydefpath # use cached value - else + if isempty(keydefpath) || isusedcreds defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") if isempty(keydefpath) && isfile(defaultkeydefpath) keydefpath = defaultkeydefpath @@ -75,6 +73,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, prompt("Private key location for '$schema$username@$host'", default=keydefpath) end end + keydefpath end # If the private key changed, invalidate the cached public key @@ -87,18 +86,16 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, ENV["SSH_PUB_KEY_PATH"] else keydefpath = creds.pubkey # check if credentials were already used - if keydefpath !== nothing && !isusedcreds - keydefpath # use cached value - else - if keydefpath === nothing || isempty(keydefpath) + keydefpath === nothing && (keydefpath = "") + if isempty(keydefpath) || isusedcreds + if isempty(keydefpath) keydefpath = privatekey*".pub" end - if isfile(keydefpath) - keydefpath - else + if !isfile(keydefpath) prompt("Public key location for '$schema$username@$host'", default=keydefpath) end end + keydefpath end creds.pubkey = publickey # save credentials diff --git a/base/libgit2/error.jl b/base/libgit2/error.jl index 823b4e3907ded..1d6c904a62e57 100644 --- a/base/libgit2/error.jl +++ b/base/libgit2/error.jl @@ -23,6 +23,7 @@ export GitError ECERTIFICATE = Cint(-17), # server certificate is invalid EAPPLIED = Cint(-18), # patch/merge has already been applied EPEEL = Cint(-19), # the requested peel operation is not possible + EEOF = Cint(-20), # Unexpted EOF PASSTHROUGH = Cint(-30), # internal only ITEROVER = Cint(-31)) # signals end of iteration diff --git a/test/TestHelpers.jl b/test/TestHelpers.jl index 609962e68e389..b57b0d10f77ee 100644 --- a/test/TestHelpers.jl +++ b/test/TestHelpers.jl @@ -16,4 +16,31 @@ Base.Terminals.hascolor(t::FakeTerminal) = t.hascolor Base.Terminals.raw!(t::FakeTerminal, raw::Bool) = t.raw = raw Base.Terminals.size(t::FakeTerminal) = (24, 80) +function open_fake_pty() + const O_RDWR = Base.Filesystem.JL_O_RDWR + const O_NOCTTY = Base.Filesystem.JL_O_NOCTTY + + fdm = ccall(:posix_openpt, Cint, (Cint,), O_RDWR|O_NOCTTY) + fdm == -1 && error("Failed to open PTY master") + rc = ccall(:grantpt, Cint, (Cint,), fdm) + rc != 0 && error("grantpt failed") + rc = ccall(:unlockpt, Cint, (Cint,), fdm) + rc != 0 && error("unlockpt") + + fds = ccall(:open, Cint, (Ptr{UInt8}, Cint), + ccall(:ptsname, Ptr{UInt8}, (Cint,), fdm), O_RDWR|O_NOCTTY) + + # slave + slave = RawFD(fds) + master = Base.TTY(RawFD(fdm); readable = true) + slave, master +end + +function with_fake_pty(f) + slave, master = open_fake_pty() + f(slave, master) + ccall(:close,Cint,(Cint,),slave) # XXX: this causes the kernel to throw away all unread data on the pty + close(master) +end + end diff --git a/test/choosetests.jl b/test/choosetests.jl index 9484d0e394f7c..d02ed1dcf3cf0 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -80,7 +80,7 @@ function choosetests(choices = []) prepend!(tests, linalgtests) end - net_required_for = ["socket", "parallel"] + net_required_for = ["socket", "parallel", "libgit2"] net_on = true try getipaddr() diff --git a/test/libgit2.jl b/test/libgit2.jl index c5ba7a9082e9d..a0466c21c7e15 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,6 +2,9 @@ #@testset "libgit2" begin +isdefined(:TestHelpers) || include(joinpath(dirname(@__FILE__), "TestHelpers.jl")) +using TestHelpers + const LIBGIT2_MIN_VER = v"0.23.0" ######### @@ -567,6 +570,180 @@ mktempdir() do dir @test creds.user == creds_user @test creds.pass == creds_pass #end + + #@testset "SSH" begin + sshd_command = "" + ssh_repo = joinpath(dir, "Example.SSH") + if !is_windows() + try + # SSHD needs to be executed by its full absolute path + sshd_command = strip(readstring(`which sshd`)) + catch + warn("Skipping SSH tests (Are `which` and `sshd` installed?)") + end + end + if !isempty(sshd_command) + mktempdir() do fakehomedir + mkdir(joinpath(fakehomedir,".ssh")) + # Unsetting the SSH agent serves two purposes. First, we make + # sure that we don't accidentally pick up an existing agent, + # and second we test that we fall back to using a key file + # if the agent isn't present. + withenv("HOME"=>fakehomedir,"SSH_AUTH_SOCK"=>nothing) do + # Generate user file, first an unencrypted one + wait(spawn(`ssh-keygen -N "" -C juliatest@localhost -f $fakehomedir/.ssh/id_rsa`)) + + # Generate host keys + wait(spawn(`ssh-keygen -f $fakehomedir/ssh_host_rsa_key -N '' -t rsa`)) + wait(spawn(`ssh-keygen -f $fakehomedir/ssh_host_dsa_key -N '' -t dsa`)) + + our_ssh_port = rand(13000:14000) # Chosen arbitrarily + + key_option = "AuthorizedKeysFile $fakehomedir/.ssh/id_rsa.pub" + pidfile_option = "PidFile $fakehomedir/sshd.pid" + sshp = agentp = nothing + logfile = tempname() + ssh_debug = false + function spawn_sshd() + debug_flags = ssh_debug ? `-d -d` : `` + _p = open(logfile, "a") do logfilestream + spawn(pipeline(pipeline(`$sshd_command + -e -f /dev/null $debug_flags + -h $fakehomedir/ssh_host_rsa_key + -h $fakehomedir/ssh_host_dsa_key -p $our_ssh_port + -o $pidfile_option + -o 'Protocol 2' + -o $key_option + -o 'UsePrivilegeSeparation no' + -o 'StrictModes no'`,STDOUT),stderr=logfilestream)) + end + # Give the SSH server 5 seconds to start up + yield(); sleep(5) + _p + end + sshp = spawn_sshd() + + TIOCSCTTY_str = "ccall(:ioctl, Void, (Cint, Cint, Int64), 0, + (is_bsd() || is_apple()) ? 0x20007461 : is_linux() ? 0x540E : + error(\"Fill in TIOCSCTTY for this OS here\"), 0)" + + # To fail rather than hang + function killer_task(p, master) + @async begin + sleep(10) + kill(p) + if isopen(master) + nb_available(master) > 0 && + write(logfile, + readavailable(master)) + close(master) + end + end + end + + try + function try_clone(challenges = []) + cmd = """ + repo = nothing + try + $TIOCSCTTY_str + reponame = "ssh://$(ENV["USER"])@localhost:$our_ssh_port$cache_repo" + repo = LibGit2.clone(reponame, "$ssh_repo") + catch err + open("$logfile","a") do f + println(f,"HOME: ",ENV["HOME"]) + println(f, err) + end + finally + finalize(repo) + end + """ + # We try to be helpful by desparately looking for + # a way to prompt the password interactively. Pretend + # to be a TTY to suppress those shenanigans. Further, we + # need to detach and change the controlling terminal with + # TIOCSCTTY, since getpass opens the controlling terminal + TestHelpers.with_fake_pty() do slave, master + err = Base.Pipe() + let p = spawn(detach( + `$(Base.julia_cmd()) --startup-file=no -e $cmd`),slave,slave,STDERR) + killer_task(p, master) + for (challenge, response) in challenges + readuntil(master, challenge) + sleep(1) + print(master, response) + end + sleep(2) + wait(p) + close(master) + end + end + @test isfile(joinpath(ssh_repo,"testfile")) + rm(ssh_repo, recursive = true) + end + + # Should use the default files, no interaction required. + try_clone() + ssh_debug && (kill(sshp); sshp = spawn_sshd()) + + # Ok, now encrypt the file and test with that (this also + # makes sure that we don't accidentally fall back to the + # unencrypted version) + wait(spawn(`ssh-keygen -p -N "xxxxx" -f $fakehomedir/.ssh/id_rsa`)) + + # Try with the encrypted file. Needs a password. + try_clone(["Passphrase"=>"xxxxx\r\n"]) + ssh_debug && (kill(sshp); sshp = spawn_sshd()) + + # Move the file. It should now ask for the location and + # then the passphrase + mv("$fakehomedir/.ssh/id_rsa","$fakehomedir/.ssh/id_rsa2") + cp("$fakehomedir/.ssh/id_rsa.pub","$fakehomedir/.ssh/id_rsa2.pub") + try_clone(["location"=>"$fakehomedir/.ssh/id_rsa2\n", + "Passphrase"=>"xxxxx\n"]) + mv("$fakehomedir/.ssh/id_rsa2","$fakehomedir/.ssh/id_rsa") + rm("$fakehomedir/.ssh/id_rsa2.pub") + + # Ok, now start an agent + agent_sock = tempname() + agentp = spawn(`ssh-agent -a $agent_sock -d`) + while stat(agent_sock).mode == 0 # Wait until the agent is started + sleep(1) + end + + # fake pty is required for the same reason as in try_clone + # above + withenv("SSH_AUTH_SOCK" => agent_sock) do + TestHelpers.with_fake_pty() do slave, master + cmd = """ + $TIOCSCTTY_str + run(pipeline(`ssh-add $fakehomedir/.ssh/id_rsa`, + stderr = DevNull)) + """ + addp = spawn(detach(`$(Base.julia_cmd()) --startup-file=no -e $cmd`), + slave, slave, STDERR) + killer_task(addp, master) + sleep(2) + write(master, "xxxxx\n") + wait(addp) + end + + # Should now use the agent + try_clone() + end + catch err + println("SSHD logfile contents follows:") + println(readstring(logfile)) + rethrow(err) + finally + rm(logfile) + sshp !== nothing && kill(sshp) + agentp !== nothing && kill(agentp) + end + end + end + end + #end end #end diff --git a/test/repl.jl b/test/repl.jl index f75411b165b58..09653bcb8422a 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -444,40 +444,25 @@ let exename = Base.julia_cmd() # Test REPL in dumb mode if !is_windows() - const O_RDWR = Base.Filesystem.JL_O_RDWR - const O_NOCTTY = Base.Filesystem.JL_O_NOCTTY - - fdm = ccall(:posix_openpt, Cint, (Cint,), O_RDWR|O_NOCTTY) - fdm == -1 && error("Failed to open PTY master") - rc = ccall(:grantpt, Cint, (Cint,), fdm) - rc != 0 && error("grantpt failed") - rc = ccall(:unlockpt, Cint, (Cint,), fdm) - rc != 0 && error("unlockpt") - - fds = ccall(:open, Cint, (Ptr{UInt8}, Cint), - ccall(:ptsname, Ptr{UInt8}, (Cint,), fdm), O_RDWR|O_NOCTTY) - - # slave - slave = RawFD(fds) - master = Base.TTY(RawFD(fdm); readable = true) - - nENV = copy(ENV) - nENV["TERM"] = "dumb" - p = spawn(setenv(`$exename --startup-file=no --quiet`,nENV),slave,slave,slave) - output = readuntil(master,"julia> ") - if ccall(:jl_running_on_valgrind,Cint,()) == 0 - # If --trace-children=yes is passed to valgrind, we will get a - # valgrind banner here, not just the prompt. - @test output == "julia> " + TestHelpers.with_fake_pty() do slave, master + + nENV = copy(ENV) + nENV["TERM"] = "dumb" + p = spawn(setenv(`$exename --startup-file=no --quiet`,nENV),slave,slave,slave) + output = readuntil(master,"julia> ") + if ccall(:jl_running_on_valgrind,Cint,()) == 0 + # If --trace-children=yes is passed to valgrind, we will get a + # valgrind banner here, not just the prompt. + @test output == "julia> " + end + write(master,"1\nquit()\n") + + wait(p) + output = readuntil(master,' ') + @test output == "1\r\nquit()\r\n1\r\n\r\njulia> " + @test nb_available(master) == 0 + end - write(master,"1\nquit()\n") - - wait(p) - output = readuntil(master,' ') - @test output == "1\r\nquit()\r\n1\r\n\r\njulia> " - @test nb_available(master) == 0 - ccall(:close,Cint,(Cint,),fds) # XXX: this causes the kernel to throw away all unread data on the pty - close(master) end # Test stream mode From a004634d855e95ad1b868e4dee3021b638e3c4d6 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 7 Aug 2016 02:03:01 -0500 Subject: [PATCH 0837/1117] Fix copy(region::Tuple) depwarn in FFTW (#17840) --- base/fft/FFTW.jl | 10 +++++----- test/fft.jl | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/base/fft/FFTW.jl b/base/fft/FFTW.jl index 509081c23b60d..fba05d2b3b3fa 100644 --- a/base/fft/FFTW.jl +++ b/base/fft/FFTW.jl @@ -459,7 +459,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), region, flags::Integer, timelimit::Real) direction = K set_timelimit($Tr, timelimit) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) dims, howmany = dims_howmany(X, Y, [size(X)...], R) plan = ccall(($(string(fftw,"_plan_guru64_dft")),$lib), PlanPtr, @@ -477,7 +477,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), @eval function (::Type{rFFTWPlan{$Tr,$FORWARD,inplace,N}}){inplace,N}(X::StridedArray{$Tr,N}, Y::StridedArray{$Tc,N}, region, flags::Integer, timelimit::Real) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) region = circshift([region...],-1) # FFTW halves last dim set_timelimit($Tr, timelimit) dims, howmany = dims_howmany(X, Y, [size(X)...], region) @@ -497,7 +497,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), @eval function (::Type{rFFTWPlan{$Tc,$BACKWARD,inplace,N}}){inplace,N}(X::StridedArray{$Tc,N}, Y::StridedArray{$Tr,N}, region, flags::Integer, timelimit::Real) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) region = circshift([region...],-1) # FFTW halves last dim set_timelimit($Tr, timelimit) dims, howmany = dims_howmany(X, Y, [size(Y)...], region) @@ -518,7 +518,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), Y::StridedArray{$Tr,N}, region, kinds, flags::Integer, timelimit::Real) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) knd = fix_kinds(region, kinds) set_timelimit($Tr, timelimit) dims, howmany = dims_howmany(X, Y, [size(X)...], region) @@ -540,7 +540,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), Y::StridedArray{$Tc,N}, region, kinds, flags::Integer, timelimit::Real) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) knd = fix_kinds(region, kinds) set_timelimit($Tr, timelimit) dims, howmany = dims_howmany(X, Y, [size(X)...], region) diff --git a/test/fft.jl b/test/fft.jl index 6aca96d874a02..6905bc85ffb30 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -3,6 +3,9 @@ # fft a = rand(8) + im*rand(8) @test norm(ifft(fft(a)) - a) < 1e-8 +@test norm(ifft(fft(a,1),1) - a) < 1e-8 +@test norm(ifft(fft(a,[1]),[1]) - a) < 1e-8 +@test norm(ifft(fft(a,(1,)),(1,)) - a) < 1e-8 m4 = [16. 2 3 13; 5 11 10 8; From d19e9cad2f62b9f57eeea925dd7eea05c17ac53c Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner <pfitzseb@physik.hu-berlin.de> Date: Sun, 7 Aug 2016 10:19:33 +0200 Subject: [PATCH 0838/1117] fix less on windows by using `more` as the default and escaping `file`. --- base/interactiveutil.jl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 7ae43796fedf0..24b5f49f727a6 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -70,9 +70,17 @@ edit(file, line::Integer) = error("could not find source file for function") # terminal pager -function less(file::AbstractString, line::Integer) - pager = get(ENV, "PAGER", "less") - run(`$pager +$(line)g $file`) +if is_windows() + function less(file::AbstractString, line::Integer) + pager = get(ENV, "PAGER", "more") + g = pager == "more" ? "" : "g" + run(Cmd(`$pager +$(line)$(g) \"$file\"`, windows_verbatim = true)) + end +else + function less(file::AbstractString, line::Integer) + pager = get(ENV, "PAGER", "less") + run(`$pager +$(line)g $file`) + end end less(file::AbstractString) = less(file, 1) From f2b892b0b59b8ed3b946b5a4d7ea07b23b3ec465 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Sun, 7 Aug 2016 16:35:56 -0700 Subject: [PATCH 0839/1117] Remove dupe digamma, fix docstrings --- base/docs/helpdb/Base.jl | 7 ------- base/special/trig.jl | 18 ++++++++++++------ doc/stdlib/math.rst | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 463fac30e923e..e84547a23a682 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -9,13 +9,6 @@ Raises a `SystemError` for `errno` with the descriptive string `sysfunc` if `ift """ systemerror -""" - digamma(x) - -Compute the digamma function of `x` (the logarithmic derivative of `gamma(x)`) -""" -digamma - """ fill!(A, x) diff --git a/base/special/trig.jl b/base/special/trig.jl index fa4c33717f784..2ad94211c590f 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -319,14 +319,20 @@ for (finv, f) in ((:sec, :cos), (:csc, :sin), (:cot, :tan), end end -for (fa, fainv, fn) in ((:asec, :acos, "secant"), (:acsc, :asin, "cosecant"), (:acot, :atan, "cotangent"), - (:asech, :acosh, "hyperbolic secant"), (:acsch, :asinh, "hyperbolic cosecant"), (:acoth, :atanh, "hyperbolic cotangent")) - name = string(fa) +for (tfa, tfainv, hfa, hfainv, fn) in ((:asec, :acos, :asech, :acosh, "secant"), + (:acsc, :asin, :acsch, :asinh, "cosecant"), + (:acot, :atan, :acoth, :atahn, "cotangent")) + tname = string(tfa) + hname = string(hfa) @eval begin @doc """ - $($name)(x) - Compute the $($fn) of `x`, where the output is in radians. """ ($fa){T<:Number}(y::T) = ($fainv)(one(T) / y) - ($fa){T<:Number}(y::AbstractArray{T}) = ($fainv)(one(T) ./ y) + $($tname)(x) + Compute the inverse $($fn) of `x`, where the output is in radians. """ ($tfa){T<:Number}(y::T) = ($tfainv)(one(T) / y) + ($tfa){T<:Number}(y::AbstractArray{T}) = ($tfainv)(one(T) ./ y) + @doc """ + $($hname)(x) + Compute the inverse hyperbolic $($fn) of `x`. """ ($hfa){T<:Number}(y::T) = ($hfainv)(one(T) / y) + ($hfa){T<:Number}(y::AbstractArray{T}) = ($hfainv)(one(T) ./ y) end end diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 7c9a7c653e808..91b6c7d926867 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -664,7 +664,7 @@ Mathematical Functions .. Docstring generated from Julia source - Compute the inverse cosecant of ``x``\ , where the output is in radians + Compute the inverse cosecant of ``x``\ , where the output is in radians. .. function:: acot(x) From 7aaf6e93cd37a31d799507f254d94e5cf07b3bdb Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 8 Aug 2016 05:09:24 -0500 Subject: [PATCH 0840/1117] Faster, indices-aware circshift (and non-allocating circshift!) Fixes #16032, fixes #17581 --- base/abstractarraymath.jl | 22 ++++++++--------- base/exports.jl | 1 + base/multidimensional.jl | 51 +++++++++++++++++++++++++++++++++++++++ doc/stdlib/arrays.rst | 16 ++++++++++-- test/arrayops.jl | 9 +++++++ test/offsetarray.jl | 1 + 6 files changed, 86 insertions(+), 14 deletions(-) diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 7f95b1f1651ea..8abe1b19d86f9 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -158,8 +158,10 @@ function flipdim(A::AbstractArray, d::Integer) return B end -circshift(a::AbstractArray, shiftamt::Real) = circshift(a, [Integer(shiftamt)]) - +function circshift(a::AbstractArray, shiftamt::Real) + circshift!(similar(a), a, (Integer(shiftamt),)) +end +circshift(a::AbstractArray, shiftamt::DimsInteger) = circshift!(similar(a), a, shiftamt) """ circshift(A, shifts) @@ -174,29 +176,25 @@ julia> b = reshape(collect(1:16), (4,4)) 3 7 11 15 4 8 12 16 -julia> circshift(b, [0,2]) +julia> circshift(b, (0,2)) 4×4 Array{Int64,2}: 9 13 1 5 10 14 2 6 11 15 3 7 12 16 4 8 -julia> circshift(b, [-1,0]) +julia> circshift(b, (-1,0)) 4×4 Array{Int64,2}: 2 6 10 14 3 7 11 15 4 8 12 16 1 5 9 13 ``` + +See also `circshift!`. """ -function circshift{T,N}(a::AbstractArray{T,N}, shiftamts) - I = () - for i=1:N - s = size(a,i) - d = i<=length(shiftamts) ? shiftamts[i] : 0 - I = tuple(I..., d==0 ? [1:s;] : mod([-d:s-1-d;], s).+1) - end - a[(I::NTuple{N,Vector{Int}})...] +function circshift(a::AbstractArray, shiftamt) + circshift!(similar(a), a, map(Integer, (shiftamt...,))) end # Uses K-B-N summation diff --git a/base/exports.jl b/base/exports.jl index b92dfe12af6b9..33fe8c6850852 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -490,6 +490,7 @@ export checkbounds, checkindex, circshift, + circshift!, clamp!, colon, conj!, diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 385e9e218f891..a6ef97fe3c87f 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -609,6 +609,57 @@ function copy!{T,N}(dest::AbstractArray{T,N}, src::AbstractArray{T,N}) dest end +function copy!(dest::AbstractArray, Rdest::CartesianRange, src::AbstractArray, Rsrc::CartesianRange) + isempty(Rdest) && return dest + size(Rdest) == size(Rsrc) || throw(ArgumentError("source and destination must have same size (got $(size(Rsrc)) and $(size(Rdest)))")) + @boundscheck checkbounds(dest, Rdest.start) + @boundscheck checkbounds(dest, Rdest.stop) + @boundscheck checkbounds(src, Rsrc.start) + @boundscheck checkbounds(src, Rsrc.stop) + deltaI = Rdest.start - Rsrc.start + for I in Rsrc + @inbounds dest[I+deltaI] = src[I] + end + dest +end + +# circshift! +circshift!(dest::AbstractArray, src, ::Tuple{}) = copy!(dest, src) +""" + circshift!(dest, src, shifts) + +Circularly shift the data in `src`, storing the result in +`dest`. `shifts` specifies the amount to shift in each dimension. + +The `dest` array must be distinct from the `src` array (they cannot +alias each other). + +See also `circshift`. +""" +@noinline function circshift!{T,N}(dest::AbstractArray{T,N}, src, shiftamt::DimsInteger) + dest === src && throw(ArgumentError("dest and src must be separate arrays")) + inds = indices(src) + indices(dest) == inds || throw(ArgumentError("indices of src and dest must match (got $inds and $(indices(dest)))")) + _circshift!(dest, (), src, (), inds, fill_to_length(shiftamt, 0, Val{N})) +end +circshift!(dest::AbstractArray, src, shiftamt) = circshift!(dest, src, (shiftamt...,)) + +@inline function _circshift!(dest, rdest, src, rsrc, + inds::Tuple{AbstractUnitRange,Vararg{Any}}, + shiftamt::Tuple{Integer,Vararg{Any}}) + ind1, d = inds[1], shiftamt[1] + s = mod(d, length(ind1)) + sf, sl = first(ind1)+s, last(ind1)-s + r1, r2 = first(ind1):sf-1, sf:last(ind1) + r3, r4 = first(ind1):sl, sl+1:last(ind1) + tinds, tshiftamt = tail(inds), tail(shiftamt) + _circshift!(dest, (rdest..., r1), src, (rsrc..., r4), tinds, tshiftamt) + _circshift!(dest, (rdest..., r2), src, (rsrc..., r3), tinds, tshiftamt) +end +# At least one of inds, shiftamt is empty +function _circshift!(dest, rdest, src, rsrc, inds, shiftamt) + copy!(dest, CartesianRange(rdest), src, CartesianRange(rsrc)) +end ### BitArrays diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index a475fd69b31ad..116433c9f3c5b 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -590,20 +590,32 @@ Indexing, Assignment, and Concatenation 3 7 11 15 4 8 12 16 - julia> circshift(b, [0,2]) + julia> circshift(b, (0,2)) 4×4 Array{Int64,2}: 9 13 1 5 10 14 2 6 11 15 3 7 12 16 4 8 - julia> circshift(b, [-1,0]) + julia> circshift(b, (-1,0)) 4×4 Array{Int64,2}: 2 6 10 14 3 7 11 15 4 8 12 16 1 5 9 13 + See also ``circshift!``\ . + +.. function:: circshift!(dest, src, shifts) + + .. Docstring generated from Julia source + + Circularly shift the data in ``src``\ , storing the result in ``dest``\ . ``shifts`` specifies the amount to shift in each dimension. + + The ``dest`` array must be distinct from the ``src`` array (they cannot alias each other). + + See also ``circshift``\ . + .. function:: find(A) .. Docstring generated from Julia source diff --git a/test/arrayops.jl b/test/arrayops.jl index 0d735436338c3..9fac28c0126ac 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -457,6 +457,15 @@ for i = tensors @test isequal(i,permutedims(ipermutedims(i,perm),perm)) end +## circshift + +@test circshift(1:5, -1) == circshift(1:5, 4) == circshift(1:5, -6) == [2,3,4,5,1] +@test circshift(1:5, 1) == circshift(1:5, -4) == circshift(1:5, 6) == [5,1,2,3,4] +a = [1:5;] +@test_throws ArgumentError Base.circshift!(a, a, 1) +b = copy(a) +@test Base.circshift!(b, a, 1) == [5,1,2,3,4] + ## unique across dim ## # All rows and columns unique diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 5ca538539ec6a..a509e17d3a8fb 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -411,6 +411,7 @@ v = OffsetArray(rand(8), (-2,)) @test rotr90(A) == OffsetArray(rotr90(parent(A)), A.offsets[[2,1]]) @test flipdim(A, 1) == OffsetArray(flipdim(parent(A), 1), A.offsets) @test flipdim(A, 2) == OffsetArray(flipdim(parent(A), 2), A.offsets) +@test circshift(A, (-1,2)) == OffsetArray(circshift(parent(A), (-1,2)), A.offsets) @test A+1 == OffsetArray(parent(A)+1, A.offsets) @test 2*A == OffsetArray(2*parent(A), A.offsets) From 0c1faa6d48f9695e0a9efc98900282aa3cf202f3 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman <benditlikeranjan@gmail.com> Date: Mon, 8 Aug 2016 16:46:22 +0530 Subject: [PATCH 0841/1117] Improve fft documentation (#17778) * Improve fft documentation * Remove whitespace * Use new notes syntax * Commit the generated .rst * Talk about cores, use CPU_CORES. * Committing generated rst (again) * `Other languages` -> `FFT libraries in other languages` * Remove redundant parts, add period * Link to section of manual * Fix doctests from changing line numbers * Fix doctests once more --- base/dft.jl | 12 ++++++++++-- doc/stdlib/math.rst | 9 +++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/base/dft.jl b/base/dft.jl index 6379968e022ea..04b9c5bf7875b 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -395,8 +395,16 @@ if Base.USE_GPL_LIBS A multidimensional FFT simply performs this operation along each transformed dimension of `A`. - Higher performance is usually possible with multi-threading. Use `FFTW.set_num_threads(np)` - to use `np` threads, if you have `np` processors. + !!! note + * Julia starts FFTW up with 1 thread by default. Higher performance is usually possible by + increasing number of threads. Use `FFTW.set_num_threads(Sys.CPU_CORES)` to use as many + threads as cores on your system. + + * This performs a multidimensional FFT by default. FFT libraries in other languages such as + Python and Octave perform a one-dimensional FFT along the first non-singleton dimension + of the array. This is worth noting while performing comparisons. For more details, + refer to the ["Noteworthy Differences from other Languages"](:ref:`man-noteworthy-differences`) + section of the manual. """ -> fft diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 91b6c7d926867..cf4a4ad72312d 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1805,9 +1805,7 @@ Signal Processing Fast Fourier transform (FFT) functions in Julia are implemented by calling functions from `FFTW -<http://www.fftw.org>`_. By default, Julia does not use multi-threaded -FFTW. Higher performance may be obtained by experimenting with -multi-threading. Use ``FFTW.set_num_threads(np)`` to use ``np`` threads. +<http://www.fftw.org>`_. .. function:: fft(A [, dims]) @@ -1826,7 +1824,10 @@ multi-threading. Use ``FFTW.set_num_threads(np)`` to use ``np`` threads. A multidimensional FFT simply performs this operation along each transformed dimension of ``A``\ . - Higher performance is usually possible with multi-threading. Use ``FFTW.set_num_threads(np)`` to use ``np`` threads, if you have ``np`` processors. + .. note:: + * Julia starts FFTW up with 1 thread by default. Higher performance is usually possible by increasing number of threads. Use ``FFTW.set_num_threads(Sys.CPU_CORES)`` to use as many threads as cores on your system. + * This performs a multidimensional FFT by default. FFT libraries in other languages such as Python and Octave perform a one-dimensional FFT along the first non-singleton dimension of the array. This is worth noting while performing comparisons. For more details, refer to the :ref:`man-noteworthy-differences` section of the manual. + .. function:: fft!(A [, dims]) From 2e9dbf2eb8fd592d028bb7b48a660a147467f616 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 8 Aug 2016 11:53:57 -0500 Subject: [PATCH 0842/1117] Support -I for I::CartesianIndex --- base/multidimensional.jl | 1 + test/arrayops.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index a6ef97fe3c87f..a6e22e481601b 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -48,6 +48,7 @@ one{N}(::CartesianIndex{N}) = one(CartesianIndex{N}) one{N}(::Type{CartesianIndex{N}}) = CartesianIndex(ntuple(x -> 1, Val{N})) # arithmetic, min/max +(-){N}(index::CartesianIndex{N}) = CartesianIndex{N}(map(-, index.I)) (+){N}(index1::CartesianIndex{N}, index2::CartesianIndex{N}) = CartesianIndex{N}(map(+, index1.I, index2.I)) (-){N}(index1::CartesianIndex{N}, index2::CartesianIndex{N}) = CartesianIndex{N}(map(-, index1.I, index2.I)) min{N}(index1::CartesianIndex{N}, index2::CartesianIndex{N}) = CartesianIndex{N}(map(min, index1.I, index2.I)) diff --git a/test/arrayops.jl b/test/arrayops.jl index 9fac28c0126ac..8feb2f410bfc6 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1261,6 +1261,7 @@ end I1 = CartesianIndex((2,3,0)) I2 = CartesianIndex((-1,5,2)) +@test -I1 == CartesianIndex((-2,-3,0)) @test I1 + I2 == CartesianIndex((1,8,2)) @test I2 + I1 == CartesianIndex((1,8,2)) @test I1 - I2 == CartesianIndex((3,-2,-2)) From d8fe0bf426f0808ee68f1b3d7ccb16393f1e0d3a Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 8 Aug 2016 11:54:17 -0500 Subject: [PATCH 0843/1117] Fix PermutedDimsArrays typo --- base/permuteddimsarray.jl | 2 +- test/arrayops.jl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index aa2944f7c4bb0..f940bd87e864b 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -17,7 +17,7 @@ immutable PermutedDimsArray{T,N,perm,iperm,AA<:AbstractArray} <: AbstractArray{T end function PermutedDimsArray{T,N}(data::AbstractArray{T,N}, perm) - length(perm) == N || throw(ArgumentError(string(p, " is not a valid permutation of dimensions 1:", N))) + length(perm) == N || throw(ArgumentError(string(perm, " is not a valid permutation of dimensions 1:", N))) iperm = invperm(perm) PermutedDimsArray{T,N,(perm...,),(iperm...,),typeof(data)}(data) end diff --git a/test/arrayops.jl b/test/arrayops.jl index 8feb2f410bfc6..5280484c95faf 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -446,7 +446,12 @@ s = view(a,:,[1,2,4],[1,5]) c = convert(Array, s) for p in ([1,2,3], [1,3,2], [2,1,3], [2,3,1], [3,1,2], [3,2,1]) @test permutedims(s, p) == permutedims(c, p) + @test Base.PermutedDimsArrays.PermutedDimsArray(s, p) == permutedims(c, p) end +@test_throws ArgumentError permutedims(a, (1,1,1)) +@test_throws ArgumentError permutedims(s, (1,1,1)) +@test_throws ArgumentError Base.PermutedDimsArrays.PermutedDimsArray(a, (1,1,1)) +@test_throws ArgumentError Base.PermutedDimsArrays.PermutedDimsArray(s, (1,1,1)) ## ipermutedims ## From 24556543edbac55adfc41dab18d989ed0bb84fc4 Mon Sep 17 00:00:00 2001 From: Christopher Rackauckas <Contact@ChrisRackauckas.com> Date: Mon, 8 Aug 2016 09:56:16 -0700 Subject: [PATCH 0844/1117] Add Gitter to README (#17878) * Add Gitter to README The Gitter is a large active community that new users have found lots of help at. I think it would be helpful to have the Gitter right here, because some of the people went to IRC for chat only to find out about the Gitter through the Gitter/IRC bot (and then switched to Gtiter for the code highlighting). I think showing the active community and the help it gives on a README will be important to lower the new-user threshold. * Removed Google+, and moved Gitter above IRC --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97601eda402ef..9bafbf8d1f1da 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ This is the GitHub repository of Julia source code, including instructions for c - **Source code:** <https://github.com/JuliaLang/julia> - **Git clone URL:** <git://github.com/JuliaLang/julia.git> - **Mailing lists:** <http://julialang.org/community/> +- **Gitter:** <https://gitter.im/JuliaLang/julia> - **IRC:** <http://webchat.freenode.net/?channels=julia> - **Code coverage:** <https://coveralls.io/r/JuliaLang/julia> @@ -42,7 +43,6 @@ developers may find the notes in [CONTRIBUTING](https://github.com/JuliaLang/jul - [**StackOverflow**](https://stackoverflow.com/questions/tagged/julia-lang) - [**Youtube**](https://www.youtube.com/channel/UC9IuUwwE2xdjQUT_LMLONoA) - [**Twitter**](https://twitter.com/JuliaLanguage) -- [**Google+**](https://plus.google.com/communities/111295162646766639102) - [**Meetup**](http://julia.meetup.com/) <a name="Currently-Supported-Platforms"/> From 86765d9937b1d374c7b4ff6858e52aa8cb187e0e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Mon, 8 Aug 2016 21:11:21 +0200 Subject: [PATCH 0845/1117] remove REPL prompt prefix before parsing in bracket paste mode (#17599) * remove REPL prompt prefix before parsing in bracket paste mode * remove created tmpdir and fix var name * use realpath to check path equality * remove prompt paste from shell> and help?> and ignore non prompted lines * add news * add option and update manual * fix typo variable name * fix rst code quote [ci skip] * split into more lines [ci skip] * update grammar and formatting [ci skip] --- NEWS.md | 7 ++++ base/REPL.jl | 31 +++++++++++++- doc/manual/interacting-with-julia.rst | 7 ++++ test/repl.jl | 60 +++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 80094d7659355..f427b4e5adf47 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,13 @@ Julia v0.6.0 Release Notes New language features --------------------- + * The REPL now supports something called *prompt pasting*. + This activates when pasting text that starts with `julia> ` into the REPL. + In that case, only expressions starting with `julia> ` are parsed, the rest are removed. + This makes it possible to paste a chunk of code that has been copied from a REPL session + without having to scrub away prompts and outputs. + This can be disabled or enabled at will with `Base.REPL.enable_promptpaste(::Bool)`. + Language changes ---------------- diff --git a/base/REPL.jl b/base/REPL.jl index 0ad9968178ce8..a3fd467861460 100644 --- a/base/REPL.jl +++ b/base/REPL.jl @@ -36,6 +36,8 @@ abstract AbstractREPL answer_color(::AbstractREPL) = "" +const JULIA_PROMPT = "julia> " + type REPLBackend "channel for AST" repl_channel::Channel @@ -207,7 +209,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef) hit_eof = false while true Base.reseteof(repl.terminal) - write(repl.terminal, "julia> ") + write(repl.terminal, JULIA_PROMPT) line = "" ast = nothing interrupted = false @@ -692,6 +694,9 @@ end repl_filename(repl, hp::REPLHistoryProvider) = "REPL[$(length(hp.history)-hp.start_idx)]" repl_filename(repl, hp) = "REPL" +const JL_PROMT_PASTE = Ref(true) +enable_promtpaste(v::Bool) = JL_PROMT_PASTE[] = v + function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_repl_keymap = Dict{Any,Any}[]) ### # @@ -723,7 +728,7 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep replc = REPLCompletionProvider(repl) # Set up the main Julia prompt - julia_prompt = Prompt("julia> "; + julia_prompt = Prompt(JULIA_PROMPT; # Copy colors from the prompt object prompt_prefix = hascolor ? repl.prompt_color : "", prompt_suffix = hascolor ? @@ -837,7 +842,29 @@ function setup_interface(repl::LineEditREPL; hascolor = repl.hascolor, extra_rep input = takebuf_string(sbuffer) oldpos = start(input) firstline = true + isprompt_paste = false while !done(input, oldpos) # loop until all lines have been executed + if JL_PROMT_PASTE[] + # Check if the next statement starts with "julia> ", in that case + # skip it. But first skip whitespace + while input[oldpos] in ('\n', ' ', '\t') + oldpos = nextind(input, oldpos) + oldpos >= sizeof(input) && return + end + # Check if input line starts with "julia> ", remove it if we are in prompt paste mode + jl_prompt_len = 7 + if (firstline || isprompt_paste) && (oldpos + jl_prompt_len <= sizeof(input) && input[oldpos:oldpos+jl_prompt_len-1] == JULIA_PROMPT) + isprompt_paste = true + oldpos += jl_prompt_len + # If we are prompt pasting and current statement does not begin with julia> , skip to next line + elseif isprompt_paste + while input[oldpos] != '\n' + oldpos = nextind(input, oldpos) + oldpos >= sizeof(input) && return + end + continue + end + end ast, pos = Base.syntax_deprecation_warnings(false) do Base.parse(input, oldpos, raise=false) end diff --git a/doc/manual/interacting-with-julia.rst b/doc/manual/interacting-with-julia.rst index 882d0c5a3cb3b..5a2f06981fa78 100644 --- a/doc/manual/interacting-with-julia.rst +++ b/doc/manual/interacting-with-julia.rst @@ -42,6 +42,13 @@ There are a number useful features unique to interactive work. In addition to sh julia> ans "12" +In Julia mode, the REPL supports something called *prompt pasting*. This activates when pasting text that starts with ``julia> `` into the REPL. +In that case, only expressions starting with ``julia> `` are parsed, others are removed. +This makes it is possible to paste a chunk of code that has been copied from a REPL session without having to scrub away prompts and outputs. +This feature is enabled by default but can be disabled or enabled at will with ``Base.REPL.enable_promptpaste(::Bool)``. +If it is enabled, you can try it out by pasting the code block above this paragraph straight into the REPL. +This feature does not work on the standard Windows command prompt due to its limitation at detecting when a paste occurs. + Help mode ~~~~~~~~~ diff --git a/test/repl.jl b/test/repl.jl index 09653bcb8422a..308f5ffe52fa1 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -397,6 +397,66 @@ begin # Try entering search mode while in custom repl mode LineEdit.enter_search(s, custom_histp, true) end + +# Test removal of prompt in bracket pasting +begin + stdin_write, stdout_read, stderr_read, repl = fake_repl() + + repl.interface = REPL.setup_interface(repl) + repl_mode = repl.interface.modes[1] + shell_mode = repl.interface.modes[2] + help_mode = repl.interface.modes[3] + + repltask = @async begin + Base.REPL.run_repl(repl) + end + + c = Condition() + + sendrepl2(cmd) = write(stdin_write,"$cmd\n notify(c)\n") + + # Test removal of prefix in single statement paste + sendrepl2("\e[200~julia> A = 2\e[201~\n") + wait(c) + @test A == 2 + + # Test removal of prefix in multiple statement paste + sendrepl2("""\e[200~ + julia> type T17599; a::Int; end + + julia> function foo(julia) + julia> 3 + end + + julia> A = 3\e[201~ + """) + wait(c) + @test A == 3 + @test foo(4) + @test T17599(3).a == 3 + @test !foo(2) + + sendrepl2("""\e[200~ + julia> goo(x) = x + 1 + error() + + julia> A = 4 + 4\e[201~ + """) + wait(c) + @test A == 4 + @test goo(4) == 5 + + # Test prefix removal only active in bracket paste mode + sendrepl2("julia = 4\n julia> 3 && (A = 1)\n") + wait(c) + @test A == 1 + + # Close repl + write(stdin_write, '\x04') + wait(repltask) +end + # Simple non-standard REPL tests if !is_windows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER stdin_write, stdout_read, stdout_read, repl = fake_repl() From 1db0d678f3a6e496a81c799ebf083c7568414dca Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 8 Aug 2016 15:51:54 -0500 Subject: [PATCH 0846/1117] Add more backtrace tests --- test/backtrace.jl | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/backtrace.jl b/test/backtrace.jl index 8267c689c7635..bdc93b000e0d9 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -96,3 +96,46 @@ let @test i1 > 0 && i2 > 0 @test b1[i1].line != b2[i2].line end + +module BackTraceTesting + +using Base.Test + +@inline bt2() = backtrace() +@inline bt1() = bt2() +bt() = bt1() + +lkup = map(StackTraces.lookup, bt()) +hasbt = hasbt2 = false +for sfs in lkup + for sf in sfs + if sf.func == :bt + hasbt = true + end + if sf.func == :bt2 + hasbt2 = true + end + end +end +@test hasbt +@test_broken hasbt2 + +function btmacro() + @time backtrace() +end +lkup = map(StackTraces.lookup, btmacro()) +hasme = hasbtmacro = false +for sfs in lkup + for sf in sfs + if sf.func == Symbol("macro expansion") + hasme = true + end + if sf.func == :btmacro + hasbtmacro = true + end + end +end +@test hasme +@test hasbtmacro + +end From 07a9508fdf9bb69530225264de5f281eec8f0b41 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 21 Jul 2016 13:01:15 -0400 Subject: [PATCH 0847/1117] hard kill all access to internal libuv callbacks this API was unused, untested, and unreliable (errors would corrupt libuv) --- base/process.jl | 53 +++++--------- base/socket.jl | 186 +++++++++++++++++++++++++----------------------- base/stream.jl | 130 +++++++++++---------------------- 3 files changed, 157 insertions(+), 212 deletions(-) diff --git a/base/process.jl b/base/process.jl index 22c39e4527f64..be39453865c4b 100644 --- a/base/process.jl +++ b/base/process.jl @@ -267,9 +267,7 @@ type Process <: AbstractPipe err::IO exitcode::Int64 termsignal::Int32 - exitcb::Callback exitnotify::Condition - closecb::Callback closenotify::Condition function Process(cmd::Cmd, handle::Ptr{Void}, in::Union{Redirectable, Ptr{Void}}, @@ -287,7 +285,7 @@ type Process <: AbstractPipe this = new(cmd, handle, in, out, err, typemin(fieldtype(Process, :exitcode)), typemin(fieldtype(Process, :termsignal)), - false, Condition(), false, Condition()) + Condition(), Condition()) finalizer(this, uvfinalize) return this end @@ -339,9 +337,6 @@ function uv_return_spawn(p::Ptr{Void}, exit_status::Int64, termsignal::Int32) proc = unsafe_pointer_to_objref(data)::Process proc.exitcode = exit_status proc.termsignal = termsignal - if isa(proc.exitcb, Function) - proc.exitcb(proc, exit_status, termsignal) - end ccall(:jl_close_uv, Void, (Ptr{Void},), proc.handle) notify(proc.exitnotify) nothing @@ -349,21 +344,18 @@ end function _uv_hook_close(proc::Process) proc.handle = C_NULL - if isa(proc.closecb, Function) - proc.closecb(proc) - end notify(proc.closenotify) end -function spawn(redirect::CmdRedirect, stdios::StdIOSet, exitcb::Callback, closecb::Callback; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) +function spawn(redirect::CmdRedirect, stdios::StdIOSet; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) spawn(redirect.cmd, (redirect.stream_no == STDIN_NO ? redirect.handle : stdios[1], redirect.stream_no == STDOUT_NO ? redirect.handle : stdios[2], redirect.stream_no == STDERR_NO ? redirect.handle : stdios[3]), - exitcb, closecb, chain=chain) + chain=chain) end -function spawn(cmds::OrCmds, stdios::StdIOSet, exitcb::Callback, closecb::Callback; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) +function spawn(cmds::OrCmds, stdios::StdIOSet; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) out_pipe = Libc.malloc(_sizeof_uv_named_pipe) in_pipe = Libc.malloc(_sizeof_uv_named_pipe) link_pipe(in_pipe, false, out_pipe, false) @@ -371,8 +363,8 @@ function spawn(cmds::OrCmds, stdios::StdIOSet, exitcb::Callback, closecb::Callba chain = Nullable(ProcessChain(stdios)) end try - spawn(cmds.a, (stdios[1], out_pipe, stdios[3]), exitcb, closecb, chain=chain) - spawn(cmds.b, (in_pipe, stdios[2], stdios[3]), exitcb, closecb, chain=chain) + spawn(cmds.a, (stdios[1], out_pipe, stdios[3]), chain=chain) + spawn(cmds.b, (in_pipe, stdios[2], stdios[3]), chain=chain) finally close_pipe_sync(out_pipe) close_pipe_sync(in_pipe) @@ -382,7 +374,7 @@ function spawn(cmds::OrCmds, stdios::StdIOSet, exitcb::Callback, closecb::Callba get(chain) end -function spawn(cmds::ErrOrCmds, stdios::StdIOSet, exitcb::Callback, closecb::Callback; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) +function spawn(cmds::ErrOrCmds, stdios::StdIOSet; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) out_pipe = Libc.malloc(_sizeof_uv_named_pipe) in_pipe = Libc.malloc(_sizeof_uv_named_pipe) link_pipe(in_pipe, false, out_pipe, false) @@ -390,8 +382,8 @@ function spawn(cmds::ErrOrCmds, stdios::StdIOSet, exitcb::Callback, closecb::Cal chain = Nullable(ProcessChain(stdios)) end try - spawn(cmds.a, (stdios[1], stdios[2], out_pipe), exitcb, closecb, chain=chain) - spawn(cmds.b, (in_pipe, stdios[2], stdios[3]), exitcb, closecb, chain=chain) + spawn(cmds.a, (stdios[1], stdios[2], out_pipe), chain=chain) + spawn(cmds.b, (in_pipe, stdios[2], stdios[3]), chain=chain) finally close_pipe_sync(out_pipe) close_pipe_sync(in_pipe) @@ -469,11 +461,9 @@ function setup_stdio(anon::Function, stdio::StdIOSet) close_err && close_stdio(err) end -function spawn(cmd::Cmd, stdios::StdIOSet, exitcb::Callback, closecb::Callback; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) +function spawn(cmd::Cmd, stdios::StdIOSet; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) loop = eventloop() pp = Process(cmd, C_NULL, stdios[1], stdios[2], stdios[3]) - pp.exitcb = exitcb - pp.closecb = closecb setup_stdio(stdios) do in, out, err pp.handle = _jl_spawn(cmd.exec[1], cmd.exec, loop, pp, in, out, err) @@ -484,39 +474,34 @@ function spawn(cmd::Cmd, stdios::StdIOSet, exitcb::Callback, closecb::Callback; pp end -function spawn(cmds::AndCmds, stdios::StdIOSet, exitcb::Callback, closecb::Callback; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) +function spawn(cmds::AndCmds, stdios::StdIOSet; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) if isnull(chain) chain = Nullable(ProcessChain(stdios)) end setup_stdio(stdios) do in, out, err - spawn(cmds.a, (in,out,err), exitcb, closecb, chain=chain) - spawn(cmds.b, (in,out,err), exitcb, closecb, chain=chain) + spawn(cmds.a, (in,out,err), chain=chain) + spawn(cmds.b, (in,out,err), chain=chain) end get(chain) end # INTERNAL -# returns a tuple of function arguments to spawn: -# (stdios, exitcb, closecb) -# | | \ The function to be called once the uv handle is closed -# | \ The function to be called once the process exits -# \ A set of up to 256 stdio instructions, where each entry can be either: +# returns stdios: +# A set of up to 256 stdio instructions, where each entry can be either: # | - An IO to be passed to the child # | - DevNull to pass /dev/null # | - An Filesystem.File object to redirect the output to # \ - A string specifying a filename to be opened -spawn_opts_swallow(stdios::StdIOSet, exitcb::Callback=false, closecb::Callback=false) = - (stdios,exitcb,closecb) +spawn_opts_swallow(stdios::StdIOSet) = (stdios,) spawn_opts_swallow(in::Redirectable=DevNull, out::Redirectable=DevNull, err::Redirectable=DevNull, args...) = - (tuple(in,out,err,args...),false,false) -spawn_opts_inherit(stdios::StdIOSet, exitcb::Callback=false, closecb::Callback=false) = - (stdios,exitcb,closecb) + ((in, out, err), args...) +spawn_opts_inherit(stdios::StdIOSet) = (stdios,) # pass original descriptors to child processes by default, because we might # have already exhausted and closed the libuv object for our standard streams. # this caused issue #8529. spawn_opts_inherit(in::Redirectable=RawFD(0), out::Redirectable=RawFD(1), err::Redirectable=RawFD(2), args...) = - (tuple(in,out,err,args...),false,false) + ((in, out, err), args...) spawn(cmds::AbstractCmd, args...; chain::Nullable{ProcessChain}=Nullable{ProcessChain}()) = spawn(cmds, spawn_opts_swallow(args...)...; chain=chain) diff --git a/base/socket.jl b/base/socket.jl index d037173ea62b5..52b2dd0178062 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -243,13 +243,9 @@ InetAddr(ip::IPAddr, port) = InetAddr{typeof(ip)}(ip, port) type TCPSocket <: LibuvStream handle::Ptr{Void} status::Int - line_buffered::Bool buffer::IOBuffer - readcb::Callback readnotify::Condition - ccb::Callback connectnotify::Condition - closecb::Callback closenotify::Condition sendbuf::Nullable{IOBuffer} lock::ReentrantLock @@ -259,11 +255,10 @@ type TCPSocket <: LibuvStream tcp = new( handle, status, - true, PipeBuffer(), - false, Condition(), - false, Condition(), - false, Condition(), + Condition(), + Condition(), + Condition(), nothing, ReentrantLock(), DEFAULT_READ_BUFFER_SZ) @@ -284,17 +279,15 @@ end type TCPServer <: LibuvServer handle::Ptr{Void} status::Int - ccb::Callback connectnotify::Condition - closecb::Callback closenotify::Condition function TCPServer(handle::Ptr{Void}, status) tcp = new( handle, status, - false, Condition(), - false, Condition()) + Condition(), + Condition()) associate_julia_struct(tcp.handle, tcp) finalizer(tcp, uvfinalize) return tcp @@ -314,12 +307,12 @@ iswritable(io::TCPSocket) = isopen(io) && io.status != StatusClosing ## VARIOUS METHODS TO BE MOVED TO BETTER LOCATION -_jl_connect_raw(sock::TCPSocket,sockaddr::Ptr{Void}) = - ccall(:jl_connect_raw,Int32,(Ptr{Void},Ptr{Void},Ptr{Void}),sock.handle,sockaddr,uv_jl_connectcb::Ptr{Void}) +_jl_connect_raw(sock::TCPSocket, sockaddr::Ptr{Void}) = + ccall(:jl_connect_raw, Int32, (Ptr{Void}, Ptr{Void}, Ptr{Void}), sock.handle, sockaddr, uv_jl_connectcb::Ptr{Void}) _jl_sockaddr_from_addrinfo(addrinfo::Ptr{Void}) = - ccall(:jl_sockaddr_from_addrinfo,Ptr{Void},(Ptr{Void},),addrinfo) -_jl_sockaddr_set_port(ptr::Ptr{Void},port::UInt16) = - ccall(:jl_sockaddr_set_port,Void,(Ptr{Void},UInt16),ptr,port) + ccall(:jl_sockaddr_from_addrinfo, Ptr{Void}, (Ptr{Void}, ), addrinfo) +_jl_sockaddr_set_port(ptr::Ptr{Void}, port::UInt16) = + ccall(:jl_sockaddr_set_port, Void, (Ptr{Void}, UInt16), ptr, port) accept(server::TCPServer) = accept(server, TCPSocket()) @@ -402,7 +395,7 @@ _bind(sock::UDPSocket, host::IPv6, port::UInt16, flags::UInt32 = UInt32(0)) = cc function bind(sock::Union{TCPServer, UDPSocket}, host::IPAddr, port::Integer; ipv6only = false, reuseaddr = false, kws...) if sock.status != StatusInit - error("$(typeof(sock)) is not initialized") + error("$(typeof(sock)) is not in initialization state") end flags = 0 if isa(host,IPv6) && ipv6only @@ -411,21 +404,21 @@ function bind(sock::Union{TCPServer, UDPSocket}, host::IPAddr, port::Integer; ip if isa(sock, UDPSocket) && reuseaddr flags |= UV_UDP_REUSEADDR end - err = _bind(sock,host,UInt16(port),UInt32(flags)) + err = _bind(sock, host, UInt16(port), UInt32(flags)) if err < 0 if err != UV_EADDRINUSE && err != UV_EACCES && err != UV_EADDRNOTAVAIL #TODO: this codepath is not currently tested - throw(UVError("bind",err)) + throw(UVError("bind", err)) else return false end end sock.status = StatusOpen isa(sock, UDPSocket) && setopt(sock; kws...) - true + return true end -bind(sock::TCPServer, addr::InetAddr) = bind(sock,addr.host,addr.port) +bind(sock::TCPServer, addr::InetAddr) = bind(sock, addr.host, addr.port) function setopt(sock::UDPSocket; multicast_loop = nothing, multicast_ttl=nothing, enable_broadcast=nothing, ttl=nothing) if sock.status == StatusUninit @@ -448,13 +441,13 @@ end alloc_buf_hook(sock::UDPSocket,size::UInt) = (Libc.malloc(size),size) function _recv_start(sock::UDPSocket) - if ccall(:uv_is_active,Cint,(Ptr{Void},),sock.handle) == 0 - uv_error("recv_start",ccall(:uv_udp_recv_start,Cint,(Ptr{Void},Ptr{Void},Ptr{Void}), - sock.handle,uv_jl_alloc_buf::Ptr{Void},uv_jl_recvcb::Ptr{Void})) + if ccall(:uv_is_active, Cint, (Ptr{Void}, ), sock.handle) == 0 + uv_error("recv_start", ccall(:uv_udp_recv_start, Cint, (Ptr{Void}, Ptr{Void}, Ptr{Void}), + sock.handle, uv_jl_alloc_buf::Ptr{Void}, uv_jl_recvcb::Ptr{Void})) end end -_recv_stop(sock::UDPSocket) = uv_error("recv_stop",ccall(:uv_udp_recv_stop,Cint,(Ptr{Void},),sock.handle)) +_recv_stop(sock::UDPSocket) = uv_error("recv_stop", ccall(:uv_udp_recv_stop, Cint, (Ptr{Void}, ), sock.handle)) function recv(sock::UDPSocket) addr, data = recvfrom(sock) @@ -467,7 +460,7 @@ function recvfrom(sock::UDPSocket) error("UDPSocket is not initialized and open") end _recv_start(sock) - stream_wait(sock,sock.recvnotify)::Tuple{Union{IPv4, IPv6}, Vector{UInt8}} + return stream_wait(sock, sock.recvnotify)::Tuple{Union{IPv4, IPv6}, Vector{UInt8}} end @@ -496,14 +489,14 @@ function uv_recvcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}, addr::Ptr nothing end -function _send(sock::UDPSocket,ipaddr::IPv4,port::UInt16,buf) - ccall(:jl_udp_send, Cint, (Ptr{Void},UInt16,UInt32,Ptr{UInt8},Csize_t,Ptr{Void}), - sock.handle,hton(port),hton(ipaddr.host),buf,sizeof(buf),uv_jl_sendcb::Ptr{Void}) +function _send(sock::UDPSocket, ipaddr::IPv4, port::UInt16, buf) + ccall(:jl_udp_send, Cint, (Ptr{Void}, UInt16, UInt32, Ptr{UInt8}, Csize_t, Ptr{Void}), + sock.handle, hton(port), hton(ipaddr.host), buf, sizeof(buf), uv_jl_sendcb::Ptr{Void}) end -function _send(sock::UDPSocket,ipaddr::IPv6,port::UInt16,buf) - ccall(:jl_udp_send6, Cint, (Ptr{Void},UInt16,Ptr{UInt128},Ptr{UInt8},Csize_t,Ptr{Void}), - sock.handle,hton(port),&hton(ipaddr.host),buf,sizeof(buf),uv_jl_sendcb::Ptr{Void}) +function _send(sock::UDPSocket, ipaddr::IPv6, port::UInt16, buf) + ccall(:jl_udp_send6, Cint, (Ptr{Void}, UInt16, Ptr{UInt128}, Ptr{UInt8}, Csize_t, Ptr{Void}), + sock.handle, hton(port), &hton(ipaddr.host), buf, sizeof(buf), uv_jl_sendcb::Ptr{Void}) end function send(sock::UDPSocket,ipaddr,port,msg) @@ -511,8 +504,8 @@ function send(sock::UDPSocket,ipaddr,port,msg) if sock.status != StatusInit && sock.status != StatusOpen error("UDPSocket is not initialized and open") end - uv_error("send",_send(sock,ipaddr,UInt16(port),msg)) - stream_wait(sock,sock.sendnotify) + uv_error("send", _send(sock, ipaddr, UInt16(port), msg)) + stream_wait(sock, sock.sendnotify) nothing end @@ -550,19 +543,19 @@ function uv_getaddrinfocb(req::Ptr{Void}, status::Cint, addrinfo::Ptr{Void}) else freeaddrinfo = addrinfo while addrinfo != C_NULL - sockaddr = ccall(:jl_sockaddr_from_addrinfo,Ptr{Void},(Ptr{Void},),addrinfo) - if ccall(:jl_sockaddr_is_ip4,Int32,(Ptr{Void},),sockaddr) == 1 - cb(IPv4(ntoh(ccall(:jl_sockaddr_host4,UInt32,(Ptr{Void},),sockaddr)))) + sockaddr = ccall(:jl_sockaddr_from_addrinfo, Ptr{Void}, (Ptr{Void},), addrinfo) + if ccall(:jl_sockaddr_is_ip4, Int32, (Ptr{Void},), sockaddr) == 1 + cb(IPv4(ntoh(ccall(:jl_sockaddr_host4, UInt32, (Ptr{Void},), sockaddr)))) break - #elseif ccall(:jl_sockaddr_is_ip6,Int32,(Ptr{Void},),sockaddr) == 1 + #elseif ccall(:jl_sockaddr_is_ip6, Int32, (Ptr{Void},), sockaddr) == 1 # host = Array{UInt128}(1) - # scope_id = ccall(:jl_sockaddr_host6,UInt32,(Ptr{Void},Ptr{UInt128}),sockaddr,host) + # scope_id = ccall(:jl_sockaddr_host6, UInt32, (Ptr{Void}, Ptr{UInt128}), sockaddr, host) # cb(IPv6(ntoh(host[1]))) # break end - addrinfo = ccall(:jl_next_from_addrinfo,Ptr{Void},(Ptr{Void},),addrinfo) + addrinfo = ccall(:jl_next_from_addrinfo, Ptr{Void}, (Ptr{Void},), addrinfo) end - ccall(:uv_freeaddrinfo,Void,(Ptr{Void},),freeaddrinfo) + ccall(:uv_freeaddrinfo, Void, (Ptr{Void},), freeaddrinfo) end Libc.free(req) nothing @@ -592,7 +585,7 @@ function getaddrinfo(host::String) r = wait(c) if isa(r,UVError) if r.code in [UV_EAI_NONAME, UV_EAI_AGAIN, UV_EAI_FAIL, UV_EAI_NODATA] - throw(DNSError(host,r.code)) + throw(DNSError(host, r.code)) elseif r.code == UV_EAI_SYSTEM throw(SystemError("uv_getaddrinfocb")) elseif r.code == UV_EAI_MEMORY @@ -612,116 +605,132 @@ function getipaddr() addr[1] = C_NULL count = zeros(Int32,1) lo_present = false - err = ccall(:jl_uv_interface_addresses,Int32,(Ptr{Ptr{UInt8}},Ptr{Int32}),addr,count) - addr, count = addr[1],count[1] + err = ccall(:jl_uv_interface_addresses, Int32, (Ptr{Ptr{UInt8}}, Ptr{Int32}), addr, count) + addr, count = addr[1], count[1] if err != 0 - ccall(:uv_free_interface_addresses,Void,(Ptr{UInt8},Int32),addr,count) - throw(UVError("getlocalip",err)) + ccall(:uv_free_interface_addresses, Void, (Ptr{UInt8}, Int32), addr, count) + throw(UVError("getlocalip", err)) end for i = 0:(count-1) current_addr = addr + i*_sizeof_uv_interface_address - if 1 == ccall(:jl_uv_interface_address_is_internal,Int32,(Ptr{UInt8},),current_addr) + if 1 == ccall(:jl_uv_interface_address_is_internal, Int32, (Ptr{UInt8},), current_addr) lo_present = true continue end - sockaddr = ccall(:jl_uv_interface_address_sockaddr,Ptr{Void},(Ptr{UInt8},),current_addr) - if ccall(:jl_sockaddr_in_is_ip4,Int32,(Ptr{Void},),sockaddr) == 1 - rv = IPv4(ntoh(ccall(:jl_sockaddr_host4,UInt32,(Ptr{Void},),sockaddr))) - ccall(:uv_free_interface_addresses,Void,(Ptr{UInt8},Int32),addr,count) + sockaddr = ccall(:jl_uv_interface_address_sockaddr, Ptr{Void}, (Ptr{UInt8},), current_addr) + if ccall(:jl_sockaddr_in_is_ip4, Int32, (Ptr{Void},), sockaddr) == 1 + rv = IPv4(ntoh(ccall(:jl_sockaddr_host4, UInt32, (Ptr{Void},), sockaddr))) + ccall(:uv_free_interface_addresses, Void, (Ptr{UInt8}, Int32), addr, count) return rv # Uncomment to enbable IPv6 - #elseif ccall(:jl_sockaddr_in_is_ip6,Int32,(Ptr{Void},),sockaddr) == 1 + #elseif ccall(:jl_sockaddr_in_is_ip6, Int32, (Ptr{Void},), sockaddr) == 1 # host = Array{UInt128}(1) - # ccall(:jl_sockaddr_host6,UInt32,(Ptr{Void},Ptr{UInt128}),sockaddrr,host) + # ccall(:jl_sockaddr_host6, UInt32, (Ptr{Void}, Ptr{UInt128}), sockaddrr, host) # return IPv6(ntoh(host[1])) end end - ccall(:uv_free_interface_addresses,Void,(Ptr{UInt8},Int32),addr,count) - lo_present ? ip"127.0.0.1" : error("No networking interface available") + ccall(:uv_free_interface_addresses, Void, (Ptr{UInt8}, Int32), addr, count) + return lo_present ? ip"127.0.0.1" : error("No networking interface available") end ## function connect!(sock::TCPSocket, host::IPv4, port::Integer) if sock.status != StatusInit - error("TCPSocket is not initialized") + error("TCPSocket is not in initialization state") end if !(0 <= port <= typemax(UInt16)) throw(ArgumentError("port out of range, must be 0 ≤ port ≤ 65535, got $port")) end - uv_error("connect",ccall(:jl_tcp4_connect,Int32,(Ptr{Void},UInt32,UInt16,Ptr{Void}), - sock.handle,hton(host.host),hton(UInt16(port)),uv_jl_connectcb::Ptr{Void})) + uv_error("connect", ccall(:jl_tcp4_connect, Int32, (Ptr{Void}, UInt32, UInt16, Ptr{Void}), + sock.handle, hton(host.host), hton(UInt16(port)), uv_jl_connectcb::Ptr{Void})) sock.status = StatusConnecting + nothing end function connect!(sock::TCPSocket, host::IPv6, port::Integer) if sock.status != StatusInit - error("TCPSocket is not initialized") + error("TCPSocket is not in initialization state") end if !(0 <= port <= typemax(UInt16)) throw(ArgumentError("port out of range, must be 0 ≤ port ≤ 65535, got $port")) end - uv_error("connect",ccall(:jl_tcp6_connect,Int32,(Ptr{Void},Ptr{UInt128},UInt16,Ptr{Void}), - sock.handle,&hton(host.host),hton(UInt16(port)),uv_jl_connectcb::Ptr{Void})) + uv_error("connect", ccall(:jl_tcp6_connect, Int32, (Ptr{Void}, Ptr{UInt128}, UInt16, Ptr{Void}), + sock.handle, &hton(host.host), hton(UInt16(port)), uv_jl_connectcb::Ptr{Void})) sock.status = StatusConnecting + nothing end connect!(sock::TCPSocket, addr::InetAddr) = connect!(sock, addr.host, addr.port) # Default Host to localhost -connect(sock::TCPSocket, port::Integer) = connect(sock,IPv4(127,0,0,1),port) -connect(port::Integer) = connect(IPv4(127,0,0,1),port) +connect(sock::TCPSocket, port::Integer) = connect(sock,IPv4(127,0,0,1), port) +connect(port::Integer) = connect(IPv4(127,0,0,1), port) # Valid connect signatures for TCP -connect(host::AbstractString, port::Integer) = connect(TCPSocket(),host,port) -connect(addr::IPAddr, port::Integer) = connect(TCPSocket(),addr,port) -connect(addr::InetAddr) = connect(TCPSocket(),addr) +connect(host::AbstractString, port::Integer) = connect(TCPSocket(), host, port) +connect(addr::IPAddr, port::Integer) = connect(TCPSocket(), addr, port) +connect(addr::InetAddr) = connect(TCPSocket(), addr) default_connectcb(sock,status) = nothing function connect!(sock::TCPSocket, host::AbstractString, port::Integer) if sock.status != StatusInit - error("TCPSocket is not initialized") + error("TCPSocket is not in initialization state") end ipaddr = getaddrinfo(host) sock.status = StatusInit connect!(sock,ipaddr,port) sock.status = StatusConnecting - sock + return sock end ## -listen(sock::LibuvServer; backlog::Integer=BACKLOG_DEFAULT) = (uv_error("listen",_listen(sock;backlog=backlog)); sock) - function listen(addr; backlog::Integer=BACKLOG_DEFAULT) sock = TCPServer() - !bind(sock,addr) && error("cannot bind to port; may already be in use or access denied") - uv_error("listen",_listen(sock;backlog=backlog)) - sock + !bind(sock, addr) && error("cannot bind to port; may already be in use or access denied") + listen(sock; backlog=backlog) + return sock +end +listen(port::Integer; backlog::Integer=BACKLOG_DEFAULT) = listen(IPv4(UInt32(0)), port; backlog=backlog) +listen(host::IPAddr, port::Integer; backlog::Integer=BACKLOG_DEFAULT) = listen(InetAddr(host, port); backlog=backlog) + +function listen(callback, server::Union{TCPSocket, UDPSocket}) + @async begin + local client = TCPSocket() + while isopen(server) + err = accept_nonblock(server, client) + if err == 0 + callback(client) + client = TCPSocket() + elseif err != UV_EAGAIN + uv_error("accept", err) + else + stream_wait(server, server.connectnotify) + end + end + end + return sock end -listen(port::Integer; backlog::Integer=BACKLOG_DEFAULT) = listen(IPv4(UInt32(0)),port;backlog=backlog) -listen(host::IPAddr, port::Integer; backlog::Integer=BACKLOG_DEFAULT) = listen(InetAddr(host,port);backlog=backlog) - -listen(cb::Callback,args...; backlog::Integer=BACKLOG_DEFAULT) = (sock=listen(args...;backlog=backlog);sock.ccb=cb;sock) -listen(cb::Callback,sock::Union{TCPSocket,UDPSocket}; backlog::Integer=BACKLOG_DEFAULT) = (sock.ccb=cb;listen(sock;backlog=backlog)) ## -function accept_nonblock(server::TCPServer,client::TCPSocket) +function accept_nonblock(server::TCPServer, client::TCPSocket) if client.status != StatusInit - error("client TCPSocket is not initialized") + error("client TCPSocket is not in initialization state") end - err = ccall(:uv_accept,Int32,(Ptr{Void},Ptr{Void}),server.handle,client.handle) + err = ccall(:uv_accept, Int32, (Ptr{Void}, Ptr{Void}), server.handle, client.handle) if err == 0 client.status = StatusOpen end - err + return err end + function accept_nonblock(server::TCPServer) client = TCPSocket() uv_error("accept", accept_nonblock(server, client)) - client + return client end ## Utility functions @@ -730,17 +739,18 @@ function listenany(host::IPAddr, default_port) addr = InetAddr(host, default_port) while true sock = TCPServer() - if bind(sock,addr) && _listen(sock) == 0 - return (addr.port,sock) + if bind(sock, addr) && trylisten(sock) == 0 + return (addr.port, sock) end close(sock) addr = InetAddr(addr.host, addr.port + 1) - if addr.port==default_port + if addr.port == default_port error("no ports available") end end end -listenany(default_port) = listenany(IPv4(UInt32(0)),default_port) + +listenany(default_port) = listenany(IPv4(UInt32(0)), default_port) function getsockname(sock::Union{TCPServer,TCPSocket}) rport = Ref{Cushort}(0) diff --git a/base/stream.jl b/base/stream.jl index 2c7ad60244ccd..d57180fe7badb 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -6,8 +6,6 @@ if is_windows() end ## types ## -typealias Callback Union{Function,Bool} - abstract IOServer abstract LibuvServer <: IOServer abstract LibuvStream <: IO @@ -102,12 +100,8 @@ type PipeEndpoint <: LibuvStream handle::Ptr{Void} status::Int buffer::IOBuffer - line_buffered::Bool - readcb::Callback readnotify::Condition - ccb::Callback connectnotify::Condition - closecb::Callback closenotify::Condition sendbuf::Nullable{IOBuffer} lock::ReentrantLock @@ -118,11 +112,11 @@ type PipeEndpoint <: LibuvStream p = new(handle, status, PipeBuffer(), - true, - false,Condition(), - false,Condition(), - false,Condition(), - nothing, ReentrantLock(), + Condition(), + Condition(), + Condition(), + nothing, + ReentrantLock(), DEFAULT_READ_BUFFER_SZ) associate_julia_struct(handle, p) finalizer(p, uvfinalize) @@ -133,15 +127,13 @@ end type PipeServer <: LibuvServer handle::Ptr{Void} status::Int - ccb::Callback connectnotify::Condition - closecb::Callback closenotify::Condition function PipeServer(handle::Ptr{Void}, status) p = new(handle, status, - false,Condition(), - false,Condition()) + Condition(), + Condition()) associate_julia_struct(p.handle, p) finalizer(p, uvfinalize) return p @@ -158,11 +150,8 @@ end type TTY <: LibuvStream handle::Ptr{Void} status::Int - line_buffered::Bool buffer::IOBuffer - readcb::Callback readnotify::Condition - closecb::Callback closenotify::Condition sendbuf::Nullable{IOBuffer} lock::ReentrantLock @@ -173,10 +162,9 @@ type TTY <: LibuvStream tty = new( handle, status, - true, PipeBuffer(), - false,Condition(), - false,Condition(), + Condition(), + Condition(), nothing, ReentrantLock(), DEFAULT_READ_BUFFER_SZ) associate_julia_struct(handle, tty) @@ -196,7 +184,6 @@ function TTY(fd::RawFD; readable::Bool = false) eventloop(), tty.handle, fd.fd, readable) uv_error("TTY", err) tty.status = StatusOpen - tty.line_buffered = false return tty end @@ -241,7 +228,6 @@ function init_stdio(handle::Ptr{Void}) else throw(ArgumentError("invalid stdio type: $t")) end - ret.line_buffered = false return ret end end @@ -400,15 +386,12 @@ function uv_connectcb(conn::Ptr{Void}, status::Cint) @assert sock.status == StatusConnecting if status >= 0 sock.status = StatusOpen - err = nothing + notify(sock.connectnotify) else sock.status = StatusInit - err = UVError("connect",status) - end - if isa(sock.ccb,Function) - sock.ccb(sock, status) + err = UVError("connect", status) + notify_error(sock.connectnotify, err) end - err===nothing ? notify(sock.connectnotify) : notify_error(sock.connectnotify, err) Libc.free(conn) nothing end @@ -417,14 +400,11 @@ end function uv_connectioncb(stream::Ptr{Void}, status::Cint) sock = @handle_as stream LibuvServer if status >= 0 - err = nothing + notify(sock.connectnotify) else - err = UVError("connection",status) - end - if isa(sock.ccb, Function) - sock.ccb(sock, status) + err = UVError("connection", status) + notify_error(sock.connectnotify, err) end - err === nothing ? notify(sock.connectnotify) : notify_error(sock.connectnotify, err) nothing end @@ -453,7 +433,6 @@ function uv_alloc_buf(handle::Ptr{Void}, size::Csize_t, buf::Ptr{Void}) end alloc_buf_hook(stream::LibuvStream, size::UInt) = alloc_request(stream.buffer, UInt(size)) - function notify_filled(buffer::IOBuffer, nread::Int, base::Ptr{Void}, len::UInt) if buffer.append buffer.size += nread @@ -462,22 +441,6 @@ function notify_filled(buffer::IOBuffer, nread::Int, base::Ptr{Void}, len::UInt) end end -function notify_filled(stream::LibuvStream, nread::Int) - more = true - while more - if isa(stream.readcb,Function) - nreadable = (stream.line_buffered ? Int(search(stream.buffer, '\n')) : nb_available(stream.buffer)) - if nreadable > 0 - more = stream.readcb(stream, nreadable) - else - more = false - end - else - more = false - end - end -end - function uv_readcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}) stream = @handle_as handle LibuvStream nread = Int(nread) @@ -502,11 +465,10 @@ function uv_readcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}) # This is a fatal connection error. Shutdown requests as per the usual # close function won't work and libuv will fail with an assertion failure ccall(:jl_forceclose_uv, Void, (Ptr{Void},), stream) - notify_error(stream.readnotify, UVError("readcb",nread)) + notify_error(stream.readnotify, UVError("read", nread)) end else notify_filled(stream.buffer, nread, base, len) - notify_filled(stream, nread) notify(stream.readnotify) end @@ -530,12 +492,10 @@ end function _uv_hook_close(uv::Union{LibuvStream, LibuvServer}) uv.handle = C_NULL uv.status = StatusClosed - if isa(uv.closecb, Function) - uv.closecb(uv) - end + # notify any listeners that exist on this libuv stream type notify(uv.closenotify) - try notify(uv.readnotify) end - try notify(uv.connectnotify) end + isdefined(uv, :readnotify) && notify(uv.readnotify) + isdefined(uv, :connectnotify) && notify(uv.connectnotify) nothing end @@ -683,22 +643,6 @@ function start_reading(stream::LibuvStream) end end -function start_reading(stream::LibuvStream, cb::Function) - failure = start_reading(stream) - stream.readcb = cb - nread = nb_available(stream.buffer) - if nread > 0 - notify_filled(stream, nread) - end - return failure_code -end - -function start_reading(stream::LibuvStream, cb::Bool) - failure_code = start_reading(stream) - stream.readcb = cb - return failure_code -end - function stop_reading(stream::LibuvStream) if stream.status == StatusActive ret = ccall(:uv_read_stop, Cint, (Ptr{Void},), stream) @@ -881,19 +825,21 @@ end function accept_nonblock(server::PipeServer,client::PipeEndpoint) if client.status != StatusInit - error(client.status == StatusUninit ? "client is not initialized" : + error(client.status == StatusUninit ? + "client is not initialized" : "client is already in use or has been closed") end - err = ccall(:uv_accept,Int32,(Ptr{Void},Ptr{Void}),server.handle,client.handle) + err = ccall(:uv_accept, Int32, (Ptr{Void}, Ptr{Void}), server.handle, client.handle) if err == 0 client.status = StatusOpen end - err + return err end + function accept_nonblock(server::PipeServer) client = init_pipe!(PipeEndpoint(); readable=true, writable=true, julia_only=true) - uv_error("accept", accept_nonblock(server,client) != 0) - client + uv_error("accept", accept_nonblock(server, client) != 0) + return client end function accept(server::LibuvServer, client::LibuvStream) @@ -901,25 +847,30 @@ function accept(server::LibuvServer, client::LibuvStream) throw(ArgumentError("server not connected, make sure \"listen\" has been called")) end while isopen(server) - err = accept_nonblock(server,client) + err = accept_nonblock(server, client) if err == 0 return client elseif err != UV_EAGAIN - uv_error("accept",err) + uv_error("accept", err) end - stream_wait(server,server.connectnotify) + stream_wait(server, server.connectnotify) end uv_error("accept", UV_ECONNABORTED) end const BACKLOG_DEFAULT = 511 -function _listen(sock::LibuvServer; backlog::Integer=BACKLOG_DEFAULT) +function listen(sock::LibuvServer; backlog::Integer=BACKLOG_DEFAULT) + uv_error("listen", trylisten(sock)) + return sock +end + +function trylisten(sock::LibuvServer; backlog::Integer=BACKLOG_DEFAULT) check_open(sock) err = ccall(:uv_listen, Cint, (Ptr{Void}, Cint, Ptr{Void}), sock, backlog, uv_jl_connectioncb::Ptr{Void}) sock.status = StatusActive - err + return err end function bind(server::PipeServer, name::AbstractString) @@ -935,15 +886,14 @@ function bind(server::PipeServer, name::AbstractString) end end server.status = StatusOpen - true + return true end function listen(path::AbstractString) sock = PipeServer() bind(sock, path) || throw(ArgumentError("could not listen on path $path")) - uv_error("listen", _listen(sock)) - sock + return listen(sock) end function connect!(sock::PipeEndpoint, path::AbstractString) @@ -952,13 +902,13 @@ function connect!(sock::PipeEndpoint, path::AbstractString) uv_req_set_data(req,C_NULL) ccall(:uv_pipe_connect, Void, (Ptr{Void}, Ptr{Void}, Cstring, Ptr{Void}), req, sock.handle, path, uv_jl_connectcb::Ptr{Void}) sock.status = StatusConnecting - sock + return sock end function connect(sock::LibuvStream, args...) connect!(sock, args...) wait_connected(sock) - sock + return sock end # Libuv will internally reset read/writability, which is uses to From d6c7667d2b037d1d4306ad0044a8650eba55c161 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman <benditlikeranjan@gmail.com> Date: Tue, 9 Aug 2016 20:22:32 +0530 Subject: [PATCH 0848/1117] Add test for @code_typed and @code_lowered (#17892) Add test for typed and lowered Check if @code_typed and @code_lowered return LambdaInfo instead of Array --- test/reflection.jl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/reflection.jl b/test/reflection.jl index 625e61c1cc1ee..cd8e7d702245e 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -533,3 +533,23 @@ if is_windows() else @test isnull(h16850) end + +# Adds test for PR #17636 +let + a = @code_typed 1 + 1 + b = @code_lowered 1 + 1 + @test isa(a, LambdaInfo) + @test isa(b, LambdaInfo) + + function thing(a::Array, b::Real) + println("thing") + end + function thing(a::AbstractArray, b::Int) + println("blah") + end + @test_throws MethodError thing(rand(10), 1) + a = @code_typed thing(rand(10), 1) + b = @code_lowered thing(rand(10), 1) + @test length(a) == 0 + @test length(b) == 0 +end From 22a61ee8f6ee712e6ce1516e7ad86caadd93fde9 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 9 Aug 2016 11:05:28 -0400 Subject: [PATCH 0849/1117] Revert "Fix `@threadcall` argument passing." --- base/threadcall.jl | 3 --- test/ccall.jl | 4 ---- 2 files changed, 7 deletions(-) diff --git a/base/threadcall.jl b/base/threadcall.jl index da877d9707109..c9da92c5a49ea 100644 --- a/base/threadcall.jl +++ b/base/threadcall.jl @@ -23,8 +23,6 @@ function without causing the main `julia` thread to become blocked. Concurrency is limited by size of the libuv thread pool, which defaults to 4 threads but can be increased by setting the `UV_THREADPOOL_SIZE` environment variable and restarting the `julia` process. - -Note that the called function should never call back into Julia. """ macro threadcall(f, rettype, argtypes, argvals...) # check for usage errors @@ -75,7 +73,6 @@ function do_threadcall(wrapper::Function, rettype::Type, argtypes::Vector, argva y = cconvert(T, x) push!(roots, y) unsafe_store!(convert(Ptr{T}, ptr), unsafe_convert(T, y)) - ptr += sizeof(T) end # create return buffer diff --git a/test/ccall.jl b/test/ccall.jl index e5cf4709d6d12..7f0792a87fdd5 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -589,10 +589,6 @@ threadcall_test_func(x) = @test threadcall_test_func(3) == 1 @test threadcall_test_func(259) == 1 -f17819(a,b) = Cint(a+b) -cf17819 = cfunction(f17819, Cint, (Cint,Cint)) -@test @threadcall(cf17819, Cint, (Cint, Cint), 1, 2) == 3 - let n=3 tids = Culong[] @sync for i in 1:10^n From e2800f278726e2b3a9d8d2c9d285852457e8dc87 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman <benditlikeranjan@gmail.com> Date: Tue, 9 Aug 2016 20:45:32 +0530 Subject: [PATCH 0850/1117] Add test for function name being assigned in function (#17835) For issue #4914 --- test/core.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/core.jl b/test/core.jl index 394f7882dd0b3..ee4605ff60704 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4461,3 +4461,11 @@ end #end #@test local_innersig(Int32(2)) == ((Int32(2), Int32(1), Int32(2)im), (Int32(2), UInt32(1))) #@test local_innersig(Int64(3)) == ((Int64(3), Int64(1), Int64(3)im), (Int64(3), UInt64(1))) + +# Issue 4914 +let + j(j) = j + @test j(1) == 1 + k(x) = (k = x; k) + @test k(1) == 1 +end From 1161480bde6028bfb45e570d9a47f9164865e96c Mon Sep 17 00:00:00 2001 From: Amit Murthy <amit.murthy@gmail.com> Date: Tue, 9 Aug 2016 21:46:20 +0530 Subject: [PATCH 0851/1117] Document `@threadcall` (#17915) --- base/docs/helpdb/Base.jl | 2 +- doc/manual/parallel-computing.rst | 25 +++++++++++++++++++++++++ doc/stdlib/parallel.rst | 13 ++++++++++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index e84547a23a682..2175bd10b0a16 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4839,7 +4839,7 @@ IntSet """ Task(func) -Create a `Task` (i.e. thread, or coroutine) to execute the given function (which must be +Create a `Task` (i.e. coroutine) to execute the given function (which must be callable with no arguments). The task exits when this function returns. """ Task diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index ca78dd0f6c4f6..9f9548ddf356b 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -1076,6 +1076,31 @@ The iteration space is split amongst the threads, after which each thread writes Note that :obj:`Threads.@threads` does not have an optional reduction parameter like :obj:`@parallel`. +@threadcall (Experimental) +-------------------------- +All I/O tasks, timers, REPL commands, etc are multiplexed onto a single OS thread via an event loop. +A patched version of libuv (http://docs.libuv.org/en/v1.x/) provides this functionality. Yield points provide +for co-operatively scheduling multiple tasks onto the same OS thread. I/O tasks and timers yield implicitly while +waiting for the event to occur. Calling `yield()` explicitly allows for other tasks to be scheduled. + +Thus, a task executing a ``ccall`` effectively prevents the Julia scheduler from executing any other +tasks till the call returns. This is true for all calls into external libraries. Exceptions are calls into +custom C code that call back into Julia (which may then yield) or C code that calls ``jl_yield()``(C equivalent of ``yield()``). + +Note that while Julia code runs on a single thread (by default), libraries used by Julia may launch their own internal +threads. For example, the BLAS library may start as many threads as there are cores on a machine. + +The ``@threadcall`` macro addresses scenarios where we do not want a ``ccall`` to block the main Julia event loop. +It schedules a C function for execution in a separate thread. A threadpool with a default size of 4 is used for this. +The size of the threadpool is controlled via environment variable UV_THREADPOOL_SIZE. While waiting for a free thread, +and during function execution once a thread is available, the requesting task (on the main Julia event loop) +yields to other tasks. Note that ``@threadcall`` does not return till the execution is complete. From a user point of +view, it is therefore a blocking call like other Julia API. + +It is very important that the called function does not call back into Julia. + +``@threadcall`` may be removed/changed in future versions of Julia. + .. rubric:: Footnotes .. [#mpi2rma] In this context, MPI refers to the MPI-1 standard. Beginning with MPI-2, the MPI standards committee introduced a new set of communication mechanisms, collectively referred to as Remote Memory Access (RMA). The motivation for adding RMA to the MPI standard was to facilitate one-sided communication patterns. For additional information on the latest MPI standard, see http://www.mpi-forum.org/docs. diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 9aeaeb22d096f..8c37e3812cb46 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -11,7 +11,7 @@ Tasks .. Docstring generated from Julia source - Create a ``Task`` (i.e. thread, or coroutine) to execute the given function (which must be callable with no arguments). The task exits when this function returns. + Create a ``Task`` (i.e. coroutine) to execute the given function (which must be callable with no arguments). The task exits when this function returns. .. function:: yieldto(task, arg = nothing) @@ -814,6 +814,17 @@ will) change in the future. For further details, see LLVM's ``fence`` instruction. +ccall using a threadpool (Experimental) +--------------------------------------- + +.. function:: @threadcall((cfunc, clib), rettype, (argtypes...), argvals...) + + .. Docstring generated from Julia source + + The ``@threadcall`` macro is called in the same way as ``ccall`` but does the work in a different thread. This is useful when you want to call a blocking C function without causing the main ``julia`` thread to become blocked. Concurrency is limited by size of the libuv thread pool, which defaults to 4 threads but can be increased by setting the ``UV_THREADPOOL_SIZE`` environment variable and restarting the ``julia`` process. + + Note that the called function should never call back into Julia. + Synchronization Primitives -------------------------- From c724db6a80c6c3d56433334fa4e68d82a7e54acf Mon Sep 17 00:00:00 2001 From: Simon Byrne <simonbyrne@gmail.com> Date: Tue, 9 Aug 2016 17:45:05 +0100 Subject: [PATCH 0852/1117] bump openlibm, fixes #17751 (#17912) --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + deps/openlibm.version | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 delete mode 100644 deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 create mode 100644 deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 create mode 100644 deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 diff --git a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 deleted file mode 100644 index 8e7d4cc7c4029..0000000000000 --- a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -3b852052db9052a3668122449ce1f82c diff --git a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 deleted file mode 100644 index a840095574b1a..0000000000000 --- a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -7125800186428a8aefea3030ff1d35f4169c29cb3ae30c2bd7965bd5e66f0958b8b33fcd2e5a30b012ecececd09ce3ce0195b5a8fc9066efd31e124cc32ec2c0 diff --git a/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 new file mode 100644 index 0000000000000..aa9ed22e8bb6d --- /dev/null +++ b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 @@ -0,0 +1 @@ +1bde34205734b9b30bd6a92768bd916b diff --git a/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 new file mode 100644 index 0000000000000..5af90ecd2b687 --- /dev/null +++ b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 @@ -0,0 +1 @@ +9a324bf989fa513b6d513794114d08a24e77a4ff15da577c950c178322e34121f9eba6bf01e1520cd12a926723cda4687bb8efd6aa9de987cd9b3a1e6230d2ce diff --git a/deps/openlibm.version b/deps/openlibm.version index ceb185d9bc9de..af5b13cbea355 100644 --- a/deps/openlibm.version +++ b/deps/openlibm.version @@ -1,2 +1,2 @@ -OPENLIBM_BRANCH=v0.5.2 -OPENLIBM_SHA1=0fa599cce845be67ed1729d9753f67e366c37b96 +OPENLIBM_BRANCH=v0.5.3 +OPENLIBM_SHA1=71e79eb6f74d3f04ce724195b8ef5846a70d281e From 7ebcab5fd553ff95d621b7ca244eeeb9c63f94cc Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 3 Aug 2016 15:20:33 -0400 Subject: [PATCH 0853/1117] fix #17785, process all keyword args left-to-right Whatever comes later, whether splatted or not, takes precedence. --- NEWS.md | 5 +++ doc/manual/functions.rst | 7 +++ src/julia-syntax.scm | 97 ++++++++++++++++++++++------------------ test/keywordargs.jl | 9 ++++ 4 files changed, 75 insertions(+), 43 deletions(-) diff --git a/NEWS.md b/NEWS.md index f427b4e5adf47..e7b5931cb99d1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -21,6 +21,9 @@ This section lists changes that do not have deprecation warnings. * Operations between `Float16` and `Integers` now return `Float16` instead of `Float32`. ([#17261]) + * Keyword arguments are processed left-to-right: if the same keyword is specified more than + once, the rightmost occurrence takes precedence ([#17785]). + Library improvements -------------------- @@ -616,6 +619,7 @@ Language tooling improvements [#17037]: https://github.com/JuliaLang/julia/issues/17037 [#17075]: https://github.com/JuliaLang/julia/issues/17075 [#17132]: https://github.com/JuliaLang/julia/issues/17132 +[#17261]: https://github.com/JuliaLang/julia/issues/17261 [#17266]: https://github.com/JuliaLang/julia/issues/17266 [#17300]: https://github.com/JuliaLang/julia/issues/17300 [#17323]: https://github.com/JuliaLang/julia/issues/17323 @@ -626,3 +630,4 @@ Language tooling improvements [#17510]: https://github.com/JuliaLang/julia/issues/17510 [#17546]: https://github.com/JuliaLang/julia/issues/17546 [#17668]: https://github.com/JuliaLang/julia/issues/17668 +[#17785]: https://github.com/JuliaLang/julia/issues/17785 diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 1b6b6883e8f14..65e1432d06f92 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -510,6 +510,13 @@ tuple, explicitly after a semicolon. For example, ``plot(x, y; ``plot(x, y, width=2)``. This is useful in situations where the keyword name is computed at runtime. +The nature of keyword arguments makes it possible to specify the same +argument more than once. For example, in the call +``plot(x, y; options..., width=2)`` it is possible that the ``options`` +structure also contains a value for ``width``. In such a case the +rightmost occurrence takes precedence; in this example, ``width`` +is certain to have the value ``2``. + .. _man-evaluation-scope-default-values: Evaluation Scope of Default Values diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index f456a68bf1ea4..6fd0b31745119 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1358,49 +1358,60 @@ ;; lower function call containing keyword arguments (define (lower-kw-call f kw pa) - (if (any (lambda (x) (and (pair? x) (eq? (car x) 'parameters))) - kw) - (error "more than one semicolon in argument list")) - (receive - (keys restkeys) (separate kwarg? kw) - (let ((keyargs (apply append - (map (lambda (a) - (if (not (symbol? (cadr a))) - (error (string "keyword argument is not a symbol: \"" - (deparse (cadr a)) "\""))) - (if (vararg? (caddr a)) - (error "splicing with \"...\" cannot be used for a keyword argument value")) - `((quote ,(cadr a)) ,(caddr a))) - keys)))) - (if (null? restkeys) - `(call (call (core kwfunc) ,f) (call (top vector_any) ,@keyargs) ,f ,@pa) - (let ((container (make-ssavalue))) - `(block - (= ,container (call (top vector_any) ,@keyargs)) - ,@(map (lambda (rk) - (let* ((k (make-ssavalue)) - (v (make-ssavalue)) - (push-expr `(ccall 'jl_array_ptr_1d_push2 Void - (tuple Any Any Any) - ,container - (|::| ,k (core Symbol)) - ,v))) - (if (vararg? rk) - `(for (= (tuple ,k ,v) ,(cadr rk)) - ,push-expr) - `(block (= (tuple ,k ,v) ,rk) - ,push-expr)))) - restkeys) - ,(if (not (null? keys)) - `(call (call (core kwfunc) ,f) ,container ,f ,@pa) - (let* ((expr_stmts (remove-argument-side-effects `(call ,f ,@pa))) - (pa (cddr (car expr_stmts))) - (stmts (cdr expr_stmts))) - `(block - ,@stmts - (if (call (top isempty) ,container) - (call ,f ,@pa) - (call (call (core kwfunc) ,f) ,container ,f ,@pa))))))))))) + (let ((container (make-ssavalue))) + (let loop ((kw kw) + (initial-kw '()) ;; keyword args before any splats + (stmts '()) + (has-kw #f)) ;; whether there are definitely >0 kwargs + (if (null? kw) + (if (null? stmts) + `(call (call (core kwfunc) ,f) (call (top vector_any) ,@(reverse initial-kw)) ,f ,@pa) + `(block + (= ,container (call (top vector_any) ,@(reverse initial-kw))) + ,@(reverse stmts) + ,(if has-kw + `(call (call (core kwfunc) ,f) ,container ,f ,@pa) + (let* ((expr_stmts (remove-argument-side-effects `(call ,f ,@pa))) + (pa (cddr (car expr_stmts))) + (stmts (cdr expr_stmts))) + `(block + ,@stmts + (if (call (top isempty) ,container) + (call ,f ,@pa) + (call (call (core kwfunc) ,f) ,container ,f ,@pa))))))) + (let ((arg (car kw))) + (cond ((and (pair? arg) (eq? (car arg) 'parameters)) + (error "more than one semicolon in argument list")) + ((kwarg? arg) + (if (not (symbol? (cadr arg))) + (error (string "keyword argument is not a symbol: \"" + (deparse (cadr arg)) "\""))) + (if (vararg? (caddr arg)) + (error "splicing with \"...\" cannot be used for a keyword argument value")) + (if (null? stmts) + (loop (cdr kw) (list* (caddr arg) `(quote ,(cadr arg)) initial-kw) stmts #t) + (loop (cdr kw) initial-kw + (cons `(ccall 'jl_array_ptr_1d_push2 Void (tuple Any Any Any) + ,container + (|::| (quote ,(cadr arg)) (core Symbol)) + ,(caddr arg)) + stmts) + #t))) + (else + (loop (cdr kw) initial-kw + (cons (let* ((k (make-ssavalue)) + (v (make-ssavalue)) + (push-expr `(ccall 'jl_array_ptr_1d_push2 Void (tuple Any Any Any) + ,container + (|::| ,k (core Symbol)) + ,v))) + (if (vararg? arg) + `(for (= (tuple ,k ,v) ,(cadr arg)) + ,push-expr) + `(block (= (tuple ,k ,v) ,arg) + ,push-expr))) + stmts) + (or has-kw (not (vararg? arg))))))))))) ;; convert e.g. A'*B to Ac_mul_B(A,B) (define (expand-transposed-op e ops) diff --git a/test/keywordargs.jl b/test/keywordargs.jl index 7a582b73645f1..65423eb4eac46 100644 --- a/test/keywordargs.jl +++ b/test/keywordargs.jl @@ -218,3 +218,12 @@ end @test f9948(x=5) == 5 @test_throws UndefVarError f9948() @test getx9948() == 3 + +# issue #17785 - handle all sources of kwargs left-to-right +g17785(; a=1, b=2) = (a, b) +let opts = (:a=>3, :b=>4) + @test g17785(; a=5, opts...) == (3, 4) + @test g17785(; opts..., a=5) == (5, 4) + @test g17785(; opts..., a=5, b=6) == (5, 6) + @test g17785(; b=0, opts..., a=5) == (5, 4) +end From fa8e40f5c772ee7673a4d20e997320850fecd414 Mon Sep 17 00:00:00 2001 From: kshyatt <kshyatt@physics.ucsb.edu> Date: Fri, 5 Aug 2016 16:59:08 -0700 Subject: [PATCH 0854/1117] More examples, cleanup of array and eigen method docs --- base/abstractarray.jl | 77 +++++++- base/array.jl | 388 ++++++++++++++++++++++++++++++++++++- base/collections.jl | 78 +++++++- base/datafmt.jl | 10 +- base/docs/helpdb/Base.jl | 322 ------------------------------ base/linalg/eigen.jl | 205 +++++++++++++++++++- base/reducedim.jl | 10 + doc/stdlib/arrays.rst | 308 ++++++++++++++++++++++++++--- doc/stdlib/collections.rst | 122 ++++++++++-- doc/stdlib/io-network.rst | 6 +- doc/stdlib/linalg.rst | 74 ++++++- 11 files changed, 1197 insertions(+), 403 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 65303a33d7b08..ef2b9fa100be3 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -26,6 +26,23 @@ function vect(X...) copy!(Array{T,1}(length(X)), X) end +""" + size(A::AbstractArray, [dim...]) + +Returns a tuple containing the dimensions of `A`. Optionally you can specify the +dimension(s) you want the length of, and get the length of that dimension, or a tuple of the +lengths of dimensions you asked for. + +```jldoctest +julia> A = ones(2,3,4); + +julia> size(A, 2) +3 + +julia> size(A,3,2) +(4,3) +``` +""" size{T,N}(t::AbstractArray{T,N}, d) = d <= N ? size(t)[d] : 1 size{N}(x, d1::Integer, d2::Integer, dx::Vararg{Integer, N}) = (size(x, d1), size(x, d2), ntuple(k->size(x, dx[k]), Val{N})...) @@ -72,9 +89,34 @@ linearindices(A::AbstractVector) = (@_inline_meta; indices1(A)) eltype{T}(::Type{AbstractArray{T}}) = T eltype{T,N}(::Type{AbstractArray{T,N}}) = T elsize{T}(::AbstractArray{T}) = sizeof(T) +""" + ndims(A::AbstractArray) -> Integer + +Returns the number of dimensions of `A`. + +```jldoctest +julia> A = ones(3,4,5); + +julia> ndims(A) +3 +``` +""" ndims{T,N}(::AbstractArray{T,N}) = N ndims{T,N}(::Type{AbstractArray{T,N}}) = N ndims{T<:AbstractArray}(::Type{T}) = ndims(supertype(T)) + +""" + length(A::AbstractArray) -> Integer + +Returns the number of elements in `A`. + +```jldoctest +julia> A = ones(3,4,5); + +julia> length(A) +60 +``` +""" length(t::AbstractArray) = prod(size(t)) _length(A::AbstractArray) = prod(map(unsafe_length, indices(A))) # circumvent missing size _length(A) = length(A) @@ -89,9 +131,19 @@ end last(a) = a[end] """ - stride(A, k) + stride(A, k::Integer) Returns the distance in memory (in number of elements) between adjacent elements in dimension `k`. + +```jldoctest +julia> A = ones(3,4,5); + +julia> stride(A,2) +3 + +julia> stride(A,3) +12 +``` """ function stride(a::AbstractArray, i::Integer) if i > ndims(a) @@ -109,6 +161,13 @@ strides{T}(A::AbstractArray{T,0}) = () strides(A) Returns a tuple of the memory strides in each dimension. + +```jldoctest +julia> A = ones(3,4,5); + +julia> strides(A) +(1,3,12) +``` """ strides(A::AbstractArray) = _strides((1,), A) _strides{T,N}(out::NTuple{N}, A::AbstractArray{T,N}) = out @@ -146,6 +205,22 @@ abstract LinearIndexing immutable LinearFast <: LinearIndexing end immutable LinearSlow <: LinearIndexing end +""" + Base.linearindexing(A) + +`linearindexing` defines how an AbstractArray most efficiently accesses its elements. If +`Base.linearindexing(A)` returns `Base.LinearFast()`, this means that linear indexing with +only one index is an efficient operation. If it instead returns `Base.LinearSlow()` (by +default), this means that the array intrinsically accesses its elements with indices +specified for every dimension. Since converting a linear index to multiple indexing +subscripts is typically very expensive, this provides a traits-based mechanism to enable +efficient generic code for all array types. + +An abstract array subtype `MyArray` that wishes to opt into fast linear indexing behaviors +should define `linearindexing` in the type-domain: + + Base.linearindexing{T<:MyArray}(::Type{T}) = Base.LinearFast() +""" linearindexing(A::AbstractArray) = linearindexing(typeof(A)) linearindexing{T<:AbstractArray}(::Type{T}) = LinearSlow() linearindexing{T<:Array}(::Type{T}) = LinearFast() diff --git a/base/array.jl b/base/array.jl index 8c753b3e3924d..fad82af049850 100644 --- a/base/array.jl +++ b/base/array.jl @@ -554,6 +554,22 @@ function insert!{T}(a::Array{T,1}, i::Integer, item) return a end +""" + deleteat!(a::Vector, i::Integer) + +Remove the item at the given `i` and return the modified `a`. Subsequent items +are shifted to fill the resulting gap. + +```jldoctest +julia> deleteat!([6, 5, 4, 3, 2, 1], 2) +5-element Array{Int64,1}: + 6 + 4 + 3 + 2 + 1 +``` +""" deleteat!(a::Vector, i::Integer) = (_deleteat!(a, i, 1); a) function deleteat!{T<:Integer}(a::Vector, r::UnitRange{T}) @@ -562,6 +578,25 @@ function deleteat!{T<:Integer}(a::Vector, r::UnitRange{T}) return a end +""" + deleteat!(a::Vector, inds) + +Remove the items at the indices given by `inds`, and return the modified `a`. +Subsequent items are shifted to fill the resulting gap. `inds` must be sorted and unique. + +```jldoctest +julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) +3-element Array{Int64,1}: + 5 + 3 + 1 + +julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) +ERROR: ArgumentError: indices must be unique and sorted + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:576 + ... +``` +""" function deleteat!(a::Vector, inds) n = length(a) s = start(inds) @@ -732,7 +767,24 @@ end ## find ## -# returns the index of the next non-zero element, or 0 if all zeros +""" + findnext(A, i::Integer) + +Find the next linear index >= `i` of a non-zero element of `A`, or `0` if not found. + +```jldoctest +julia> A = [0 0; 1 0] +2×2 Array{Int64,2}: + 0 0 + 1 0 + +julia> findnext(A,1) +2 + +julia> findnext(A,3) +0 +``` +""" function findnext(A, start::Integer) for i = start:length(A) if A[i] != 0 @@ -741,9 +793,43 @@ function findnext(A, start::Integer) end return 0 end + +""" + findfirst(A) + +Return the linear index of the first non-zero value in `A` (determined by `A[i]!=0`). +Returns `0` if no such value is found. + +```jldoctest +julia> A = [0 0; 1 0] +2×2 Array{Int64,2}: + 0 0 + 1 0 + +julia> findfirst(A) +2 +``` +""" findfirst(A) = findnext(A, 1) -# returns the index of the next matching element +""" + findnext(A, v, i::Integer) + +Find the next linear index >= `i` of an element of `A` equal to `v` (using `==`), or `0` if not found. + +```jldoctest +julia> A = [1 4; 2 2] +2×2 Array{Int64,2}: + 1 4 + 2 2 + +julia> findnext(A,4,4) +0 + +julia> findnext(A,4,3) +3 +``` +""" function findnext(A, v, start::Integer) for i = start:length(A) if A[i] == v @@ -752,9 +838,45 @@ function findnext(A, v, start::Integer) end return 0 end +""" + findfirst(A, v) + +Return the linear index of the first element equal to `v` in `A`. +Returns `0` if `v` is not found. + +```jldoctest +julia> A = [4 6; 2 2] +2×2 Array{Int64,2}: + 4 6 + 2 2 + +julia> findfirst(A,2) +2 + +julia> findfirst(A,3) +0 +``` +""" findfirst(A, v) = findnext(A, v, 1) -# returns the index of the next element for which the function returns true +""" + findnext(predicate::Function, A, i::Integer) + +Find the next linear index >= `i` of an element of `A` for which `predicate` returns `true`, or `0` if not found. + +```jldoctest +julia> A = [1 4; 2 2] +2×2 Array{Int64,2}: + 1 4 + 2 2 + +julia> findnext(isodd, A, 1) +1 + +julia> findnext(isodd, A, 2) +0 +``` +""" function findnext(testf::Function, A, start::Integer) for i = start:length(A) if testf(A[i]) @@ -763,35 +885,193 @@ function findnext(testf::Function, A, start::Integer) end return 0 end + +""" + findfirst(predicate::Function, A) + +Return the linear index of the first element of `A` for which `predicate` returns `true`. +Returns `0` if there is no such element. + +```jldoctest +julia> A = [1 4; 2 2] +2×2 Array{Int64,2}: + 1 4 + 2 2 + +julia> findfirst(iseven, A) +2 + +julia> findfirst(x -> x>10, A) +0 +``` +""" findfirst(testf::Function, A) = findnext(testf, A, 1) -# returns the index of the previous non-zero element, or 0 if all zeros +""" + findprev(A, i::Integer) + +Find the previous linear index <= `i` of a non-zero element of `A`, or `0` if not found. + +```jldoctest +julia> A = [0 0; 1 2] +2×2 Array{Int64,2}: + 0 0 + 1 2 + +julia> findprev(A,2) +2 + +julia> findprev(A,1) +0 +``` +""" function findprev(A, start::Integer) for i = start:-1:1 A[i] != 0 && return i end return 0 end + +""" + findlast(A) + +Return the linear index of the last non-zero value in `A` (determined by `A[i]!=0`). +Returns `0` if there is no non-zero value in `A`. + +```jldoctest +julia> A = [1 0; 1 0] +2×2 Array{Int64,2}: + 1 0 + 1 0 + +julia> findlast(A) +2 + +julia> A = zeros(2,2) +2×2 Array{Float64,2}: + 0.0 0.0 + 0.0 0.0 + +julia> findlast(A) +0 +``` +""" findlast(A) = findprev(A, length(A)) -# returns the index of the matching element, or 0 if no matching +""" + findprev(A, v, i::Integer) + +Find the previous linear index <= `i` of an element of `A` equal to `v` (using `==`), or `0` if not found. + +```jldoctest +julia> A = [0 0; 1 2] +2×2 Array{Int64,2}: + 0 0 + 1 2 + +julia> findprev(A, 1, 4) +2 + +julia> findprev(A, 1, 1) +0 +``` +""" function findprev(A, v, start::Integer) for i = start:-1:1 A[i] == v && return i end return 0 end + +""" + findlast(A, v) + +Return the linear index of the last element equal to `v` in `A`. +Returns `0` if there is no element of `A` equal to `v`. + +```jldoctest +julia> A = [1 2; 2 1] +2×2 Array{Int64,2}: + 1 2 + 2 1 + +julia> findlast(A,1) +4 + +julia> findlast(A,2) +3 + +julia> findlast(A,3) +0 +``` +""" findlast(A, v) = findprev(A, v, length(A)) -# returns the index of the previous element for which the function returns true, or zero if it never does +""" + findprev(predicate::Function, A, i::Integer) + +Find the previous linear index <= `i` of an element of `A` for which `predicate` returns `true`, or +`0` if not found. + +```jldoctest +julia> A = [4 6; 1 2] +2×2 Array{Int64,2}: + 4 6 + 1 2 + +julia> findprev(isodd, A, 1) +0 + +julia> findprev(isodd, A, 3) +2 +``` +""" function findprev(testf::Function, A, start::Integer) for i = start:-1:1 testf(A[i]) && return i end return 0 end + +""" + findlast(predicate::Function, A) + +Return the linear index of the last element of `A` for which `predicate` returns `true`. +Returns `0` if there is no such element. + +```jldoctest +julia> A = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> findlast(isodd, A) +2 + +julia> findlast(x -> x > 5, A) +0 +``` +""" findlast(testf::Function, A) = findprev(testf, A, length(A)) +""" + find(f::Function, A) + +Return a vector `I` of the linear indexes of `A` where `f(A[I])` returns `true`. +If there are no such elements of `A`, find returns an empty array. + +```jldoctest +julia> A = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> find(isodd,A) +2-element Array{Int64,1}: + 1 + 2 +``` +""" function find(testf::Function, A) # use a dynamic-length array to store the indexes, then copy to a non-padded # array for the return @@ -806,6 +1086,25 @@ function find(testf::Function, A) return I end +""" + find(A) + +Return a vector of the linear indexes of the non-zeros in `A` (determined by `A[i]!=0`). A +common use of this is to convert a boolean array to an array of indexes of the `true` +elements. If there are no non-zero elements of `A`, `find` returns an empty array. + +```jldoctest +julia> A = [true false; false true] +2×2 Array{Bool,2}: + true false + false true + +julia> find(A) +2-element Array{Int64,1}: + 1 + 4 +``` +""" function find(A) nnzA = countnz(A) I = Vector{Int}(nnzA) @@ -824,6 +1123,32 @@ find(testf::Function, x::Number) = !testf(x) ? Array{Int,1}(0) : [1] findn(A::AbstractVector) = find(A) +""" + findn(A) + +Return a vector of indexes for each dimension giving the locations of the non-zeros in `A` +(determined by `A[i]!=0`). +If there are no non-zero elements of `A`, `findn` returns a 2-tuple of empty arrays. + +```jldoctest +julia> A = [1 2 0; 0 0 3; 0 4 0] +3×3 Array{Int64,2}: + 1 2 0 + 0 0 3 + 0 4 0 + +julia> findn(A) +([1,1,3,2],[1,2,2,3]) + +julia> A = zeros(2,2) +2×2 Array{Float64,2}: + 0.0 0.0 + 0.0 0.0 + +julia> findn(A) +(Int64[],Int64[]) +``` +""" function findn(A::AbstractMatrix) nnzA = countnz(A) I = similar(A, Int, nnzA) @@ -839,6 +1164,23 @@ function findn(A::AbstractMatrix) return (I, J) end +""" + findnz(A) + +Return a tuple `(I, J, V)` where `I` and `J` are the row and column indexes of the non-zero +values in matrix `A`, and `V` is a vector of the non-zero values. + +```jldoctest +julia> A = [1 2 0; 0 0 3; 0 4 0] +3×3 Array{Int64,2}: + 1 2 0 + 0 0 3 + 0 4 0 + +julia> findnz(A) +([1,1,3,2],[1,2,2,3],[1,2,4,3]) +``` +""" function findnz{T}(A::AbstractMatrix{T}) nnzA = countnz(A) I = zeros(Int, nnzA) @@ -921,6 +1263,8 @@ end indmax(itr) -> Integer Returns the index of the maximum element in a collection. +The collection must not be empty. + ```jldoctest julia> indmax([8,0.1,-9,pi]) 1 @@ -932,6 +1276,8 @@ indmax(a) = findmax(a)[2] indmin(itr) -> Integer Returns the index of the minimum element in a collection. +The collection must not be empty. + ```jldoctest julia> indmin([8,0.1,-9,pi]) 3 @@ -1091,6 +1437,22 @@ function union(vs...) ret end # setdiff only accepts two args + +""" + setdiff(a, b) + +Construct the set of elements in `a` but not `b`. Maintains order with arrays. Note that +both arguments must be collections, and both will be iterated over. In particular, +`setdiff(set,element)` where `element` is a potential member of `set`, will not work in +general. + +```jldoctest +julia> setdiff([1,2,3],[3,4,5]) +2-element Array{Int64,1}: + 1 + 2 +``` +""" function setdiff(a, b) args_type = promote_type(eltype(a), eltype(b)) bset = Set(b) @@ -1111,4 +1473,18 @@ end # store counts with a Dict. symdiff(a) = a symdiff(a, b) = union(setdiff(a,b), setdiff(b,a)) +""" + symdiff(a, b, rest...) + +Construct the symmetric difference of elements in the passed in sets or arrays. +Maintains order with arrays. + +```jldoctest +julia> symdiff([1,2,3],[3,4,5],[4,5,6]) +3-element Array{Int64,1}: + 1 + 2 + 6 +``` +""" symdiff(a, b, rest...) = symdiff(a, symdiff(b, rest...)) diff --git a/base/collections.jl b/base/collections.jl index 545112e25ba8b..caeebf673fa86 100644 --- a/base/collections.jl +++ b/base/collections.jl @@ -109,7 +109,7 @@ end # Turn an arbitrary array into a binary min-heap in linear time. """ - heapify!(v, [ord]) + heapify!(v, ord::Ordering=Forward) In-place [`heapify`](:func:`heapify`). """ @@ -121,16 +121,49 @@ function heapify!(xs::AbstractArray, o::Ordering=Forward) end """ - heapify(v, [ord]) + heapify(v, ord::Ordering=Forward) Returns a new vector in binary heap order, optionally using the given ordering. +```jldoctest +julia> a = [1,3,4,5,2]; + +julia> Base.Collections.heapify(a) +5-element Array{Int64,1}: + 1 + 2 + 4 + 5 + 3 + +julia> Base.Collections.heapify(a, Base.Order.Reverse) +5-element Array{Int64,1}: + 5 + 3 + 4 + 1 + 2 +``` """ heapify(xs::AbstractArray, o::Ordering=Forward) = heapify!(copymutable(xs), o) """ - isheap(v, [ord]) + isheap(v, ord::Ordering=Forward) Return `true` if an array is heap-ordered according to the given order. + +```jldoctest +julia> a = [1,2,3] +3-element Array{Int64,1}: + 1 + 2 + 3 + +julia> Base.Collections.isheap(a,Base.Order.Forward) +true + +julia> Base.Collections.isheap(a,Base.Order.Reverse) +false +``` """ function isheap(xs::AbstractArray, o::Ordering=Forward) for i in 1:div(length(xs), 2) @@ -157,6 +190,14 @@ the default comparison for `V`. A `PriorityQueue` acts like a `Dict`, mapping values to their priorities, with the addition of a `dequeue!` function to remove the lowest priority element. + +```jldoctest +julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 +``` """ type PriorityQueue{K,V,O<:Ordering} <: Associative{K,V} # Binary heap of (element, priority) pairs. @@ -304,6 +345,21 @@ end enqueue!(pq, k, v) Insert the a key `k` into a priority queue `pq` with priority `v`. + +```jldoctest +julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + +julia> Base.Collections.enqueue!(a, "d", 4) +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 4 entries: + "c" => 1 + "b" => 3 + "a" => 2 + "d" => 4 +``` """ function enqueue!{K,V}(pq::PriorityQueue{K,V}, key, value) if haskey(pq, key) @@ -319,6 +375,22 @@ end dequeue!(pq) Remove and return the lowest priority key from a priority queue. + +```jldoctest +julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + +julia> Base.Collections.dequeue!(a) +"c" + +julia> a +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 2 entries: + "b" => 3 + "a" => 2 +``` """ function dequeue!(pq::PriorityQueue) x = pq.xs[1] diff --git a/base/datafmt.jl b/base/datafmt.jl index 7016d09a3d6d3..13efa2c337277 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -690,11 +690,11 @@ function writedlm(fname::AbstractString, a, dlm; opts...) end """ - writedlm(f, A, dl='\\t'; opts) + writedlm(f, A, delim='\\t'; opts) -Write `A` (a vector, matrix or an iterable collection of iterable rows) as text to `f` -(either a filename string or an `IO` stream) using the given delimiter `delim` (which -defaults to tab, but can be any printable Julia object, typically a `Char` or +Write `A` (a vector, matrix, or an iterable collection of iterable rows) as text to `f` +(either a filename string or an [`IO`](:class:`IO`) stream) using the given delimiter +`delim` (which defaults to tab, but can be any printable Julia object, typically a `Char` or `AbstractString`). For example, two vectors `x` and `y` of the same length can be written as two columns of @@ -705,7 +705,7 @@ writedlm(io, a; opts...) = writedlm(io, a, '\t'; opts...) """ writecsv(filename, A; opts) -Equivalent to `writedlm` with `delim` set to comma. +Equivalent to [`writedlm`](:func:`writedlm`) with `delim` set to comma. """ writecsv(io, a; opts...) = writedlm(io, a, ','; opts...) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 2175bd10b0a16..3a669992c71f3 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -263,14 +263,6 @@ The text is assumed to be encoded in UTF-8. """ readlines -""" - findnz(A) - -Return a tuple `(I, J, V)` where `I` and `J` are the row and column indexes of the non-zero -values in matrix `A`, and `V` is a vector of the non-zero values. -""" -findnz - """ foldl(op, v0, itr) @@ -1639,13 +1631,6 @@ Bitwise and. """ & -""" - eigmax(A) - -Returns the largest eigenvalue of `A`. -""" -eigmax - """ PipeBuffer() @@ -2113,21 +2098,6 @@ themselves in another collection. The result is of the preceding example is equi """ append! -""" - find(A) - -Return a vector of the linear indexes of the non-zeros in `A` (determined by `A[i]!=0`). A -common use of this is to convert a boolean array to an array of indexes of the `true` -elements. -""" -find(A) - -""" - find(f,A) - -Return a vector of the linear indexes of `A` where `f` returns `true`. -""" -find(f, A) """ ctranspose(A) @@ -2309,24 +2279,6 @@ Multiply elements of an array over the given dimensions. """ prod(A, dims) -""" - Base.linearindexing(A) - -`linearindexing` defines how an AbstractArray most efficiently accesses its elements. If -`Base.linearindexing(A)` returns `Base.LinearFast()`, this means that linear indexing with -only one index is an efficient operation. If it instead returns `Base.LinearSlow()` (by -default), this means that the array intrinsically accesses its elements with indices -specified for every dimension. Since converting a linear index to multiple indexing -subscripts is typically very expensive, this provides a traits-based mechanism to enable -efficient generic code for all array types. - -An abstract array subtype `MyArray` that wishes to opt into fast linear indexing behaviors -should define `linearindexing` in the type-domain: - - Base.linearindexing{T<:MyArray}(::Type{T}) = Base.LinearFast() -""" -Base.linearindexing - """ isqrt(n) @@ -3215,14 +3167,6 @@ results `A[ks...]`, where `ks` goes over the positions in the broadcast. """ broadcast_getindex -""" - findn(A) - -Return a vector of indexes for each dimension giving the locations of the non-zeros in `A` -(determined by `A[i]!=0`). -""" -findn - """ invoke(f, (types...), args...) @@ -3380,13 +3324,6 @@ Display an informational message. Argument `msg` is a string describing the info """ info -""" - eigmin(A) - -Returns the smallest eigenvalue of `A`. -""" -eigmin - """ ltoh(x) @@ -4188,13 +4125,6 @@ last argument optionally specifies a size beyond which the buffer may not be gro """ IOBuffer(data=?) -""" - findmax(A, dims) -> (maxval, index) - -For an array input, returns the value and index of the maximum over the given dimensions. -""" -findmax(A,dims) - """ tempname() @@ -4274,45 +4204,6 @@ Compute the inverse hyperbolic tangent of `x`. """ atanh -""" - deleteat!(collection, index) - -Remove the item at the given `index` and return the modified `collection`. Subsequent items -are shifted to fill the resulting gap. - -```jldoctest -julia> deleteat!([6, 5, 4, 3, 2, 1], 2) -5-element Array{Int64,1}: - 6 - 4 - 3 - 2 - 1 -``` -""" -deleteat!(collection, index::Integer) - -""" - deleteat!(collection, itr) - -Remove the items at the indices given by `itr`, and return the modified `collection`. -Subsequent items are shifted to fill the resulting gap. `itr` must be sorted and unique. - -```jldoctest -julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) -3-element Array{Int64,1}: - 5 - 3 - 1 - -julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) -ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:575 - ... -``` -""" -deleteat!(collection, itr) - """ read(stream::IO, T) @@ -4398,14 +4289,6 @@ after the end of the string. """ nextind -""" - symdiff(s1,s2...) - -Construct the symmetric difference of elements in the passed in sets or arrays. Maintains -order with arrays. -""" -symdiff - """ eta(x) @@ -4727,13 +4610,6 @@ Convert a hexadecimal string to the floating point number it represents. """ hex2num -""" - ndims(A) -> Integer - -Returns the number of dimensions of `A`. -""" -ndims - """ ishermitian(A) -> Bool @@ -5061,16 +4937,6 @@ Show a descriptive representation of an exception object. """ showerror -""" - setdiff(s1,s2) - -Construct the set of elements in `s1` but not `s2`. Maintains order with arrays. Note that -both arguments must be collections, and both will be iterated over. In particular, -`setdiff(set,element)` where `element` is a potential member of `set`, will not work in -general. -""" -setdiff - """ error(message::AbstractString) @@ -5883,28 +5749,6 @@ Return ``x^{1/3}``. The prefix operator `∛` is equivalent to `cbrt`. """ cbrt -""" - findprev(A, i) - -Find the previous index <= `i` of a non-zero element of `A`, or `0` if not found. -""" -findprev(A,i) - -""" - findprev(predicate, A, i) - -Find the previous index <= `i` of an element of `A` for which `predicate` returns `true`, or -`0` if not found. -""" -findprev(predicate::Function,A,i) - -""" - findprev(A, v, i) - -Find the previous index <= `i` of an element of `A` equal to `v` (using `==`), or `0` if not found. -""" -findprev(A,v,i) - """ matchall(r::Regex, s::AbstractString[, overlap::Bool=false]) -> Vector{AbstractString} @@ -6110,30 +5954,6 @@ all elements of the string. """ ispunct -""" - size(A, [dim...]) - -Returns a tuple containing the dimensions of `A`. Optionally you can specify the -dimension(s) you want the length of, and get the length of that dimension, or a tuple of the -lengths of dimensions you asked for. - - julia> A = rand(2,3,4); - - julia> size(A, 2) - 3 - - julia> size(A,3,2) - (4,3) -""" -size - -""" - findmin(A, dims) -> (minval, index) - -For an array input, returns the value and index of the minimum over the given dimensions. -""" -findmin(A,dims) - """ ismount(path) -> Bool @@ -6156,13 +5976,6 @@ Boolean not. """ Base.:(!) -""" - length(A) -> Integer - -Returns the number of elements in `A`. -""" -length(::AbstractArray) - """ length(collection) -> Integer @@ -6302,27 +6115,6 @@ returning a `Future` to the result. """ :@spawn -""" - findfirst(A) - -Return the index of the first non-zero value in `A` (determined by `A[i]!=0`). -""" -findfirst(A) - -""" - findfirst(A,v) - -Return the index of the first element equal to `v` in `A`. -""" -findfirst(A,v) - -""" - findfirst(predicate, A) - -Return the index of the first element of `A` for which `predicate` returns `true`. -""" -findfirst - """ promote_rule(type1, type2) @@ -6821,48 +6613,6 @@ false """ isbits -""" - findlast(A) - -Return the index of the last non-zero value in `A` (determined by `A[i]!=0`). -""" -findlast(A) - -""" - findlast(A, v) - -Return the index of the last element equal to `v` in `A`. -""" -findlast(A,v) - -""" - findlast(predicate, A) - -Return the index of the last element of `A` for which `predicate` returns `true`. -""" -findlast(::Function, A) - -""" - findnext(A, i) - -Find the next index >= `i` of a non-zero element of `A`, or `0` if not found. -""" -findnext - -""" - findnext(predicate, A, i) - -Find the next index >= `i` of an element of `A` for which `predicate` returns `true`, or `0` if not found. -""" -findnext(::Function,A,i) - -""" - findnext(A, v, i) - -Find the next index >= `i` of an element of `A` equal to `v` (using `==`), or `0` if not found. -""" -findnext(A,v,i) - """ angle(z) @@ -6909,18 +6659,6 @@ an array of the results `f(as...)` for each position. """ broadcast -""" - eigvecs(A, [eigvals,][permute=true,][scale=true]) -> Matrix - -Returns a matrix `M` whose columns are the eigenvectors of `A`. (The `k`th eigenvector can -be obtained from the slice `M[:, k]`.) The `permute` and `scale` keywords are the same as -for [`eigfact`](:func:`eigfact`). - -For [`SymTridiagonal`](:class:`SymTridiagonal`) matrices, if the optional vector of -eigenvalues `eigvals` is specified, returns the specific corresponding eigenvectors. -""" -eigvecs - """ ntoh(x) @@ -7364,38 +7102,6 @@ unsigned without checking for negative values. """ unsigned -""" - eigfact(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> Eigen - -Computes the eigenvalue decomposition of `A`, returning an `Eigen` factorization object `F` -which contains the eigenvalues in `F[:values]` and the eigenvectors in the columns of the -matrix `F[:vectors]`. (The `k`th eigenvector can be obtained from the slice `F[:vectors][:, k]`.) - -The following functions are available for `Eigen` objects: `inv`, `det`. - -If `A` is [`Symmetric`](:class:`Symmetric`), [`Hermitian`](:class:`Hermitian`) or -[`SymTridiagonal`](:class:`SymTridiagonal`), it is possible to calculate only a subset of -the eigenvalues by specifying either a [`UnitRange`](:class:`UnitRange`) `irange` covering -indices of the sorted eigenvalues or a pair `vl` and `vu` for the lower and upper boundaries -of the eigenvalues. - -For general nonsymmetric matrices it is possible to specify how the matrix is balanced -before the eigenvector calculation. The option `permute=true` permutes the matrix to become -closer to upper triangular, and `scale=true` scales the matrix by its diagonal elements to -make rows and columns more equal in norm. The default is `true` for both options. -""" -eigfact(A,?,?,?,?) - -""" - eigfact(A, B) -> GeneralizedEigen - -Computes the generalized eigenvalue decomposition of `A` and `B`, returning a -`GeneralizedEigen` factorization object `F` which contains the generalized eigenvalues in -`F[:values]` and the generalized eigenvectors in the columns of the matrix `F[:vectors]`. -(The `k`th generalized eigenvector can be obtained from the slice `F[:vectors][:, k]`.) -""" -eigfact(A,B) - """ mkdir(path, [mode]) @@ -7404,7 +7110,6 @@ modified by the current file creation mask. """ mkdir - """ midpoints(e) @@ -7465,33 +7170,6 @@ Letter: Lowercase. """ islower -""" - eig(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> D, V - -Computes eigenvalues and eigenvectors of `A`. See [`eigfact`](:func:`eigfact`) for details -on the `permute` and `scale` keyword arguments. The eigenvectors are returned columnwise. - -```jldoctest -julia> eig([1.0 0.0 0.0; 0.0 3.0 0.0; 0.0 0.0 18.0]) -([1.0,3.0,18.0], -[1.0 0.0 0.0; 0.0 1.0 0.0; 0.0 0.0 1.0]) -``` - -`eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the -factorization to a tuple; where possible, using [`eigfact`](:func:`eigfact`) is recommended. -""" -eig(A,?,?,?) - -""" - eig(A, B) -> D, V - -Computes generalized eigenvalues and vectors of `A` with respect to `B`. - -`eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the -factorization to a tuple; where possible, using [`eigfact`](:func:`eigfact`) is recommended. -""" -eig(A,B) - """ exp2(x) diff --git a/base/linalg/eigen.jl b/base/linalg/eigen.jl index 6e68c72d4b30f..93e9468fd24a7 100644 --- a/base/linalg/eigen.jl +++ b/base/linalg/eigen.jl @@ -54,6 +54,27 @@ function eigfact!{T<:BlasComplex}(A::StridedMatrix{T}; permute::Bool=true, scale ishermitian(A) && return eigfact!(Hermitian(A)) return Eigen(LAPACK.geevx!(permute ? (scale ? 'B' : 'P') : (scale ? 'S' : 'N'), 'N', 'V', 'N', A)[[2,4]]...) end + +""" + eigfact(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> Eigen + +Computes the eigenvalue decomposition of `A`, returning an [`Eigen`](:obj:`Eigen`) factorization object `F` +which contains the eigenvalues in `F[:values]` and the eigenvectors in the columns of the +matrix `F[:vectors]`. (The `k`th eigenvector can be obtained from the slice `F[:vectors][:, k]`.) + +The following functions are available for `Eigen` objects: [`inv`](:func:`inv`), [`det`](:func:`det`), and [`isposdef`](:func:`isposdef`). + +If `A` is [`Symmetric`](:class:`Symmetric`), [`Hermitian`](:class:`Hermitian`) or +[`SymTridiagonal`](:class:`SymTridiagonal`), it is possible to calculate only a subset of +the eigenvalues by specifying either a [`UnitRange`](:class:`UnitRange`) `irange` covering +indices of the sorted eigenvalues or a pair `vl` and `vu` for the lower and upper boundaries +of the eigenvalues. + +For general nonsymmetric matrices it is possible to specify how the matrix is balanced +before the eigenvector calculation. The option `permute=true` permutes the matrix to become +closer to upper triangular, and `scale=true` scales the matrix by its diagonal elements to +make rows and columns more equal in norm. The default is `true` for both options. +""" function eigfact{T}(A::StridedMatrix{T}; permute::Bool=true, scale::Bool=true) S = promote_type(Float32, typeof(one(T)/norm(one(T)))) eigfact!(copy_oftype(A, S), permute = permute, scale = scale) @@ -64,12 +85,40 @@ function eig(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=tr F = eigfact(A, permute=permute, scale=scale) F.values, F.vectors end + +""" + eig(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> D, V + +Computes eigenvalues (`D`) and eigenvectors (`V`) of `A`. +See [`eigfact`](:func:`eigfact`) for details on the +`irange`, `vl`, and `vu` arguments +and the `permute` and `scale` keyword arguments. +The eigenvectors are returned columnwise. + +```jldoctest +julia> eig([1.0 0.0 0.0; 0.0 3.0 0.0; 0.0 0.0 18.0]) +([1.0,3.0,18.0], +[1.0 0.0 0.0; 0.0 1.0 0.0; 0.0 0.0 1.0]) +``` + +`eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the +factorization to a tuple; where possible, using [`eigfact`](:func:`eigfact`) is recommended. +""" function eig(A::AbstractMatrix, args...) F = eigfact(A, args...) F.values, F.vectors end -#Calculates eigenvectors +""" + eigvecs(A, [eigvals,][permute=true,][scale=true]) -> Matrix + +Returns a matrix `M` whose columns are the eigenvectors of `A`. (The `k`th eigenvector can +be obtained from the slice `M[:, k]`.) The `permute` and `scale` keywords are the same as +for [`eigfact`](:func:`eigfact`). + +For [`SymTridiagonal`](:class:`SymTridiagonal`) matrices, if the optional vector of +eigenvalues `eigvals` is specified, returns the specific corresponding eigenvectors. +""" eigvecs(A::Union{Number, AbstractMatrix}; permute::Bool=true, scale::Bool=true) = eigvecs(eigfact(A, permute=permute, scale=scale)) eigvecs{T,V,S,U}(F::Union{Eigen{T,V,S,U}, GeneralizedEigen{T,V,S,U}}) = F[:vectors]::S @@ -77,10 +126,9 @@ eigvecs{T,V,S,U}(F::Union{Eigen{T,V,S,U}, GeneralizedEigen{T,V,S,U}}) = F[:vecto eigvals{T,V,S,U}(F::Union{Eigen{T,V,S,U}, GeneralizedEigen{T,V,S,U}}) = F[:values]::U """ - eigvals!(A,[irange,][vl,][vu]) -> values -Same as `eigvals`, but saves space by overwriting the input `A` (and `B`), instead of creating a copy. +Same as [`eigvals`](:func:`eigvals`), but saves space by overwriting the input `A`, instead of creating a copy. """ function eigvals!{T<:BlasReal}(A::StridedMatrix{T}; permute::Bool=true, scale::Bool=true) issymmetric(A) && return eigvals!(Symmetric(A)) @@ -100,8 +148,37 @@ function eigvals{T<:Number}(x::T; kwargs...) return imag(val) == 0 ? [real(val)] : [val] end -# TO DO: Put message about not being able to sort complex numbers back in! -#Computes maximum and minimum eigenvalue +""" + eigmax(A; permute::Bool=true, scale::Bool=true) + +Returns the largest eigenvalue of `A`. +The option `permute=true` permutes the matrix to become +closer to upper triangular, and `scale=true` scales the matrix by its diagonal elements to +make rows and columns more equal in norm. +Note that if the eigenvalues of `A` are complex, +this method will fail, since complex numbers cannot +be sorted. + +```jldoctest +julia> A = [0 im; -im 0] +2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + 0-1im 0+0im + +julia> eigmax(A) +1.0 + +julia> A = [0 im; -1 0] +2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + -1+0im 0+0im + +julia> eigmax(A) +ERROR: DomainError: + in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:108 + in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:106 +``` +""" function eigmax(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=true) v = eigvals(A, permute = permute, scale = scale) if eltype(v)<:Complex @@ -109,6 +186,38 @@ function eigmax(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool end maximum(v) end + +""" + eigmin(A; permute::Bool=true, scale::Bool=true) + +Returns the smallest eigenvalue of `A`. +The option `permute=true` permutes the matrix to become +closer to upper triangular, and `scale=true` scales the matrix by its diagonal elements to +make rows and columns more equal in norm. +Note that if the eigenvalues of `A` are complex, +this method will fail, since complex numbers cannot +be sorted. + +```jldoctest +julia> A = [0 im; -im 0] +2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + 0-1im 0+0im + +julia> eigmin(A) +-1.0 + +julia> A = [0 im; -1 0] +2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + -1+0im 0+0im + +julia> eigmin(A) +ERROR: DomainError: + in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:115 + in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:113 +``` +""" function eigmin(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=true) v = eigvals(A, permute = permute, scale = scale) if eltype(v)<:Complex @@ -149,6 +258,15 @@ function eigfact!{T<:BlasComplex}(A::StridedMatrix{T}, B::StridedMatrix{T}) alpha, beta, _, vr = LAPACK.ggev!('N', 'V', A, B) return GeneralizedEigen(alpha./beta, vr) end + +""" + eigfact(A, B) -> GeneralizedEigen + +Computes the generalized eigenvalue decomposition of `A` and `B`, returning a +`GeneralizedEigen` factorization object `F` which contains the generalized eigenvalues in +`F[:values]` and the generalized eigenvectors in the columns of the matrix `F[:vectors]`. +(The `k`th generalized eigenvector can be obtained from the slice `F[:vectors][:, k]`.) +""" function eigfact{TA,TB}(A::AbstractMatrix{TA}, B::AbstractMatrix{TB}) S = promote_type(Float32, typeof(one(TA)/norm(one(TA))),TB) return eigfact!(copy_oftype(A, S), copy_oftype(B, S)) @@ -156,6 +274,30 @@ end eigfact(A::Number, B::Number) = eigfact(fill(A,1,1), fill(B,1,1)) +""" + eig(A, B) -> D, V + +Computes generalized eigenvalues (`D`) and vectors (`V`) of `A` with respect to `B`. + +`eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the +factorization to a tuple; where possible, using [`eigfact`](:func:`eigfact`) is recommended. + +```jldoctest +julia> A = [1 0; 0 -1] +2×2 Array{Int64,2}: + 1 0 + 0 -1 + +julia> B = [0 1; 1 0] +2×2 Array{Int64,2}: + 0 1 + 1 0 + +julia> eig(A, B) +(Complex{Float64}[0.0+1.0im,0.0-1.0im], +Complex{Float64}[0.0-1.0im 0.0+1.0im; -1.0-0.0im -1.0+0.0im]) +``` +""" function eig(A::AbstractMatrix, B::AbstractMatrix) F = eigfact(A,B) F.values, F.vectors @@ -165,6 +307,11 @@ function eig(A::Number, B::Number) F.values, F.vectors end +""" + eigvals!(A, B) -> values + +Same as [`eigvals`](:func:`eigvals`), but saves space by overwriting the input `A` (and `B`), instead of creating copies. +""" function eigvals!{T<:BlasReal}(A::StridedMatrix{T}, B::StridedMatrix{T}) issymmetric(A) && isposdef(B) && return eigvals!(Symmetric(A), Symmetric(B)) alphar, alphai, beta, vl, vr = LAPACK.ggev!('N', 'N', A, B) @@ -175,11 +322,57 @@ function eigvals!{T<:BlasComplex}(A::StridedMatrix{T}, B::StridedMatrix{T}) alpha, beta, vl, vr = LAPACK.ggev!('N', 'N', A, B) alpha./beta end + +""" + eigvals(A, B) -> values + +Computes the generalized eigenvalues of `A` and `B`. + +```jldoctest +julia> A = [1 0; 0 -1] +2×2 Array{Int64,2}: + 1 0 + 0 -1 + +julia> B = [0 1; 1 0] +2×2 Array{Int64,2}: + 0 1 + 1 0 + +julia> eigvals(A,B) +2-element Array{Complex{Float64},1}: + 0.0+1.0im + 0.0-1.0im +``` +""" function eigvals{TA,TB}(A::AbstractMatrix{TA}, B::AbstractMatrix{TB}) S = promote_type(Float32, typeof(one(TA)/norm(one(TA))),TB) return eigvals!(copy_oftype(A, S), copy_oftype(B, S)) end +""" + eigvecs(A, B) -> Matrix + +Returns a matrix `M` whose columns are the generalized eigenvectors of `A` and `B`. (The `k`th eigenvector can +be obtained from the slice `M[:, k]`.) + +```jldoctest +julia> A = [1 0; 0 -1] +2×2 Array{Int64,2}: + 1 0 + 0 -1 + +julia> B = [0 1; 1 0] +2×2 Array{Int64,2}: + 0 1 + 1 0 + +julia> eigvecs(A, B) +2×2 Array{Complex{Float64},2}: + 0.0-1.0im 0.0+1.0im + -1.0-0.0im -1.0+0.0im +``` +""" eigvecs(A::AbstractMatrix, B::AbstractMatrix) = eigvecs(eigfact(A, B)) # Conversion methods @@ -189,4 +382,4 @@ convert(::Type{AbstractMatrix}, F::Eigen) = F.vectors * Diagonal(F.values) / F.v convert(::Type{AbstractArray}, F::Eigen) = convert(AbstractMatrix, F) convert(::Type{Matrix}, F::Eigen) = convert(Array, convert(AbstractArray, F)) convert(::Type{Array}, F::Eigen) = convert(Matrix, F) -full(F::Eigen) = convert(Array, F) \ No newline at end of file +full(F::Eigen) = convert(Array, F) diff --git a/base/reducedim.jl b/base/reducedim.jl index 047cc9af80e92..95770ea0abafd 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -380,6 +380,11 @@ function findmin!{R}(rval::AbstractArray{R}, findminmax!(<, initarray!(rval, scalarmin, init), rind, A) end +""" + findmin(A, region) -> (minval, index) + +For an array input, returns the value and index of the minimum over the given region. +""" function findmin{T}(A::AbstractArray{T}, region) if isempty(A) return (similar(A, reduced_dims0(A, region)), @@ -402,6 +407,11 @@ function findmax!{R}(rval::AbstractArray{R}, findminmax!(>, initarray!(rval, scalarmax, init), rind, A) end +""" + findmax(A, region) -> (maxval, index) + +For an array input, returns the value and index of the maximum over the given region. +""" function findmax{T}(A::AbstractArray{T}, region) if isempty(A) return (similar(A, reduced_dims0(A,region)), diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 116433c9f3c5b..669b08626b7ee 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -9,21 +9,28 @@ Basic functions --------------- -.. function:: ndims(A) -> Integer +.. function:: ndims(A::AbstractArray) -> Integer .. Docstring generated from Julia source Returns the number of dimensions of ``A``\ . -.. function:: size(A, [dim...]) + .. doctest:: + + julia> A = ones(3,4,5); + + julia> ndims(A) + 3 + +.. function:: size(A::AbstractArray, [dim...]) .. Docstring generated from Julia source Returns a tuple containing the dimensions of ``A``\ . Optionally you can specify the dimension(s) you want the length of, and get the length of that dimension, or a tuple of the lengths of dimensions you asked for. - .. code-block:: julia + .. doctest:: - julia> A = rand(2,3,4); + julia> A = ones(2,3,4); julia> size(A, 2) 3 @@ -43,12 +50,19 @@ Basic functions Returns the valid range of indices for array ``A`` along dimension ``d``\ . -.. function:: length(A) -> Integer +.. function:: length(A::AbstractArray) -> Integer .. Docstring generated from Julia source Returns the number of elements in ``A``\ . + .. doctest:: + + julia> A = ones(3,4,5); + + julia> length(A) + 60 + .. function:: eachindex(A...) .. Docstring generated from Julia source @@ -116,18 +130,35 @@ Basic functions Convert an array to its complex conjugate in-place. -.. function:: stride(A, k) +.. function:: stride(A, k::Integer) .. Docstring generated from Julia source Returns the distance in memory (in number of elements) between adjacent elements in dimension ``k``\ . + .. doctest:: + + julia> A = ones(3,4,5); + + julia> stride(A,2) + 3 + + julia> stride(A,3) + 12 + .. function:: strides(A) .. Docstring generated from Julia source Returns a tuple of the memory strides in each dimension. + .. doctest:: + + julia> A = ones(3,4,5); + + julia> strides(A) + (1,3,12) + .. function:: ind2sub(dims, index) -> subscripts .. Docstring generated from Julia source @@ -620,19 +651,62 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Return a vector of the linear indexes of the non-zeros in ``A`` (determined by ``A[i]!=0``\ ). A common use of this is to convert a boolean array to an array of indexes of the ``true`` elements. + Return a vector of the linear indexes of the non-zeros in ``A`` (determined by ``A[i]!=0``\ ). A common use of this is to convert a boolean array to an array of indexes of the ``true`` elements. If there are no non-zero elements of ``A``\ , ``find`` returns an empty array. -.. function:: find(f,A) + .. doctest:: + + julia> A = [true false; false true] + 2×2 Array{Bool,2}: + true false + false true + + julia> find(A) + 2-element Array{Int64,1}: + 1 + 4 + +.. function:: find(f::Function, A) .. Docstring generated from Julia source - Return a vector of the linear indexes of ``A`` where ``f`` returns ``true``\ . + Return a vector ``I`` of the linear indexes of ``A`` where ``f(A[I])`` returns ``true``\ . If there are no such elements of ``A``\ , find returns an empty array. + + .. doctest:: + + julia> A = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> find(isodd,A) + 2-element Array{Int64,1}: + 1 + 2 .. function:: findn(A) .. Docstring generated from Julia source - Return a vector of indexes for each dimension giving the locations of the non-zeros in ``A`` (determined by ``A[i]!=0``\ ). + Return a vector of indexes for each dimension giving the locations of the non-zeros in ``A`` (determined by ``A[i]!=0``\ ). If there are no non-zero elements of ``A``\ , ``findn`` returns a 2-tuple of empty arrays. + + .. doctest:: + + julia> A = [1 2 0; 0 0 3; 0 4 0] + 3×3 Array{Int64,2}: + 1 2 0 + 0 0 3 + 0 4 0 + + julia> findn(A) + ([1,1,3,2],[1,2,2,3]) + + julia> A = zeros(2,2) + 2×2 Array{Float64,2}: + 0.0 0.0 + 0.0 0.0 + + julia> findn(A) + (Int64[],Int64[]) .. function:: findnz(A) @@ -640,77 +714,249 @@ Indexing, Assignment, and Concatenation Return a tuple ``(I, J, V)`` where ``I`` and ``J`` are the row and column indexes of the non-zero values in matrix ``A``\ , and ``V`` is a vector of the non-zero values. + .. doctest:: + + julia> A = [1 2 0; 0 0 3; 0 4 0] + 3×3 Array{Int64,2}: + 1 2 0 + 0 0 3 + 0 4 0 + + julia> findnz(A) + ([1,1,3,2],[1,2,2,3],[1,2,4,3]) + .. function:: findfirst(A) .. Docstring generated from Julia source - Return the index of the first non-zero value in ``A`` (determined by ``A[i]!=0``\ ). + Return the linear index of the first non-zero value in ``A`` (determined by ``A[i]!=0``\ ). Returns ``0`` if no such value is found. + + .. doctest:: -.. function:: findfirst(A,v) + julia> A = [0 0; 1 0] + 2×2 Array{Int64,2}: + 0 0 + 1 0 + + julia> findfirst(A) + 2 + +.. function:: findfirst(A, v) .. Docstring generated from Julia source - Return the index of the first element equal to ``v`` in ``A``\ . + Return the linear index of the first element equal to ``v`` in ``A``\ . Returns ``0`` if ``v`` is not found. + + .. doctest:: + + julia> A = [4 6; 2 2] + 2×2 Array{Int64,2}: + 4 6 + 2 2 + + julia> findfirst(A,2) + 2 + + julia> findfirst(A,3) + 0 -.. function:: findfirst(predicate, A) +.. function:: findfirst(predicate::Function, A) .. Docstring generated from Julia source - Return the index of the first element of ``A`` for which ``predicate`` returns ``true``\ . + Return the linear index of the first element of ``A`` for which ``predicate`` returns ``true``\ . Returns ``0`` if there is no such element. + + .. doctest:: + + julia> A = [1 4; 2 2] + 2×2 Array{Int64,2}: + 1 4 + 2 2 + + julia> findfirst(iseven, A) + 2 + + julia> findfirst(x -> x>10, A) + 0 .. function:: findlast(A) .. Docstring generated from Julia source - Return the index of the last non-zero value in ``A`` (determined by ``A[i]!=0``\ ). + Return the linear index of the last non-zero value in ``A`` (determined by ``A[i]!=0``\ ). Returns ``0`` if there is no non-zero value in ``A``\ . + + .. doctest:: + + julia> A = [1 0; 1 0] + 2×2 Array{Int64,2}: + 1 0 + 1 0 + + julia> findlast(A) + 2 + + julia> A = zeros(2,2) + 2×2 Array{Float64,2}: + 0.0 0.0 + 0.0 0.0 + + julia> findlast(A) + 0 .. function:: findlast(A, v) .. Docstring generated from Julia source - Return the index of the last element equal to ``v`` in ``A``\ . + Return the linear index of the last element equal to ``v`` in ``A``\ . Returns ``0`` if there is no element of ``A`` equal to ``v``\ . -.. function:: findlast(predicate, A) + .. doctest:: + + julia> A = [1 2; 2 1] + 2×2 Array{Int64,2}: + 1 2 + 2 1 + + julia> findlast(A,1) + 4 + + julia> findlast(A,2) + 3 + + julia> findlast(A,3) + 0 + +.. function:: findlast(predicate::Function, A) .. Docstring generated from Julia source - Return the index of the last element of ``A`` for which ``predicate`` returns ``true``\ . + Return the linear index of the last element of ``A`` for which ``predicate`` returns ``true``\ . Returns ``0`` if there is no such element. -.. function:: findnext(A, i) + .. doctest:: + + julia> A = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> findlast(isodd, A) + 2 + + julia> findlast(x -> x > 5, A) + 0 + +.. function:: findnext(A, i::Integer) .. Docstring generated from Julia source - Find the next index >= ``i`` of a non-zero element of ``A``\ , or ``0`` if not found. + Find the next linear index >= ``i`` of a non-zero element of ``A``\ , or ``0`` if not found. + + .. doctest:: + + julia> A = [0 0; 1 0] + 2×2 Array{Int64,2}: + 0 0 + 1 0 + + julia> findnext(A,1) + 2 -.. function:: findnext(predicate, A, i) + julia> findnext(A,3) + 0 + +.. function:: findnext(predicate::Function, A, i::Integer) .. Docstring generated from Julia source - Find the next index >= ``i`` of an element of ``A`` for which ``predicate`` returns ``true``\ , or ``0`` if not found. + Find the next linear index >= ``i`` of an element of ``A`` for which ``predicate`` returns ``true``\ , or ``0`` if not found. + + .. doctest:: + + julia> A = [1 4; 2 2] + 2×2 Array{Int64,2}: + 1 4 + 2 2 + + julia> findnext(isodd, A, 1) + 1 -.. function:: findnext(A, v, i) + julia> findnext(isodd, A, 2) + 0 + +.. function:: findnext(A, v, i::Integer) .. Docstring generated from Julia source - Find the next index >= ``i`` of an element of ``A`` equal to ``v`` (using ``==``\ ), or ``0`` if not found. + Find the next linear index >= ``i`` of an element of ``A`` equal to ``v`` (using ``==``\ ), or ``0`` if not found. + + .. doctest:: + + julia> A = [1 4; 2 2] + 2×2 Array{Int64,2}: + 1 4 + 2 2 -.. function:: findprev(A, i) + julia> findnext(A,4,4) + 0 + + julia> findnext(A,4,3) + 3 + +.. function:: findprev(A, i::Integer) .. Docstring generated from Julia source - Find the previous index <= ``i`` of a non-zero element of ``A``\ , or ``0`` if not found. + Find the previous linear index <= ``i`` of a non-zero element of ``A``\ , or ``0`` if not found. + + .. doctest:: + + julia> A = [0 0; 1 2] + 2×2 Array{Int64,2}: + 0 0 + 1 2 -.. function:: findprev(predicate, A, i) + julia> findprev(A,2) + 2 + + julia> findprev(A,1) + 0 + +.. function:: findprev(predicate::Function, A, i::Integer) .. Docstring generated from Julia source - Find the previous index <= ``i`` of an element of ``A`` for which ``predicate`` returns ``true``\ , or ``0`` if not found. + Find the previous linear index <= ``i`` of an element of ``A`` for which ``predicate`` returns ``true``\ , or ``0`` if not found. + + .. doctest:: + + julia> A = [4 6; 1 2] + 2×2 Array{Int64,2}: + 4 6 + 1 2 + + julia> findprev(isodd, A, 1) + 0 -.. function:: findprev(A, v, i) + julia> findprev(isodd, A, 3) + 2 + +.. function:: findprev(A, v, i::Integer) .. Docstring generated from Julia source - Find the previous index <= ``i`` of an element of ``A`` equal to ``v`` (using ``==``\ ), or ``0`` if not found. + Find the previous linear index <= ``i`` of an element of ``A`` equal to ``v`` (using ``==``\ ), or ``0`` if not found. + + .. doctest:: + + julia> A = [0 0; 1 2] + 2×2 Array{Int64,2}: + 0 0 + 1 2 + + julia> findprev(A, 1, 4) + 2 + + julia> findprev(A, 1, 1) + 0 .. function:: permutedims(A, perm) diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 7d3fba881b555..a8b4285b754c4 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -480,7 +480,7 @@ Iterable Collections .. Docstring generated from Julia source - Returns the index of the maximum element in a collection. + Returns the index of the maximum element in a collection. The collection must not be empty. .. doctest:: @@ -491,7 +491,7 @@ Iterable Collections .. Docstring generated from Julia source - Returns the index of the minimum element in a collection. + Returns the index of the minimum element in a collection. The collection must not be empty. .. doctest:: @@ -509,11 +509,11 @@ Iterable Collections julia> findmax([8,0.1,-9,pi]) (8.0,1) -.. function:: findmax(A, dims) -> (maxval, index) +.. function:: findmax(A, region) -> (maxval, index) .. Docstring generated from Julia source - For an array input, returns the value and index of the maximum over the given dimensions. + For an array input, returns the value and index of the maximum over the given region. .. function:: findmin(itr) -> (x, index) @@ -526,11 +526,11 @@ Iterable Collections julia> findmin([8,0.1,-9,pi]) (-9.0,3) -.. function:: findmin(A, dims) -> (minval, index) +.. function:: findmin(A, region) -> (minval, index) .. Docstring generated from Julia source - For an array input, returns the value and index of the minimum over the given dimensions. + For an array input, returns the value and index of the minimum over the given region. .. function:: findmax!(rval, rind, A, [init=true]) -> (maxval, index) @@ -1162,11 +1162,18 @@ Set-Like Collections Construct the intersection of two or more sets. Maintains order and multiplicity of the first argument for arrays and ranges. -.. function:: setdiff(s1,s2) +.. function:: setdiff(a, b) .. Docstring generated from Julia source - Construct the set of elements in ``s1`` but not ``s2``\ . Maintains order with arrays. Note that both arguments must be collections, and both will be iterated over. In particular, ``setdiff(set,element)`` where ``element`` is a potential member of ``set``\ , will not work in general. + Construct the set of elements in ``a`` but not ``b``\ . Maintains order with arrays. Note that both arguments must be collections, and both will be iterated over. In particular, ``setdiff(set,element)`` where ``element`` is a potential member of ``set``\ , will not work in general. + + .. doctest:: + + julia> setdiff([1,2,3],[3,4,5]) + 2-element Array{Int64,1}: + 1 + 2 .. function:: setdiff!(s, iterable) @@ -1174,12 +1181,20 @@ Set-Like Collections Remove each element of ``iterable`` from set ``s`` in-place. -.. function:: symdiff(s1,s2...) +.. function:: symdiff(a, b, rest...) .. Docstring generated from Julia source Construct the symmetric difference of elements in the passed in sets or arrays. Maintains order with arrays. + .. doctest:: + + julia> symdiff([1,2,3],[3,4,5],[4,5,6]) + 3-element Array{Int64,1}: + 1 + 2 + 6 + .. function:: symdiff!(s, n) .. Docstring generated from Julia source @@ -1332,11 +1347,11 @@ Dequeues 2 1 -.. function:: deleteat!(collection, index) +.. function:: deleteat!(a::Vector, i::Integer) .. Docstring generated from Julia source - Remove the item at the given ``index`` and return the modified ``collection``\ . Subsequent items are shifted to fill the resulting gap. + Remove the item at the given ``i`` and return the modified ``a``\ . Subsequent items are shifted to fill the resulting gap. .. doctest:: @@ -1348,11 +1363,11 @@ Dequeues 2 1 -.. function:: deleteat!(collection, itr) +.. function:: deleteat!(a::Vector, inds) .. Docstring generated from Julia source - Remove the items at the indices given by ``itr``\ , and return the modified ``collection``\ . Subsequent items are shifted to fill the resulting gap. ``itr`` must be sorted and unique. + Remove the items at the indices given by ``inds``\ , and return the modified ``a``\ . Subsequent items are shifted to fill the resulting gap. ``inds`` must be sorted and unique. .. doctest:: @@ -1364,7 +1379,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:575 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:576 ... .. function:: splice!(collection, index, [replacement]) -> item @@ -1527,18 +1542,57 @@ changed efficiently. A ``PriorityQueue`` acts like a ``Dict``\ , mapping values to their priorities, with the addition of a ``dequeue!`` function to remove the lowest priority element. + .. doctest:: + + julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + .. function:: enqueue!(pq, k, v) .. Docstring generated from Julia source Insert the a key ``k`` into a priority queue ``pq`` with priority ``v``\ . + .. doctest:: + + julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + + julia> Base.Collections.enqueue!(a, "d", 4) + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 4 entries: + "c" => 1 + "b" => 3 + "a" => 2 + "d" => 4 + .. function:: dequeue!(pq) .. Docstring generated from Julia source Remove and return the lowest priority key from a priority queue. + .. doctest:: + + julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + + julia> Base.Collections.dequeue!(a) + "c" + + julia> a + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 2 entries: + "b" => 3 + "a" => 2 + .. function:: peek(pq) .. Docstring generated from Julia source @@ -1575,24 +1629,58 @@ lower level functions for performing binary heap operations on arrays. Each function takes an optional ordering argument. If not given, default ordering is used, so that elements popped from the heap are given in ascending order. -.. function:: heapify(v, [ord]) +.. function:: heapify(v, ord::Ordering=Forward) .. Docstring generated from Julia source Returns a new vector in binary heap order, optionally using the given ordering. -.. function:: heapify!(v, [ord]) + .. doctest:: + + julia> a = [1,3,4,5,2]; + + julia> Base.Collections.heapify(a) + 5-element Array{Int64,1}: + 1 + 2 + 4 + 5 + 3 + + julia> Base.Collections.heapify(a, Base.Order.Reverse) + 5-element Array{Int64,1}: + 5 + 3 + 4 + 1 + 2 + +.. function:: heapify!(v, ord::Ordering=Forward) .. Docstring generated from Julia source In-place :func:`heapify`\ . -.. function:: isheap(v, [ord]) +.. function:: isheap(v, ord::Ordering=Forward) .. Docstring generated from Julia source Return ``true`` if an array is heap-ordered according to the given order. + .. doctest:: + + julia> a = [1,2,3] + 3-element Array{Int64,1}: + 1 + 2 + 3 + + julia> Base.Collections.isheap(a,Base.Order.Forward) + true + + julia> Base.Collections.isheap(a,Base.Order.Reverse) + false + .. function:: heappush!(v, x, [ord]) .. Docstring generated from Julia source diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 6f4737b133498..4e747354dda2a 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -624,11 +624,11 @@ Text I/O The columns are assumed to be separated by one or more whitespaces. The end of line delimiter is taken as ``\n``\ . If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. -.. function:: writedlm(f, A, dl='\\t'; opts) +.. function:: writedlm(f, A, delim='\\t'; opts) .. Docstring generated from Julia source - Write ``A`` (a vector, matrix or an iterable collection of iterable rows) as text to ``f`` (either a filename string or an ``IO`` stream) using the given delimiter ``delim`` (which defaults to tab, but can be any printable Julia object, typically a ``Char`` or ``AbstractString``\ ). + Write ``A`` (a vector, matrix, or an iterable collection of iterable rows) as text to ``f`` (either a filename string or an :class:`IO` stream) using the given delimiter ``delim`` (which defaults to tab, but can be any printable Julia object, typically a ``Char`` or ``AbstractString``\ ). For example, two vectors ``x`` and ``y`` of the same length can be written as two columns of tab-delimited text to ``f`` by either ``writedlm(f, [x y])`` or by ``writedlm(f, zip(x, y))``\ . @@ -642,7 +642,7 @@ Text I/O .. Docstring generated from Julia source - Equivalent to ``writedlm`` with ``delim`` set to comma. + Equivalent to :func:`writedlm` with ``delim`` set to comma. .. function:: Base64EncodePipe(ostream) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 8352695c8b3c5..2bd41341c74fb 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -580,7 +580,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Computes eigenvalues and eigenvectors of ``A``\ . See :func:`eigfact` for details on the ``permute`` and ``scale`` keyword arguments. The eigenvectors are returned columnwise. + Computes eigenvalues (``D``\ ) and eigenvectors (``V``\ ) of ``A``\ . See :func:`eigfact` for details on the ``irange``\ , ``vl``\ , and ``vu`` arguments and the ``permute`` and ``scale`` keyword arguments. The eigenvectors are returned columnwise. .. doctest:: @@ -594,10 +594,26 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Computes generalized eigenvalues and vectors of ``A`` with respect to ``B``\ . + Computes generalized eigenvalues (``D``\ ) and vectors (``V``\ ) of ``A`` with respect to ``B``\ . ``eig`` is a wrapper around :func:`eigfact`\ , extracting all parts of the factorization to a tuple; where possible, using :func:`eigfact` is recommended. + .. doctest:: + + julia> A = [1 0; 0 -1] + 2×2 Array{Int64,2}: + 1 0 + 0 -1 + + julia> B = [0 1; 1 0] + 2×2 Array{Int64,2}: + 0 1 + 1 0 + + julia> eig(A, B) + (Complex{Float64}[0.0+1.0im,0.0-1.0im], + Complex{Float64}[0.0-1.0im 0.0+1.0im; -1.0-0.0im -1.0+0.0im]) + .. function:: eigvals(A,[irange,][vl,][vu]) -> values .. Docstring generated from Julia source @@ -610,19 +626,59 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Same as ``eigvals``\ , but saves space by overwriting the input ``A`` (and ``B``\ ), instead of creating a copy. + Same as :func:`eigvals`\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. -.. function:: eigmax(A) +.. function:: eigmax(A; permute::Bool=true, scale::Bool=true) .. Docstring generated from Julia source - Returns the largest eigenvalue of ``A``\ . + Returns the largest eigenvalue of ``A``\ . The option ``permute=true`` permutes the matrix to become closer to upper triangular, and ``scale=true`` scales the matrix by its diagonal elements to make rows and columns more equal in norm. Note that if the eigenvalues of ``A`` are complex, this method will fail, since complex numbers cannot be sorted. + + .. doctest:: + + julia> A = [0 im; -im 0] + 2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + 0-1im 0+0im + + julia> eigmax(A) + 1.0 -.. function:: eigmin(A) + julia> A = [0 im; -1 0] + 2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + -1+0im 0+0im + + julia> eigmax(A) + ERROR: DomainError: + in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:108 + in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:106 + +.. function:: eigmin(A; permute::Bool=true, scale::Bool=true) .. Docstring generated from Julia source - Returns the smallest eigenvalue of ``A``\ . + Returns the smallest eigenvalue of ``A``\ . The option ``permute=true`` permutes the matrix to become closer to upper triangular, and ``scale=true`` scales the matrix by its diagonal elements to make rows and columns more equal in norm. Note that if the eigenvalues of ``A`` are complex, this method will fail, since complex numbers cannot be sorted. + + .. doctest:: + + julia> A = [0 im; -im 0] + 2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + 0-1im 0+0im + + julia> eigmin(A) + -1.0 + + julia> A = [0 im; -1 0] + 2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + -1+0im 0+0im + + julia> eigmin(A) + ERROR: DomainError: + in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:115 + in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:113 .. function:: eigvecs(A, [eigvals,][permute=true,][scale=true]) -> Matrix @@ -636,9 +692,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Computes the eigenvalue decomposition of ``A``\ , returning an ``Eigen`` factorization object ``F`` which contains the eigenvalues in ``F[:values]`` and the eigenvectors in the columns of the matrix ``F[:vectors]``\ . (The ``k``\ th eigenvector can be obtained from the slice ``F[:vectors][:, k]``\ .) + Computes the eigenvalue decomposition of ``A``\ , returning an :obj:`Eigen` factorization object ``F`` which contains the eigenvalues in ``F[:values]`` and the eigenvectors in the columns of the matrix ``F[:vectors]``\ . (The ``k``\ th eigenvector can be obtained from the slice ``F[:vectors][:, k]``\ .) - The following functions are available for ``Eigen`` objects: ``inv``\ , ``det``\ . + The following functions are available for ``Eigen`` objects: :func:`inv`\ , :func:`det`\ , and :func:`isposdef`\ . If ``A`` is :class:`Symmetric`\ , :class:`Hermitian` or :class:`SymTridiagonal`\ , it is possible to calculate only a subset of the eigenvalues by specifying either a :class:`UnitRange` ``irange`` covering indices of the sorted eigenvalues or a pair ``vl`` and ``vu`` for the lower and upper boundaries of the eigenvalues. From 0deabbdc89b6b493369e19ffa8486a0f8dfee2f7 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 4 Aug 2016 06:20:24 -0700 Subject: [PATCH 0855/1117] excluse asc signature files from checksum listing --- contrib/prepare_release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/prepare_release.sh b/contrib/prepare_release.sh index f8d50353b7344..9f4e0f2bacb90 100755 --- a/contrib/prepare_release.sh +++ b/contrib/prepare_release.sh @@ -56,8 +56,8 @@ cp julia-$version-win32.exe julia-$majmin-latest-win32.exe echo "Note: if windows code signing is not working on the buildbots, then the" echo "checksums need to be re-calculated after the binaries are manually signed!" -shasum -a 256 julia-$version* | grep -v sha256 | grep -v md5 > julia-$version.sha256 -md5sum julia-$version* | grep -v sha256 | grep -v md5 > julia-$version.md5 +shasum -a 256 julia-$version* | grep -v -e sha256 -e md5 -e asc > julia-$version.sha256 +md5sum julia-$version* | grep -v -e sha256 -e md5 -e asc > julia-$version.md5 gpg -u julia --armor --detach-sig julia-$version-full.tar.gz gpg -u julia --armor --detach-sig julia-$version.tar.gz From 5588d523de05fcf47804a9b619f4dad728b07e9a Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 4 Aug 2016 03:15:48 -0700 Subject: [PATCH 0856/1117] Use sphinx 1.4.5 to fix a linkcheck unicode bug with valgrind doc site --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index c5091f3764c04..0749d73a459a9 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -30,7 +30,7 @@ $(ACTIVATE): touch -c $@ $(SPHINX_BUILD): $(SRCDIR)/requirements.txt $(ACTIVATE) - . $(ACTIVATE) && pip install sphinx==1.3.1 \ + . $(ACTIVATE) && pip install sphinx==1.4.5 \ && pip install -r $< touch -c $@ From 11034271bf3acab553aae6b7d37dd8e60e8598fa Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 6 Aug 2016 02:31:59 -0700 Subject: [PATCH 0857/1117] Update url's in docs to fix `make -C doc linkcheck` --- doc/conf.py | 3 +++ doc/devdocs/backtraces.rst | 2 +- doc/manual/calling-c-and-fortran-code.rst | 4 ++-- doc/manual/dates.rst | 2 +- doc/manual/getting-started.rst | 2 +- .../integers-and-floating-point-numbers.rst | 4 ++-- doc/manual/packages.rst | 18 +++++++++--------- doc/manual/parallel-computing.rst | 2 +- doc/manual/performance-tips.rst | 2 +- doc/stdlib/pkg.rst | 2 +- 10 files changed, 22 insertions(+), 19 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 934e10d79bb41..53dd883c7cd78 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -96,6 +96,9 @@ primary_domain = 'jl' highlight_language = 'julia' +# Flaky links to ignore in linkcheck - permissions or empty returns +linkcheck_ignore = ['https://www.appveyor.com', + 'https://bugs.kde.org/show_bug.cgi\?id=136779'] # -- Options for HTML output --------------------------------------------------- diff --git a/doc/devdocs/backtraces.rst b/doc/devdocs/backtraces.rst index 6dbe87dd3fe4e..518643397283b 100644 --- a/doc/devdocs/backtraces.rst +++ b/doc/devdocs/backtraces.rst @@ -104,4 +104,4 @@ A few terms have been used as shorthand in this guide: * ``<julia_root>`` refers to the root directory of the Julia source tree; e.g. it should contain folders such as ``base``, ``deps``, ``src``, ``test``, etc..... .. _gist: https://gist.github.com -.. _issue: https://github.com/JuliaLang/julia/issues?state=open +.. _issue: https://github.com/JuliaLang/julia/issues?q=is%3Aopen diff --git a/doc/manual/calling-c-and-fortran-code.rst b/doc/manual/calling-c-and-fortran-code.rst index cc4fdea8a935e..535a14e8409d9 100644 --- a/doc/manual/calling-c-and-fortran-code.rst +++ b/doc/manual/calling-c-and-fortran-code.rst @@ -41,7 +41,7 @@ functions in the Julia runtime, or functions in an application linked to Julia. By default, Fortran compilers `generate mangled names -<https://en.wikipedia.org/wiki/Name_mangling#Name_mangling_in_Fortran>`_ +<https://en.wikipedia.org/wiki/Name_mangling#Fortran>`_ (for example, converting function names to lowercase or uppercase, often appending an underscore), and so to call a Fortran function via :func:`ccall` you must pass the mangled identifier corresponding to the rule @@ -1094,7 +1094,7 @@ More About Callbacks -------------------- For more details on how to pass callbacks to C libraries, see this -`blog post <http://julialang.org/blog/2013/05/callback/>`_. +`blog post <http://julialang.org/blog/2013/05/callback>`_. C++ --- diff --git a/doc/manual/dates.rst b/doc/manual/dates.rst index e23df78283255..0ede9544cf97f 100644 --- a/doc/manual/dates.rst +++ b/doc/manual/dates.rst @@ -238,7 +238,7 @@ Similarly for the :func:`monthname` function, a mapping of ``locale=>Dict{Int,St TimeType-Period Arithmetic -------------------------- -It's good practice when using any language/date framework to be familiar with how date-period arithmetic is handled as there are some `tricky issues <http://codeblog.jonskeet.uk/2010/12/01/the-joys-of-date-time-arithmetic>`_ to deal with (though much less so for day-precision types). +It's good practice when using any language/date framework to be familiar with how date-period arithmetic is handled as there are some `tricky issues <https://codeblog.jonskeet.uk/2010/12/01/the-joys-of-date-time-arithmetic/>`_ to deal with (though much less so for day-precision types). The :mod:`Dates` module approach tries to follow the simple principle of trying to change as little as possible when doing :class:`Period` arithmetic. This approach is also often known as *calendrical* arithmetic or what you would probably guess if someone were to ask you the same calculation in a conversation. Why all the fuss about this? Let's take a classic example: add 1 month to January 31st, 2014. What's the answer? Javascript will say `March 3 <http://www.markhneedham.com/blog/2009/01/07/javascript-add-a-month-to-a-date/>`_ (assumes 31 days). PHP says `March 2 <http://stackoverflow.com/questions/5760262/php-adding-months-to-a-date-while-not-exceeding-the-last-day-of-the-month>`_ (assumes 30 days). The fact is, there is no right answer. In the :mod:`Dates` module, it gives the result of February 28th. How does it figure that out? I like to think of the classic 7-7-7 gambling game in casinos. diff --git a/doc/manual/getting-started.rst b/doc/manual/getting-started.rst index d159c8731e658..d89a4bd61ad1b 100644 --- a/doc/manual/getting-started.rst +++ b/doc/manual/getting-started.rst @@ -162,7 +162,7 @@ In addition to this manual, there are various other resources that may help new users get started with Julia: - `Julia and IJulia cheatsheet <http://math.mit.edu/~stevenj/Julia-cheatsheet.pdf>`_ -- `Learn Julia in a few minutes <http://learnxinyminutes.com/docs/julia/>`_ +- `Learn Julia in a few minutes <https://learnxinyminutes.com/docs/julia/>`_ - `Learn Julia the Hard Way <https://github.com/chrisvoncsefalvay/learn-julia-the-hard-way>`_ - `Julia by Example <http://samuelcolvin.github.io/JuliaByExample/>`_ - `Hands-on Julia <https://github.com/dpsanders/hands_on_julia>`_ diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index 0e79f2feb4952..36279871cc0f5 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -554,11 +554,11 @@ computation, and also in the following references: - For even more extensive documentation of the history of, rationale for, and issues with floating-point numbers, as well as discussion of many other topics in numerical computing, see the `collected writings - <http://www.cs.berkeley.edu/~wkahan/>`_ of `William Kahan + <https://people.eecs.berkeley.edu/~wkahan/>`_ of `William Kahan <https://en.wikipedia.org/wiki/William_Kahan>`_, commonly known as the "Father of Floating-Point". Of particular interest may be `An Interview with the Old Man of Floating-Point - <http://www.cs.berkeley.edu/~wkahan/ieee754status/754story.html>`_. + <https://people.eecs.berkeley.edu/~wkahan/ieee754status/754story.html>`_. .. _man-arbitrary-precision-arithmetic: diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index a2db87603aa00..747fd950653dd 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -196,7 +196,7 @@ To get the latest and greatest versions of all your packages, just do :func:`Pkg The first step of updating packages is to pull new changes to ``~/.julia/v0.4/METADATA`` and see if any new registered package versions have been published. After this, :func:`Pkg.update` attempts to update packages that are checked out on a branch and not dirty (i.e. no changes have been made to files tracked by git) by pulling changes from the package's upstream repository. -Upstream changes will only be applied if no merging or rebasing is necessary – i.e. if the branch can be `"fast-forwarded" <http://git-scm.com/book/en/Git-Branching-Basic-Branching-and-Merging>`_. +Upstream changes will only be applied if no merging or rebasing is necessary – i.e. if the branch can be `"fast-forwarded" <https://git-scm.com/book/en/v2/Git-Branching-Basic-Branching-and-Merging>`_. If the branch cannot be fast-forwarded, it is assumed that you're working on it and will update the repository yourself. Finally, the update process recomputes an optimal set of package versions to have installed to satisfy your top-level requirements and the requirements of "fixed" packages. @@ -360,7 +360,7 @@ We recommend that you create a `free account <https://github.com/join>`_ on GitH where ``USERNAME`` is your actual GitHub user name. Once you do this, the package manager knows your GitHub user name and can configure things accordingly. -You should also `upload <https://github.com/settings/ssh>`_ your public SSH key to GitHub and set up an `SSH agent <http://linux.die.net/man/1/ssh-agent>`_ on your development machine so that you can push changes with minimal hassle. +You should also `upload <https://github.com/login?return_to=https%3A%2F%2Fgithub.com%2Fsettings%2Fssh>`_ your public SSH key to GitHub and set up an `SSH agent <http://linux.die.net/man/1/ssh-agent>`_ on your development machine so that you can push changes with minimal hassle. In the future, we will make this system extensible and support other common git hosting options like `BitBucket <https://bitbucket.org>`_ and allow developers to choose their favorite. Since the package development functions has been moved to the `PkgDev <https://github.com/JuliaLang/PkgDev.jl>`_ package, you need to run ``Pkg.add("PkgDev"); import PkgDev`` to access the functions starting with ``PkgDev.`` in the document below. @@ -455,7 +455,7 @@ are several possible approaches, here is one that is widely used: ``fixbar``). By creating a branch, you ensure that you can easily go back and forth between your new work and the current ``master`` branch (see - `<http://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell>`_). + `<https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell>`_). If you forget to do this step until after you've already made some changes, don't worry: see :ref:`more detail about branching @@ -486,7 +486,7 @@ are several possible approaches, here is one that is widely used: ``src/`` folder. -- Commit your changes: see `<http://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository>`_. +- Commit your changes: see `<https://git-scm.com/book/en/v2/Git-Basics-Recording-Changes-to-the-Repository>`_. - Submit your changes: From the Julia prompt, type :func:`PkgDev.submit("Foo") <PkgDev.submit>`. This will push your changes to your @@ -555,7 +555,7 @@ following procedure: until you have resolved the problems, or you may lose your changes. - *Reset* ``master`` (your current branch) back to an earlier state with ``git reset --hard origin/master`` (see - `<http://git-scm.com/blog/2011/07/11/reset.html>`_). + `<https://git-scm.com/blog/2011/07/11/reset.html>`_). This requires a bit more familiarity with git, so it's much better to get in the habit of creating a branch at the outset. @@ -583,7 +583,7 @@ quite simple but your commit history looks like this:: This gets into the territory of more advanced git usage, and you're encouraged to do some reading -(`<http://git-scm.com/book/en/v2/Git-Branching-Rebasing>`_). However, +(`<https://git-scm.com/book/en/v2/Git-Branching-Rebasing>`_). However, a brief summary of the procedure is as follows: - To protect yourself from error, start from your ``fixbar`` branch @@ -767,7 +767,7 @@ different license, you can ask us to add it to the package generator, or just pi If you created a GitHub account and configured git to know about it, :func:`PkgDev.generate` will set an appropriate origin URL for you. It will also automatically generate a ``.travis.yml`` file for using the `Travis <https://travis-ci.org>`_ automated -testing service, and an ``appveyor.yml`` file for using `AppVeyor <http://appveyor.com>`_. You will have to enable testing on +testing service, and an ``appveyor.yml`` file for using `AppVeyor <https://www.appveyor.com>`_. You will have to enable testing on the Travis and AppVeyor websites for your package repository, but once you've done that, it will already have working tests. Of course, all the default testing does is verify that ``using FooBar`` in Julia works. @@ -840,7 +840,7 @@ on GitHub, push your changes to your fork, and open a pull request:: then you may have encountered an issue from using the GitHub API on multiple systems. The solution is to delete the "Julia Package Manager" personal access token `from your Github account - <https://github.com/settings/tokens>`_ and try again. + <https://github.com/login?return_to=https%3A%2F%2Fgithub.com%2Fsettings%2Ftokens>`_ and try again. Other failures may require you to circumvent :func:`PkgDev.publish` by `creating a pull request on GitHub @@ -915,7 +915,7 @@ that copy exists, you can push your local changes to your copy (just like any other GitHub project). -1. go to `<https://github.com/JuliaLang/METADATA.jl/fork>`_ and create your own +1. go to `<https://github.com/login?return_to=https%3A%2F%2Fgithub.com%2FJuliaLang%2FMETADATA.jl%2Ffork>`_ and create your own fork. 2. add your fork as a remote repository for the METADATA diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 9f9548ddf356b..d551f1e288a66 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -15,7 +15,7 @@ it's fairly obvious that a given CPU will have fastest access to the RAM within the same computer (node). Perhaps more surprisingly, similar issues are relevant on a typical multicore laptop, due to differences in the speed of main memory and the -`cache <http://www.akkadia.org/drepper/cpumemory.pdf>`_. Consequently, a +`cache <https://www.akkadia.org/drepper/cpumemory.pdf>`_. Consequently, a good multiprocessing environment should allow control over the "ownership" of a chunk of memory by a particular CPU. Julia provides a multiprocessing environment based on message passing to allow programs diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index e5aabd82f53c0..70d3bb9820b6b 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -770,7 +770,7 @@ code. Some run-time benchmarks comparing (1) type dispatch, (2) dictionary lookup, and (3) a "switch" statement can be found `on the mailing list -<https://groups.google.com/d/msg/julia-users/jUMu9A3QKQQ/qjgVWr7vAwAJ>`_. +<https://groups.google.com/forum/#!msg/julia-users/jUMu9A3QKQQ/qjgVWr7vAwAJ>`_. Perhaps even worse than the run-time impact is the compile-time impact: Julia will compile specialized functions for each different diff --git a/doc/stdlib/pkg.rst b/doc/stdlib/pkg.rst index f0d455b9e0829..348505145fbb7 100644 --- a/doc/stdlib/pkg.rst +++ b/doc/stdlib/pkg.rst @@ -7,7 +7,7 @@ All package manager functions are defined in the ``Pkg`` module. None of the ``Pkg`` module's functions are exported; to use them, you'll need to prefix each function call with an explicit ``Pkg.``, e.g. ``Pkg.status()`` or ``Pkg.dir()``. -Functions for package development (e.g. ``tag``, ``publish``, etc.) have been moved to the `PkgDev <https://github.com/JuliaLang/PkgDev.jl>`_ package. See `PkgDev README <https://github.com/JuliaLang/PkgDev.jl/blob/master/README.md#usage>`_ for the documentation of those functions. +Functions for package development (e.g. ``tag``, ``publish``, etc.) have been moved to the `PkgDev <https://github.com/JuliaLang/PkgDev.jl>`_ package. See `PkgDev README <https://github.com/JuliaLang/PkgDev.jl/blob/master/README.md>`_ for the documentation of those functions. .. function:: dir() -> AbstractString From 54d5a446ee46e671834dedb83dc693d8a4fa5ad4 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 6 Aug 2016 02:33:25 -0700 Subject: [PATCH 0858/1117] Update 0.4 links and Pkg output to 0.5 (backport this commit) --- README.md | 4 ++-- doc/manual/packages.rst | 40 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9bafbf8d1f1da..d893f1b32c036 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,9 @@ First, acquire the source code by cloning the git repository: Be sure to also configure your system to use the appropriate proxy settings, e.g. by setting the `https_proxy` and `http_proxy` variables.) -By default you will be building the latest unstable version of Julia. However, most users should use the most recent stable version of Julia, which is currently the `0.4` series of releases. You can get this version by changing to the Julia directory and running +By default you will be building the latest unstable version of Julia. However, most users should use the most recent stable version of Julia, which is currently the `0.5` series of releases. You can get this version by changing to the Julia directory and running - git checkout release-0.4 + git checkout release-0.5 Now run `make` to build the `julia` executable. To perform a parallel build, use `make -j N` and supply the maximum number of concurrent processes. (See [Platform Specific Build Notes](https://github.com/JuliaLang/julia#platform-specific-build-notes) for details.) When compiled the first time, it will automatically download and build its [external dependencies](#Required-Build-Tools-External-Libraries). diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index 747fd950653dd..e32565dafcf72 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -22,7 +22,7 @@ The :func:`Pkg.status` function prints out a summary of the state of packages yo Initially, you'll have no packages installed:: julia> Pkg.status() - INFO: Initializing package repository /Users/stefan/.julia/v0.4 + INFO: Initializing package repository /Users/stefan/.julia/v0.5 INFO: Cloning METADATA from git://github.com/JuliaLang/METADATA.jl No packages installed. @@ -56,7 +56,7 @@ This means that you tell it what you want and it figures out what versions to in So rather than installing a package, you just add it to the list of requirements and then "resolve" what needs to be installed. In particular, this means that if some package had been installed because it was needed by a previous version of something you wanted, and a newer version doesn't have that requirement anymore, updating will actually remove that package. -Your package requirements are in the file ``~/.julia/v0.4/REQUIRE``. +Your package requirements are in the file ``~/.julia/v0.5/REQUIRE``. You can edit this file by hand and then call :func:`Pkg.resolve` to install, upgrade or remove packages to optimally satisfy the requirements, or you can do :func:`Pkg.edit`, which will open ``REQUIRE`` in your editor (configured via the ``EDITOR`` or ``VISUAL`` environment variables), and then automatically call :func:`Pkg.resolve` afterwards if necessary. If you only want to add or remove the requirement for a single package, you can also use the non-interactive :func:`Pkg.add` and :func:`Pkg.rm` commands, which add or remove a single requirement to ``REQUIRE`` and then call :func:`Pkg.resolve`. @@ -81,15 +81,15 @@ You can add a package to the list of requirements with the :func:`Pkg.add` funct - NumericExtensions 0.2.17 - Stats 0.2.6 -What this is doing is first adding ``Distributions`` to your ``~/.julia/v0.4/REQUIRE`` file:: +What this is doing is first adding ``Distributions`` to your ``~/.julia/v0.5/REQUIRE`` file:: - $ cat ~/.julia/v0.4/REQUIRE + $ cat ~/.julia/v0.5/REQUIRE Distributions It then runs :func:`Pkg.resolve` using these new requirements, which leads to the conclusion that the ``Distributions`` package should be installed since it is required but not installed. -As stated before, you can accomplish the same thing by editing your ``~/.julia/v0.4/REQUIRE`` file by hand and then running :func:`Pkg.resolve` yourself:: +As stated before, you can accomplish the same thing by editing your ``~/.julia/v0.5/REQUIRE`` file by hand and then running :func:`Pkg.resolve` yourself:: - $ echo UTF16 >> ~/.julia/v0.4/REQUIRE + $ echo UTF16 >> ~/.julia/v0.5/REQUIRE julia> Pkg.resolve() INFO: Cloning cache of UTF16 from git://github.com/nolta/UTF16.jl.git @@ -175,7 +175,7 @@ To install an unregistered package, use :func:`Pkg.clone(url) <Pkg.clone>`, wher Resolving deltas: 100% (8/8), done. By convention, Julia repository names end with ``.jl`` (the additional ``.git`` indicates a "bare" git repository), which keeps them from colliding with repositories for other languages, and also makes Julia packages easy to find in search engines. -When packages are installed in your ``.julia/v0.4`` directory, however, the extension is redundant so we leave it off. +When packages are installed in your ``.julia/v0.5`` directory, however, the extension is redundant so we leave it off. If unregistered packages contain a ``REQUIRE`` file at the top of their source tree, that file will be used to determine which registered packages the unregistered package depends on, and they will automatically be installed. Unregistered packages participate in the same version resolution logic as registered packages, so installed package versions will be adjusted as necessary to satisfy the requirements of both registered and unregistered packages. @@ -194,7 +194,7 @@ To get the latest and greatest versions of all your packages, just do :func:`Pkg INFO: Upgrading Distributions: v0.2.8 => v0.2.10 INFO: Upgrading Stats: v0.2.7 => v0.2.8 -The first step of updating packages is to pull new changes to ``~/.julia/v0.4/METADATA`` and see if any new registered package versions have been published. +The first step of updating packages is to pull new changes to ``~/.julia/v0.5/METADATA`` and see if any new registered package versions have been published. After this, :func:`Pkg.update` attempts to update packages that are checked out on a branch and not dirty (i.e. no changes have been made to files tracked by git) by pulling changes from the package's upstream repository. Upstream changes will only be applied if no merging or rebasing is necessary – i.e. if the branch can be `"fast-forwarded" <https://git-scm.com/book/en/v2/Git-Branching-Basic-Branching-and-Merging>`_. If the branch cannot be fast-forwarded, it is assumed that you're working on it and will update the repository yourself. @@ -207,7 +207,7 @@ A package is considered fixed if it is one of the following: 3. **Dirty:** changes have been made to files in the repo. If any of these are the case, the package manager cannot freely change the installed version of the package, so its requirements must be satisfied by whatever other package versions it picks. -The combination of top-level requirements in ``~/.julia/v0.4/REQUIRE`` and the requirement of fixed packages are used to determine what should be installed. +The combination of top-level requirements in ``~/.julia/v0.5/REQUIRE`` and the requirement of fixed packages are used to determine what should be installed. You can also update only a subset of the installed packages, by providing arguments to the `Pkg.update` function. In that case, only the packages provided as arguments and their dependencies will be updated:: @@ -720,7 +720,7 @@ Suppose you want to create a new Julia package called ``FooBar``. To get starte name of a license that the package generator knows about:: julia> PkgDev.generate("FooBar","MIT") - INFO: Initializing FooBar repo: /Users/stefan/.julia/v0.4/FooBar + INFO: Initializing FooBar repo: /Users/stefan/.julia/v0.5/FooBar INFO: Origin: git://github.com/StefanKarpinski/FooBar.jl.git INFO: Generating LICENSE.md INFO: Generating README.md @@ -732,10 +732,10 @@ name of a license that the package generator knows about:: INFO: Generating .gitignore INFO: Committing FooBar generated files -This creates the directory ``~/.julia/v0.4/FooBar``, initializes it as a git repository, generates a bunch of files +This creates the directory ``~/.julia/v0.5/FooBar``, initializes it as a git repository, generates a bunch of files that all packages should have, and commits them to the repository:: - $ cd ~/.julia/v0.4/FooBar && git show --stat + $ cd ~/.julia/v0.5/FooBar && git show --stat commit 84b8e266dae6de30ab9703150b3bf771ec7b6285 Author: Stefan Karpinski <stefan@karpinski.org> @@ -763,7 +763,7 @@ that all packages should have, and commits them to the repository:: At the moment, the package manager knows about the MIT "Expat" License, indicated by ``"MIT"``, the Simplified BSD License, indicated by ``"BSD"``, and version 2.0 of the Apache Software License, indicated by ``"ASL"``. If you want to use a different license, you can ask us to add it to the package generator, or just pick one of these three and then modify the -``~/.julia/v0.4/PACKAGE/LICENSE.md`` file after it has been generated. +``~/.julia/v0.5/PACKAGE/LICENSE.md`` file after it has been generated. If you created a GitHub account and configured git to know about it, :func:`PkgDev.generate` will set an appropriate origin URL for you. It will also automatically generate a ``.travis.yml`` file for using the `Travis <https://travis-ci.org>`_ automated @@ -799,9 +799,9 @@ of ``METADATA`` using :func:`PkgDev.register`:: INFO: Registering FooBar at git://github.com/StefanKarpinski/FooBar.jl.git INFO: Committing METADATA for FooBar -This creates a commit in the ``~/.julia/v0.4/METADATA`` repo:: +This creates a commit in the ``~/.julia/v0.5/METADATA`` repo:: - $ cd ~/.julia/v0.4/METADATA && git show + $ cd ~/.julia/v0.5/METADATA && git show commit 9f71f4becb05cadacb983c54a72eed744e5c019d Author: Stefan Karpinski <stefan@karpinski.org> @@ -857,12 +857,12 @@ register it with the :func:`PkgDev.tag` command:: This tags ``v0.0.1`` in the ``FooBar`` repo:: - $ cd ~/.julia/v0.4/FooBar && git tag + $ cd ~/.julia/v0.5/FooBar && git tag v0.0.1 It also creates a new version entry in your local ``METADATA`` repo for ``FooBar``:: - $ cd ~/.julia/v0.4/FooBar && git show + $ cd ~/.julia/v0.5/FooBar && git show commit de77ee4dc0689b12c5e8b574aef7f70e8b311b0e Author: Stefan Karpinski <stefan@karpinski.org> Date: Wed Oct 16 23:06:18 2013 -0400 @@ -922,7 +922,7 @@ fork. repository on your local computer (in the terminal where USERNAME is your github username):: - cd ~/.julia/v0.4/METADATA + cd ~/.julia/v0.5/METADATA git remote add USERNAME https://github.com/USERNAME/METADATA.jl.git 3. push your changes to your fork:: @@ -938,7 +938,7 @@ Fixing Package Requirements If you need to fix the registered requirements of an already-published package version, you can do so just by editing the metadata for that version, which will still have the same commit hash – the hash associated with a version is permanent:: - $ cd ~/.julia/v0.4/METADATA/FooBar/versions/0.0.1 && cat requires + $ cd ~/.julia/v0.5/METADATA/FooBar/versions/0.0.1 && cat requires julia 0.3- $ vi requires @@ -951,7 +951,7 @@ When you fix the requirements in ``METADATA`` for a previous version of a packag Requirements Specification -------------------------- -The ``~/.julia/v0.4/REQUIRE`` file, the ``REQUIRE`` file inside packages, and the ``METADATA`` package ``requires`` files use a simple line-based format to express the ranges of package versions which need to be installed. Package ``REQUIRE`` and ``METADATA requires`` files should also include the range of versions of ``julia`` the package is expected to work with. +The ``~/.julia/v0.5/REQUIRE`` file, the ``REQUIRE`` file inside packages, and the ``METADATA`` package ``requires`` files use a simple line-based format to express the ranges of package versions which need to be installed. Package ``REQUIRE`` and ``METADATA requires`` files should also include the range of versions of ``julia`` the package is expected to work with. Here's how these files are parsed and interpreted. From 89bb3ca782282d699c47ba18daf3e439498ebd42 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 6 Aug 2016 02:44:41 -0700 Subject: [PATCH 0859/1117] Update Pkg paths in docs to 0.6 (do not backport this commit) --- doc/manual/packages.rst | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index e32565dafcf72..eeb4ff147fb5f 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -22,7 +22,7 @@ The :func:`Pkg.status` function prints out a summary of the state of packages yo Initially, you'll have no packages installed:: julia> Pkg.status() - INFO: Initializing package repository /Users/stefan/.julia/v0.5 + INFO: Initializing package repository /Users/stefan/.julia/v0.6 INFO: Cloning METADATA from git://github.com/JuliaLang/METADATA.jl No packages installed. @@ -56,7 +56,7 @@ This means that you tell it what you want and it figures out what versions to in So rather than installing a package, you just add it to the list of requirements and then "resolve" what needs to be installed. In particular, this means that if some package had been installed because it was needed by a previous version of something you wanted, and a newer version doesn't have that requirement anymore, updating will actually remove that package. -Your package requirements are in the file ``~/.julia/v0.5/REQUIRE``. +Your package requirements are in the file ``~/.julia/v0.6/REQUIRE``. You can edit this file by hand and then call :func:`Pkg.resolve` to install, upgrade or remove packages to optimally satisfy the requirements, or you can do :func:`Pkg.edit`, which will open ``REQUIRE`` in your editor (configured via the ``EDITOR`` or ``VISUAL`` environment variables), and then automatically call :func:`Pkg.resolve` afterwards if necessary. If you only want to add or remove the requirement for a single package, you can also use the non-interactive :func:`Pkg.add` and :func:`Pkg.rm` commands, which add or remove a single requirement to ``REQUIRE`` and then call :func:`Pkg.resolve`. @@ -81,15 +81,15 @@ You can add a package to the list of requirements with the :func:`Pkg.add` funct - NumericExtensions 0.2.17 - Stats 0.2.6 -What this is doing is first adding ``Distributions`` to your ``~/.julia/v0.5/REQUIRE`` file:: +What this is doing is first adding ``Distributions`` to your ``~/.julia/v0.6/REQUIRE`` file:: - $ cat ~/.julia/v0.5/REQUIRE + $ cat ~/.julia/v0.6/REQUIRE Distributions It then runs :func:`Pkg.resolve` using these new requirements, which leads to the conclusion that the ``Distributions`` package should be installed since it is required but not installed. -As stated before, you can accomplish the same thing by editing your ``~/.julia/v0.5/REQUIRE`` file by hand and then running :func:`Pkg.resolve` yourself:: +As stated before, you can accomplish the same thing by editing your ``~/.julia/v0.6/REQUIRE`` file by hand and then running :func:`Pkg.resolve` yourself:: - $ echo UTF16 >> ~/.julia/v0.5/REQUIRE + $ echo UTF16 >> ~/.julia/v0.6/REQUIRE julia> Pkg.resolve() INFO: Cloning cache of UTF16 from git://github.com/nolta/UTF16.jl.git @@ -175,7 +175,7 @@ To install an unregistered package, use :func:`Pkg.clone(url) <Pkg.clone>`, wher Resolving deltas: 100% (8/8), done. By convention, Julia repository names end with ``.jl`` (the additional ``.git`` indicates a "bare" git repository), which keeps them from colliding with repositories for other languages, and also makes Julia packages easy to find in search engines. -When packages are installed in your ``.julia/v0.5`` directory, however, the extension is redundant so we leave it off. +When packages are installed in your ``.julia/v0.6`` directory, however, the extension is redundant so we leave it off. If unregistered packages contain a ``REQUIRE`` file at the top of their source tree, that file will be used to determine which registered packages the unregistered package depends on, and they will automatically be installed. Unregistered packages participate in the same version resolution logic as registered packages, so installed package versions will be adjusted as necessary to satisfy the requirements of both registered and unregistered packages. @@ -194,7 +194,7 @@ To get the latest and greatest versions of all your packages, just do :func:`Pkg INFO: Upgrading Distributions: v0.2.8 => v0.2.10 INFO: Upgrading Stats: v0.2.7 => v0.2.8 -The first step of updating packages is to pull new changes to ``~/.julia/v0.5/METADATA`` and see if any new registered package versions have been published. +The first step of updating packages is to pull new changes to ``~/.julia/v0.6/METADATA`` and see if any new registered package versions have been published. After this, :func:`Pkg.update` attempts to update packages that are checked out on a branch and not dirty (i.e. no changes have been made to files tracked by git) by pulling changes from the package's upstream repository. Upstream changes will only be applied if no merging or rebasing is necessary – i.e. if the branch can be `"fast-forwarded" <https://git-scm.com/book/en/v2/Git-Branching-Basic-Branching-and-Merging>`_. If the branch cannot be fast-forwarded, it is assumed that you're working on it and will update the repository yourself. @@ -207,7 +207,7 @@ A package is considered fixed if it is one of the following: 3. **Dirty:** changes have been made to files in the repo. If any of these are the case, the package manager cannot freely change the installed version of the package, so its requirements must be satisfied by whatever other package versions it picks. -The combination of top-level requirements in ``~/.julia/v0.5/REQUIRE`` and the requirement of fixed packages are used to determine what should be installed. +The combination of top-level requirements in ``~/.julia/v0.6/REQUIRE`` and the requirement of fixed packages are used to determine what should be installed. You can also update only a subset of the installed packages, by providing arguments to the `Pkg.update` function. In that case, only the packages provided as arguments and their dependencies will be updated:: @@ -720,7 +720,7 @@ Suppose you want to create a new Julia package called ``FooBar``. To get starte name of a license that the package generator knows about:: julia> PkgDev.generate("FooBar","MIT") - INFO: Initializing FooBar repo: /Users/stefan/.julia/v0.5/FooBar + INFO: Initializing FooBar repo: /Users/stefan/.julia/v0.6/FooBar INFO: Origin: git://github.com/StefanKarpinski/FooBar.jl.git INFO: Generating LICENSE.md INFO: Generating README.md @@ -732,10 +732,10 @@ name of a license that the package generator knows about:: INFO: Generating .gitignore INFO: Committing FooBar generated files -This creates the directory ``~/.julia/v0.5/FooBar``, initializes it as a git repository, generates a bunch of files +This creates the directory ``~/.julia/v0.6/FooBar``, initializes it as a git repository, generates a bunch of files that all packages should have, and commits them to the repository:: - $ cd ~/.julia/v0.5/FooBar && git show --stat + $ cd ~/.julia/v0.6/FooBar && git show --stat commit 84b8e266dae6de30ab9703150b3bf771ec7b6285 Author: Stefan Karpinski <stefan@karpinski.org> @@ -763,7 +763,7 @@ that all packages should have, and commits them to the repository:: At the moment, the package manager knows about the MIT "Expat" License, indicated by ``"MIT"``, the Simplified BSD License, indicated by ``"BSD"``, and version 2.0 of the Apache Software License, indicated by ``"ASL"``. If you want to use a different license, you can ask us to add it to the package generator, or just pick one of these three and then modify the -``~/.julia/v0.5/PACKAGE/LICENSE.md`` file after it has been generated. +``~/.julia/v0.6/PACKAGE/LICENSE.md`` file after it has been generated. If you created a GitHub account and configured git to know about it, :func:`PkgDev.generate` will set an appropriate origin URL for you. It will also automatically generate a ``.travis.yml`` file for using the `Travis <https://travis-ci.org>`_ automated @@ -799,9 +799,9 @@ of ``METADATA`` using :func:`PkgDev.register`:: INFO: Registering FooBar at git://github.com/StefanKarpinski/FooBar.jl.git INFO: Committing METADATA for FooBar -This creates a commit in the ``~/.julia/v0.5/METADATA`` repo:: +This creates a commit in the ``~/.julia/v0.6/METADATA`` repo:: - $ cd ~/.julia/v0.5/METADATA && git show + $ cd ~/.julia/v0.6/METADATA && git show commit 9f71f4becb05cadacb983c54a72eed744e5c019d Author: Stefan Karpinski <stefan@karpinski.org> @@ -857,12 +857,12 @@ register it with the :func:`PkgDev.tag` command:: This tags ``v0.0.1`` in the ``FooBar`` repo:: - $ cd ~/.julia/v0.5/FooBar && git tag + $ cd ~/.julia/v0.6/FooBar && git tag v0.0.1 It also creates a new version entry in your local ``METADATA`` repo for ``FooBar``:: - $ cd ~/.julia/v0.5/FooBar && git show + $ cd ~/.julia/v0.6/FooBar && git show commit de77ee4dc0689b12c5e8b574aef7f70e8b311b0e Author: Stefan Karpinski <stefan@karpinski.org> Date: Wed Oct 16 23:06:18 2013 -0400 @@ -922,7 +922,7 @@ fork. repository on your local computer (in the terminal where USERNAME is your github username):: - cd ~/.julia/v0.5/METADATA + cd ~/.julia/v0.6/METADATA git remote add USERNAME https://github.com/USERNAME/METADATA.jl.git 3. push your changes to your fork:: @@ -938,7 +938,7 @@ Fixing Package Requirements If you need to fix the registered requirements of an already-published package version, you can do so just by editing the metadata for that version, which will still have the same commit hash – the hash associated with a version is permanent:: - $ cd ~/.julia/v0.5/METADATA/FooBar/versions/0.0.1 && cat requires + $ cd ~/.julia/v0.6/METADATA/FooBar/versions/0.0.1 && cat requires julia 0.3- $ vi requires @@ -951,7 +951,7 @@ When you fix the requirements in ``METADATA`` for a previous version of a packag Requirements Specification -------------------------- -The ``~/.julia/v0.5/REQUIRE`` file, the ``REQUIRE`` file inside packages, and the ``METADATA`` package ``requires`` files use a simple line-based format to express the ranges of package versions which need to be installed. Package ``REQUIRE`` and ``METADATA requires`` files should also include the range of versions of ``julia`` the package is expected to work with. +The ``~/.julia/v0.6/REQUIRE`` file, the ``REQUIRE`` file inside packages, and the ``METADATA`` package ``requires`` files use a simple line-based format to express the ranges of package versions which need to be installed. Package ``REQUIRE`` and ``METADATA requires`` files should also include the range of versions of ``julia`` the package is expected to work with. Here's how these files are parsed and interpreted. From 9f997bd7862191d08ba57e7fa100960e342bc79c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 5 Aug 2016 14:37:25 -0400 Subject: [PATCH 0860/1117] use inferred type instead of `Union{}` for empty comprehensions fixes #17811 --- base/array.jl | 15 ++++++++------- base/dict.jl | 11 +++++++++-- base/set.jl | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/base/array.jl b/base/array.jl index fad82af049850..a24a7773477a7 100644 --- a/base/array.jl +++ b/base/array.jl @@ -283,14 +283,10 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown) end if isdefined(Core, :Inference) - function _default_eltype(itrt::ANY) - rt = Core.Inference.return_type(first, Tuple{itrt}) - return isleaftype(rt) ? rt : Union{} - end + _default_eltype(itrt::ANY) = Core.Inference.return_type(first, Tuple{itrt}) else - _default_eltype(itr::ANY) = Union{} + _default_eltype(itr::ANY) = Any end -_default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T _array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) _array_for(T, itr, ::HasShape) = similar(Array{T}, indices(itr)) @@ -354,7 +350,12 @@ function collect_to!{T}(dest::AbstractArray{T}, itr, offs, st) return dest end -function grow_to!(dest, itr, st = start(itr)) +function grow_to!(dest, itr) + out = grow_to!(similar(dest,Union{}), itr, start(itr)) + return isempty(out) ? dest : out +end + +function grow_to!(dest, itr, st) T = eltype(dest) while !done(itr, st) el, st = next(itr, st) diff --git a/base/dict.jl b/base/dict.jl index 8990bb198c1a2..2ea7e983b1c85 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -380,11 +380,18 @@ end dict_with_eltype{K,V}(kv, ::Type{Tuple{K,V}}) = Dict{K,V}(kv) dict_with_eltype{K,V}(kv, ::Type{Pair{K,V}}) = Dict{K,V}(kv) -dict_with_eltype(kv, t) = grow_to!(Dict{Union{},Union{}}(), kv) +dict_with_eltype{K,V}(::Type{Pair{K,V}}) = Dict{K,V}() +dict_with_eltype(::Type) = Dict() +dict_with_eltype(kv, t) = grow_to!(dict_with_eltype(_default_eltype(typeof(kv))), kv) # this is a special case due to (1) allowing both Pairs and Tuples as elements, # and (2) Pair being invariant. a bit annoying. -function grow_to!{K,V}(dest::Associative{K,V}, itr, st = start(itr)) +function grow_to!(dest::Associative, itr) + out = grow_to!(similar(dest, Pair{Union{},Union{}}), itr, start(itr)) + return isempty(out) ? dest : out +end + +function grow_to!{K,V}(dest::Associative{K,V}, itr, st) while !done(itr, st) (k,v), st = next(itr, st) if isa(k,K) && isa(v,V) diff --git a/base/set.jl b/base/set.jl index cafa9b66eeac1..804fffa8a4d3f 100644 --- a/base/set.jl +++ b/base/set.jl @@ -10,7 +10,7 @@ Set() = Set{Any}() Set(itr) = Set{eltype(itr)}(itr) function Set(g::Generator) T = _default_eltype(typeof(g)) - T === Union{} && return grow_to!(Set{T}(), g) + (isleaftype(T) || T === Union{}) || return grow_to!(Set{T}(), g) return Set{T}(g) end From 451a1733627f5db96811ce30647e5cd340748448 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman <benditlikeranjan@gmail.com> Date: Tue, 9 Aug 2016 20:27:00 +0530 Subject: [PATCH 0861/1117] test for #17811, type of e.g. `abs(Integer[])` (#17891) This tests that the result type of an empty comprehension is big enough to hold the possible results. --- test/inference.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/inference.jl b/test/inference.jl index dc8677df4d71e..2615e9cc4847a 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -274,3 +274,11 @@ end let f(x) = (x===nothing) ? 1 : 1.0 @test Base.return_types(f, (Void,)) == Any[Int] end + +# Issue #17811 +let I = Integer[] + I = abs(I) + @test typeof(I) == Array{Any,1} + push!(I, 1) + @test I == Any[1] +end From ac1d42d7c24382cee13f6ba8261c46d929ab7777 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 9 Aug 2016 15:17:54 -0400 Subject: [PATCH 0862/1117] fix non-ORCJIT build --- src/jitlayers.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/jitlayers.h b/src/jitlayers.h index a87138f46c10a..a4d41dc680f17 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -37,17 +37,6 @@ extern PassManager *jl_globalPM; #include <llvm/Target/TargetMachine.h> #endif -#ifdef LLVM40 -typedef JITSymbol JL_JITSymbol; -// The type that is similar to SymbolInfo on LLVM 4.0 is actually -// `JITEvaluatedSymbol`. However, we only use this type when a JITSymbol -// is expected. -typedef JITSymbol JL_SymbolInfo; -#else -typedef orc::JITSymbol JL_JITSymbol; -typedef RuntimeDyld::SymbolInfo JL_SymbolInfo; -#endif - extern "C" { extern int globalUnique; } @@ -163,6 +152,17 @@ static inline void add_named_global(GlobalValue *gv, T *addr, bool dllimport = t void jl_init_jit(Type *T_pjlvalue_); #ifdef USE_ORCJIT +#ifdef LLVM40 +typedef JITSymbol JL_JITSymbol; +// The type that is similar to SymbolInfo on LLVM 4.0 is actually +// `JITEvaluatedSymbol`. However, we only use this type when a JITSymbol +// is expected. +typedef JITSymbol JL_SymbolInfo; +#else +typedef orc::JITSymbol JL_JITSymbol; +typedef RuntimeDyld::SymbolInfo JL_SymbolInfo; +#endif + class JuliaOJIT { // Custom object emission notification handler for the JuliaOJIT // TODO: hook up RegisterJITEventListener, instead of hard-coding the GDB and JuliaListener targets From 763dd109d7cfa8f6f8543149fb1ecc0a15d542d6 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 9 Aug 2016 12:30:08 -0700 Subject: [PATCH 0863/1117] doc formatting and add back threadcall note --- base/threadcall.jl | 2 ++ doc/manual/parallel-computing.rst | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/base/threadcall.jl b/base/threadcall.jl index c9da92c5a49ea..be3e99f324cb8 100644 --- a/base/threadcall.jl +++ b/base/threadcall.jl @@ -23,6 +23,8 @@ function without causing the main `julia` thread to become blocked. Concurrency is limited by size of the libuv thread pool, which defaults to 4 threads but can be increased by setting the `UV_THREADPOOL_SIZE` environment variable and restarting the `julia` process. + +Note that the called function should never call back into Julia. """ macro threadcall(f, rettype, argtypes, argvals...) # check for usage errors diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index d551f1e288a66..588a364ff997a 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -1081,21 +1081,21 @@ Note that :obj:`Threads.@threads` does not have an optional reduction parameter All I/O tasks, timers, REPL commands, etc are multiplexed onto a single OS thread via an event loop. A patched version of libuv (http://docs.libuv.org/en/v1.x/) provides this functionality. Yield points provide for co-operatively scheduling multiple tasks onto the same OS thread. I/O tasks and timers yield implicitly while -waiting for the event to occur. Calling `yield()` explicitly allows for other tasks to be scheduled. +waiting for the event to occur. Calling ``yield()`` explicitly allows for other tasks to be scheduled. Thus, a task executing a ``ccall`` effectively prevents the Julia scheduler from executing any other tasks till the call returns. This is true for all calls into external libraries. Exceptions are calls into -custom C code that call back into Julia (which may then yield) or C code that calls ``jl_yield()``(C equivalent of ``yield()``). +custom C code that call back into Julia (which may then yield) or C code that calls ``jl_yield()`` (C equivalent of ``yield()``). Note that while Julia code runs on a single thread (by default), libraries used by Julia may launch their own internal threads. For example, the BLAS library may start as many threads as there are cores on a machine. The ``@threadcall`` macro addresses scenarios where we do not want a ``ccall`` to block the main Julia event loop. It schedules a C function for execution in a separate thread. A threadpool with a default size of 4 is used for this. -The size of the threadpool is controlled via environment variable UV_THREADPOOL_SIZE. While waiting for a free thread, +The size of the threadpool is controlled via environment variable ``UV_THREADPOOL_SIZE``. While waiting for a free thread, and during function execution once a thread is available, the requesting task (on the main Julia event loop) yields to other tasks. Note that ``@threadcall`` does not return till the execution is complete. From a user point of -view, it is therefore a blocking call like other Julia API. +view, it is therefore a blocking call like other Julia APIs. It is very important that the called function does not call back into Julia. From 818917194388876b772b1100d1e542710c8e256d Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 9 Aug 2016 16:06:01 -0400 Subject: [PATCH 0864/1117] fix spawn test to check exit status of async tasks --- test/spawn.jl | 173 +++++++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 78 deletions(-) diff --git a/test/spawn.jl b/test/spawn.jl index cfff6b6b6425a..21ce279b5eb3e 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -83,27 +83,32 @@ end @test_broken success(ignorestatus(falsecmd & falsecmd)) # STDIN Redirection -file = tempname() -run(pipeline(`$echo hello world`, file)) -@test readstring(pipeline(file, catcmd)) == "hello world\n" -@test open(readstring, pipeline(file, catcmd), "r") == "hello world\n" -rm(file) +let file = tempname() + run(pipeline(`$echo hello world`, file)) + @test readstring(pipeline(file, catcmd)) == "hello world\n" + @test open(readstring, pipeline(file, catcmd), "r") == "hello world\n" + rm(file) +end # Stream Redirection if !is_windows() # WINNT reports operation not supported on socket (ENOTSUP) for this test - local r = Channel(1), port, server, sock, client - @async begin + local r = Channel(1), port, server, sock, client, t1, t2 + t1 = @async begin port, server = listenany(2326) put!(r, port) client = accept(server) @test readstring(pipeline(client, catcmd)) == "hello world\n" close(server) + return true end - @async begin + t2 = @async begin sock = connect(fetch(r)) run(pipeline(`$echo hello world`, sock)) close(sock) + return true end + @test wait(t1) + @test wait(t2) end @test readstring(setenv(`$shcmd -c "echo \$TEST"`,["TEST=Hello World"])) == "Hello World\n" @@ -111,77 +116,87 @@ end @test readstring(setenv(`$shcmd -c "echo \$TEST"`,"TEST"=>"Hello World")) == "Hello World\n" @test (withenv("TEST"=>"Hello World") do readstring(`$shcmd -c "echo \$TEST"`); end) == "Hello World\n" -pathA = readchomp(setenv(`$shcmd -c "pwd -P"`;dir="..")) -pathB = readchomp(setenv(`$shcmd -c "cd .. && pwd -P"`)) -if is_windows() - # on windows, sh returns posix-style paths that are not valid according to ispath - @test pathA == pathB -else - @test Base.samefile(pathA, pathB) +let pathA = readchomp(setenv(`$shcmd -c "pwd -P"`;dir="..")), + pathB = readchomp(setenv(`$shcmd -c "cd .. && pwd -P"`)) + if is_windows() + # on windows, sh returns posix-style paths that are not valid according to ispath + @test pathA == pathB + else + @test Base.samefile(pathA, pathB) + end end -# Here we test that if we close a stream with pending writes, we don't lose the writes. -str = "" -for i=1:1000 - str = "$str\n $(randstring(10))" -end -stdout, stdin, proc = readandwrite(`$catcmd -`) -write(stdin, str) -close(stdin) -str2 = readstring(stdout) -@test str2 == str - -# This test hangs if the end of run walk across uv streams calls shutdown on a stream that is shutting down. -file = tempname() -open(pipeline(`$catcmd -`, file), "w") do io - write(io, str) +let str = "", stdin, stdout, proc, str2, file + for i = 1:1000 + str = "$str\n $(randstring(10))" + end + + # Here we test that if we close a stream with pending writes, we don't lose the writes. + stdout, stdin, proc = readandwrite(`$catcmd -`) + write(stdin, str) + close(stdin) + str2 = readstring(stdout) + @test str2 == str + + # This test hangs if the end-of-run-walk-across-uv-streams calls shutdown on a stream that is shutting down. + file = tempname() + open(pipeline(`$catcmd -`, file), "w") do io + write(io, str) + end + rm(file) end -rm(file) # issue #3373 # fixing up Conditions after interruptions -r = Channel(1) -t = @async begin - try - wait(r) +let r, t + r = Channel(1) + t = @async begin + try + wait(r) + end + p = spawn(`$sleepcmd 1`); wait(p) + @test p.exitcode == 0 + return true end - p = spawn(`$sleepcmd 1`); wait(p) - @test p.exitcode == 0 + yield() + schedule(t, InterruptException(), error=true) + yield() + put!(r,11) + yield() + @test wait(t) end -yield() -schedule(t, InterruptException(), error=true) -yield() -put!(r,11) -yield() # Test marking of IO -r = Channel(1) -@async begin - port, server = listenany(2327) - put!(r, port) - client = accept(server) - write(client, "Hello, world!\n") - write(client, "Goodbye, world...\n") - close(server) +let r, t, sock + r = Channel(1) + t = @async begin + port, server = listenany(2327) + put!(r, port) + client = accept(server) + write(client, "Hello, world!\n") + write(client, "Goodbye, world...\n") + close(server) + return true + end + sock = connect(fetch(r)) + mark(sock) + @test ismarked(sock) + @test readline(sock) == "Hello, world!\n" + @test readline(sock) == "Goodbye, world...\n" + @test reset(sock) == 0 + @test !ismarked(sock) + mark(sock) + @test ismarked(sock) + @test readline(sock) == "Hello, world!\n" + unmark(sock) + @test !ismarked(sock) + @test_throws ArgumentError reset(sock) + @test !unmark(sock) + @test readline(sock) == "Goodbye, world...\n" + #@test eof(sock) ## doesn't work + close(sock) + @test wait(t) end -sock = connect(fetch(r)) -mark(sock) -@test ismarked(sock) -@test readline(sock) == "Hello, world!\n" -@test readline(sock) == "Goodbye, world...\n" -@test reset(sock) == 0 -@test !ismarked(sock) -mark(sock) -@test ismarked(sock) -@test readline(sock) == "Hello, world!\n" -unmark(sock) -@test !ismarked(sock) -@test_throws ArgumentError reset(sock) -@test !unmark(sock) -@test readline(sock) == "Goodbye, world...\n" -#@test eof(sock) ## doesn't work -close(sock) - # issue #4535 exename = Base.julia_cmd() if valgrind_off @@ -210,16 +225,18 @@ yield() @test consume(ducer) == 2 # redirect_* -OLD_STDOUT = STDOUT -fname = tempname() -f = open(fname,"w") -redirect_stdout(f) -println("Hello World") -redirect_stdout(OLD_STDOUT) -close(f) -@test "Hello World\n" == readstring(fname) -@test is(OLD_STDOUT,STDOUT) -rm(fname) +let OLD_STDOUT = STDOUT, + fname = tempname(), + f = open(fname,"w") + + redirect_stdout(f) + println("Hello World") + redirect_stdout(OLD_STDOUT) + close(f) + @test "Hello World\n" == readstring(fname) + @test is(OLD_STDOUT,STDOUT) + rm(fname) +end # Test that redirecting an IOStream does not crash the process let fname = tempname() From 587a504aa922e49c13f2a9d17c3a14b03776650a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 27 Jul 2016 16:25:50 -0400 Subject: [PATCH 0865/1117] fix #13529, slowdown with large number of async `sleep` calls The problem was performance degradation of ObjectIdDict with many deleted items. The table needs to be rehashed after a large number of deletions. --- base/dict.jl | 24 +++++++++++++++++++----- src/dump.c | 2 +- src/julia_internal.h | 2 +- src/table.c | 14 ++++++++------ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 2ea7e983b1c85..d66b93377c7f8 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -247,7 +247,8 @@ push!(t::Associative, p::Pair, q::Pair, r::Pair...) = push!(push!(push!(t, p), q type ObjectIdDict <: Associative{Any,Any} ht::Vector{Any} - ObjectIdDict() = new(Vector{Any}(32)) + ndel::Int + ObjectIdDict() = new(Vector{Any}(32), 0) function ObjectIdDict(itr) d = ObjectIdDict() @@ -266,7 +267,16 @@ end similar(d::ObjectIdDict) = ObjectIdDict() +function rehash!(t::ObjectIdDict, newsz = length(t.ht)) + t.ht = ccall(:jl_idtable_rehash, Any, (Any, Csize_t), t.ht, newsz) + t +end + function setindex!(t::ObjectIdDict, v::ANY, k::ANY) + if t.ndel >= ((3*length(t.ht))>>2) + rehash!(t, max(length(t.ht)>>1, 32)) + t.ndel = 0 + end t.ht = ccall(:jl_eqtable_put, Array{Any,1}, (Any, Any, Any), t.ht, k, v) return t end @@ -274,8 +284,12 @@ end get(t::ObjectIdDict, key::ANY, default::ANY) = ccall(:jl_eqtable_get, Any, (Any, Any, Any), t.ht, key, default) -pop!(t::ObjectIdDict, key::ANY, default::ANY) = - ccall(:jl_eqtable_pop, Any, (Any, Any, Any), t.ht, key, default) +function pop!(t::ObjectIdDict, key::ANY, default::ANY) + val = ccall(:jl_eqtable_pop, Any, (Any, Any, Any), t.ht, key, default) + # TODO: this can underestimate `ndel` + val === default || (t.ndel += 1) + return val +end function pop!(t::ObjectIdDict, key::ANY) val = pop!(t, key, secret_table_token) @@ -283,11 +297,11 @@ function pop!(t::ObjectIdDict, key::ANY) end function delete!(t::ObjectIdDict, key::ANY) - ccall(:jl_eqtable_pop, Any, (Any, Any), t.ht, key) + pop!(t, key, secret_table_token) t end -empty!(t::ObjectIdDict) = (t.ht = Vector{Any}(length(t.ht)); t) +empty!(t::ObjectIdDict) = (t.ht = Vector{Any}(length(t.ht)); t.ndel = 0; t) _oidd_nextind(a, i) = reinterpret(Int,ccall(:jl_eqtable_nextind, Csize_t, (Any, Csize_t), a, i)) diff --git a/src/dump.c b/src/dump.c index 7157d89bb5a36..d46fb1b624d7a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1815,7 +1815,7 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) case 1: { // rehash ObjectIdDict jl_array_t **a = (jl_array_t**)v; // Assume *a don't need a write barrier - jl_idtable_rehash(a, jl_array_len(*a)); + *a = jl_idtable_rehash(*a, jl_array_len(*a)); jl_gc_wb(v, *a); break; } diff --git a/src/julia_internal.h b/src/julia_internal.h index 2c7105b62e53c..8346c247e5808 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -425,7 +425,7 @@ int32_t jl_get_llvm_gv(jl_value_t *p); int32_t jl_assign_functionID(/*llvm::Function*/void *function); // the first argument to jl_idtable_rehash is used to return a value // make sure it is rooted if it is used after the function returns -void jl_idtable_rehash(jl_array_t **pa, size_t newsz); +JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); diff --git a/src/table.c b/src/table.c index 5bed49efbfe26..37738f9816d71 100644 --- a/src/table.c +++ b/src/table.c @@ -10,13 +10,13 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key); -void jl_idtable_rehash(jl_array_t **pa, size_t newsz) +JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz) { // Assume *pa don't need a write barrier // pa doesn't have to be a GC slot but *pa needs to be rooted - size_t sz = jl_array_len(*pa); + size_t sz = jl_array_len(a); size_t i; - void **ol = (void**)(*pa)->data; + void **ol = (void**)a->data; jl_array_t *newa = jl_alloc_vec_any(newsz); // keep the original array in the original slot since we need `ol` // to be valid in the loop below. @@ -25,16 +25,16 @@ void jl_idtable_rehash(jl_array_t **pa, size_t newsz) if (ol[i+1] != NULL) { (*jl_table_lookup_bp(&newa, ol[i])) = ol[i+1]; jl_gc_wb(newa, ol[i+1]); - // it is however necessary here because allocation + // it is however necessary here because allocation // can (and will) occur in a recursive call inside table_lookup_bp } } - *pa = newa; // we do not check the write barrier here // because pa always points to a C stack location // (see jl_eqtable_put and jl_finalize_deserializer) // it should be changed if this assumption no longer holds JL_GC_POP(); + return newa; } static void **jl_table_lookup_bp(jl_array_t **pa, void *key) @@ -44,6 +44,7 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key) jl_array_t *a = *pa; size_t orig, index, iter; size_t newsz, sz = hash_size(a); + assert(sz >= 1); size_t maxprobe = max_probe(sz); void **tab = (void**)a->data; @@ -81,7 +82,7 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key) newsz = HT_N_INLINE; else newsz = sz<<2; - jl_idtable_rehash(pa, newsz); + *pa = jl_idtable_rehash(*pa, newsz); a = *pa; tab = (void**)a->data; @@ -98,6 +99,7 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key) static void **jl_table_peek_bp(jl_array_t *a, void *key) { size_t sz = hash_size(a); + assert(sz >= 1); size_t maxprobe = max_probe(sz); void **tab = (void**)a->data; uint_t hv = keyhash((jl_value_t*)key); From 30ce6539d36db1a5ccbf7eeaaae40c6e82efc25b Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Fri, 5 Aug 2016 20:51:43 -0400 Subject: [PATCH 0866/1117] Refactor credentials yet again Now, we never prompt for credentials unless we consider the credentials invalid (we tried three times and gave up), (note exceptions are ssh keys which we know have to exist for ssh credentials to work). Also, if through interactive prompting credentials get changed, they get reset to make sure that we do the retries. This fixes the original issue of not prompting when a prompt would have been required as well as being nicer in the face of typos when entering credentials. Fixes certain clone operations over HTTPS. --- base/libgit2/callbacks.jl | 263 +++++++++++++++++++------------------- base/libgit2/libgit2.jl | 19 ++- base/libgit2/types.jl | 41 +++--- 3 files changed, 163 insertions(+), 160 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 0143125c3c2f7..a932923bbfe6c 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -48,90 +48,97 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, err == 0 && return Cint(0) end - # if username is not provided, then prompt for it - username = if username_ptr == Cstring(C_NULL) - uname = creds.user # check if credentials were already used - uname !== nothing && !isusedcreds ? uname : prompt("Username for '$schema$host'") - else - unsafe_string(username_ptr) - end - creds.user = username # save credentials - isempty(username) && return Cint(Error.EAUTH) + if creds.prompt_if_incorrect + # if username is not provided, then prompt for it + username = if username_ptr == Cstring(C_NULL) + uname = creds.user # check if credentials were already used + !isusedcreds ? uname : prompt("Username for '$schema$host'", default=uname) + else + unsafe_string(username_ptr) + end + isempty(username) && return Cint(Error.EAUTH) - # For SSH we need a private key location - privatekey = if haskey(ENV,"SSH_KEY_PATH") - ENV["SSH_KEY_PATH"] - else - keydefpath = creds.prvkey # check if credentials were already used - keydefpath === nothing && (keydefpath = "") - if isempty(keydefpath) || isusedcreds - defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") - if isempty(keydefpath) && isfile(defaultkeydefpath) - keydefpath = defaultkeydefpath - else - keydefpath = - prompt("Private key location for '$schema$username@$host'", default=keydefpath) + # For SSH we need a private key location + privatekey = if haskey(ENV,"SSH_KEY_PATH") + ENV["SSH_KEY_PATH"] + else + keydefpath = creds.prvkey # check if credentials were already used + keydefpath === nothing && (keydefpath = "") + if isempty(keydefpath) || isusedcreds + defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") + if isempty(keydefpath) && isfile(defaultkeydefpath) + keydefpath = defaultkeydefpath + else + keydefpath = + prompt("Private key location for '$schema$username@$host'", default=keydefpath) + end end + keydefpath end - keydefpath - end - # If the private key changed, invalidate the cached public key - (privatekey != creds.prvkey) && - (creds.pubkey = "") - creds.prvkey = privatekey # save credentials + # If the private key changed, invalidate the cached public key + (privatekey != creds.prvkey) && + (creds.pubkey = "") - # For SSH we need a public key location, look for environment vars SSH_* as well - publickey = if haskey(ENV,"SSH_PUB_KEY_PATH") - ENV["SSH_PUB_KEY_PATH"] - else - keydefpath = creds.pubkey # check if credentials were already used - keydefpath === nothing && (keydefpath = "") - if isempty(keydefpath) || isusedcreds - if isempty(keydefpath) - keydefpath = privatekey*".pub" - end - if !isfile(keydefpath) - prompt("Public key location for '$schema$username@$host'", default=keydefpath) + # For SSH we need a public key location, look for environment vars SSH_* as well + publickey = if haskey(ENV,"SSH_PUB_KEY_PATH") + ENV["SSH_PUB_KEY_PATH"] + else + keydefpath = creds.pubkey # check if credentials were already used + keydefpath === nothing && (keydefpath = "") + if isempty(keydefpath) || isusedcreds + if isempty(keydefpath) + keydefpath = privatekey*".pub" + end + if !isfile(keydefpath) + prompt("Public key location for '$schema$username@$host'", default=keydefpath) + end end + keydefpath end - keydefpath - end - creds.pubkey = publickey # save credentials - passphrase_required = true - if !isfile(privatekey) - warn("Private key not found") - else - # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" - open(privatekey) do f - passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") + passphrase_required = true + if !isfile(privatekey) + warn("Private key not found") + else + # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" + open(privatekey) do f + passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") + end end - end - passphrase = if haskey(ENV,"SSH_KEY_PASS") - ENV["SSH_KEY_PASS"] - else - passdef = creds.pass # check if credentials were already used - passdef === nothing && (passdef = "") - if passphrase_required && (isempty(passdef) || isusedcreds) - if is_windows() - passdef = Base.winprompt( - "Your SSH Key requires a password, please enter it now:", - "Passphrase required", privatekey; prompt_username = false) - isnull(passdef) && return Cint(Error.EAUTH) - passdef = Base.get(passdef)[2] - else - passdef = prompt("Passphrase for $privatekey", password=true) + passphrase = if haskey(ENV,"SSH_KEY_PASS") + ENV["SSH_KEY_PASS"] + else + passdef = creds.pass # check if credentials were already used + passdef === nothing && (passdef = "") + if passphrase_required && (isempty(passdef) || isusedcreds) + if is_windows() + passdef = Base.winprompt( + "Your SSH Key requires a password, please enter it now:", + "Passphrase required", privatekey; prompt_username = false) + isnull(passdef) && return Cint(Error.EAUTH) + passdef = Base.get(passdef)[2] + else + passdef = prompt("Passphrase for $privatekey", password=true) + end end + passdef end - passdef + (creds.user != username) || (creds.pass != userpass) || + (creds.prvkey != privatekey) || (creds.pubkey != publickey) && reset!(creds) + + creds.user = username # save credentials + creds.prvkey = privatekey # save credentials + creds.pubkey = publickey # save credentials + creds.pass = passphrase + else + isusedcreds && return Cint(Error.EAUTH) end - creds.pass = passphrase err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), - libgit2credptr, username, publickey, privatekey, passphrase) + libgit2credptr, creds.user, creds.pubkey, creds.prvkey, creds.pass) return err end @@ -139,33 +146,36 @@ function authenticate_userpass(creds::UserPasswordCredentials, libgit2credptr::P schema, host, urlusername) isusedcreds = checkused!(creds) - username = creds.user - userpass = creds.pass - if is_windows() - if username === nothing || userpass === nothing || isusedcreds - res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", - username === nothing || isempty(username) ? - urlusername : username; prompt_username = true) - isnull(res) && return Cint(Error.EAUTH) - username, userpass = Base.get(res) - end - else - if username === nothing || isusedcreds - username = prompt("Username for '$schema$host'", default = urlusername) - end - - if userpass === nothing || isusedcreds + if creds.prompt_if_incorrect + username = creds.user + userpass = creds.pass + (username === nothing) && (username = "") + (userpass === nothing) && (userpass = "") + if is_windows() + if isempty(username) || isempty(userpass) || isusedcreds + res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", + username === nothing || isempty(username) ? + urlusername : username; prompt_username = true) + isnull(res) && return Cint(Error.EAUTH) + username, userpass = Base.get(res) + end + elseif isusedcreds + username = prompt("Username for '$schema$host'", default = isempty(username) ? + urlusername : username) userpass = prompt("Password for '$schema$username@$host'", password=true) end - end - creds.user = username # save credentials - creds.pass = userpass # save credentials + (creds.user != username) || (creds.pass != userpass) && reset!(creds) + creds.user = username # save credentials + creds.pass = userpass # save credentials - isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) + isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) + else + isusedcreds && return Cint(Error.EAUTH) + end err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring), - libgit2credptr, username, userpass) + libgit2credptr, creds.user, creds.pass) err == 0 && return Cint(0) end @@ -211,58 +221,41 @@ function credentials_callback(libgit2credptr::Ptr{Ptr{Void}}, url_ptr::Cstring, schema = schema === nothing ? "" : schema*"://" # get credentials object from payload pointer - creds = nothing - creds_are_temp = true - if payload_ptr != C_NULL - tmpobj = unsafe_pointer_to_objref(payload_ptr) - if isa(tmpobj, AbstractCredentials) - creds = tmpobj - creds_are_temp = false + @assert payload_ptr != C_NULL + creds = unsafe_pointer_to_objref(payload_ptr) + explicit = !isnull(creds[]) && !isa(Base.get(creds[]), CachedCredentials) + # use ssh key or ssh-agent + if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) + sshcreds = get_creds!(creds, "ssh://$host", reset!(SSHCredentials(true), -1)) + if isa(sshcreds, SSHCredentials) + err = authenticate_ssh(sshcreds, libgit2credptr, username_ptr, schema, host) + err == 0 && return err end end - try - # use ssh key or ssh-agent - if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) - sshcreds = get_creds!(creds, "ssh://$host", SSHCredentials()) - if isa(sshcreds, SSHCredentials) - creds = sshcreds # To make sure these get cleaned up below - err = authenticate_ssh(creds, libgit2credptr, username_ptr, schema, host) - err == 0 && return err - end - end - - if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) - defaultcreds = UserPasswordCredentials() - credid = "$schema$host" - upcreds = get_creds!(creds, credid, defaultcreds) - # If there were stored SSH credentials, but we ended up here that must - # mean that something went wrong. Replace the SSH credentials by user/pass - # credentials - if !isa(upcreds, UserPasswordCredentials) - upcreds = defaultcreds - isa(creds, CachedCredentials) && (creds.creds[credid] = upcreds) - end - creds = upcreds # To make sure these get cleaned up below - return authenticate_userpass(creds, libgit2credptr, schema, host, urlusername) + if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) + defaultcreds = reset!(UserPasswordCredentials(true), -1) + credid = "$schema$host" + upcreds = get_creds!(creds, credid, defaultcreds) + # If there were stored SSH credentials, but we ended up here that must + # mean that something went wrong. Replace the SSH credentials by user/pass + # credentials + if !isa(upcreds, UserPasswordCredentials) + upcreds = defaultcreds + isa(Base.get(creds[]), CachedCredentials) && (Base.get(creds[]).creds[credid] = upcreds) end + return authenticate_userpass(upcreds, libgit2credptr, schema, host, urlusername) + end - # No authentication method we support succeeded. The most likely cause is - # that explicit credentials were passed in, but said credentials are incompatible - # with the remote host. - if err == 0 - if (creds != nothing && !isa(creds, CachedCredentials)) - warn("The explicitly provided credentials were incompatible with " * - "the server's supported authentication methods") - end - err = Cint(Error.EAUTH) - end - finally - # if credentials are not passed back to caller via payload, - # then zero any passwords immediately. - if creds_are_temp && creds !== nothing - securezero!(creds) + # No authentication method we support succeeded. The most likely cause is + # that explicit credentials were passed in, but said credentials are incompatible + # with the remote host. + if err == 0 + if explicit + warn("The explicitly provided credentials were incompatible with " * + "the server's supported authentication methods") end + err = Cint(Error.EAUTH) end return Cint(err) end diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index fe9b08c87594c..321e974694966 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -143,18 +143,23 @@ function set_remote_url(path::AbstractString, url::AbstractString; remote::Abstr end end +function make_payload{P<:AbstractCredentials}(payload::Nullable{P}) + Ref{Nullable{AbstractCredentials}}(payload) +end + """ git fetch [<url>|<repository>] [<refspecs>]""" -function fetch{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; +function fetch{T<:AbstractString, P<:AbstractCredentials}(repo::GitRepo; remote::AbstractString="origin", remoteurl::AbstractString="", refspecs::Vector{T}=AbstractString[], - payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) + payload::Nullable{P}=Nullable{AbstractCredentials}()) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else GitRemoteAnon(repo, remoteurl) end try + payload = make_payload(payload) fo = FetchOptions(callbacks=RemoteCallbacks(credentials_cb(), payload)) fetch(rmt, refspecs, msg="from $(url(rmt))", options = fo) finally @@ -163,18 +168,19 @@ function fetch{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; end """ git push [<url>|<repository>] [<refspecs>]""" -function push{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; +function push{T<:AbstractString, P<:AbstractCredentials}(repo::GitRepo; remote::AbstractString="origin", remoteurl::AbstractString="", refspecs::Vector{T}=AbstractString[], force::Bool=false, - payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) + payload::Nullable{P}=Nullable{AbstractCredentials}()) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else GitRemoteAnon(repo, remoteurl) end try + payload = make_payload(payload) push_opts=PushOptions(callbacks=RemoteCallbacks(credentials_cb(), payload)) push(rmt, refspecs, force=force, options=push_opts) finally @@ -303,13 +309,14 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; end """ git clone [-b <branch>] [--bare] <url> <dir> """ -function clone{P<:AbstractPayload}(repo_url::AbstractString, repo_path::AbstractString; +function clone{P<:AbstractCredentials}(repo_url::AbstractString, repo_path::AbstractString; branch::AbstractString="", isbare::Bool = false, remote_cb::Ptr{Void} = C_NULL, - payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) + payload::Nullable{P}=Nullable{AbstractCredentials}()) # setup clone options lbranch = Base.cconvert(Cstring, branch) + payload = make_payload(payload) fetch_opts=FetchOptions(callbacks = RemoteCallbacks(credentials_cb(), payload)) clone_opts = CloneOptions( bare = Cint(isbare), diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index c5ce76c79fb8c..36b01fa0f46ac 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -50,11 +50,9 @@ function Base.finalize(buf::Buffer) return buf_ptr[] end -"Abstract payload type for callback functions" -abstract AbstractPayload - "Abstract credentials payload" -abstract AbstractCredentials <: AbstractPayload +abstract AbstractCredentials + "Checks if credentials were used" checkused!(p::AbstractCredentials) = true checkused!(p::Void) = false @@ -172,13 +170,8 @@ RemoteCallbacks(; sideband_progress::Ptr{Void} = C_NULL, transport, payload) -function RemoteCallbacks{P<:AbstractPayload}(credentials::Ptr{Void}, payload::Nullable{P}) - if isnull(payload) - RemoteCallbacks(credentials=credentials_cb()) - else - payload_ptr = pointer_from_objref(Base.get(payload)) - RemoteCallbacks(credentials=credentials_cb(), payload=payload_ptr) - end +function RemoteCallbacks(credentials::Ptr{Void}, payload::Ref{Nullable{AbstractCredentials}}) + RemoteCallbacks(credentials=credentials_cb(), payload=pointer_from_objref(payload)) end if LibGit2.version() >= v"0.24.0" @@ -698,13 +691,14 @@ import Base.securezero! type UserPasswordCredentials <: AbstractCredentials user::String pass::String + prompt_if_incorrect::Bool # Whether to allow interactive prompting if the credentials are incorrect count::Int # authentication failure protection count - function UserPasswordCredentials(u::AbstractString,p::AbstractString) - c = new(u,p,3) + function UserPasswordCredentials(u::AbstractString,p::AbstractString,prompt_if_incorrect::Bool=false) + c = new(u,p,prompt_if_incorrect,3) finalizer(c, securezero!) return c end - UserPasswordCredentials() = UserPasswordCredentials("","") + UserPasswordCredentials(prompt_if_incorrect::Bool=false) = UserPasswordCredentials("","",prompt_if_incorrect) end function securezero!(cred::UserPasswordCredentials) @@ -721,14 +715,15 @@ type SSHCredentials <: AbstractCredentials pubkey::String prvkey::String usesshagent::String # used for ssh-agent authentication + prompt_if_incorrect::Bool # Whether to allow interactive prompting if the credentials are incorrect count::Int - function SSHCredentials(u::AbstractString,p::AbstractString) - c = new(u,p,"","","Y",3) + function SSHCredentials(u::AbstractString,p::AbstractString,prompt_if_incorrect::Bool=false) + c = new(u,p,"","","Y",prompt_if_incorrect,3) finalizer(c, securezero!) return c end - SSHCredentials() = SSHCredentials("","") + SSHCredentials(prompt_if_incorrect::Bool=false) = SSHCredentials("","",prompt_if_incorrect) end function securezero!(cred::SSHCredentials) securezero!(cred.user) @@ -752,13 +747,21 @@ function checkused!(p::Union{UserPasswordCredentials, SSHCredentials}) p.count -= 1 return false end -reset!(p::Union{UserPasswordCredentials, SSHCredentials}, cnt::Int=3) = (p.count = cnt) -reset!(p::CachedCredentials) = foreach(reset!, values(p.cred)) +reset!(p::Union{UserPasswordCredentials, SSHCredentials}, cnt::Int=3) = (p.count = cnt; p) +reset!(p::CachedCredentials) = (foreach(reset!, values(p.cred)); p) "Obtain the cached credentials for the given host+protocol (credid), or return and store the default if not found" get_creds!(collection::CachedCredentials, credid, default) = get!(collection.cred, credid, default) get_creds!(creds::AbstractCredentials, credid, default) = creds get_creds!(creds::Void, credid, default) = default +function get_creds!(creds::Ref{Nullable{AbstractCredentials}}, credid, default) + if isnull(creds[]) + creds[] = Nullable{AbstractCredentials}(default) + return default + else + get_creds!(Base.get(creds[]), credid, default) + end +end function securezero!(p::CachedCredentials) foreach(securezero!, values(p.cred)) From 867131b21fe0c8f6f93428b1a5347d4dbf7be1c9 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer@college.harvard.edu> Date: Tue, 9 Aug 2016 10:05:51 -0400 Subject: [PATCH 0867/1117] Don't print an error message if ssh-agent fails Since we'll just fall back to the keys anyway. --- base/libgit2/callbacks.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index a932923bbfe6c..91a0b8d4ca42b 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -32,12 +32,8 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, if errcls != Error.None # Check if we used ssh-agent if creds.usesshagent == "U" - println("ERROR: $errmsg ssh-agent") creds.usesshagent = "E" # reported ssh-agent error, disables ssh agent use for the future - else - println("ERROR: $errmsg") end - flush(STDOUT) end # first try ssh-agent if credentials support its usage From 2c9ff3be550cd124031d87d9abfcd4e9fa133305 Mon Sep 17 00:00:00 2001 From: pabloferz <pabloferz@yahoo.com.mx> Date: Tue, 9 Aug 2016 21:01:50 +0200 Subject: [PATCH 0868/1117] Improve inferability of promote_op --- base/abstractarray.jl | 10 +++++----- base/arraymath.jl | 24 ++++++++++++------------ base/promotion.jl | 23 +++++++++++------------ 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index ef2b9fa100be3..2a292343bc9d0 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1653,11 +1653,11 @@ end # These are needed because map(eltype, As) is not inferrable promote_eltype_op(::Any) = (@_pure_meta; Bottom) -promote_eltype_op(op, A) = (@_pure_meta; _promote_op(op, eltype(A))) -promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; _promote_op(op, T)) -promote_eltype_op{T}(op, ::AbstractArray{T}, A) = (@_pure_meta; _promote_op(op, T, eltype(A))) -promote_eltype_op{T}(op, A, ::AbstractArray{T}) = (@_pure_meta; _promote_op(op, eltype(A), T)) -promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; _promote_op(op, R, S)) +promote_eltype_op(op, A) = (@_pure_meta; promote_op(op, eltype(A))) +promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T)) +promote_eltype_op{T}(op, ::AbstractArray{T}, A) = (@_pure_meta; promote_op(op, T, eltype(A))) +promote_eltype_op{T}(op, A, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, eltype(A), T)) +promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S)) promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_eltype_op(op, promote_eltype_op(op, A, B), C, D...)) ## 1 argument diff --git a/base/arraymath.jl b/base/arraymath.jl index 0de1bd39486f5..171d301fb48e9 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -46,8 +46,8 @@ promote_array_type{S<:Integer}(::typeof(.\), ::Type{S}, ::Type{Bool}, T::Type) = promote_array_type{S<:Integer}(F, ::Type{S}, ::Type{Bool}, T::Type) = T for f in (:+, :-, :div, :mod, :&, :|, :$) - @eval ($f)(A::AbstractArray, B::AbstractArray) = - _elementwise($f, promote_op($f, eltype(A), eltype(B)), A, B) + @eval ($f){R,S}(A::AbstractArray{R}, B::AbstractArray{S}) = + _elementwise($f, promote_op($f, R, S), A, B) end function _elementwise(op, ::Type{Any}, A::AbstractArray, B::AbstractArray) promote_shape(A, B) # check size compatibility @@ -63,21 +63,21 @@ end for f in (:.+, :.-, :.*, :./, :.\, :.^, :.÷, :.%, :.<<, :.>>, :div, :mod, :rem, :&, :|, :$) @eval begin - function ($f)(A::Number, B::AbstractArray) - P = promote_op($f, typeof(A), eltype(B)) - T = promote_array_type($f, typeof(A), eltype(B), P) - T === Any && return [($f)(A, b) for b in B] - F = similar(B, T) + function ($f){T}(A::Number, B::AbstractArray{T}) + R = promote_op($f, typeof(A), T) + S = promote_array_type($f, typeof(A), T, R) + S === Any && return [($f)(A, b) for b in B] + F = similar(B, S) for (iF, iB) in zip(eachindex(F), eachindex(B)) @inbounds F[iF] = ($f)(A, B[iB]) end return F end - function ($f)(A::AbstractArray, B::Number) - P = promote_op($f, eltype(A), typeof(B)) - T = promote_array_type($f, typeof(B), eltype(A), P) - T === Any && return [($f)(a, B) for a in A] - F = similar(A, T) + function ($f){T}(A::AbstractArray{T}, B::Number) + R = promote_op($f, T, typeof(B)) + S = promote_array_type($f, typeof(B), T, R) + S === Any && return [($f)(a, B) for a in A] + F = similar(A, S) for (iF, iA) in zip(eachindex(F), eachindex(A)) @inbounds F[iF] = ($f)(A[iA], B) end diff --git a/base/promotion.jl b/base/promotion.jl index 449dae5add81a..4952126dc8cd8 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -220,19 +220,17 @@ minmax(x::Real, y::Real) = minmax(promote(x, y)...) # "Promotion" that takes a function into account. These are meant to be # used mainly by broadcast methods, so it is advised against overriding them if isdefined(Core, :Inference) - function _promote_op(op, T::Type) + function _promote_op(op, T::ANY) G = Tuple{Generator{Tuple{T},typeof(op)}} - R = Core.Inference.return_type(first, G) - return isleaftype(R) ? R : Any + return Core.Inference.return_type(first, G) end - function _promote_op(op, R::Type, S::Type) + function _promote_op(op, R::ANY, S::ANY) F = typeof(a -> op(a...)) G = Tuple{Generator{Zip2{Tuple{R},Tuple{S}},F}} - T = Core.Inference.return_type(first, G) - return isleaftype(T) ? T : Any + return Core.Inference.return_type(first, G) end else - _promote_op(::Any...) = (@_pure_meta; Any) + _promote_op(::ANY...) = (@_pure_meta; Any) end _default_type(T::Type) = (@_pure_meta; T) @@ -240,14 +238,15 @@ promote_op(::Any...) = (@_pure_meta; Any) promote_op(T::Type, ::Any) = (@_pure_meta; T) promote_op(T::Type, ::Type) = (@_pure_meta; T) # To handle ambiguities # Promotion that tries to preserve non-concrete types -function promote_op(f, S::Type) +function promote_op{S}(f, ::Type{S}) T = _promote_op(f, _default_type(S)) - return isleaftype(S) ? T : typejoin(S, T) + isleaftype(S) && return isleaftype(T) ? T : Any + return typejoin(S, T) end -function promote_op(f, R::Type, S::Type) +function promote_op{R,S}(f, ::Type{R}, ::Type{S}) T = _promote_op(f, _default_type(R), _default_type(S)) - isleaftype(R) && return isleaftype(S) ? T : typejoin(S, T) - return isleaftype(S) ? typejoin(R, T) : typejoin(R, S, T) + isleaftype(R) && isleaftype(S) && return isleaftype(T) ? T : Any + return typejoin(R, S, T) end ## catch-alls to prevent infinite recursion when definitions are missing ## From 5842445350ab0bb0abe29975acc87105a9613e92 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Tue, 9 Aug 2016 17:54:51 -0700 Subject: [PATCH 0869/1117] Revert "Temporary ugly hack to download old versions of winrpm gcc dll's" (#17906) This reverts commit 97f18418b257bdcf625a5384687389b048e9c2ab. ref #15521 --- Makefile | 20 +++----------------- contrib/windows/msys_build.sh | 5 +++++ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 7f0ef928f902f..59d3b34fc9977 100644 --- a/Makefile +++ b/Makefile @@ -471,7 +471,7 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) + cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) ifeq ($(USE_GPL_LIBS), 1) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ cp busybox.exe $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) @@ -600,14 +600,7 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686)) $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920.exe && \ 7z x -y 7z920.exe 7z.exe 7z.dll && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.2 \ - "mingw32-libexpat1 mingw32-zlib1" && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libgfortran3-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libquadmath0-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libstdc%2B%2B6-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libgcc_s_sjlj1-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libssp0-5.3.0-1.1.noarch.rpm && \ - for i in *.rpm; do 7z x -y $$i; done && \ - for i in *.cpio; do 7z x -y $$i; done && \ + "mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \ cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . else ifeq ($(ARCH),x86_64) cd $(JULIAHOME)/dist-extras && \ @@ -616,14 +609,7 @@ else ifeq ($(ARCH),x86_64) mv _7z.dll 7z.dll && \ mv _7z.exe 7z.exe && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.2 \ - "mingw64-libexpat1 mingw64-zlib1" && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libgfortran3-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libquadmath0-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libstdc%2B%2B6-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libgcc_s_seh1-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libssp0-5.3.0-1.1.noarch.rpm && \ - for i in *.rpm; do 7z x -y $$i; done && \ - for i in *.cpio; do 7z x -y $$i; done && \ + "mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \ cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . else $(error no win-extras target for ARCH=$(ARCH)) diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 034127a9d7536..ae343ae1dc85b 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -104,6 +104,11 @@ echo "override PCRE_INCL_PATH =" >> Make.user # Remove libjulia.dll if it was copied from downloaded binary rm -f usr/bin/libjulia.dll rm -f usr/bin/libjulia-debug.dll +rm -f usr/bin/libgcc_s_s*-1.dll +rm -f usr/bin/libgfortran-3.dll +rm -f usr/bin/libquadmath-0.dll +rm -f usr/bin/libssp-0.dll +rm -f usr/bin/libstdc++-6.dll if [ -z "$USEMSVC" ]; then if [ -z "`which ${CROSS_COMPILE}gcc 2>/dev/null`" -o -n "$APPVEYOR" ]; then From cf5952e72f80b4bd0d6dc55b296b94557df30c54 Mon Sep 17 00:00:00 2001 From: Art <wildart@users.noreply.github.com> Date: Wed, 10 Aug 2016 02:20:02 -0400 Subject: [PATCH 0870/1117] fix incorrect bignum rng buffer size [fix #17772] (#17874) --- deps/mbedtls.mk | 6 +-- deps/patches/libssh2-mbedtls.patch | 85 ++++++++++++++++++++++-------- deps/patches/mbedtls-config.patch | 27 ---------- 3 files changed, 63 insertions(+), 55 deletions(-) delete mode 100644 deps/patches/mbedtls-config.patch diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index ce0396a80f895..150a39f2a1196 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -35,15 +35,11 @@ $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt: $(SRCDIR)/srccache/$(MBEDTLS_S $(TAR) -C $(dir $@) --strip-components 1 -xf $< touch -c $@ -$(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied: | $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt - cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls-config.patch - echo 1 > $@ - $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied: | $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt cd $(SRCDIR)/srccache/$(MBEDTLS_SRC)/include/mbedtls && patch -p0 -f < $(SRCDIR)/patches/mbedtls-ssl.h.patch echo 1 > $@ -$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied +$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(MBEDTLS_OPTS) diff --git a/deps/patches/libssh2-mbedtls.patch b/deps/patches/libssh2-mbedtls.patch index 2311d1e210b49..d7cdeef91a44b 100644 --- a/deps/patches/libssh2-mbedtls.patch +++ b/deps/patches/libssh2-mbedtls.patch @@ -159,10 +159,10 @@ index e85aecd..366d007 100644 diff --git a/src/mbedtls.c b/src/mbedtls.c new file mode 100644 -index 0000000..98bc549 +index 0000000..1d181e1 --- /dev/null +++ b/src/mbedtls.c -@@ -0,0 +1,570 @@ +@@ -0,0 +1,606 @@ +#include "libssh2_priv.h" + +#ifdef LIBSSH2_MBEDTLS /* compile only if we build with mbedtls */ @@ -277,10 +277,10 @@ index 0000000..98bc549 + if(!ret) + ret = mbedtls_cipher_finish(ctx, output + olen, &finish_olen); + -+ olen += finish_olen; -+ -+ if (!ret) ++ if (!ret) { ++ olen += finish_olen; + memcpy(block, output, olen); ++ } + + _libssh2_mbedtls_safe_free(output, osize); + } @@ -306,6 +306,9 @@ index 0000000..98bc549 + int ret, hmac; + + md_info = mbedtls_md_info_from_type(mdtype); ++ if(!md_info) ++ return 0; ++ + hmac = key == NULL ? 0 : 1; + + mbedtls_md_init(ctx); @@ -339,6 +342,9 @@ index 0000000..98bc549 + int ret; + + md_info = mbedtls_md_info_from_type(mdtype); ++ if(!md_info) ++ return 0; ++ + ret = mbedtls_md(md_info, data, datalen, hash); + + return ret == 0 ? 0 : -1; @@ -362,17 +368,47 @@ index 0000000..98bc549 + return bignum; +} + -+void -+_libssh2_mbedtls_bignum_free(_libssh2_bn *bn) ++int ++_libssh2_mbedtls_bignum_random(_libssh2_bn *bn, int bits, int top, int bottom) +{ -+ if (bn) -+ { -+ mbedtls_mpi_free(bn); -+#ifdef LIBSSH2_CLEAR_MEMORY -+ memset(bn, 0, sizeof(_libssh2_bn)); -+#endif ++ size_t len; ++ int err; ++ int i; ++ ++ if (!bn || bits <= 0) ++ return -1; ++ ++ len = (bits + 7) >> 3; ++ err = mbedtls_mpi_fill_random(bn, len, mbedtls_ctr_drbg_random, &_libssh2_mbedtls_ctr_drbg); ++ if (err) ++ return -1; ++ ++ /* Zero unsued bits above the most significant bit*/ ++ for(i=len*8-1;bits<=i;--i) { ++ err = mbedtls_mpi_set_bit(bn, i, 0); ++ if (err) ++ return -1; ++ } ++ ++ /* If `top` is -1, the most significant bit of the random number can be zero. ++ If top is 0, the most significant bit of the random number is set to 1, ++ and if top is 1, the two most significant bits of the number will be set ++ to 1, so that the product of two such random numbers will always have 2*bits length. ++ */ ++ for(i=0;i<=top;++i) { ++ err = mbedtls_mpi_set_bit(bn, bits-i-1, 1); ++ if (err) ++ return -1; ++ } ++ ++ /* make odd by setting first bit in least significant byte */ ++ if (bottom) { ++ err = mbedtls_mpi_set_bit(bn, 0, 1); ++ if (err) ++ return -1; + } -+ mbedtls_free(bn); ++ ++ return 0; +} + + @@ -464,7 +500,7 @@ index 0000000..98bc549 + mbedtls_pk_init(&pkey); + + ret = mbedtls_pk_parse_keyfile(&pkey, filename, (char *)passphrase); -+ if( ret != 0 ) ++ if( ret != 0 || mbedtls_pk_get_type(&pkey) != MBEDTLS_PK_RSA) + { + mbedtls_pk_free(&pkey); + mbedtls_rsa_free(*rsa); @@ -498,7 +534,7 @@ index 0000000..98bc549 + + ret = mbedtls_pk_parse_key(&pkey, (unsigned char *)filedata, + filedata_len, NULL, 0); -+ if( ret != 0 ) ++ if( ret != 0 || mbedtls_pk_get_type(&pkey) != MBEDTLS_PK_RSA) + { + mbedtls_pk_free(&pkey); + mbedtls_rsa_free(*rsa); @@ -529,7 +565,7 @@ index 0000000..98bc549 + return -1; /* failure */ + + ret = mbedtls_rsa_pkcs1_verify(rsa, NULL, NULL, MBEDTLS_RSA_PUBLIC, -+ MBEDTLS_MD_SHA1, sig_len, hash, sig); ++ MBEDTLS_MD_SHA1, SHA_DIGEST_LENGTH, hash, sig); + + return (ret == 0) ? 0 : -1; +} @@ -735,10 +771,10 @@ index 0000000..98bc549 +#endif /* LIBSSH2_MBEDTLS */ diff --git a/src/mbedtls.h b/src/mbedtls.h new file mode 100644 -index 0000000..f594575 +index 0000000..248583e --- /dev/null +++ b/src/mbedtls.h -@@ -0,0 +1,368 @@ +@@ -0,0 +1,371 @@ +#include <stdlib.h> +#include <string.h> + @@ -980,8 +1016,8 @@ index 0000000..f594575 + _libssh2_mbedtls_bignum_init() +#define _libssh2_bn_init_from_bin() \ + _libssh2_mbedtls_bignum_init() -+#define _libssh2_bn_rand(bn, bytes, top, bottom) \ -+ mbedtls_mpi_fill_random(bn, bytes, mbedtls_ctr_drbg_random, &_libssh2_mbedtls_ctr_drbg) ++#define _libssh2_bn_rand(bn, bits, top, bottom) \ ++ _libssh2_mbedtls_bignum_random(bn, bits, top, bottom) +#define _libssh2_bn_mod_exp(r, a, p, m, ctx) \ + mbedtls_mpi_exp_mod(r, a, p, m, NULL) +#define _libssh2_bn_set_word(bn, word) \ @@ -995,7 +1031,7 @@ index 0000000..f594575 +#define _libssh2_bn_bits(bn) \ + mbedtls_mpi_bitlen(bn) +#define _libssh2_bn_free(bn) \ -+ _libssh2_mbedtls_bignum_free(bn) ++ mbedtls_mpi_free(bn) + + +/*******************************************************************/ @@ -1044,6 +1080,9 @@ index 0000000..f594575 +_libssh2_mbedtls_bignum_free(_libssh2_bn *bn); + +int ++_libssh2_mbedtls_bignum_random(_libssh2_bn *bn, int bits, int top, int bottom); ++ ++int +_libssh2_mbedtls_rsa_new(libssh2_rsa_ctx **rsa, + const unsigned char *edata, + unsigned long elen, @@ -1106,4 +1145,4 @@ index 0000000..f594575 + size_t *pubkeydata_len, + const char *privatekeydata, + size_t privatekeydata_len, -+ const char *passphrase); ++ const char *passphrase); \ No newline at end of file diff --git a/deps/patches/mbedtls-config.patch b/deps/patches/mbedtls-config.patch deleted file mode 100644 index 1f774ec61ac33..0000000000000 --- a/deps/patches/mbedtls-config.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git include/mbedtls/config.h include/mbedtls/config.h -index 0efee04..787cd8c 100644 ---- include/mbedtls/config.h -+++ include/mbedtls/config.h -@@ -2434,19 +2434,19 @@ - - /* MPI / BIGNUM options */ - //#define MBEDTLS_MPI_WINDOW_SIZE 6 /**< Maximum windows size used. */ --//#define MBEDTLS_MPI_MAX_SIZE 1024 /**< Maximum number of bytes for usable MPIs. */ -+#define MBEDTLS_MPI_MAX_SIZE 2048 /**< Maximum number of bytes for usable MPIs. */ - - /* CTR_DRBG options */ - //#define MBEDTLS_CTR_DRBG_ENTROPY_LEN 48 /**< Amount of entropy used per seed by default (48 with SHA-512, 32 with SHA-256) */ - //#define MBEDTLS_CTR_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ - //#define MBEDTLS_CTR_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ --//#define MBEDTLS_CTR_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -+#define MBEDTLS_CTR_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ - //#define MBEDTLS_CTR_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ - - /* HMAC_DRBG options */ - //#define MBEDTLS_HMAC_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ - //#define MBEDTLS_HMAC_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ --//#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -+#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ - //#define MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ - - /* ECP options */ From ea748d653731d6edb159a1ab381855a2330dda0b Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 30 Jul 2016 08:04:26 +0800 Subject: [PATCH 0871/1117] Add more `try`-`catch` and sigatomic for top-level code/new tasks So that we don't need to run `jl_exit` in strange (signal handler) context due to missing exception handler. --- base/task.jl | 3 +++ src/julia.h | 1 + src/task.c | 22 ++++++++++++++++------ ui/repl.c | 7 ++++++- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/base/task.jl b/base/task.jl index cc06115a23554..3f3fc253e7338 100644 --- a/base/task.jl +++ b/base/task.jl @@ -141,6 +141,7 @@ suppress_excp_printing(t::Task) = isa(t.storage, ObjectIdDict) ? get(get_task_tl # runtime system hook called when a task finishes function task_done_hook(t::Task) + # `finish_task` sets `sigatomic` before entering this function err = (t.state == :failed) result = t.result handled = false @@ -188,6 +189,8 @@ function task_done_hook(t::Task) end end end + # Clear sigatomic before waiting + sigatomic_end() wait() end diff --git a/src/julia.h b/src/julia.h index 3e3b2284300bf..5ee59917b999f 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1465,6 +1465,7 @@ JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg); JL_DLLEXPORT void JL_NORETURN jl_throw(jl_value_t *e); JL_DLLEXPORT void JL_NORETURN jl_rethrow(void); JL_DLLEXPORT void JL_NORETURN jl_rethrow_other(jl_value_t *e); +JL_DLLEXPORT void JL_NORETURN jl_no_exc_handler(jl_value_t *e); #ifdef JULIA_ENABLE_THREADING static inline void jl_lock_frame_push(jl_mutex_t *lock) diff --git a/src/task.c b/src/task.c index b1cf229bd1248..105e80d34d579 100644 --- a/src/task.c +++ b/src/task.c @@ -188,6 +188,7 @@ static jl_function_t *task_done_hook_func=NULL; static void JL_NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) { jl_ptls_t ptls = jl_get_ptls_states(); + JL_SIGATOMIC_BEGIN(); if (t->exception != jl_nothing) t->state = failed_sym; else @@ -211,7 +212,12 @@ static void JL_NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) } if (task_done_hook_func != NULL) { jl_value_t *args[2] = {task_done_hook_func, (jl_value_t*)t}; - jl_apply(args, 2); + JL_TRY { + jl_apply(args, 2); + } + JL_CATCH { + jl_no_exc_handler(jl_exception_in_transit); + } } gc_debug_critical_error(); abort(); @@ -509,6 +515,14 @@ static void init_task(jl_task_t *t, char *stack) #endif /* !COPY_STACKS */ jl_timing_block_t *jl_pop_timing_block(jl_timing_block_t *cur_block); +JL_DLLEXPORT JL_NORETURN void jl_no_exc_handler(jl_value_t *e) +{ + jl_printf(JL_STDERR, "fatal: error thrown and no exception handler available.\n"); + jl_static_show(JL_STDERR, e); + jl_printf(JL_STDERR, "\n"); + jlbacktrace(); + jl_exit(1); +} // yield to exception handler void JL_NORETURN throw_internal(jl_value_t *e) @@ -532,11 +546,7 @@ void JL_NORETURN throw_internal(jl_value_t *e) jl_longjmp(eh->eh_ctx, 1); } else { - jl_printf(JL_STDERR, "fatal: error thrown and no exception handler available.\n"); - jl_static_show(JL_STDERR, e); - jl_printf(JL_STDERR, "\n"); - jlbacktrace(); - jl_exit(1); + jl_no_exc_handler(e); } assert(0); } diff --git a/ui/repl.c b/ui/repl.c index b0ae9c03c1ccd..826d23fb0dd7e 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -109,7 +109,12 @@ static NOINLINE int true_main(int argc, char *argv[]) (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("_start")) : NULL; if (start_client) { - jl_apply(&start_client, 1); + JL_TRY { + jl_apply(&start_client, 1); + } + JL_CATCH { + jl_no_exc_handler(jl_exception_in_transit); + } return 0; } From e25d21ec8343cf7bbff411ba21824785631eb3c8 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 30 Jul 2016 23:22:35 +0800 Subject: [PATCH 0872/1117] Implement `jl_call_in_ctx` on unix. Use it to make sure that `jl_rethrow` and `jl_exit` are running on the right thread and right stack when an exception/exit is caused by a signal. Fix #17706 --- src/julia_internal.h | 3 - src/signals-mach.c | 86 ++++++++++++++----- src/signals-unix.c | 192 +++++++++++++++++++++++++++++++++++-------- src/threading.c | 8 -- 4 files changed, 226 insertions(+), 63 deletions(-) diff --git a/src/julia_internal.h b/src/julia_internal.h index 8346c247e5808..6eec5c076d43a 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -49,9 +49,6 @@ void jl_call_tracer(tracer_cb callback, jl_value_t *tracee); extern size_t jl_page_size; extern jl_function_t *jl_typeinf_func; -#if defined(JL_USE_INTEL_JITEVENTS) -extern unsigned sig_stack_size; -#endif JL_DLLEXPORT extern int jl_lineno; JL_DLLEXPORT extern const char *jl_filename; diff --git a/src/signals-mach.c b/src/signals-mach.c index b168f6cd70a11..cd7cb1b826557 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -15,7 +15,7 @@ #include <sys/_structs.h> #endif -static void attach_exception_port(thread_port_t thread); +static void attach_exception_port(thread_port_t thread, int segv_only); #ifdef JULIA_ENABLE_THREADING // low 16 bits are the thread id, the next 8 bits are the original gc_state @@ -99,7 +99,7 @@ static void allocate_segv_handler() } pthread_attr_destroy(&attr); for (int16_t tid = 0;tid < jl_n_threads;tid++) { - attach_exception_port(pthread_mach_thread_np(jl_all_tls_states[tid]->system_id)); + attach_exception_port(pthread_mach_thread_np(jl_all_tls_states[tid]->system_id), 0); } } @@ -120,7 +120,21 @@ enum x86_trap_flags { PAGE_PRESENT = 0x1 }; -void jl_throw_in_thread(int tid, mach_port_t thread, jl_value_t *exception) +static void jl_call_in_state(jl_ptls_t ptls2, x86_thread_state64_t *state, + void (*fptr)(void)) +{ + uint64_t rsp = (uint64_t)ptls2->signal_stack + sig_stack_size; + assert(rsp % 16 == 0); + + // push (null) $RIP onto the stack + rsp -= sizeof(void*); + *(void**)rsp = NULL; + + state->__rsp = rsp; // set stack pointer + state->__rip = (uint64_t)fptr; // "call" the function +} + +static void jl_throw_in_thread(int tid, mach_port_t thread, jl_value_t *exception) { unsigned int count = MACHINE_THREAD_STATE_COUNT; x86_thread_state64_t state; @@ -131,18 +145,9 @@ void jl_throw_in_thread(int tid, mach_port_t thread, jl_value_t *exception) ptls2->bt_size = rec_backtrace_ctx(ptls2->bt_data, JL_MAX_BT_SIZE, (bt_context_t*)&state); ptls2->exception_in_transit = exception; - - uint64_t rsp = (uint64_t)ptls2->signal_stack + sig_stack_size; - rsp &= -16; // ensure 16-byte alignment - - // push (null) $RIP onto the stack - rsp -= sizeof(void*); - *(void**)rsp = NULL; - - state.__rsp = rsp; // set stack pointer - state.__rip = (uint64_t)&jl_rethrow; // "call" the function - - ret = thread_set_state(thread, x86_THREAD_STATE64, (thread_state_t)&state, count); + jl_call_in_state(ptls2, &state, &jl_rethrow); + ret = thread_set_state(thread, x86_THREAD_STATE64, + (thread_state_t)&state, count); HANDLE_MACH_ERROR("thread_set_state",ret); } @@ -185,6 +190,11 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, jl_ptls_t ptls2 = &jl_tls_states; tid = 0; #endif + if (exception == EXC_ARITHMETIC) { + jl_throw_in_thread(tid, thread, jl_diverror_exception); + return KERN_SUCCESS; + } + assert(exception == EXC_BAD_ACCESS); kern_return_t ret = thread_get_state(thread, x86_EXCEPTION_STATE64, (thread_state_t)&exc_state, &exc_count); HANDLE_MACH_ERROR("thread_get_state", ret); uint64_t fault_addr = exc_state.__faultvaddr; @@ -237,11 +247,14 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, } } -static void attach_exception_port(thread_port_t thread) +static void attach_exception_port(thread_port_t thread, int segv_only) { kern_return_t ret; // http://www.opensource.apple.com/source/xnu/xnu-2782.1.97/osfmk/man/thread_set_exception_ports.html - ret = thread_set_exception_ports(thread, EXC_MASK_BAD_ACCESS, segv_port, EXCEPTION_DEFAULT, MACHINE_THREAD_STATE); + exception_mask_t mask = EXC_MASK_BAD_ACCESS; + if (!segv_only) + mask |= EXC_MASK_ARITHMETIC; + ret = thread_set_exception_ports(thread, mask, segv_port, EXCEPTION_DEFAULT, MACHINE_THREAD_STATE); HANDLE_MACH_ERROR("thread_set_exception_ports", ret); } @@ -283,7 +296,7 @@ static void jl_try_deliver_sigint(void) kern_return_t ret = thread_suspend(thread); HANDLE_MACH_ERROR("thread_suspend", ret); - // This aborts `sleep` and other syscall. + // This aborts `sleep` and other syscalls. ret = thread_abort(thread); HANDLE_MACH_ERROR("thread_abort", ret); @@ -304,6 +317,41 @@ static void jl_try_deliver_sigint(void) HANDLE_MACH_ERROR("thread_resume", ret); } +static void jl_exit_thread0(int exitstate) +{ + jl_ptls_t ptls2 = jl_all_tls_states[0]; + mach_port_t thread = pthread_mach_thread_np(ptls2->system_id); + kern_return_t ret = thread_suspend(thread); + HANDLE_MACH_ERROR("thread_suspend", ret); + + // This aborts `sleep` and other syscalls. + ret = thread_abort(thread); + HANDLE_MACH_ERROR("thread_abort", ret); + + unsigned int count = MACHINE_THREAD_STATE_COUNT; + x86_thread_state64_t state; + ret = thread_get_state(thread, x86_THREAD_STATE64, + (thread_state_t)&state, &count); + + void (*exit_func)(int) = &_exit; + if (thread0_exit_count <= 1) { + exit_func = &jl_exit; + } + else if (thread0_exit_count == 2) { + exit_func = &exit; + } + + // First integer argument. Not portable but good enough =) + state.__rdi = exitstate; + jl_call_in_state(ptls2, &state, (void (*)(void))exit_func); + ret = thread_set_state(thread, x86_THREAD_STATE64, + (thread_state_t)&state, count); + HANDLE_MACH_ERROR("thread_set_state",ret); + + ret = thread_resume(thread); + HANDLE_MACH_ERROR("thread_resume", ret); +} + static int profile_started = 0; mach_timespec_t timerprof; static pthread_t profiler_thread; @@ -363,7 +411,7 @@ void *mach_profile_listener(void *arg) (void)arg; int i; const int max_size = 512; - attach_exception_port(mach_thread_self()); + attach_exception_port(mach_thread_self(), 1); #ifdef LIBOSXUNWIND mach_profiler_thread = mach_thread_self(); #endif diff --git a/src/signals-unix.c b/src/signals-unix.c index 9557dd0a3bd60..9bf9bdb01f734 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -28,14 +28,10 @@ #define HAVE_TIMER #endif -#if defined(JL_USE_INTEL_JITEVENTS) -unsigned sig_stack_size = SIGSTKSZ; -#elif defined(_CPU_AARCH64_) -// The default SIGSTKSZ causes stack overflow in libunwind. -#define sig_stack_size (1 << 16) -#else -#define sig_stack_size SIGSTKSZ -#endif +// 8M signal stack, same as default stack size and enough +// for reasonable finalizers. +// Should also be enough for parallel GC when we have it =) +#define sig_stack_size (8 * 1024 * 1024) static bt_context_t *jl_to_bt_context(void *sigctx) { @@ -46,17 +42,88 @@ static bt_context_t *jl_to_bt_context(void *sigctx) #endif } -static void JL_NORETURN jl_throw_in_ctx(jl_value_t *e, void *sigctx) +static int thread0_exit_count = 0; + +static inline __attribute__((unused)) uintptr_t jl_get_rsp_from_ctx(const void *_ctx) +{ +#if defined(_OS_LINUX_) && defined(_CPU_X86_64_) + const ucontext_t *ctx = (const ucontext_t*)_ctx; + return ctx->uc_mcontext.gregs[REG_RSP]; +#elif defined(_OS_LINUX_) && defined(_CPU_X86_) + const ucontext_t *ctx = (const ucontext_t*)_ctx; + return ctx->uc_mcontext.gregs[REG_ESP]; +#elif defined(_OS_LINUX_) && defined(_CPU_AARCH64_) + const ucontext_t *ctx = (const ucontext_t*)_ctx; + return ctx->uc_mcontext.sp; +#elif defined(_OS_LINUX_) && defined(_CPU_ARM_) + const ucontext_t *ctx = (const ucontext_t*)_ctx; + return ctx->uc_mcontext.arm_sp; +#elif defined(_OS_DARWIN_) + const ucontext64_t *ctx = (const ucontext64_t*)_ctx; + return ctx->uc_mcontext64->__ss.__rsp; +#else + // TODO Add support for FreeBSD and PowerPC(64)? + return 0; +#endif +} + +static void jl_call_in_ctx(jl_ptls_t ptls, void (*fptr)(void), void *_ctx) +{ + // Modifying the ucontext should work but there is concern that + // sigreturn orientated programming mitigation can work against us + // by rejecting ucontext that is modified. + // The current (staged) implementation in the Linux Kernel only + // checks that the syscall is made in the signal handler and that + // the ucontext address is valid. Hopefully the value of the ucontext + // will not be part of the validation... + uintptr_t rsp = (uintptr_t)ptls->signal_stack + sig_stack_size; + assert(rsp % 16 == 0); +#if defined(_OS_LINUX_) && defined(_CPU_X86_64_) + ucontext_t *ctx = (ucontext_t*)_ctx; + rsp -= sizeof(void*); + *(void**)rsp = NULL; + ctx->uc_mcontext.gregs[REG_RSP] = rsp; + ctx->uc_mcontext.gregs[REG_RIP] = (uintptr_t)fptr; +#elif defined(_OS_LINUX_) && defined(_CPU_X86_) + ucontext_t *ctx = (ucontext_t*)_ctx; + rsp -= sizeof(void*); + *(void**)rsp = NULL; + ctx->uc_mcontext.gregs[REG_ESP] = rsp; + ctx->uc_mcontext.gregs[REG_EIP] = (uintptr_t)fptr; +#elif defined(_OS_LINUX_) && defined(_CPU_AARCH64_) + ucontext_t *ctx = (ucontext_t*)_ctx; + ctx->uc_mcontext.sp = rsp; + ctx->uc_mcontext.regs[29] = 0; // Clear link register (x29) + ctx->uc_mcontext.pc = (uintptr_t)fptr; +#elif defined(_OS_LINUX_) && defined(_CPU_ARM_) + ucontext_t *ctx = (ucontext_t*)_ctx; + ctx->uc_mcontext.arm_sp = rsp; + ctx->uc_mcontext.arm_lr = 0; // Clear link register + ctx->uc_mcontext.arm_pc = (uintptr_t)fptr; +#elif defined(_OS_DARWIN_) + // Only used for SIGFPE. + // This doesn't seems to be reliable when the SIGFPE is generated + // from a divide-by-zero exception, which is now handled by + // `catch_exception_raise`. It works fine when a signal is recieved + // due to `kill`/`raise` though. + ucontext64_t *ctx = (ucontext64_t*)_ctx; + rsp -= sizeof(void*); + *(void**)rsp = NULL; + ctx->uc_mcontext64->__ss.__rsp = rsp; + ctx->uc_mcontext64->__ss.__rip = (uintptr_t)fptr; +#else + // TODO Add support for FreeBSD and PowerPC(64)? + fptr(); +#endif +} + +static void jl_throw_in_ctx(jl_ptls_t ptls, jl_value_t *e, void *sigctx) { - jl_ptls_t ptls = jl_get_ptls_states(); if (!ptls->safe_restore) ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, jl_to_bt_context(sigctx)); ptls->exception_in_transit = e; - // TODO throw the error by modifying sigctx for supported platforms - // This will avoid running the atexit handler on the signal stack - // if no excepiton handler is registered. - jl_rethrow(); + jl_call_in_ctx(ptls, &jl_rethrow, sigctx); } static pthread_t signals_thread; @@ -104,6 +171,19 @@ static void jl_unblock_signal(int sig) #include <signals-mach.c> #else +static int is_addr_on_sigstack(jl_ptls_t ptls, void *ptr) +{ + // One guard page for signal_stack. + return !((char*)ptr < (char*)ptls->signal_stack - jl_page_size || + (char*)ptr > (char*)ptls->signal_stack + sig_stack_size); +} + +static int jl_is_on_sigstack(jl_ptls_t ptls, void *ptr, void *context) +{ + return (is_addr_on_sigstack(ptls, ptr) && + is_addr_on_sigstack(ptls, (void*)jl_get_rsp_from_ctx(context))); +} + static void segv_handler(int sig, siginfo_t *info, void *context) { jl_ptls_t ptls = jl_get_ptls_states(); @@ -117,27 +197,36 @@ static void segv_handler(int sig, siginfo_t *info, void *context) if (ptls->tid != 0) return; #endif - if (jl_get_ptls_states()->defer_signal) { + if (ptls->defer_signal) { jl_safepoint_defer_sigint(); } else if (jl_safepoint_consume_sigint()) { jl_clear_force_sigint(); - jl_throw_in_ctx(jl_interrupt_exception, context); + jl_throw_in_ctx(ptls, jl_interrupt_exception, context); } return; } - if (ptls->safe_restore || is_addr_on_stack(jl_get_ptls_states(), info->si_addr)) { // stack overflow, or restarting jl_ + if (ptls->safe_restore || is_addr_on_stack(ptls, info->si_addr)) { // stack overflow, or restarting jl_ jl_unblock_signal(sig); - jl_throw_in_ctx(jl_stackovf_exception, context); + jl_throw_in_ctx(ptls, jl_stackovf_exception, context); + } + else if (jl_is_on_sigstack(ptls, info->si_addr, context)) { + // This mainly happens when one of the finalizers during final cleanup + // on the signal stack has a deep/infinite recursion. + // There isn't anything more we can do + // (we are already corrupting that stack running this function) + // so just call `_exit` to terminate immediately. + jl_safe_printf("ERROR: Signal stack overflow, exit\n"); + _exit(sig + 128); } else if (sig == SIGSEGV && info->si_code == SEGV_ACCERR) { // writing to read-only memory (e.g., mmap) jl_unblock_signal(sig); - jl_throw_in_ctx(jl_readonlymemory_exception, context); + jl_throw_in_ctx(ptls, jl_readonlymemory_exception, context); } else { #ifdef SEGV_EXCEPTION jl_unblock_signal(sig); - jl_throw_in_ctx(jl_segv_exception, context); + jl_throw_in_ctx(ptls, jl_segv_exception, context); #else sigdie_handler(sig, info, context); #endif @@ -199,11 +288,39 @@ static void jl_try_deliver_sigint(void) pthread_kill(ptls2->system_id, SIGUSR2); } +// Write only by signal handling thread, read only by main thread +// no sync necessary. +static int thread0_exit_state = 0; +static void jl_exit_thread0_cb(void) +{ + // This can get stuck if it happens at an unfortunate spot + // (unavoidable due to its async nature). + // Try harder to exit each time if we get multiple exit requests. + if (thread0_exit_count <= 1) { + jl_exit(thread0_exit_state); + } + else if (thread0_exit_count == 2) { + exit(thread0_exit_state); + } + else { + _exit(thread0_exit_state); + } +} + +static void jl_exit_thread0(int state) +{ + jl_ptls_t ptls2 = jl_all_tls_states[0]; + thread0_exit_state = state; + jl_atomic_store_release(&ptls2->signal_request, 3); + pthread_kill(ptls2->system_id, SIGUSR2); +} + // request: // 0: nothing // 1: get state -// 3: throw sigint if `!defer_signal && io_wait` or if force throw threshold +// 2: throw sigint if `!defer_signal && io_wait` or if force throw threshold // is reached +// 3: exit with `thread0_exit_state` void usr2_handler(int sig, siginfo_t *info, void *ctx) { jl_ptls_t ptls = jl_get_ptls_states(); @@ -229,9 +346,13 @@ void usr2_handler(int sig, siginfo_t *info, void *ctx) jl_safe_printf("WARNING: Force throwing a SIGINT\n"); // Force a throw jl_clear_force_sigint(); - jl_throw_in_ctx(jl_interrupt_exception, ctx); + jl_throw_in_ctx(ptls, jl_interrupt_exception, ctx); } } + else if (request == 3) { + jl_unblock_signal(sig); + jl_call_in_ctx(ptls, jl_exit_thread0_cb, ctx); + } } #if defined(HAVE_TIMER) @@ -407,6 +528,15 @@ static void *signal_listener(void *arg) critical |= (sig == SIGUSR1 && !profile); #endif + int doexit = critical; +#ifdef SIGINFO + if (sig == SIGINFO) + doexit = 0; +#else + if (sig == SIGUSR1) + doexit = 0; +#endif + bt_size = 0; // sample each thread, round-robin style in reverse order // (so that thread zero gets notified last) @@ -446,18 +576,13 @@ static void *signal_listener(void *arg) // and must be thread-safe, but not necessarily signal-handler safe if (critical) { jl_critical_error(sig, NULL, bt_data, &bt_size); - // FIXME - // It is unsafe to run the exit handler on this thread - // (this thread is not managed and has a rather limited stack space) - // try harder to run this on a managed thread. -#ifdef SIGINFO - if (sig != SIGINFO) -#else - if (sig != SIGUSR1) -#endif - jl_exit(128 + sig); + if (doexit) { + thread0_exit_count++; + jl_exit_thread0(128 + sig); + } } } + return NULL; } void restore_signals(void) @@ -482,8 +607,9 @@ void restore_signals(void) void fpe_handler(int sig, siginfo_t *info, void *context) { (void)info; + jl_ptls_t ptls = jl_get_ptls_states(); jl_unblock_signal(sig); - jl_throw_in_ctx(jl_diverror_exception, context); + jl_throw_in_ctx(ptls, jl_diverror_exception, context); } void jl_install_default_signal_handlers(void) diff --git a/src/threading.c b/src/threading.c index 6b1d373c7f1f6..664a28df8880a 100644 --- a/src/threading.c +++ b/src/threading.c @@ -777,14 +777,6 @@ void jl_init_threading(void) static jl_ptls_t _jl_all_tls_states; jl_all_tls_states = &_jl_all_tls_states; jl_n_threads = 1; - -#if defined(__linux__) && defined(JL_USE_INTEL_JITEVENTS) - if (jl_using_intel_jitevents) - // Intel VTune Amplifier needs at least 64k for alternate stack. - if (SIGSTKSZ < 1<<16) - sig_stack_size = 1<<16; -#endif - ti_init_master_thread(); } From 86e17ae4b4e5788342d0ee96c716c20748b1edf2 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 10 Aug 2016 02:48:52 -0400 Subject: [PATCH 0873/1117] Patch is picky about newlines at end of file --- deps/patches/libssh2-mbedtls.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/patches/libssh2-mbedtls.patch b/deps/patches/libssh2-mbedtls.patch index d7cdeef91a44b..63856f33a9e79 100644 --- a/deps/patches/libssh2-mbedtls.patch +++ b/deps/patches/libssh2-mbedtls.patch @@ -1145,4 +1145,4 @@ index 0000000..248583e + size_t *pubkeydata_len, + const char *privatekeydata, + size_t privatekeydata_len, -+ const char *passphrase); \ No newline at end of file ++ const char *passphrase); From 8bd6f0f4d8239518282e8e334791d4719b06f821 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer+github@college.harvard.edu> Date: Wed, 10 Aug 2016 07:17:12 +0000 Subject: [PATCH 0874/1117] Set an RPATH on all libraries requiring fortran in binary-dist (#17901) --- Makefile | 2 +- contrib/fixup-libgfortran.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 59d3b34fc9977..fd2d8edb109cc 100644 --- a/Makefile +++ b/Makefile @@ -453,7 +453,7 @@ endif @$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/Makefile install cp $(JULIAHOME)/LICENSE.md $(BUILDROOT)/julia-$(JULIA_COMMIT) ifneq ($(OS), WINNT) - -$(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir) + -$(CUSTOM_LD_LIBRARY_PATH) PATH=$(PATH):$(build_depsbindir) $(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir) endif ifeq ($(OS), Linux) -$(JULIAHOME)/contrib/fixup-libstdc++.sh $(DESTDIR)$(libdir) $(DESTDIR)$(private_libdir) diff --git a/contrib/fixup-libgfortran.sh b/contrib/fixup-libgfortran.sh index a0d52c7f9b5f3..a7ee7025b9a7f 100755 --- a/contrib/fixup-libgfortran.sh +++ b/contrib/fixup-libgfortran.sh @@ -93,3 +93,12 @@ if [ "$UNAME" = "Darwin" ]; then done done fi + +if [ "$UNAME" = "Linux" ]; then + cd $private_libdir + for file in openlibm quadmath gfortran openblas arpack lapack openspecfun; do + for dylib in $(ls lib$file*.so* 2>/dev/null); do + patchelf --set-rpath \$ORIGIN $dylib + done + done +fi From e2b83ae4ee24f72a82796f629a47778c0efed9d0 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Wed, 10 Aug 2016 03:18:17 -0400 Subject: [PATCH 0875/1117] Remove redundant uplo argument in chol family. When using Hermitian (#17909) and Symmetric, it is redundant to also have an uplo argument and occasionally, it also gave the wrong result. This can therefore be considered a bugfix. --- base/deprecated.jl | 3 ++ base/linalg/cholesky.jl | 97 ++++++++++++++++++++++------------------- doc/stdlib/linalg.rst | 12 ++--- test/linalg/cholesky.jl | 10 ++--- 4 files changed, 66 insertions(+), 56 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 50f2386804559..7dffd61acc9ba 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -775,6 +775,9 @@ function transpose(x) return x end +@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol, ::Type{Val{false}}) cholfact!(A, Val{false}) +@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol = :U) cholfact!(A) + # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index e961ef09f5a57..ce799f327b782 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -121,8 +121,10 @@ non_hermitian_error(f) = throw(ArgumentError("matrix is not symmetric/" * # chol!. Destructive methods for computing Cholesky factor of real symmetric or Hermitian # matrix -chol!(A::Hermitian) = _chol!(A.data, UpperTriangular) -chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = _chol!(A.data, UpperTriangular) +chol!(A::Hermitian) = + _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) +chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = + _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) function chol!(A::StridedMatrix) ishermitian(A) || non_hermitian_error("chol!") return _chol!(A, UpperTriangular) @@ -135,14 +137,22 @@ end function chol(A::Hermitian) T = promote_type(typeof(chol(one(eltype(A)))), Float32) AA = similar(A, T, size(A)) - copy!(AA, A.data) - chol!(Hermitian(AA)) + if A.uplo == 'U' + copy!(AA, A.data) + else + Base.ccopy!(AA, A.data) + end + chol!(Hermitian(AA, :U)) end function chol{T<:Real,S<:AbstractMatrix}(A::Symmetric{T,S}) TT = promote_type(typeof(chol(one(T))), Float32) AA = similar(A, TT, size(A)) - copy!(AA, A.data) - chol!(Hermitian(AA)) + if A.uplo == 'U' + copy!(AA, A.data) + else + Base.ccopy!(AA, A.data) + end + chol!(Hermitian(AA, :U)) end ## for StridedMatrices, check that matrix is symmetric/Hermitian @@ -170,15 +180,15 @@ chol(x::Number, args...) = _chol!(x, nothing) # cholfact!. Destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -function cholfact!(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) - if uplo == :U +function cholfact!(A::Hermitian, ::Type{Val{false}}) + if A.uplo == :U Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') end end -function cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) - if uplo == :U +function cholfact!{T<:Real,S}(A::Symmetric{T,S}, ::Type{Val{false}}) + if A.uplo == :U Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') @@ -187,7 +197,7 @@ end ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky + cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the factorisation @@ -196,37 +206,36 @@ integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo, Val{false}) + return cholfact!(Hermitian(A, uplo), Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact!(A::Hermitian, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) -cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) +cholfact!(A::Hermitian) = cholfact!(A, Val{false}) +cholfact!{T<:Real,S}(A::Symmetric{T,S}) = cholfact!(A, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact!(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo) + return cholfact!(Hermitian(A, uplo)) end ## With pivoting ### BLAS/LAPACK element types function cholfact!{T<:BlasReal,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, - uplo::Symbol, ::Type{Val{true}}; tol = 0.0) - uplochar = char_uplo(uplo) - AA, piv, rank, info = LAPACK.pstrf!(uplochar, A.data, tol) - return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, uplochar, piv, rank, tol, info) + ::Type{Val{true}}; tol = 0.0) + AA, piv, rank, info = LAPACK.pstrf!(A.uplo, A.data, tol) + return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, A.uplo, piv, rank, tol, info) end ### Non BLAS/LAPACK element types (generic). Since generic fallback for pivoted Cholesky ### is not implemented yet we throw an error -cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, ::Type{Val{true}}; +cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = throw(ArgumentError("generic pivoted Cholesky factorization is not implemented yet")) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the @@ -235,26 +244,25 @@ e.g. for integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo, Val{true}; tol = tol) + return cholfact!(Hermitian(A, uplo), Val{true}; tol = tol) end - - # cholfact. Non-destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) +cholfact(A::Hermitian, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, uplo::Symbol, Val{false}) -> Cholesky + cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky Compute the Cholesky factorization of a dense symmetric positive definite matrix `A` -and return a `Cholesky` factorization. -The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `Cholesky` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` +`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, +the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `Cholesky` objects: `size`, `\\`, `inv`, `det`. @@ -262,36 +270,35 @@ A `PosDefException` exception is thrown in case the matrix is not positive defin """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo, Val{false}) + return cholfact(Hermitian(A, uplo), Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact(A::Hermitian, uplo::Symbol = :U) = cholfact(A, uplo, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol = :U) = - cholfact(A, uplo, Val{false}) +cholfact(A::Hermitian) = cholfact(A, Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = cholfact(A, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo) + return cholfact(Hermitian(A, uplo)) end ## With pivoting -cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) = +cholfact(A::Hermitian, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - uplo, Val{true}; tol = tol) -cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, - ::Type{Val{true}}; tol = 0.0) = + Val{true}; tol = tol) +cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - uplo, Val{true}; tol = tol) + Val{true}; tol = tol) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix `A` -and return a `CholeskyPivoted` factorization. -The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `CholeskyPivoted` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` +`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, +the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `PivotedCholesky` objects: `size`, `\\`, `inv`, `det`, and `rank`. @@ -300,7 +307,7 @@ For negative values, the tolerance is the machine precision. """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo, Val{true}; tol = tol) + return cholfact(Hermitian(A, uplo), Val{true}; tol = tol) end ## Number diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 2bd41341c74fb..349869b5a5214 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -308,17 +308,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the square root of a non-negative number ``x``\ . -.. function:: cholfact(A, uplo::Symbol, Val{false}) -> Cholesky +.. function:: cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky .. Docstring generated from Julia source - Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. + Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. -.. function:: cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source - Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. + Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. .. function:: cholfact(A; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor @@ -344,13 +344,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. -.. function:: cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky +.. function:: cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky .. Docstring generated from Julia source The same as ``cholfact``\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. An ``InexactError`` exception is thrown if the factorisation produces a number not representable by the element type of ``A``\ , e.g. for integer types. -.. function:: cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index 00e4696b4f12b..bb0657c799838 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -74,15 +74,15 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test full(lapd) ≈ apd l = lapd[:L] @test l*l' ≈ apd - @test triu(capd.factors) ≈ lapd[:U] - @test tril(lapd.factors) ≈ capd[:L] + @test capd[:U] ≈ lapd[:U] + @test lapd[:L] ≈ capd[:L] if eltya <: Real capds = cholfact(apds) - lapds = cholfact(apds, :L) + lapds = cholfact(full(apds), :L) ls = lapds[:L] @test ls*ls' ≈ apd - @test triu(capds.factors) ≈ lapds[:U] - @test tril(lapds.factors) ≈ capds[:L] + @test capds[:U] ≈ lapds[:U] + @test lapds[:L] ≈ capds[:L] end #pivoted upper Cholesky From 4ba2497de6b91d97d4bfaaaffcbb5f7b9770c4f9 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Wed, 10 Aug 2016 03:20:14 -0400 Subject: [PATCH 0876/1117] Define conj and conj! for for Symmetric and Hermitian. (#17827) Fixes #17780 --- base/linalg/symmetric.jl | 3 +++ test/linalg/symmetric.jl | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index 40898a9aa2789..dead41a2b93c3 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -114,6 +114,9 @@ end ctranspose(A::Hermitian) = A trace(A::Hermitian) = real(trace(A.data)) +Base.conj(A::HermOrSym) = typeof(A)(conj(A.data), A.uplo) +Base.conj!(A::HermOrSym) = typeof(A)(conj!(A.data), A.uplo) + #tril/triu function tril(A::Hermitian, k::Integer=0) if A.uplo == 'U' && k <= 0 diff --git a/test/linalg/symmetric.jl b/test/linalg/symmetric.jl index 3a33105b5372f..55f815f500fcb 100644 --- a/test/linalg/symmetric.jl +++ b/test/linalg/symmetric.jl @@ -236,3 +236,17 @@ let A = Symmetric(randn(5,5)) B = -A @test A + B ≈ zeros(5,5) end + +# 17780 +let a = randn(2,2) + a = a'a + b = complex(a,a) + c = Symmetric(b) + @test conj(c) == conj(Array(c)) + cc = copy(c) + @test conj!(c) == conj(Array(cc)) + c = Hermitian(b + b') + @test conj(c) == conj(Array(c)) + cc = copy(c) + @test conj!(c) == conj(Array(c)) +end From a554434ca5dedd92a84d5776d4b195c58a3047b9 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman <benditlikeranjan@gmail.com> Date: Wed, 10 Aug 2016 15:22:05 +0530 Subject: [PATCH 0877/1117] Add cross-release compatibility note to CONTRIBUTING.md (#17911) * Add cross-release compatibility note to CONTRIBUTING.md * tag -> specify --- CONTRIBUTING.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index adf70e7efbfee..dbd0fe9179efa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,6 +60,22 @@ Julia has a built-in [package manager](https://github.com/JuliaLang/METADATA.jl) For developers who need to wrap C libraries so that they can be called from Julia, the [Clang.jl](https://github.com/ihnorton/Clang.jl) package can help generate the wrappers automatically from the C header files. +### Package Compatibility Across Releases + +Sometimes, you might find that while your package works +on the current release, it might not work on the upcoming release or nightly. +This is due to the fact that some Julia functions (after some discussion) +could be deprecated or removed altogether. This may cause your package to break or +throw a number of deprecation warnings on usage. Therefore it is highly recommended +to port your package to latest Julia release. + +However, porting a package to the latest release may cause the package to break on +earlier Julia releases. To maintain compatibility across releases, use +[`Compat.jl`](https://github.com/JuliaLang/Compat.jl/). Find the fix for your package +from the README, and specify the minimum version of Compat that provides the fix +in your REQUIRE file. To tag the correct minimum version, refer to +[this guide](https://github.com/JuliaLang/Compat.jl/#tagging-the-correct-minimum-version-of-compat). + ### Writing tests There are never enough tests. Track [code coverage at Coveralls](https://coveralls.io/r/JuliaLang/julia), and help improve it. From f950371afa6c6db7ab987f0ec125d0190e1b336c Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 10 Aug 2016 02:57:07 -0700 Subject: [PATCH 0878/1117] reword second tag -> find in CONTRIBUTING.md ref #17911 [ci skip] --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbd0fe9179efa..a2040fbf5ef75 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ However, porting a package to the latest release may cause the package to break earlier Julia releases. To maintain compatibility across releases, use [`Compat.jl`](https://github.com/JuliaLang/Compat.jl/). Find the fix for your package from the README, and specify the minimum version of Compat that provides the fix -in your REQUIRE file. To tag the correct minimum version, refer to +in your REQUIRE file. To find the correct minimum version, refer to [this guide](https://github.com/JuliaLang/Compat.jl/#tagging-the-correct-minimum-version-of-compat). ### Writing tests From 8d64e51c212168246ffd5d642d3a665cbd87b588 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Wed, 10 Aug 2016 13:10:12 +0200 Subject: [PATCH 0879/1117] improve heuristic for when to refresh line in the REPL (#17868) * improve heuristic for when to refresh line * fix trailing whitespace... * use indent when not on first line * remove unused argument and change local var to not share name with function --- base/LineEdit.jl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index a0a45e63cc73b..9298c4dee2bc2 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -447,10 +447,19 @@ function edit_replace(s, from, to, str) end function edit_insert(s::PromptState, c) + buf = s.input_buffer + function line_size() + p = position(buf) + seek(buf, rsearch(buf.data, '\n', p)) + ls = p - position(buf) + seek(buf, p) + return ls + end str = string(c) - edit_insert(s.input_buffer, str) - if !('\n' in str) && eof(s.input_buffer) && - ((position(s.input_buffer) + sizeof(s.p.prompt) + sizeof(str) - 1) < width(terminal(s))) + edit_insert(buf, str) + offset = s.ias.curs_row == 1 ? sizeof(s.p.prompt) : s.indent + if !('\n' in str) && eof(buf) && + ((line_size() + offset + sizeof(str) - 1) < width(terminal(s))) # Avoid full update when appending characters to the end # and an update of curs_row isn't necessary (conservatively estimated) write(terminal(s), str) From d93e7bb1213f7c54032812ab1ea945408f03624a Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Wed, 10 Aug 2016 14:54:52 +0200 Subject: [PATCH 0880/1117] Fix trailing whitespace in code literal rst apparently requires that the closing double backticks for code literals appear at the end of a "word", rather than preceded by a space as markdown allows. This means that ``` ``julia> `` ``` doesn't parse correctly. The fix is to just remove the trailing space and explicitly mention the need for a space after the `>`. --- NEWS.md | 4 ++-- doc/manual/interacting-with-julia.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index e7b5931cb99d1..773f693c1d33d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,8 +5,8 @@ New language features --------------------- * The REPL now supports something called *prompt pasting*. - This activates when pasting text that starts with `julia> ` into the REPL. - In that case, only expressions starting with `julia> ` are parsed, the rest are removed. + This activates when pasting text that starts with `julia> ` into the REPL. + In that case, only expressions starting with `julia> ` are parsed, the rest are removed. This makes it possible to paste a chunk of code that has been copied from a REPL session without having to scrub away prompts and outputs. This can be disabled or enabled at will with `Base.REPL.enable_promptpaste(::Bool)`. diff --git a/doc/manual/interacting-with-julia.rst b/doc/manual/interacting-with-julia.rst index 5a2f06981fa78..996e84a0701cd 100644 --- a/doc/manual/interacting-with-julia.rst +++ b/doc/manual/interacting-with-julia.rst @@ -42,8 +42,8 @@ There are a number useful features unique to interactive work. In addition to sh julia> ans "12" -In Julia mode, the REPL supports something called *prompt pasting*. This activates when pasting text that starts with ``julia> `` into the REPL. -In that case, only expressions starting with ``julia> `` are parsed, others are removed. +In Julia mode, the REPL supports something called *prompt pasting*. This activates when pasting text that starts with ``julia> `` into the REPL. +In that case, only expressions starting with ``julia> `` are parsed, others are removed. This makes it is possible to paste a chunk of code that has been copied from a REPL session without having to scrub away prompts and outputs. This feature is enabled by default but can be disabled or enabled at will with ``Base.REPL.enable_promptpaste(::Bool)``. If it is enabled, you can try it out by pasting the code block above this paragraph straight into the REPL. From 9ab14ea0f49c6cf90be27788730c7cddc590c945 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 10 Aug 2016 15:40:48 -0400 Subject: [PATCH 0881/1117] fix missing line break in precompile error message --- src/dump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dump.c b/src/dump.c index d46fb1b624d7a..cfd8cd7ba48ea 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1752,7 +1752,7 @@ static int read_verify_mod_list(ios_t *s) if (m->uuid != uuid) { jl_printf(JL_STDERR, "WARNING: Module %s uuid did not match cache file\n" - " This is likely because module %s does not support" + " This is likely because module %s does not support\n" " precompilation but is imported by a module that does.\n", name, name); return 0; From 6b0e8b56d2261912832a4700fd938f3b39e64d8b Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 10 Aug 2016 15:45:54 -0400 Subject: [PATCH 0882/1117] remove some unnecessary anonymous functions Transform `x->f(x)` to `f`. --- base/abstractarray.jl | 2 +- base/linalg/arpack.jl | 6 +++--- base/multidimensional.jl | 2 +- base/pkg/types.jl | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 2a292343bc9d0..a4b6cbe6bda0b 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -59,7 +59,7 @@ Returns the tuple of valid indices for array `A`. """ function indices(A) @_inline_meta - map(s->OneTo(s), size(A)) + map(OneTo, size(A)) end # Performance optimization: get rid of a branch on `d` in `indices(A, diff --git a/base/linalg/arpack.jl b/base/linalg/arpack.jl index 9b27aa3bc019b..26a0c55e1dbb6 100644 --- a/base/linalg/arpack.jl +++ b/base/linalg/arpack.jl @@ -114,15 +114,15 @@ function eupd_wrapper(T, n::Integer, sym::Bool, cmplx::Bool, bmat::String, select = Array{BlasInt}(ncv) info = zeros(BlasInt, 1) - dmap = x->abs(x) + dmap = abs if iparam[7] == 3 # shift-and-invert dmap = x->abs(1./(x-sigma)) elseif which == "LR" || which == "LA" || which == "BE" - dmap = x->real(x) + dmap = real elseif which == "SR" || which == "SA" dmap = x->-real(x) elseif which == "LI" - dmap = x->imag(x) + dmap = imag elseif which == "SI" dmap = x->-imag(x) end diff --git a/base/multidimensional.jl b/base/multidimensional.jl index a6e22e481601b..0185edeae8182 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -79,7 +79,7 @@ end CartesianRange{N}(index::CartesianIndex{N}) = CartesianRange(one(index), index) CartesianRange(::Tuple{}) = CartesianRange{CartesianIndex{0}}(CartesianIndex{0}(()),CartesianIndex{0}(())) CartesianRange{N}(sz::NTuple{N,Int}) = CartesianRange(CartesianIndex(sz)) -CartesianRange{N}(rngs::NTuple{N,Union{Integer,AbstractUnitRange}}) = CartesianRange(CartesianIndex(map(r->first(r), rngs)), CartesianIndex(map(r->last(r), rngs))) +CartesianRange{N}(rngs::NTuple{N,Union{Integer,AbstractUnitRange}}) = CartesianRange(CartesianIndex(map(first, rngs)), CartesianIndex(map(last, rngs))) ndims(R::CartesianRange) = length(R.start) ndims{I<:CartesianIndex}(::Type{CartesianRange{I}}) = length(I) diff --git a/base/pkg/types.jl b/base/pkg/types.jl index 152b3e5af0c24..9d5556a370b66 100644 --- a/base/pkg/types.jl +++ b/base/pkg/types.jl @@ -37,7 +37,7 @@ end VersionSet(versions::VersionNumber...) = VersionSet(VersionNumber[versions...]) show(io::IO, s::VersionSet) = join(io, s.intervals, " ∪ ") -isempty(s::VersionSet) = all(i->isempty(i), s.intervals) +isempty(s::VersionSet) = all(isempty, s.intervals) in(v::VersionNumber, s::VersionSet) = any(i->in(v,i), s.intervals) function intersect(A::VersionSet, B::VersionSet) ivals = vec([ intersect(a,b) for a in A.intervals, b in B.intervals ]) From c7306bc6cc8b58f58625b7831b5c258d962267a1 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 5 Jul 2016 01:38:14 -0400 Subject: [PATCH 0883/1117] sweep unused types from caches in sysimg write --- src/dump.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/dump.c b/src/dump.c index cfd8cd7ba48ea..58905f0f3136f 100644 --- a/src/dump.c +++ b/src/dump.c @@ -472,7 +472,7 @@ static int module_in_worklist(jl_module_t *mod) return 0; } -static int jl_prune_tcache(jl_typemap_entry_t *ml, void *closure) +static int jl_prune_specializations(jl_typemap_entry_t *ml, void *closure) { jl_value_t *ret = ml->func.value; if (jl_is_lambda_info(ret) && @@ -858,7 +858,7 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) // go through the t-func cache, replacing ASTs with just return // types for abstract argument types. these ASTs are generally // not needed (e.g. they don't get inlined). - jl_typemap_visitor(*tf, jl_prune_tcache, NULL); + jl_typemap_visitor(*tf, jl_prune_specializations, NULL); } jl_serialize_value(s, tf->unknown); jl_serialize_value(s, (jl_value_t*)m->name); @@ -1900,6 +1900,20 @@ static void jl_init_restored_modules(jl_array_t *init_order) // --- entry points --- +// remove cached types not referenced in the stream +static void jl_prune_type_cache(jl_svec_t *cache) +{ + size_t l = jl_svec_len(cache), ins = 0, i; + for(i=0; i < l; i++) { + jl_value_t *ti = jl_svecref(cache, i); + if (ti == NULL) break; + if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND || jl_get_llvm_gv(ti) != 0) + jl_svecset(cache, ins++, ti); + } + if (i > ins) + memset(&jl_svec_data(cache)[ins], 0, (i-ins)*sizeof(jl_value_t*)); +} + static void jl_save_system_image_to_stream(ios_t *f) { jl_gc_collect(1); // full @@ -1932,6 +1946,10 @@ static void jl_save_system_image_to_stream(ios_t *f) jl_serialize_value(&s, jl_typeinf_func); jl_serialize_value(&s, jl_type_type->name->mt); + jl_prune_type_cache(jl_tuple_typename->cache); + jl_prune_type_cache(jl_tuple_typename->linearcache); + jl_prune_type_cache(jl_type_type->name->cache); + intptr_t i; for (i = 0; i < builtin_types.len; i++) { jl_serialize_value(&s, ((jl_datatype_t*)builtin_types.items[i])->name->cache); From 0b2172329246d5cff68328ebab820209da82255b Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 5 Jul 2016 18:36:57 -0400 Subject: [PATCH 0884/1117] avoid making copies of `Tuple{Vararg{Any}}` --- src/jltypes.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/jltypes.c b/src/jltypes.c index 74f54e98eb768..6dd59eecdc88d 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2175,6 +2175,10 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i if (istuple && ntp > 0 && jl_is_vararg_type(iparams[ntp - 1])) { // normalize Tuple{..., Vararg{Int, 3}} to Tuple{..., Int, Int, Int} jl_value_t *va = iparams[ntp - 1]; + // return same `Tuple` object for types equal to it + if (ntp == 1 && jl_tparam0(va) == (jl_value_t*)jl_any_type && + jl_tparam1(va) == jl_tparam1(jl_tparam0(jl_anytuple_type))) + return (jl_value_t*)jl_anytuple_type; if (jl_is_long(jl_tparam1(va))) { ssize_t nt = jl_unbox_long(jl_tparam1(va)); if (nt < 0) From a474f7171555045c9ec2ef32b8186b2159b232cd Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Wed, 10 Aug 2016 17:39:13 -0400 Subject: [PATCH 0885/1117] fix BigInt `ndigits` test See #16766. `ndigits(big(0),1)` doesn't raise an error, but we were sometimes (randomly, rarely) testing that it does. This change makes the test reliable. The issue of how this case should actually behave is still open. --- test/bigint.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/bigint.jl b/test/bigint.jl index 999b0c42cf6dd..0478b28f8b286 100644 --- a/test/bigint.jl +++ b/test/bigint.jl @@ -278,7 +278,11 @@ ndigits_mismatch(n) = ndigits(n) != ndigits(BigInt(n)) ndigits(rand(big(-999:999)), rand(63:typemax(Int))) ndigits(rand(big(-999:999)), big(2)^rand(2:999)) -@test_throws DomainError ndigits(rand(big(-999:999)), rand(typemin(Int):1)) +for i in big([-20:-1;1:20]) + for b in -10:1 + @test_throws DomainError ndigits(i, b) + end +end # conversion from float @test BigInt(2.0) == BigInt(2.0f0) == BigInt(big(2.0)) == 2 From ad0f8a7f2662a0bc37b60a345a4d9a0bb810f062 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Tue, 2 Aug 2016 15:48:29 +0800 Subject: [PATCH 0886/1117] Optimize exception frames allocation * Make llvm-gcroot a ModulePass * Use llvm-gcroot to calculate the minimum number of exception frames necessary. --- src/codegen.cpp | 35 ++---- src/llvm-gcroot.cpp | 289 +++++++++++++++++++++++++++++++++++++++----- src/llvm-ptls.cpp | 13 -- 3 files changed, 270 insertions(+), 67 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 55b8aafba0118..259a66fbacdee 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -381,6 +381,7 @@ static Function *gcroot_func; static Function *gckill_func; static Function *jlcall_frame_func; static Function *gcroot_flush_func; +static Function *except_enter_func; static std::vector<Type *> two_pvalue_llvmt; static std::vector<Type *> three_pvalue_llvmt; @@ -505,7 +506,6 @@ typedef struct { std::vector<bool> ssavalue_assigned; std::map<int, jl_arrayvar_t> *arrayvars; std::map<int, BasicBlock*> *labels; - std::map<int, Value*> *handlers; jl_module_t *module; jl_lambda_info_t *linfo; const char *name; @@ -3259,18 +3259,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) else if (head == enter_sym) { assert(jl_is_long(args[0])); int labl = jl_unbox_long(args[0]); - Value *jbuf = builder.CreateGEP((*ctx->handlers)[labl], - ConstantInt::get(T_size,0)); - builder.CreateCall(prepare_call(jlenter_func), jbuf); -#ifndef _OS_WINDOWS_ -#ifdef LLVM37 - CallInst *sj = builder.CreateCall(prepare_call(setjmp_func), { jbuf, ConstantInt::get(T_int32,0) }); -#else - CallInst *sj = builder.CreateCall2(prepare_call(setjmp_func), jbuf, ConstantInt::get(T_int32,0)); -#endif -#else - CallInst *sj = builder.CreateCall(prepare_call(setjmp_func), jbuf); -#endif + CallInst *sj = builder.CreateCall(prepare_call(except_enter_func)); // We need to mark this on the call site as well. See issue #6757 sj->setCanReturnTwice(); Value *isz = builder.CreateICmpEQ(sj, ConstantInt::get(T_int32,0)); @@ -3892,11 +3881,9 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func //jl_printf(JL_STDOUT, "\n"); std::map<int, jl_arrayvar_t> arrayvars; std::map<int, BasicBlock*> labels; - std::map<int, Value*> handlers; jl_codectx_t ctx = {}; ctx.arrayvars = &arrayvars; ctx.labels = &labels; - ctx.handlers = &handlers; ctx.module = lam->def ? lam->def->module : ptls->current_module; ctx.linfo = lam; ctx.name = jl_symbol_name(lam->def ? lam->def->name : anonymous_sym); @@ -4329,18 +4316,6 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func allocate_gc_frame(b0, &ctx); // step 8. allocate space for exception handler contexts - for(i=0; i < stmtslen; i++) { - jl_value_t *stmt = jl_array_ptr_ref(stmts,i); - if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == enter_sym) { - int labl = jl_unbox_long(jl_exprarg(stmt,0)); - AllocaInst *handlr = - builder.CreateAlloca(T_int8, - ConstantInt::get(T_int32, - sizeof(jl_handler_t))); - handlr->setAlignment(16); - handlers[labl] = handlr; - } - } // step 9. allocate local variables slots // must be in the first basic block for the llvm mem2reg pass to work @@ -5533,6 +5508,12 @@ static void init_julia_llvm_env(Module *m) "julia.gcroot_flush", m); add_named_global(gcroot_flush_func, (void*)NULL, /*dllimport*/false); + except_enter_func = Function::Create(FunctionType::get(T_int32, false), + Function::ExternalLinkage, + "julia.except_enter", m); + except_enter_func->addFnAttr(Attribute::ReturnsTwice); + add_named_global(except_enter_func, (void*)NULL, /*dllimport*/false); + // set up optimization passes #ifdef LLVM37 // No DataLayout pass needed anymore. diff --git a/src/llvm-gcroot.cpp b/src/llvm-gcroot.cpp index 5c9da2b175eb2..b722199548f6f 100644 --- a/src/llvm-gcroot.cpp +++ b/src/llvm-gcroot.cpp @@ -76,9 +76,8 @@ static void tbaa_decorate_gcframe(Instruction *inst, std::set<Instruction*> &visited, MDNode *tbaa_gcframe) { - if (visited.find(inst) != visited.end()) + if (visited.insert(inst).second) return; - visited.insert(inst); #ifdef LLVM35 Value::user_iterator I = inst->user_begin(), E = inst->user_end(); #else @@ -116,20 +115,23 @@ static void tbaa_decorate_gcframe(Instruction *inst, MDNode *tbaa_gcframe) class JuliaGCAllocator { public: - JuliaGCAllocator(CallInst *ptlsStates, Type *T_pjlvalue, MDNode *tbaa) : - F(*ptlsStates->getParent()->getParent()), + JuliaGCAllocator(Function &F, CallInst *ptlsStates, + Type *T_pjlvalue, MDNode *tbaa) : + F(F), M(*F.getParent()), T_int1(Type::getInt1Ty(F.getContext())), T_int8(Type::getInt8Ty(F.getContext())), T_int32(Type::getInt32Ty(F.getContext())), T_int64(Type::getInt64Ty(F.getContext())), - V_null(Constant::getNullValue(T_pjlvalue)), + V_null(T_pjlvalue ? Constant::getNullValue(T_pjlvalue) : nullptr), ptlsStates(ptlsStates), - gcframe(new AllocaInst(T_pjlvalue, ConstantInt::get(T_int32, 0))), + gcframe(ptlsStates ? new AllocaInst(T_pjlvalue, ConstantInt::get(T_int32, 0)) : nullptr), gcroot_func(M.getFunction("julia.gc_root_decl")), gckill_func(M.getFunction("julia.gc_root_kill")), jlcall_frame_func(M.getFunction("julia.jlcall_frame_decl")), gcroot_flush_func(M.getFunction("julia.gcroot_flush")), + except_enter_func(M.getFunction("julia.except_enter")), + jlleave_func(M.getFunction("jl_pop_handler")), tbaa_gcframe(tbaa) { /* Algorithm sketch: @@ -138,10 +140,12 @@ class JuliaGCAllocator { * Propagate liveness from each basic block to its predecessors * Allocate argument slot for each jlcall frame */ + if (gcframe) { #ifdef JL_DEBUG_BUILD - gcframe->setName("gcrootframe"); + gcframe->setName("gcrootframe"); #endif - gcframe->insertAfter(ptlsStates); + gcframe->insertAfter(ptlsStates); + } } private: @@ -158,6 +162,8 @@ class JuliaGCAllocator { Function *const gckill_func; Function *const jlcall_frame_func; Function *const gcroot_flush_func; + Function *const except_enter_func; + Function *const jlleave_func; MDNode *const tbaa_gcframe; Instruction *get_pgcstack(Instruction *ptlsStates); @@ -171,10 +177,189 @@ class JuliaGCAllocator { std::map<BasicBlock*, std::map<frame_register, liveness::id> > &bb_uses, std::map<BasicBlock*, SmallBitVector> ®s_used); void rearrangeRoots(); + void lowerHandlers(); public: void allocate_frame(); }; +struct HandlerData { + // Pairs of <jl_pop_handle call inst>, number of pops left after popping + // this frame. + std::vector<std::pair<CallInst*,uint64_t>> leaves; + // enters that are directly nested in this frame + std::set<CallInst*> nested; + std::unique_ptr<std::vector<CallInst*>> parent_vec; + CallInst *parent{nullptr}; + bool processed{false}; +}; + +void JuliaGCAllocator::lowerHandlers() +{ + if (!except_enter_func) + return; + auto jlenter_func = M.getNamedValue("jl_enter_handler"); + auto setjmp_func = M.getNamedValue(jl_setjmp_name); + // Collect all exception enters + std::map<CallInst*,HandlerData> handlers; + for (auto &BB: F) { + for (auto &I: BB) { + auto call = dyn_cast<CallInst>(&I); + if (call && call->getCalledValue() == except_enter_func) { + handlers[call] = HandlerData(); + } + } + } + + if (handlers.empty()) + return; + + typedef std::map<CallInst*,HandlerData>::iterator hdlr_iter_t; + // For each exception enter, compute the life time of the enter, find + // the corresponding leaves and collect a list of nested exception frames. + // This assumes the exception frames has simple structure. E.g. + // there's no recursion and different frames do no share the same leave. + std::function<void(hdlr_iter_t)> process_handler = [&] (hdlr_iter_t hdlr) { + auto enter = hdlr->first; + auto &data = hdlr->second; + if (data.processed) + return; + data.processed = true; + std::vector<BasicBlock::iterator> frontier = { + ++BasicBlock::iterator(enter) + }; + // Start from the enter, the enter is live until it hits a leave + // or the control flow terminates. + std::set<Instruction*> visited; + auto add_inst = [&] (BasicBlock::iterator it) { + if (visited.insert(&*it).second) { + frontier.push_back(it); + } + }; + while (!frontier.empty()) { + BasicBlock::iterator IT = frontier.back(); + Instruction *I = &*IT; + ++IT; + frontier.pop_back(); + if (I->isTerminator()) { + BasicBlock *bb = I->getParent(); + for (auto S = succ_begin(bb), E = succ_end(bb); S != E; S++) + add_inst(S->getFirstInsertionPt()); + continue; + } + auto call = dyn_cast<CallInst>(I); + if (!call) { + add_inst(IT); + continue; + } + auto callee = call->getCalledValue(); + if (callee == except_enter_func) { + // Nested frame, make sure the frame is processed and skip to + // its leaves. + auto other_it = handlers.find(call); + assert(other_it != handlers.end()); + process_handler(other_it); + auto &other_data = other_it->second; + data.nested.insert(call); + // Record a reference to parent frame, there should be only + // one parent in most cases although more than one parent is + // also possible if LLVM split the enter call. + if (__unlikely(other_data.parent_vec)) { + other_data.parent_vec->push_back(enter); + } + else if (__unlikely(other_data.parent)) { + other_data.parent_vec.reset( + new std::vector<CallInst*>{other_data.parent, enter}); + } + else { + other_data.parent = enter; + } + for (auto leave: other_it->second.leaves) { + if (leave.second > 0) { + data.leaves.push_back(std::make_pair(leave.first, + leave.second - 1)); + } + else { + add_inst(++BasicBlock::iterator(leave.first)); + } + } + } + else if (callee == jlleave_func) { + // Leave, record how many pops are left on this leave. + assert(call->getNumArgOperands() == 1); + auto arg = cast<ConstantInt>(call->getArgOperand(0)); + uint64_t pop_count = arg->getLimitedValue(); + assert(pop_count >= 1); + data.leaves.push_back(std::make_pair(call, pop_count - 1)); + } + else { + // Otherwise, go to the next instruction. + add_inst(IT); + } + } + }; + + // Make sure all frames are processed. + for (auto IT = handlers.begin(), E = handlers.end(); IT != E; ++IT) + process_handler(IT); + + std::set<CallInst*> processing; + Value *handler_sz = ConstantInt::get(T_int32, sizeof(jl_handler_t)); + // Now allocate the stack slots. + // At each iteration, we allocate a new handler and assign all the remaining + // frames that doesn't have a non-processed child to this handler. + Instruction *firstInst = &F.getEntryBlock().front(); + while (!handlers.empty()) { + processing.clear(); + auto buff = new AllocaInst(T_int8, handler_sz, "", firstInst); + buff->setAlignment(16); + // Collect the list of frames to process. + for (auto &hdlr: handlers) { + auto enter = hdlr.first; + auto &nested = hdlr.second.nested; + if (nested.empty()) { + processing.insert(enter); + } + } + // There shouldn't be loops so there has to be leafs + assert(!processing.empty()); + for (auto enter: processing) { + // Lower the enter + auto new_enter = CallInst::Create(jlenter_func, buff, "", enter); +#ifndef _OS_WINDOWS_ + // For LLVM 3.3 compatibility + Value *args[] = {buff, ConstantInt::get(T_int32,0)}; + auto sj = CallInst::Create(setjmp_func, args, "", enter); +#else + auto sj = CallInst::Create(setjmp_func, buff, "", enter); +#endif + // We need to mark this on the call site as well. See issue #6757 + sj->setCanReturnTwice(); +#ifdef LLVM36 + if (auto dbg = enter->getMetadata(LLVMContext::MD_dbg)) { + new_enter->setMetadata(LLVMContext::MD_dbg, dbg); + sj->setMetadata(LLVMContext::MD_dbg, dbg); + } +#else + (void)new_enter; +#endif + // Remove it from all the parents. + auto it = handlers.find(enter); + if (__unlikely(it->second.parent_vec)) { + for (auto parent: *it->second.parent_vec) { + handlers.find(parent)->second.nested.erase(enter); + } + } + else if (it->second.parent) { + handlers.find(it->second.parent)->second.nested.erase(enter); + } + // Delete it. + handlers.erase(it); + enter->replaceAllUsesWith(sj); + enter->eraseFromParent(); + } + } +} + Instruction *JuliaGCAllocator::get_pgcstack(Instruction *ptlsStates) { Constant *offset = ConstantInt::getSigned(T_int32, offsetof(jl_tls_states_t, pgcstack) / sizeof(void*)); @@ -528,6 +713,9 @@ void JuliaGCAllocator::rearrangeRoots() void JuliaGCAllocator::allocate_frame() { + lowerHandlers(); + if (!ptlsStates) + return; Instruction *last_gcframe_inst = gcframe; collapseRedundantRoots(); rearrangeRoots(); @@ -916,29 +1104,84 @@ void JuliaGCAllocator::allocate_frame() #endif } -struct LowerGCFrame: public FunctionPass { +struct LowerGCFrame: public ModulePass { static char ID; LowerGCFrame(MDNode *_tbaa_gcframe=nullptr) - : FunctionPass(ID), + : ModulePass(ID), tbaa_gcframe(_tbaa_gcframe) {} private: MDNode *tbaa_gcframe; // One `LLVMContext` only - bool runOnFunction(Function &F) override; + void runOnFunction(Module *M, Function &F, Function *ptls_getter, + Type *T_pjlvalue); + bool runOnModule(Module &M) override; }; -bool LowerGCFrame::runOnFunction(Function &F) +static void eraseFunction(Module &M, const char *name) { - Module *M = F.getParent(); + if (Function *f = M.getFunction(name)) { + f->eraseFromParent(); + } +} - Function *ptls_getter = M->getFunction("jl_get_ptls_states"); - if (!ptls_getter) - return true; +static void ensure_enter_function(Module &M) +{ + auto T_int8 = Type::getInt8Ty(M.getContext()); + auto T_pint8 = PointerType::get(T_int8, 0); + auto T_void = Type::getVoidTy(M.getContext()); + auto T_int32 = Type::getInt32Ty(M.getContext()); + if (!M.getNamedValue("jl_enter_handler")) { + std::vector<Type*> ehargs(0); + ehargs.push_back(T_pint8); + Function::Create(FunctionType::get(T_void, ehargs, false), + Function::ExternalLinkage, "jl_enter_handler", &M); + } + if (!M.getNamedValue(jl_setjmp_name)) { + std::vector<Type*> args2(0); + args2.push_back(T_pint8); +#ifndef _OS_WINDOWS_ + args2.push_back(T_int32); +#endif + Function::Create(FunctionType::get(T_int32, args2, false), + Function::ExternalLinkage, jl_setjmp_name, &M) + ->addFnAttr(Attribute::ReturnsTwice); + } +} + +bool LowerGCFrame::runOnModule(Module &M) +{ + Function *ptls_getter = M.getFunction("jl_get_ptls_states"); + ensure_enter_function(M); + FunctionType *functype = nullptr; + Type *T_pjlvalue = nullptr; + if (ptls_getter) { + functype = ptls_getter->getFunctionType(); + auto T_ppjlvalue = + cast<PointerType>(functype->getReturnType())->getElementType(); + T_pjlvalue = cast<PointerType>(T_ppjlvalue)->getElementType(); + } + for (auto F = M.begin(), E = M.end(); F != E; ++F) { + if (F->isDeclaration()) + continue; + runOnFunction(&M, *F, ptls_getter, T_pjlvalue); + } + + // Cleanup for GC frame lowering. + eraseFunction(M, "julia.gc_root_decl"); + eraseFunction(M, "julia.gc_root_kill"); + eraseFunction(M, "julia.jlcall_frame_decl"); + eraseFunction(M, "julia.gcroot_flush"); + eraseFunction(M, "julia.except_enter"); + return true; +} - CallInst *ptlsStates = NULL; +void LowerGCFrame::runOnFunction(Module *M, Function &F, Function *ptls_getter, + Type *T_pjlvalue) +{ + CallInst *ptlsStates = nullptr; for (auto I = F.getEntryBlock().begin(), E = F.getEntryBlock().end(); - I != E; ++I) { + ptls_getter && I != E; ++I) { if (CallInst *callInst = dyn_cast<CallInst>(&*I)) { if (callInst->getCalledValue() == ptls_getter) { ptlsStates = callInst; @@ -946,16 +1189,8 @@ bool LowerGCFrame::runOnFunction(Function &F) } } } - if (!ptlsStates) - return true; - - FunctionType *functype = ptls_getter->getFunctionType(); - auto T_ppjlvalue = - cast<PointerType>(functype->getReturnType())->getElementType(); - auto T_pjlvalue = cast<PointerType>(T_ppjlvalue)->getElementType(); - JuliaGCAllocator allocator(ptlsStates, T_pjlvalue, tbaa_gcframe); + JuliaGCAllocator allocator(F, ptlsStates, T_pjlvalue, tbaa_gcframe); allocator.allocate_frame(); - return true; } char LowerGCFrame::ID = 0; diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 019975cc3ba05..91d588243fcf5 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -163,21 +163,8 @@ void LowerPTLS::runOnFunction(LLVMContext &ctx, Module &M, Function *F, #endif } -static void eraseFunction(Module &M, const char *name) -{ - if (Function *f = M.getFunction(name)) { - f->eraseFromParent(); - } -} - bool LowerPTLS::runOnModule(Module &M) { - // Cleanup for GC frame lowering. - eraseFunction(M, "julia.gc_root_decl"); - eraseFunction(M, "julia.gc_root_kill"); - eraseFunction(M, "julia.jlcall_frame_decl"); - eraseFunction(M, "julia.gcroot_flush"); - Function *ptls_getter = M.getFunction("jl_get_ptls_states"); if (!ptls_getter) return true; From 67d881bf01a0dc252931e287dfb7071aaac47292 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Tue, 9 Aug 2016 17:47:48 -0500 Subject: [PATCH 0887/1117] Support non-1 indices and fix type problems in DFT (fixes #17896) --- base/dft.jl | 39 ++++++++++++++------ base/multidimensional.jl | 78 ++++++++++++++++++++++++++++++++++++++++ test/fft.jl | 8 +++++ test/offsetarray.jl | 33 ++++++++++++++++- 4 files changed, 147 insertions(+), 11 deletions(-) diff --git a/base/dft.jl b/base/dft.jl index 04b9c5bf7875b..700bbc68c2dec 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -20,22 +20,41 @@ export fft, ifft, bfft, fft!, ifft!, bfft!, plan_fft, plan_ifft, plan_bfft, plan_fft!, plan_ifft!, plan_bfft!, rfft, irfft, brfft, plan_rfft, plan_irfft, plan_brfft -complexfloat{T<:AbstractFloat}(x::AbstractArray{Complex{T}}) = x +typealias FFTWFloat Union{Float32,Float64} +fftwfloat(x) = _fftwfloat(float(x)) +_fftwfloat{T<:FFTWFloat}(::Type{T}) = T +_fftwfloat(::Type{Float16}) = Float32 +_fftwfloat{T}(::Type{T}) = error("type $T not supported") +_fftwfloat{T}(x::T) = _fftwfloat(T)(x) + +complexfloat{T<:FFTWFloat}(x::StridedArray{Complex{T}}) = x +realfloat{T<:FFTWFloat}(x::StridedArray{T}) = x # return an Array, rather than similar(x), to avoid an extra copy for FFTW # (which only works on StridedArray types). -complexfloat{T<:Complex}(x::AbstractArray{T}) = copy!(Array{typeof(float(one(T)))}(size(x)), x) -complexfloat{T<:AbstractFloat}(x::AbstractArray{T}) = copy!(Array{typeof(complex(one(T)))}(size(x)), x) -complexfloat{T<:Real}(x::AbstractArray{T}) = copy!(Array{typeof(complex(float(one(T))))}(size(x)), x) +complexfloat{T<:Complex}(x::AbstractArray{T}) = copy1(typeof(fftwfloat(one(T))), x) +complexfloat{T<:Real}(x::AbstractArray{T}) = copy1(typeof(complex(fftwfloat(one(T)))), x) + +realfloat{T<:Real}(x::AbstractArray{T}) = copy1(typeof(fftwfloat(one(T))), x) + +# copy to a 1-based array, using circular permutation +function copy1{T}(::Type{T}, x) + y = Array{T}(map(length, indices(x))) + Base.circcopy!(y, x) +end + +to1(x::AbstractArray) = _to1(indices(x), x) +_to1(::Tuple{Base.OneTo,Vararg{Base.OneTo}}, x) = x +_to1(::Tuple, x) = copy1(eltype(x), x) # implementations only need to provide plan_X(x, region) # for X in (:fft, :bfft, ...): for f in (:fft, :bfft, :ifft, :fft!, :bfft!, :ifft!, :rfft) pf = Symbol("plan_", f) @eval begin - $f(x::AbstractArray) = $pf(x) * x - $f(x::AbstractArray, region) = $pf(x, region) * x - $pf(x::AbstractArray; kws...) = $pf(x, 1:ndims(x); kws...) + $f(x::AbstractArray) = (y = to1(x); $pf(y) * y) + $f(x::AbstractArray, region) = (y = to1(x); $pf(y, region) * y) + $pf(x::AbstractArray; kws...) = (y = to1(x); $pf(y, 1:ndims(y); kws...)) end end @@ -187,11 +206,11 @@ for f in (:fft, :bfft, :ifft) $pf{T<:Union{Integer,Rational}}(x::AbstractArray{Complex{T}}, region; kws...) = $pf(complexfloat(x), region; kws...) end end -rfft{T<:Union{Integer,Rational}}(x::AbstractArray{T}, region=1:ndims(x)) = rfft(float(x), region) -plan_rfft{T<:Union{Integer,Rational}}(x::AbstractArray{T}, region; kws...) = plan_rfft(float(x), region; kws...) +rfft{T<:Union{Integer,Rational}}(x::AbstractArray{T}, region=1:ndims(x)) = rfft(realfloat(x), region) +plan_rfft(x::AbstractArray, region; kws...) = plan_rfft(realfloat(x), region; kws...) # only require implementation to provide *(::Plan{T}, ::Array{T}) -*{T}(p::Plan{T}, x::AbstractArray) = p * copy!(Array{T}(size(x)), x) +*{T}(p::Plan{T}, x::AbstractArray) = p * copy1(T, x) # Implementations should also implement A_mul_B!(Y, plan, X) so as to support # pre-allocated output arrays. We don't define * in terms of A_mul_B! diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 0185edeae8182..fe4f37a6d60fa 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -645,6 +645,22 @@ See also `circshift`. end circshift!(dest::AbstractArray, src, shiftamt) = circshift!(dest, src, (shiftamt...,)) +# For each dimension, we copy the first half of src to the second half +# of dest, and the second half of src to the first half of dest. This +# uses a recursive bifurcation strategy so that these splits can be +# encoded by ranges, which means that we need only one call to `mod` +# per dimension rather than one call per index. +# `rdest` and `rsrc` are tuples-of-ranges that grow one dimension at a +# time; when all the dimensions have been filled in, you call `copy!` +# for that block. In other words, in two dimensions schematically we +# have the following call sequence (--> means a call): +# circshift!(dest, src, shiftamt) --> +# _circshift!(dest, src, ("first half of dim1",)) --> +# _circshift!(dest, src, ("first half of dim1", "first half of dim2")) --> copy! +# _circshift!(dest, src, ("first half of dim1", "second half of dim2")) --> copy! +# _circshift!(dest, src, ("second half of dim1",)) --> +# _circshift!(dest, src, ("second half of dim1", "first half of dim2")) --> copy! +# _circshift!(dest, src, ("second half of dim1", "second half of dim2")) --> copy! @inline function _circshift!(dest, rdest, src, rsrc, inds::Tuple{AbstractUnitRange,Vararg{Any}}, shiftamt::Tuple{Integer,Vararg{Any}}) @@ -662,6 +678,68 @@ function _circshift!(dest, rdest, src, rsrc, inds, shiftamt) copy!(dest, CartesianRange(rdest), src, CartesianRange(rsrc)) end +# circcopy! +""" + circcopy!(dest, src) + +Copy `src` to `dest`, indexing each dimension modulo its length. +`src` and `dest` must have the same size, but can be offset in +their indices; any offset results in a (circular) wraparound. If the +arrays have overlapping indices, then on the domain of the overlap +`dest` agrees with `src`. + +```julia +julia> src = reshape(collect(1:16), (4,4)) +4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + +julia> dest = OffsetArray{Int}((0:3,2:5)) + +julia> circcopy!(dest, src) +OffsetArrays.OffsetArray{Int64,2,Array{Int64,2}} with indices 0:3×2:5: + 8 12 16 4 + 5 9 13 1 + 6 10 14 2 + 7 11 15 3 + +julia> dest[1:3,2:4] == src[1:3,2:4] +true +``` +""" +function circcopy!(dest, src) + dest === src && throw(ArgumentError("dest and src must be separate arrays")) + indssrc, indsdest = indices(src), indices(dest) + if (szsrc = map(length, indssrc)) != (szdest = map(length, indsdest)) + throw(DimensionMismatch("src and dest must have the same sizes (got $szsrc and $szdest)")) + end + shift = map((isrc, idest)->first(isrc)-first(idest), indssrc, indsdest) + all(x->x==0, shift) && return copy!(dest, src) + _circcopy!(dest, (), indsdest, src, (), indssrc) +end + +# This uses the same strategy described above for _circshift! +@inline function _circcopy!(dest, rdest, indsdest::Tuple{AbstractUnitRange,Vararg{Any}}, + src, rsrc, indssrc::Tuple{AbstractUnitRange,Vararg{Any}}) + indd1, inds1 = indsdest[1], indssrc[1] + l = length(indd1) + s = mod(first(inds1)-first(indd1), l) + sdf = first(indd1)+s + rd1, rd2 = first(indd1):sdf-1, sdf:last(indd1) + ssf = last(inds1)-s + rs1, rs2 = first(inds1):ssf, ssf+1:last(inds1) + tindsd, tindss = tail(indsdest), tail(indssrc) + _circcopy!(dest, (rdest..., rd1), tindsd, src, (rsrc..., rs2), tindss) + _circcopy!(dest, (rdest..., rd2), tindsd, src, (rsrc..., rs1), tindss) +end + +# At least one of indsdest, indssrc are empty (and both should be, since we've checked) +function _circcopy!(dest, rdest, indsdest, src, rsrc, indssrc) + copy!(dest, CartesianRange(rdest), src, CartesianRange(rsrc)) +end + ### BitArrays ## getindex diff --git a/test/fft.jl b/test/fft.jl index 6905bc85ffb30..b30e05ff09a5a 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -326,3 +326,11 @@ for x in (randn(10),randn(10,12)) # note: inference doesn't work for plan_fft_ since the # algorithm steps are included in the CTPlan type end + +# issue #17896 +a = rand(5) +@test fft(a) == fft(view(a,:)) == fft(view(a, 1:5)) == fft(view(a, [1:5;])) +@test rfft(a) == rfft(view(a,:)) == rfft(view(a, 1:5)) == rfft(view(a, [1:5;])) +a16 = convert(Vector{Float16}, a) +@test fft(a16) == fft(view(a16,:)) == fft(view(a16, 1:5)) == fft(view(a16, [1:5;])) +@test rfft(a16) == rfft(view(a16,:)) == rfft(view(a16, 1:5)) == rfft(view(a16, [1:5;])) diff --git a/test/offsetarray.jl b/test/offsetarray.jl index a509e17d3a8fb..dac5aad7995ca 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -411,10 +411,41 @@ v = OffsetArray(rand(8), (-2,)) @test rotr90(A) == OffsetArray(rotr90(parent(A)), A.offsets[[2,1]]) @test flipdim(A, 1) == OffsetArray(flipdim(parent(A), 1), A.offsets) @test flipdim(A, 2) == OffsetArray(flipdim(parent(A), 2), A.offsets) -@test circshift(A, (-1,2)) == OffsetArray(circshift(parent(A), (-1,2)), A.offsets) @test A+1 == OffsetArray(parent(A)+1, A.offsets) @test 2*A == OffsetArray(2*parent(A), A.offsets) @test A+A == OffsetArray(parent(A)+parent(A), A.offsets) @test A.*A == OffsetArray(parent(A).*parent(A), A.offsets) + +@test circshift(A, (-1,2)) == OffsetArray(circshift(parent(A), (-1,2)), A.offsets) + +src = reshape(collect(1:16), (4,4)) +dest = OffsetArray(Array{Int}(4,4), (-1,1)) +circcopy!(dest, src) +@test parent(dest) == [8 12 16 4; 5 9 13 1; 6 10 14 2; 7 11 15 3] +@test dest[1:3,2:4] == src[1:3,2:4] + +e = eye(5) +a = [e[:,1], e[:,2], e[:,3], e[:,4], e[:,5]] +a1 = zeros(5) +c = [ones(Complex{Float64}, 5), + exp(-2*pi*im*(0:4)/5), + exp(-4*pi*im*(0:4)/5), + exp(-6*pi*im*(0:4)/5), + exp(-8*pi*im*(0:4)/5)] +for s = -5:5 + for i = 1:5 + thisa = OffsetArray(a[i], (s,)) + thisc = c[mod1(i+s+5,5)] + @test_approx_eq fft(thisa) thisc + @test_approx_eq fft(thisa, 1) thisc + @test_approx_eq ifft(fft(thisa)) circcopy!(a1, thisa) + @test_approx_eq ifft(fft(thisa, 1), 1) circcopy!(a1, thisa) + @test_approx_eq rfft(thisa) thisc[1:3] + @test_approx_eq rfft(thisa, 1) thisc[1:3] + @test_approx_eq irfft(rfft(thisa, 1), 5, 1) a1 + @test_approx_eq irfft(rfft(thisa, 1), 5, 1) a1 + end end + +end # let From f218351efd3b9c7b001c78f65b5d9e0b6b39fa99 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Tue, 9 Aug 2016 17:48:08 -0500 Subject: [PATCH 0888/1117] Export circcopy! and add it to the RST manual --- base/exports.jl | 1 + doc/stdlib/arrays.rst | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/base/exports.jl b/base/exports.jl index 33fe8c6850852..da164c187ed9a 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -489,6 +489,7 @@ export cat, checkbounds, checkindex, + circcopy!, circshift, circshift!, clamp!, diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 669b08626b7ee..4fc6612761c2f 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -647,6 +647,33 @@ Indexing, Assignment, and Concatenation See also ``circshift``\ . +.. function:: circcopy!(dest, src) + + .. Docstring generated from Julia source + + Copy ``src`` to ``dest``\ , indexing each dimension modulo its length. ``src`` and ``dest`` must have the same size, but can be offset in their indices; any offset results in a (circular) wraparound. If the arrays have overlapping indices, then on the domain of the overlap ``dest`` agrees with ``src``\ . + + .. code-block:: julia + + julia> src = reshape(collect(1:16), (4,4)) + 4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + + julia> dest = OffsetArray{Int}((0:3,2:5)) + + julia> circcopy!(dest, src) + OffsetArrays.OffsetArray{Int64,2,Array{Int64,2}} with indices 0:3×2:5: + 8 12 16 4 + 5 9 13 1 + 6 10 14 2 + 7 11 15 3 + + julia> dest[1:3,2:4] == src[1:3,2:4] + true + .. function:: find(A) .. Docstring generated from Julia source From cfaaee84dc54413af22311b1826f8d6f4ccf53d5 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Wed, 10 Aug 2016 21:02:26 -0500 Subject: [PATCH 0889/1117] Move OffsetArray definition to TestHelpers --- test/TestHelpers.jl | 97 ++++++++++++++++++++++++++++++++++++++ test/offsetarray.jl | 110 +++----------------------------------------- 2 files changed, 104 insertions(+), 103 deletions(-) diff --git a/test/TestHelpers.jl b/test/TestHelpers.jl index b57b0d10f77ee..fef9a3089d6aa 100644 --- a/test/TestHelpers.jl +++ b/test/TestHelpers.jl @@ -43,4 +43,101 @@ function with_fake_pty(f) close(master) end +# OffsetArrays (arrays with indexing that doesn't start at 1) + +# This test file is designed to exercise support for generic indexing, +# even though offset arrays aren't implemented in Base. + +module OAs + +using Base: Indices, LinearSlow, LinearFast, tail + +export OffsetArray + +immutable OffsetArray{T,N,AA<:AbstractArray} <: AbstractArray{T,N} + parent::AA + offsets::NTuple{N,Int} +end +typealias OffsetVector{T,AA<:AbstractArray} OffsetArray{T,1,AA} + +OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::NTuple{N,Int}) = OffsetArray{T,N,typeof(A)}(A, offsets) +OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::Vararg{Int,N}) = OffsetArray(A, offsets) + +(::Type{OffsetArray{T,N}}){T,N}(inds::Indices{N}) = OffsetArray{T,N,Array{T,N}}(Array{T,N}(map(length, inds)), map(indsoffset, inds)) +(::Type{OffsetArray{T}}){T,N}(inds::Indices{N}) = OffsetArray{T,N}(inds) + +Base.linearindexing{T<:OffsetArray}(::Type{T}) = Base.linearindexing(parenttype(T)) +parenttype{T,N,AA}(::Type{OffsetArray{T,N,AA}}) = AA +parenttype(A::OffsetArray) = parenttype(typeof(A)) + +Base.parent(A::OffsetArray) = A.parent + +errmsg(A) = error("size not supported for arrays with indices $(indices(A)); see http://docs.julialang.org/en/latest/devdocs/offset-arrays/") +Base.size(A::OffsetArray) = errmsg(A) +Base.size(A::OffsetArray, d) = errmsg(A) +Base.eachindex(::LinearSlow, A::OffsetArray) = CartesianRange(indices(A)) +Base.eachindex(::LinearFast, A::OffsetVector) = indices(A, 1) + +# Implementations of indices and indices1. Since bounds-checking is +# performance-critical and relies on indices, these are usually worth +# optimizing thoroughly. +@inline Base.indices(A::OffsetArray, d) = 1 <= d <= length(A.offsets) ? indices(parent(A))[d] + A.offsets[d] : (1:1) +@inline Base.indices(A::OffsetArray) = _indices(indices(parent(A)), A.offsets) # would rather use ntuple, but see #15276 +@inline _indices(inds, offsets) = (inds[1]+offsets[1], _indices(tail(inds), tail(offsets))...) +_indices(::Tuple{}, ::Tuple{}) = () +Base.indices1{T}(A::OffsetArray{T,0}) = 1:1 # we only need to specialize this one + +function Base.similar(A::OffsetArray, T::Type, dims::Dims) + B = similar(parent(A), T, dims) +end +function Base.similar(A::AbstractArray, T::Type, inds::Tuple{UnitRange,Vararg{UnitRange}}) + B = similar(A, T, map(length, inds)) + OffsetArray(B, map(indsoffset, inds)) +end + +Base.similar(f::Union{Function,DataType}, shape::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(f(map(length, shape)), map(indsoffset, shape)) + +Base.reshape(A::AbstractArray, inds::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(reshape(A, map(length, inds)), map(indsoffset, inds)) + +@inline function Base.getindex{T,N}(A::OffsetArray{T,N}, I::Vararg{Int,N}) + checkbounds(A, I...) + @inbounds ret = parent(A)[offset(A.offsets, I)...] + ret +end +@inline function Base._getindex(::LinearFast, A::OffsetVector, i::Int) + checkbounds(A, i) + @inbounds ret = parent(A)[offset(A.offsets, (i,))[1]] + ret +end +@inline function Base._getindex(::LinearFast, A::OffsetArray, i::Int) + checkbounds(A, i) + @inbounds ret = parent(A)[i] + ret +end +@inline function Base.setindex!{T,N}(A::OffsetArray{T,N}, val, I::Vararg{Int,N}) + checkbounds(A, I...) + @inbounds parent(A)[offset(A.offsets, I)...] = val + val +end +@inline function Base._setindex!(::LinearFast, A::OffsetVector, val, i::Int) + checkbounds(A, i) + @inbounds parent(A)[offset(A.offsets, (i,))[1]] = val + val +end +@inline function Base._setindex!(::LinearFast, A::OffsetArray, val, i::Int) + checkbounds(A, i) + @inbounds parent(A)[i] = val + val +end + +# Computing a shifted index (subtracting the offset) +offset{N}(offsets::NTuple{N,Int}, inds::NTuple{N,Int}) = _offset((), offsets, inds) +_offset(out, ::Tuple{}, ::Tuple{}) = out +@inline _offset(out, offsets, inds) = _offset((out..., inds[1]-offsets[1]), Base.tail(offsets), Base.tail(inds)) + +indsoffset(r::Range) = first(r) - 1 +indsoffset(i::Integer) = 0 + +end + end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index dac5aad7995ca..2136e4dd580e0 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -1,103 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -# OffsetArrays (arrays with indexing that doesn't start at 1) - -# This test file is designed to exercise support for generic indexing, -# even though offset arrays aren't implemented in Base. - -module OAs - -using Base: Indices, LinearSlow, LinearFast, tail - -export OffsetArray - -immutable OffsetArray{T,N,AA<:AbstractArray} <: AbstractArray{T,N} - parent::AA - offsets::NTuple{N,Int} -end -typealias OffsetVector{T,AA<:AbstractArray} OffsetArray{T,1,AA} - -OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::NTuple{N,Int}) = OffsetArray{T,N,typeof(A)}(A, offsets) -OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::Vararg{Int,N}) = OffsetArray(A, offsets) - -(::Type{OffsetArray{T,N}}){T,N}(inds::Indices{N}) = OffsetArray{T,N,Array{T,N}}(Array{T,N}(map(length, inds)), map(indsoffset, inds)) -(::Type{OffsetArray{T}}){T,N}(inds::Indices{N}) = OffsetArray{T,N}(inds) - -Base.linearindexing{T<:OffsetArray}(::Type{T}) = Base.linearindexing(parenttype(T)) -parenttype{T,N,AA}(::Type{OffsetArray{T,N,AA}}) = AA -parenttype(A::OffsetArray) = parenttype(typeof(A)) - -Base.parent(A::OffsetArray) = A.parent - -errmsg(A) = error("size not supported for arrays with indices $(indices(A)); see http://docs.julialang.org/en/latest/devdocs/offset-arrays/") -Base.size(A::OffsetArray) = errmsg(A) -Base.size(A::OffsetArray, d) = errmsg(A) -Base.eachindex(::LinearSlow, A::OffsetArray) = CartesianRange(indices(A)) -Base.eachindex(::LinearFast, A::OffsetVector) = indices(A, 1) - -# Implementations of indices and indices1. Since bounds-checking is -# performance-critical and relies on indices, these are usually worth -# optimizing thoroughly. -@inline Base.indices(A::OffsetArray, d) = 1 <= d <= length(A.offsets) ? indices(parent(A))[d] + A.offsets[d] : (1:1) -@inline Base.indices(A::OffsetArray) = _indices(indices(parent(A)), A.offsets) # would rather use ntuple, but see #15276 -@inline _indices(inds, offsets) = (inds[1]+offsets[1], _indices(tail(inds), tail(offsets))...) -_indices(::Tuple{}, ::Tuple{}) = () -Base.indices1{T}(A::OffsetArray{T,0}) = 1:1 # we only need to specialize this one - -function Base.similar(A::OffsetArray, T::Type, dims::Dims) - B = similar(parent(A), T, dims) -end -function Base.similar(A::AbstractArray, T::Type, inds::Tuple{UnitRange,Vararg{UnitRange}}) - B = similar(A, T, map(length, inds)) - OffsetArray(B, map(indsoffset, inds)) -end - -Base.similar(f::Union{Function,DataType}, shape::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(f(map(length, shape)), map(indsoffset, shape)) - -Base.reshape(A::AbstractArray, inds::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(reshape(A, map(length, inds)), map(indsoffset, inds)) - -@inline function Base.getindex{T,N}(A::OffsetArray{T,N}, I::Vararg{Int,N}) - checkbounds(A, I...) - @inbounds ret = parent(A)[offset(A.offsets, I)...] - ret -end -@inline function Base._getindex(::LinearFast, A::OffsetVector, i::Int) - checkbounds(A, i) - @inbounds ret = parent(A)[offset(A.offsets, (i,))[1]] - ret -end -@inline function Base._getindex(::LinearFast, A::OffsetArray, i::Int) - checkbounds(A, i) - @inbounds ret = parent(A)[i] - ret -end -@inline function Base.setindex!{T,N}(A::OffsetArray{T,N}, val, I::Vararg{Int,N}) - checkbounds(A, I...) - @inbounds parent(A)[offset(A.offsets, I)...] = val - val -end -@inline function Base._setindex!(::LinearFast, A::OffsetVector, val, i::Int) - checkbounds(A, i) - @inbounds parent(A)[offset(A.offsets, (i,))[1]] = val - val -end -@inline function Base._setindex!(::LinearFast, A::OffsetArray, val, i::Int) - checkbounds(A, i) - @inbounds parent(A)[i] = val - val -end - -# Computing a shifted index (subtracting the offset) -offset{N}(offsets::NTuple{N,Int}, inds::NTuple{N,Int}) = _offset((), offsets, inds) -_offset(out, ::Tuple{}, ::Tuple{}) = out -@inline _offset(out, offsets, inds) = _offset((out..., inds[1]-offsets[1]), Base.tail(offsets), Base.tail(inds)) - -indsoffset(r::Range) = first(r) - 1 -indsoffset(i::Integer) = 0 - -end - -using OAs +isdefined(:TestHelpers) || include(joinpath(dirname(@__FILE__), "TestHelpers.jl")) +using TestHelpers.OAs let # Basics @@ -219,11 +123,11 @@ cmp_showf(Base.print_matrix, io, OffsetArray(rand(5,5), (10,-9))) # rows&c cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,5), (10,-9))) # columns fit cmp_showf(Base.print_matrix, io, OffsetArray(rand(5,10^3), (10,-9))) # rows fit cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,10^3), (10,-9))) # neither fits -targets1 = ["0-dimensional OAs.OffsetArray{Float64,0,Array{Float64,0}}:\n1.0", - "OAs.OffsetArray{Float64,1,Array{Float64,1}} with indices 2:2:\n 1.0", - "OAs.OffsetArray{Float64,2,Array{Float64,2}} with indices 2:2×3:3:\n 1.0", - "OAs.OffsetArray{Float64,3,Array{Float64,3}} with indices 2:2×3:3×4:4:\n[:, :, 4] =\n 1.0", - "OAs.OffsetArray{Float64,4,Array{Float64,4}} with indices 2:2×3:3×4:4×5:5:\n[:, :, 4, 5] =\n 1.0"] +targets1 = ["0-dimensional TestHelpers.OAs.OffsetArray{Float64,0,Array{Float64,0}}:\n1.0", + "TestHelpers.OAs.OffsetArray{Float64,1,Array{Float64,1}} with indices 2:2:\n 1.0", + "TestHelpers.OAs.OffsetArray{Float64,2,Array{Float64,2}} with indices 2:2×3:3:\n 1.0", + "TestHelpers.OAs.OffsetArray{Float64,3,Array{Float64,3}} with indices 2:2×3:3×4:4:\n[:, :, 4] =\n 1.0", + "TestHelpers.OAs.OffsetArray{Float64,4,Array{Float64,4}} with indices 2:2×3:3×4:4×5:5:\n[:, :, 4, 5] =\n 1.0"] targets2 = ["(1.0,1.0)", "([1.0],[1.0])", "(\n[1.0],\n\n[1.0])", From afa3354b84d6f6fdacdfd99d065cc81d0a37fbbd Mon Sep 17 00:00:00 2001 From: Mus M <mus_mo@outlook.com> Date: Thu, 11 Aug 2016 00:51:09 -0400 Subject: [PATCH 0890/1117] fix output of getaddrinfo in networking and streams (#17931) fix output of getaddrinfo in networking and streams documentation for v0.5. --- doc/manual/networking-and-streams.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/networking-and-streams.rst b/doc/manual/networking-and-streams.rst index cbaa39eba5985..de39508cdd6a6 100644 --- a/doc/manual/networking-and-streams.rst +++ b/doc/manual/networking-and-streams.rst @@ -260,5 +260,5 @@ allows you to do things like:: At the base of this functionality is :func:`getaddrinfo`, which will do the appropriate address resolution:: julia> getaddrinfo("google.com") - IPv4(74.125.226.225) + ip"74.125.226.225" From 1a25969fd3e905fe66d4c5f272daad69fff34e5e Mon Sep 17 00:00:00 2001 From: ranjanan <benditlikeranjan@gmail.com> Date: Thu, 11 Aug 2016 12:11:05 +0530 Subject: [PATCH 0891/1117] Add test for PR #17803 The PR fixes jl_static_show for bitstypes --- test/spawn.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/spawn.jl b/test/spawn.jl index 21ce279b5eb3e..38682d22d4604 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -428,3 +428,11 @@ if is_unix() end end end + +# Test for PR 17803 +let p=Pipe() + Base.link_pipe(p; julia_only_read=true, julia_only_write=true) + ccall(:jl_static_show, Void, (Ptr{Void}, Any), p.in, "world") + @async close(p.in) + @test readstring(p.out) == "\"world\"" +end From c12b3d1fcb5bfe0d390a329f219d4456d2d6aa1c Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 11 Aug 2016 02:10:44 -0400 Subject: [PATCH 0892/1117] Fix doctest line numbers after #17882 (cherry picked from commit 84ad7c720cb0aaaecb226388eb9c00a5a2ba2041) [ci skip] --- base/array.jl | 2 +- base/linalg/eigen.jl | 10 ++++++---- doc/stdlib/collections.rst | 2 +- doc/stdlib/linalg.rst | 10 ++++++---- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/base/array.jl b/base/array.jl index a24a7773477a7..4eba2f531ad78 100644 --- a/base/array.jl +++ b/base/array.jl @@ -594,7 +594,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:576 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:611 ... ``` """ diff --git a/base/linalg/eigen.jl b/base/linalg/eigen.jl index 93e9468fd24a7..21067ac34e024 100644 --- a/base/linalg/eigen.jl +++ b/base/linalg/eigen.jl @@ -175,8 +175,9 @@ julia> A = [0 im; -1 0] julia> eigmax(A) ERROR: DomainError: - in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:108 - in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:106 + in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:186 + in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:184 + ... ``` """ function eigmax(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=true) @@ -214,8 +215,9 @@ julia> A = [0 im; -1 0] julia> eigmin(A) ERROR: DomainError: - in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:115 - in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:113 + in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:226 + in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:224 + ... ``` """ function eigmin(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=true) diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index a8b4285b754c4..59abf1b78aba2 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1379,7 +1379,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:576 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:611 ... .. function:: splice!(collection, index, [replacement]) -> item diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 349869b5a5214..ff1a4808cd8d5 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -651,8 +651,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f julia> eigmax(A) ERROR: DomainError: - in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:108 - in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:106 + in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:186 + in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:184 + ... .. function:: eigmin(A; permute::Bool=true, scale::Bool=true) @@ -677,8 +678,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f julia> eigmin(A) ERROR: DomainError: - in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:115 - in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:113 + in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:226 + in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:224 + ... .. function:: eigvecs(A, [eigvals,][permute=true,][scale=true]) -> Matrix From 228ed993bde40f974f702a980df775b13f385580 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 11 Aug 2016 16:47:07 +0800 Subject: [PATCH 0893/1117] Fix typo in tbaa_decorate_gcframe --- src/llvm-gcroot.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llvm-gcroot.cpp b/src/llvm-gcroot.cpp index b722199548f6f..cf1a545e20dc1 100644 --- a/src/llvm-gcroot.cpp +++ b/src/llvm-gcroot.cpp @@ -76,7 +76,7 @@ static void tbaa_decorate_gcframe(Instruction *inst, std::set<Instruction*> &visited, MDNode *tbaa_gcframe) { - if (visited.insert(inst).second) + if (!visited.insert(inst).second) return; #ifdef LLVM35 Value::user_iterator I = inst->user_begin(), E = inst->user_end(); From 1ab15ae7d54b0d0be5042d3affa0cfd4c3690d8b Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 11 Aug 2016 08:30:47 -0500 Subject: [PATCH 0894/1117] Fix StackOverflowError in zip_iteratorsize --- base/iterator.jl | 1 + test/functional.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/base/iterator.jl b/base/iterator.jl index 3d74fd5da57db..09dca34b880e4 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -61,6 +61,7 @@ zip_iteratorsize(a, b) = and_iteratorsize(a,b) # as `and_iteratorsize` but inher zip_iteratorsize(::HasLength, ::IsInfinite) = HasLength() zip_iteratorsize(::HasShape, ::IsInfinite) = HasLength() zip_iteratorsize(a::IsInfinite, b) = zip_iteratorsize(b,a) +zip_iteratorsize(a::IsInfinite, b::IsInfinite) = IsInfinite() immutable Zip1{I} <: AbstractZipIterator diff --git a/test/functional.jl b/test/functional.jl index b495ddb715847..5ed8f3ce51db8 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -185,6 +185,7 @@ end @test Base.iteratorsize(repeated(0, 5)) == Base.HasLength() @test Base.iteratoreltype(repeated(0)) == Base.HasEltype() @test Base.iteratoreltype(repeated(0, 5)) == Base.HasEltype() +@test Base.iteratorsize(zip(repeated(0), repeated(0))) == Base.IsInfinite() # product From 1a1362fba368a16cf20a32d7b417c9932b700c1d Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 11 Aug 2016 10:49:53 -0400 Subject: [PATCH 0895/1117] slightly improve error message for unsupported kw arguments also fix some code formatting --- base/replutil.jl | 3 ++- src/julia-syntax.scm | 5 +++-- test/keywordargs.jl | 2 +- test/replutil.jl | 24 ++++++++++++------------ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/base/replutil.jl b/base/replutil.jl index 3c26308f06b64..fbd14f79b4608 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -533,7 +533,8 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs::Vector=Any[]) end if !isempty(unexpected) Base.with_output_color(:red, buf) do buf - print(buf, " got an unsupported keyword argument \"", join(unexpected, "\", \""), "\"") + plur = length(unexpected) > 1 ? "s" : "" + print(buf, " got unsupported keyword argument$plur \"", join(unexpected, "\", \""), "\"") end end end diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 6fd0b31745119..f32dd65d9d82c 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -522,8 +522,9 @@ ,else))) (if (null? restkw) ;; if no rest kw, give error for unrecognized - `(call (top kwerr) ,kw ,@(map arg-name pargl),@(if (null? vararg) '() - (list `(... ,(arg-name (car vararg)))))) + `(call (top kwerr) ,kw ,@(map arg-name pargl) + ,@(if (null? vararg) '() + (list `(... ,(arg-name (car vararg)))))) ;; otherwise add to rest keywords `(ccall 'jl_array_ptr_1d_push Void (tuple Any Any) ,rkw (tuple ,elt diff --git a/test/keywordargs.jl b/test/keywordargs.jl index 65423eb4eac46..706a3a326a6e0 100644 --- a/test/keywordargs.jl +++ b/test/keywordargs.jl @@ -9,7 +9,7 @@ kwf1(ones; tens=0, hundreds=0) = ones + 10*tens + 100*hundreds @test kwf1(3, tens=7, hundreds=2) == 273 @test_throws MethodError kwf1() # no method, too few args -@test_throws MethodError kwf1(1, z=0) # unsupported keyword +@test_throws MethodError kwf1(1, z=0) # unsupported keyword @test_throws MethodError kwf1(1, 2) # no method, too many positional args # keyword args plus varargs diff --git a/test/replutil.jl b/test/replutil.jl index 4ad026f3a8f80..54e1f2a7aaf2c 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -127,19 +127,19 @@ showerror(buf, m_error) error_out3 = takebuf_string(buf) if Base.have_color - @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") + @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1)\e[1m\e[31m got unsupported keyword argument \"y\"\e[0m") @test contains(error_out, "method_c6(\e[1m\e[31m::Any\e[0m; y)$cfile$(c6line + 2)") - @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") - @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") + @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2)\e[1m\e[31m got unsupported keyword argument \"x\"\e[0m") + @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2)\e[1m\e[31m got unsupported keyword argument \"y\"\e[0m") @test contains(error_out2, "method_c6_in_module(\e[1m\e[31m::Any\e[0m; y)$cfile$(c6mline + 3)") - @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") + @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3)\e[1m\e[31m got unsupported keyword argument \"x\"\e[0m") else - @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1) got an unsupported keyword argument \"y\"") + @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1) got unsupported keyword argument \"y\"") @test contains(error_out, "method_c6(!Matched::Any; y)$cfile$(c6line + 2)") - @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2) got an unsupported keyword argument \"x\"") - @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2) got an unsupported keyword argument \"y\"") + @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2) got unsupported keyword argument \"x\"") + @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2) got unsupported keyword argument \"y\"") @test contains(error_out2, "method_c6_in_module(!Matched::Any; y)$cfile$(c6mline + 3)") - @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3) got an unsupported keyword argument \"x\"") + @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3) got unsupported keyword argument \"x\"") end c7line = @__LINE__ + 1 @@ -150,16 +150,16 @@ test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c7(::Any, ::Any; c8line = @__LINE__ + 1 method_c8(a, b; y=1, w=1) = a Base.show_method_candidates(buf, MethodError(method_c8, (1, 1)), [(:x, 1), (:y, 2), (:z, 1), (:w, 1)]) -test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line\e[1m\e[31m got an unsupported keyword argument \"x\", \"z\"\e[0m\e[0m", - "\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line got an unsupported keyword argument \"x\", \"z\"") +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line\e[1m\e[31m got unsupported keyword arguments \"x\", \"z\"\e[0m\e[0m", + "\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line got unsupported keyword arguments \"x\", \"z\"") ac15639line = @__LINE__ addConstraint_15639(c::Int32) = c addConstraint_15639(c::Int64; uncset=nothing) = addConstraint_15639(Int32(c), uncset=uncset) Base.show_method_candidates(buf, MethodError(addConstraint_15639, (Int32(1),)), [(:uncset, nothing)]) -test_have_color(buf, "\e[0m\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1)\e[1m\e[31m got an unsupported keyword argument \"uncset\"\e[0m\n addConstraint_15639(\e[1m\e[31m::Int64\e[0m; uncset)$cfile$(ac15639line + 2)\e[0m", - "\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1) got an unsupported keyword argument \"uncset\"\n addConstraint_15639(!Matched::Int64; uncset)$cfile$(ac15639line + 2)") +test_have_color(buf, "\e[0m\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1)\e[1m\e[31m got unsupported keyword argument \"uncset\"\e[0m\n addConstraint_15639(\e[1m\e[31m::Int64\e[0m; uncset)$cfile$(ac15639line + 2)\e[0m", + "\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1) got unsupported keyword argument \"uncset\"\n addConstraint_15639(!Matched::Int64; uncset)$cfile$(ac15639line + 2)") macro except_str(expr, err_type) return quote From 1bc40e0caec82d909e544e81e7adf41e38710d61 Mon Sep 17 00:00:00 2001 From: ranjanan <benditlikeranjan@gmail.com> Date: Thu, 11 Aug 2016 20:55:46 +0530 Subject: [PATCH 0896/1117] Change test to Int128(-1) --- test/spawn.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/spawn.jl b/test/spawn.jl index 38682d22d4604..5379300b83ed4 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -432,7 +432,7 @@ end # Test for PR 17803 let p=Pipe() Base.link_pipe(p; julia_only_read=true, julia_only_write=true) - ccall(:jl_static_show, Void, (Ptr{Void}, Any), p.in, "world") + ccall(:jl_static_show, Void, (Ptr{Void}, Any), p.in, Int128(-1)) @async close(p.in) - @test readstring(p.out) == "\"world\"" + @test readstring(p.out) == "Int128(0xffffffffffffffffffffffffffffffff)" end From 9e05bd62646ebbe4883ed692cd70979f2552a689 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Thu, 11 Aug 2016 17:34:46 +0200 Subject: [PATCH 0897/1117] Remove two cross-references * Replace a malformed link in `write` docstring with a literal `write` rather than a real cross-reference, since that would just cross-reference itself. Also move `write` docstring inline. * Remove cross-reference for `<<` from inside a literal since text inside a literal can't contain a link. A cross-reference to `<<` already appears at the end of the docstring. --- base/docs/helpdb/Base.jl | 15 --------------- base/io.jl | 14 ++++++++++++++ base/operators.jl | 2 +- doc/stdlib/io-network.rst | 2 +- doc/stdlib/math.rst | 2 +- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 3a669992c71f3..b7748cff42198 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2912,21 +2912,6 @@ Squared absolute value of `x`. """ abs2 -""" - write(stream::IO, x) - write(filename::AbstractString, x) - -Write the canonical binary representation of a value to the given I/O stream or file. -Returns the number of bytes written into the stream. - -You can write multiple values with the same :func:`write` call. i.e. the following are -equivalent: - - write(stream, x, y...) - write(stream, x) + write(stream, y...) -""" -write - """ sizehint!(s, n) diff --git a/base/io.jl b/base/io.jl index 29aa202314678..9d65ae5e5e1d4 100644 --- a/base/io.jl +++ b/base/io.jl @@ -23,6 +23,20 @@ function iswritable end function copy end function eof end +""" + write(stream::IO, x) + write(filename::AbstractString, x) + +Write the canonical binary representation of a value to the given I/O stream or file. +Returns the number of bytes written into the stream. + +You can write multiple values with the same `write` call. i.e. the following are equivalent: + + write(stream, x, y...) + write(stream, x) + write(stream, y...) +""" +function write end + read(s::IO, ::Type{UInt8}) = error(typeof(s)," does not support byte I/O") write(s::IO, x::UInt8) = error(typeof(s)," does not support byte I/O") diff --git a/base/operators.jl b/base/operators.jl index 31faf73b1d77f..be5ccf5c5354f 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -230,7 +230,7 @@ end Unsigned right bit shift operator, `x >>> n`. For `n >= 0`, the result is `x` shifted right by `n` bits, where `n >= 0`, filling with `0`s. For `n < 0`, this -is equivalent to `x [<<](:func:`<<`) -n`]. +is equivalent to `x << -n`. For `Unsigned` integer types, this is equivalent to [`>>`](:func:`>>`). For `Signed` integer types, this is equivalent to `signed(unsigned(x) >> n)`. diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 4e747354dda2a..67a1b123c57b6 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -134,7 +134,7 @@ General I/O Write the canonical binary representation of a value to the given I/O stream or file. Returns the number of bytes written into the stream. - You can write multiple values with the same :func:``write`` call. i.e. the following are equivalent: + You can write multiple values with the same ``write`` call. i.e. the following are equivalent: .. code-block:: julia diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index cf4a4ad72312d..206f9e38733f7 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -284,7 +284,7 @@ Mathematical Operators .. Docstring generated from Julia source - Unsigned right bit shift operator, ``x >>> n``\ . For ``n >= 0``\ , the result is ``x`` shifted right by ``n`` bits, where ``n >= 0``\ , filling with ``0``\ s. For ``n < 0``\ , this is equivalent to ``x [<<](:func:``\ <<``) -n``\ ]. + Unsigned right bit shift operator, ``x >>> n``\ . For ``n >= 0``\ , the result is ``x`` shifted right by ``n`` bits, where ``n >= 0``\ , filling with ``0``\ s. For ``n < 0``\ , this is equivalent to ``x << -n``\ . For ``Unsigned`` integer types, this is equivalent to :func:`>>`\ . For ``Signed`` integer types, this is equivalent to ``signed(unsigned(x) >> n)``\ . From 5cb68c64d1012237d446c828019e718989a186f4 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 10 Aug 2016 18:16:20 -0400 Subject: [PATCH 0898/1117] improve correctness of fieldtype_tfunc Vararg is only exact if in covariant position also make getfield_tfunc monotonic for the case where the type has one field, to avoid the same bug fix #16530 --- base/inference.jl | 21 ++++++++++++++++++--- src/jltypes.c | 6 +++--- test/inference.jl | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 4022da0726a60..92874f6f7b813 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -408,7 +408,7 @@ function limit_type_depth(t::ANY, d::Int, cov::Bool, vars) else return t end - if inexact && !isvarargtype(R) + if inexact && (!cov || !isvarargtype(R)) R = TypeVar(:_,R) push!(vars, R) end @@ -467,12 +467,15 @@ function getfield_tfunc(s0::ANY, name) end end snames = s.name.names - for i=1:length(snames) + for i = 1:length(snames) if is(snames[i],fld) R = s.types[i] if isempty(s.parameters) return R, true else + # conservatively limit the type depth here, + # since the UnionAll type bound is otherwise incorrect + # in the current type system typ = limit_type_depth(R, 0, true, filter!(x->isa(x,TypeVar), Any[s.parameters...])) return typ, isleaftype(s) && typeseq(typ, R) @@ -493,8 +496,20 @@ function getfield_tfunc(s0::ANY, name) return Bottom, true end return s.types[i], false + elseif isempty(s.types) + return Bottom, true + elseif length(s.types) == 1 && isempty(s.parameters) + return s.types[1], true else - return reduce(tmerge, Bottom, map(unwrapva,s.types)) #=Union{s.types...}=#, false + R = reduce(tmerge, Bottom, map(unwrapva,s.types)) #=Union{s.types...}=# + # do the same limiting as the known-symbol case to preserve type-monotonicity + if isempty(s.parameters) + return R, typeseq(R, s.types[1]) + else + typ = limit_type_depth(R, 0, true, + filter!(x->isa(x,TypeVar), Any[s.parameters...])) + return typ, isleaftype(s) && typeseq(typ, R) + end end end add_tfunc(getfield, 2, 2, (s,name)->getfield_tfunc(s,name)[1]) diff --git a/src/jltypes.c b/src/jltypes.c index 6dd59eecdc88d..a0e4409b8745a 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -83,7 +83,7 @@ static int jl_has_typevars__(jl_value_t *v, int incl_wildcard, jl_value_t **p, s jl_has_typevars__(((jl_tvar_t*)v)->lb, incl_wildcard, p, np)) return 1; if (p != NULL) { - for(i=0; i < np; i++) { + for (i = 0; i < np; i++) { if (v == p[i]) return 1; } @@ -102,7 +102,7 @@ static int jl_has_typevars__(jl_value_t *v, int incl_wildcard, jl_value_t **p, s } else if (jl_is_datatype(v)) { if (is_unspec((jl_datatype_t*)v)) - return 0; + return 0; // TODO: fix expect in this case if (p == NULL) { if (incl_wildcard) expect = ((jl_datatype_t*)v)->haswildcard; @@ -118,7 +118,7 @@ static int jl_has_typevars__(jl_value_t *v, int incl_wildcard, jl_value_t **p, s return 0; } size_t l = jl_svec_len(t); - for(i=0; i < l; i++) { + for (i = 0; i < l; i++) { jl_value_t *elt = jl_svecref(t, i); if (elt != v) { if (jl_has_typevars__(elt, incl_wildcard, p, np)) { diff --git a/test/inference.jl b/test/inference.jl index 2615e9cc4847a..1d99d6539bc14 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -282,3 +282,27 @@ let I = Integer[] push!(I, 1) @test I == Any[1] end + +# issue #16530 +type Foo16530a{dim} + c::Vector{NTuple{dim, Float64}} + d::Vector +end +type Foo16530b{dim} + c::Vector{NTuple{dim, Float64}} +end +f16530a() = fieldtype(Foo16530a, :c) +f16530a(c) = fieldtype(Foo16530a, c) +f16530b() = fieldtype(Foo16530b, :c) +f16530b(c) = fieldtype(Foo16530b, c) + +let T = Array{Tuple{Vararg{Float64,TypeVar(:dim)}},1}, + TTlim = Type{TypeVar(:_,Array{TypeVar(:_,Tuple),1})} + + @test f16530a() == T + @test f16530a(:c) == T + @test Base.return_types(f16530a, ()) == Any[TTlim] + @test Base.return_types(f16530b, ()) == Any[TTlim] + @test Base.return_types(f16530b, (Symbol,)) == Any[TTlim] +end +@test f16530a(:d) == Vector From 7dcca5ceb1790100274d9258439ba61502c0a42b Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 10 Aug 2016 15:33:22 -0400 Subject: [PATCH 0899/1117] preserve functionality of special ^ inliner in linear IR --- base/inference.jl | 53 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 92874f6f7b813..8ef4403ef703a 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2812,8 +2812,6 @@ function mk_tuplecall(args, sv::InferenceState) e end -const corenumtype = Union{Int32,Int64,Float32,Float64} - function inlining_pass!(linfo::LambdaInfo, sv::InferenceState) eargs = linfo.code i = 1 @@ -2834,6 +2832,8 @@ function inlining_pass!(linfo::LambdaInfo, sv::InferenceState) end end +const corenumtype = Union{Int32, Int64, Float32, Float64} + function inlining_pass(e::Expr, sv, linfo) if e.head === :method # avoid running the inlining pass on function definitions @@ -2923,20 +2923,41 @@ function inlining_pass(e::Expr, sv, linfo) end end - if sv.inlining && isdefined(Main, :Base) && - ((isdefined(Main.Base, :^) && is(f, Main.Base.:^)) || - (isdefined(Main.Base, :.^) && is(f, Main.Base.:.^))) - if length(e.args) == 3 && isa(e.args[3],Union{Int32,Int64}) - a1 = e.args[2] - basenumtype = Union{corenumtype, Main.Base.Complex64, Main.Base.Complex128, Main.Base.Rational} - if isa(a1,basenumtype) || ((isa(a1,Symbol) || isa(a1,Slot) || isa(a1,SSAValue)) && - exprtype(a1,sv) ⊑ basenumtype) - if e.args[3]==2 - e.args = Any[GlobalRef(Main.Base,:*), a1, a1] - f = Main.Base.:*; ft = abstract_eval_constant(f) - elseif e.args[3]==3 - e.args = Any[GlobalRef(Main.Base,:*), a1, a1, a1] - f = Main.Base.:*; ft = abstract_eval_constant(f) + if sv.inlining + if isdefined(Main, :Base) && + ((isdefined(Main.Base, :^) && is(f, Main.Base.:^)) || + (isdefined(Main.Base, :.^) && is(f, Main.Base.:.^))) && + length(e.args) == 3 + + a2 = e.args[3] + if isa(a2, Symbol) || isa(a2, Slot) || isa(a2, SSAValue) + ta2 = exprtype(a2, sv) + if isa(ta2, Const) + a2 = ta2.val + end + end + + square = (a2 === Int32(2) || a2 === Int64(2)) + triple = (a2 === Int32(3) || a2 === Int64(3)) + if square || triple + a1 = e.args[2] + basenumtype = Union{corenumtype, Main.Base.Complex64, Main.Base.Complex128, Main.Base.Rational} + if isa(a1, basenumtype) || ((isa(a1, Symbol) || isa(a1, Slot) || isa(a1, SSAValue)) && + exprtype(a1, sv) ⊑ basenumtype) + if square + e.args = Any[GlobalRef(Main.Base,:*), a1, a1] + res = inlining_pass(e, sv, linfo) + else + e.args = Any[GlobalRef(Main.Base,:*), Expr(:call, GlobalRef(Main.Base,:*), a1, a1), a1] + res = inlining_pass(e, sv, linfo) + end + if isa(res, Tuple) + if isa(res[2], Array) && !isempty(res[2]) + append!(stmts, res[2]) + end + res = res[1] + end + return (res, stmts) end end end From 671b6f11ec2404374567dc95bb6e4879f85f0204 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 10 Aug 2016 15:56:19 -0400 Subject: [PATCH 0900/1117] add model of special case ^ inliner to inference edges required for the inliner to work, since all of the logic for computing edges is supposed to be handle by the inference step fix #17759 --- base/inference.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/base/inference.jl b/base/inference.jl index 8ef4403ef703a..779e99d16efe4 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1052,6 +1052,23 @@ function abstract_call(f::ANY, fargs, argtypes::Vector{Any}, vtypes::VarTable, s return Type end + if sv.inlining + # need to model the special inliner for ^ + # to ensure we have added the same edge + if isdefined(Main, :Base) && + ((isdefined(Main.Base, :^) && is(f, Main.Base.:^)) || + (isdefined(Main.Base, :.^) && is(f, Main.Base.:.^))) && + length(argtypes) == 3 && (argtypes[3] ⊑ Int32 || argtypes[3] ⊑ Int64) + + a1 = argtypes[2] + basenumtype = Union{corenumtype, Main.Base.Complex64, Main.Base.Complex128, Main.Base.Rational} + if a1 ⊑ basenumtype + ftimes = Main.Base.:* + ta1 = widenconst(a1) + abstract_call_gf_by_type(ftimes, Tuple{typeof(ftimes), ta1, ta1}, sv) + end + end + end return abstract_call_gf_by_type(f, atype, sv) end From 69023b9ec781af499f4b9cb8bbe2aa5a90fc5d22 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 8 Aug 2016 13:52:43 -0400 Subject: [PATCH 0901/1117] add mem fence-post verification code defining MEMFENCE will tell the gc to validate all object tags (before they get freed) to attempt to detect out-of-bounds writes that corrupted a object tag or the freelist --- src/gc-debug.c | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ src/gc.c | 15 +++++-- src/gc.h | 9 ++++ src/options.h | 5 +++ 4 files changed, 144 insertions(+), 4 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index 3536eaaf75095..880c3e9235799 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -4,6 +4,12 @@ #include <inttypes.h> #include <stdio.h> +// re-include assert.h without NDEBUG, +// so that we can always use the assert macro in this file +// for use under their respective enable flags +#undef NDEBUG +#include <assert.h> + #ifdef __cplusplus extern "C" { #endif @@ -250,6 +256,119 @@ void gc_verify(jl_ptls_t ptls) } #endif +#ifdef MEMFENCE +static uint8_t freelist_map[GC_PAGE_SZ / sizeof(void*) / 8]; +static int freelist_zerod; +void gc_verify_tags(void) +{ + // verify the freelist chains look valid + for (int t_i = 0; t_i < jl_n_threads; t_i++) { + jl_ptls_t ptls2 = jl_all_tls_states[t_i]; + for (int i = 0; i < JL_GC_N_POOLS; i++) { + // for all pools, iterate its freelist + jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; + jl_taggedvalue_t *next = p->freelist; + jl_taggedvalue_t *last = NULL; + char *allocating = gc_page_data(next); + while (next) { + // and assert that the freelist values aren't gc-marked + assert(next->bits.gc == 0); + // TODO: verify they are ordered and on the right byte boundaries + if (gc_page_data(next) != gc_page_data(last)) { + // and verify that the chain looks valid + jl_gc_pagemeta_t *pg = page_metadata(next); + assert(pg->osize == p->osize); + if (gc_page_data(next) != allocating) { + // when not currently allocating on this page, fl_begin_offset should be correct + assert(next == page_pfl_beg(pg)); + } + } + last = next; + next = next->next; + } + } + } + + + // verify that all the objects on every page are either valid julia objects + // or are part of the freelist or are on the allocated half of a page + for (int region_i = 0; region_i < REGION_COUNT; region_i++) { + if (!regions[region_i].pages) + break; + region_t *region = ®ions[region_i]; + for (int pg_i = 0; pg_i <= region->ub; pg_i++) { + uint32_t line = region->allocmap[pg_i]; + if (line) { + for (int j = 0; j < 32; j++) { + if ((line >> j) & 1) { + // for all pages in use + jl_gc_pagemeta_t *pg = ®ion->meta[pg_i*32 + j]; + int p_n = pg->pool_n; + int t_n = pg->thread_n; + jl_ptls_t ptls2 = jl_all_tls_states[t_n]; + jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; + int osize = pg->osize; + char *data = pg->data; + char *page_begin = data + GC_PAGE_OFFSET; + jl_taggedvalue_t *v = (jl_taggedvalue_t*)page_begin; + char *lim = data + GC_PAGE_SZ - osize; + // reset the freelist map to zero + if (!freelist_zerod) { + memset(freelist_map, 0, sizeof(freelist_map)); + freelist_zerod = 1; + } + // check for p in new newpages list + jl_taggedvalue_t *halfpages = p->newpages; + while (halfpages) { + char *cur_page = gc_page_data((char*)halfpages - 1); + if (cur_page == data) { + lim = (char*)halfpages - 1; + break; + } + halfpages = *(jl_taggedvalue_t**)cur_page; + } + // compute the freelist_map + if (pg->nfree) { + jl_taggedvalue_t *next = NULL; + if (gc_page_data(p->freelist) == data) { + // currently allocating on this page + next = p->freelist; + assert(page_metadata(next)->osize == osize); + freelist_zerod = 0; + } + else if (pg->fl_begin_offset != (uint16_t)-1) { + // part of free list exists on this page + next = page_pfl_beg(pg); + freelist_zerod = 0; + } + assert(halfpages || next); + while (gc_page_data(next) == data) { + int obj_idx = (((char*)next) - page_begin) / sizeof(void*); + freelist_map[obj_idx / 8] |= 1 << (obj_idx % 7); + next = next->next; + } + } + // validate all of the tags on the page + while ((char*)v <= lim) { + int obj_idx = (((char*)v) - page_begin) / sizeof(void*); + int in_freelist = freelist_map[obj_idx / 8] & (1 << (obj_idx % 7)); + if (!in_freelist) { + jl_value_t *dt = jl_typeof(jl_valueof(v)); + // 0x10 (type) and 0x20 (singleton) are used by the incremental serializer to invalidate objects + if (dt != (jl_value_t*)jl_buff_tag && v->header != 0x10 && v->header != 0x20) { + assert(jl_typeof(dt) == (jl_value_t*)jl_datatype_type); + } + } + v = (jl_taggedvalue_t*)((char*)v + osize); + } + } + } + } + } + } +} +#endif + #ifdef GC_DEBUG_ENV JL_DLLEXPORT jl_gc_debug_env_t jl_gc_debug_env = { 0, 0, diff --git a/src/gc.c b/src/gc.c index b5ac77e6a7c4d..c95a9fbc62d59 100644 --- a/src/gc.c +++ b/src/gc.c @@ -975,8 +975,14 @@ static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_t assert(!freedall); pg->has_marked = has_marked; pg->has_young = has_young; - pg->fl_begin_offset = pfl_begin ? (char*)pfl_begin - data : (uint16_t)-1; - pg->fl_end_offset = pfl_begin ? (char*)pfl - data : (uint16_t)-1; + if (pfl_begin) { + pg->fl_begin_offset = (char*)pfl_begin - data; + pg->fl_end_offset = (char*)pfl - data; + } + else { + pg->fl_begin_offset = -1; + pg->fl_end_offset = -1; + } pg->nfree = pg_nfree; if (sweep_full) { @@ -1054,7 +1060,7 @@ static void gc_sweep_pool(int sweep_full) // update metadata of pages that were pointed to by freelist or newpages from a pool // i.e. pages being the current allocation target - for (int t_i = 0;t_i < jl_n_threads;t_i++) { + for (int t_i = 0; t_i < jl_n_threads; t_i++) { jl_ptls_t ptls2 = jl_all_tls_states[t_i]; for (int i = 0; i < JL_GC_N_POOLS; i++) { jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; @@ -1087,7 +1093,7 @@ static void gc_sweep_pool(int sweep_full) // null out terminal pointers of free lists - for (int t_i = 0;t_i < jl_n_threads;t_i++) { + for (int t_i = 0; t_i < jl_n_threads; t_i++) { for (int i = 0; i < JL_GC_N_POOLS; i++) { *pfl[t_i * JL_GC_N_POOLS + i] = NULL; } @@ -1787,6 +1793,7 @@ static void _jl_gc_collect(jl_ptls_t ptls, int full) sweep_weak_refs(); gc_sweep_other(ptls, sweep_full); gc_scrub(); + gc_verify_tags(); gc_sweep_pool(sweep_full); // sweeping is over // 6. if it is a quick sweep, put back the remembered objects in queued state diff --git a/src/gc.h b/src/gc.h index 8051e1bc7424f..6f95dc63c175e 100644 --- a/src/gc.h +++ b/src/gc.h @@ -346,6 +346,15 @@ STATIC_INLINE void gc_time_count_mallocd_array(int bits) estimate_freed, sweep_full) #endif +#ifdef MEMFENCE +void gc_verify_tags(void); +#else +static inline void gc_verify_tags(void) +{ +} +#endif + + #ifdef GC_VERIFY extern jl_value_t *lostval; void gc_verify(jl_ptls_t ptls); diff --git a/src/options.h b/src/options.h index 1f5af35b02edd..c48f729b6865e 100644 --- a/src/options.h +++ b/src/options.h @@ -36,6 +36,11 @@ // catch invalid accesses. // #define MEMDEBUG +// with MEMFENCE, the object pool headers are verified during sweep +// to help detect corruption due to fence-post write errors +// #define MEMFENCE + + // GC_VERIFY force a full verification gc along with every quick gc to ensure no // reachable memory is freed #ifndef GC_VERIFY From 24c371d35686503f9ccc82e9f4e88b3c11d7eb0f Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 11 Aug 2016 14:16:11 -0500 Subject: [PATCH 0902/1117] Expand docs on enumerate's relationship with indexing --- base/iterator.jl | 10 ++++++---- doc/stdlib/collections.rst | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/base/iterator.jl b/base/iterator.jl index 09dca34b880e4..715ff753bb0fa 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -21,10 +21,12 @@ end """ enumerate(iter) -An iterator that yields `(i, x)` where `i` is an index starting at 1, and -`x` is the `i`th value from the given iterator. It's useful when you need -not only the values `x` over which you are iterating, but also the index `i` -of the iterations. +An iterator that yields `(i, x)` where `i` is a counter starting at 1, +and `x` is the `i`th value from the given iterator. It's useful when +you need not only the values `x` over which you are iterating, but +also the number of iterations so far. Note that `i` may not be valid +for indexing `iter`; it's also possible that `x != iter[i]`, if `iter` +has indices that do not start at 1. ```jldoctest julia> a = ["a", "b", "c"]; diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 59abf1b78aba2..3c2eac2380a99 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -81,7 +81,7 @@ type. .. Docstring generated from Julia source - An iterator that yields ``(i, x)`` where ``i`` is an index starting at 1, and ``x`` is the ``i``\ th value from the given iterator. It's useful when you need not only the values ``x`` over which you are iterating, but also the index ``i`` of the iterations. + An iterator that yields ``(i, x)`` where ``i`` is a counter starting at 1, and ``x`` is the ``i``\ th value from the given iterator. It's useful when you need not only the values ``x`` over which you are iterating, but also the number of iterations so far. Note that ``i`` may not be valid for indexing ``iter``\ ; it's also possible that ``x != iter[i]``\ , if ``iter`` has indices that do not start at 1. .. doctest:: From acfed7c75e73ffd99a8974b301a8d5d00e314203 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Thu, 11 Aug 2016 16:32:08 -0500 Subject: [PATCH 0903/1117] Generalize getindex for AbstractUnitRanges It turns out we were returning a Vector{Int} for operations that should return a Range --- base/range.jl | 5 +++-- test/ranges.jl | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/range.jl b/base/range.jl index caca080802553..00b2b217c1f6a 100644 --- a/base/range.jl +++ b/base/range.jl @@ -499,10 +499,11 @@ end getindex(r::Range, ::Colon) = copy(r) -function getindex{T<:Integer}(r::UnitRange, s::AbstractUnitRange{T}) +function getindex{T<:Integer}(r::AbstractUnitRange, s::AbstractUnitRange{T}) @_inline_meta @boundscheck checkbounds(r, s) - st = oftype(r.start, r.start + first(s)-1) + f = first(r) + st = oftype(f, f + first(s)-1) range(st, length(s)) end diff --git a/test/ranges.jl b/test/ranges.jl index c92ebeb385bbf..b22db9b77a1b4 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -755,6 +755,7 @@ r = Base.OneTo(3) @test minimum(r) == 1 @test maximum(r) == 3 @test r[2] == 2 +@test r[2:3] === 2:3 @test_throws BoundsError r[4] @test_throws BoundsError r[0] @test r+1 === 2:4 From efa8c17a7424bc90eb66d340bc912b5f1eef7d68 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Thu, 11 Aug 2016 21:17:18 -0400 Subject: [PATCH 0904/1117] Revert "Remove redundant uplo argument in chol family. When using Hermitian (#17909)" This reverts commit d992f3dd10602ef52d8c297ebb7c7396a579c963. --- base/deprecated.jl | 3 -- base/linalg/cholesky.jl | 97 +++++++++++++++++++---------------------- doc/stdlib/linalg.rst | 12 ++--- test/linalg/cholesky.jl | 10 ++--- 4 files changed, 56 insertions(+), 66 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 7dffd61acc9ba..50f2386804559 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -775,9 +775,6 @@ function transpose(x) return x end -@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol, ::Type{Val{false}}) cholfact!(A, Val{false}) -@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol = :U) cholfact!(A) - # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index ce799f327b782..e961ef09f5a57 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -121,10 +121,8 @@ non_hermitian_error(f) = throw(ArgumentError("matrix is not symmetric/" * # chol!. Destructive methods for computing Cholesky factor of real symmetric or Hermitian # matrix -chol!(A::Hermitian) = - _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) -chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = - _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) +chol!(A::Hermitian) = _chol!(A.data, UpperTriangular) +chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = _chol!(A.data, UpperTriangular) function chol!(A::StridedMatrix) ishermitian(A) || non_hermitian_error("chol!") return _chol!(A, UpperTriangular) @@ -137,22 +135,14 @@ end function chol(A::Hermitian) T = promote_type(typeof(chol(one(eltype(A)))), Float32) AA = similar(A, T, size(A)) - if A.uplo == 'U' - copy!(AA, A.data) - else - Base.ccopy!(AA, A.data) - end - chol!(Hermitian(AA, :U)) + copy!(AA, A.data) + chol!(Hermitian(AA)) end function chol{T<:Real,S<:AbstractMatrix}(A::Symmetric{T,S}) TT = promote_type(typeof(chol(one(T))), Float32) AA = similar(A, TT, size(A)) - if A.uplo == 'U' - copy!(AA, A.data) - else - Base.ccopy!(AA, A.data) - end - chol!(Hermitian(AA, :U)) + copy!(AA, A.data) + chol!(Hermitian(AA)) end ## for StridedMatrices, check that matrix is symmetric/Hermitian @@ -180,15 +170,15 @@ chol(x::Number, args...) = _chol!(x, nothing) # cholfact!. Destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -function cholfact!(A::Hermitian, ::Type{Val{false}}) - if A.uplo == :U +function cholfact!(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) + if uplo == :U Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') end end -function cholfact!{T<:Real,S}(A::Symmetric{T,S}, ::Type{Val{false}}) - if A.uplo == :U +function cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) + if uplo == :U Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') @@ -197,7 +187,7 @@ end ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky + cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the factorisation @@ -206,36 +196,37 @@ integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A, uplo), Val{false}) + return cholfact!(Hermitian(A), uplo, Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact!(A::Hermitian) = cholfact!(A, Val{false}) -cholfact!{T<:Real,S}(A::Symmetric{T,S}) = cholfact!(A, Val{false}) +cholfact!(A::Hermitian, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) +cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact!(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A, uplo)) + return cholfact!(Hermitian(A), uplo) end ## With pivoting ### BLAS/LAPACK element types function cholfact!{T<:BlasReal,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, - ::Type{Val{true}}; tol = 0.0) - AA, piv, rank, info = LAPACK.pstrf!(A.uplo, A.data, tol) - return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, A.uplo, piv, rank, tol, info) + uplo::Symbol, ::Type{Val{true}}; tol = 0.0) + uplochar = char_uplo(uplo) + AA, piv, rank, info = LAPACK.pstrf!(uplochar, A.data, tol) + return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, uplochar, piv, rank, tol, info) end ### Non BLAS/LAPACK element types (generic). Since generic fallback for pivoted Cholesky ### is not implemented yet we throw an error -cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; +cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) = throw(ArgumentError("generic pivoted Cholesky factorization is not implemented yet")) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the @@ -244,25 +235,26 @@ e.g. for integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A, uplo), Val{true}; tol = tol) + return cholfact!(Hermitian(A), uplo, Val{true}; tol = tol) end + + # cholfact. Non-destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -cholfact(A::Hermitian, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) +cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky + cholfact(A, uplo::Symbol, Val{false}) -> Cholesky Compute the Cholesky factorization of a dense symmetric positive definite matrix `A` -and return a `Cholesky` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` -`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, -the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `Cholesky` factorization. +The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `Cholesky` objects: `size`, `\\`, `inv`, `det`. @@ -270,35 +262,36 @@ A `PosDefException` exception is thrown in case the matrix is not positive defin """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A, uplo), Val{false}) + return cholfact(Hermitian(A), uplo, Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact(A::Hermitian) = cholfact(A, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = cholfact(A, Val{false}) +cholfact(A::Hermitian, uplo::Symbol = :U) = cholfact(A, uplo, Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol = :U) = + cholfact(A, uplo, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A, uplo)) + return cholfact(Hermitian(A), uplo) end ## With pivoting -cholfact(A::Hermitian, ::Type{Val{true}}; tol = 0.0) = +cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - Val{true}; tol = tol) -cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = + uplo, Val{true}; tol = tol) +cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, + ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - Val{true}; tol = tol) + uplo, Val{true}; tol = tol) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix `A` -and return a `CholeskyPivoted` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` -`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, -the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `CholeskyPivoted` factorization. +The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `PivotedCholesky` objects: `size`, `\\`, `inv`, `det`, and `rank`. @@ -307,7 +300,7 @@ For negative values, the tolerance is the machine precision. """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A, uplo), Val{true}; tol = tol) + return cholfact(Hermitian(A), uplo, Val{true}; tol = tol) end ## Number diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index ff1a4808cd8d5..0a06293817286 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -308,17 +308,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the square root of a non-negative number ``x``\ . -.. function:: cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky +.. function:: cholfact(A, uplo::Symbol, Val{false}) -> Cholesky .. Docstring generated from Julia source - Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. + Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. -.. function:: cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source - Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. + Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. .. function:: cholfact(A; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor @@ -344,13 +344,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. -.. function:: cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky +.. function:: cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky .. Docstring generated from Julia source The same as ``cholfact``\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. An ``InexactError`` exception is thrown if the factorisation produces a number not representable by the element type of ``A``\ , e.g. for integer types. -.. function:: cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index bb0657c799838..00e4696b4f12b 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -74,15 +74,15 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test full(lapd) ≈ apd l = lapd[:L] @test l*l' ≈ apd - @test capd[:U] ≈ lapd[:U] - @test lapd[:L] ≈ capd[:L] + @test triu(capd.factors) ≈ lapd[:U] + @test tril(lapd.factors) ≈ capd[:L] if eltya <: Real capds = cholfact(apds) - lapds = cholfact(full(apds), :L) + lapds = cholfact(apds, :L) ls = lapds[:L] @test ls*ls' ≈ apd - @test capds[:U] ≈ lapds[:U] - @test lapds[:L] ≈ capds[:L] + @test triu(capds.factors) ≈ lapds[:U] + @test tril(lapds.factors) ≈ capds[:L] end #pivoted upper Cholesky From 29b2d6e776dde0d5549e47dba325056ef3f84a00 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Thu, 11 Aug 2016 21:31:58 -0400 Subject: [PATCH 0905/1117] Remove redundant uplo argument in chol family. When using Hermitian and Symmetric, it is redundant to also have an uplo argument and occasionally, it also gave the wrong result. This can therefore be considered a bugfix. --- base/deprecated.jl | 3 ++ base/linalg/cholesky.jl | 97 ++++++++++++++++++++++------------------- doc/stdlib/linalg.rst | 12 ++--- test/linalg/cholesky.jl | 2 +- 4 files changed, 62 insertions(+), 52 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 50f2386804559..7dffd61acc9ba 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -775,6 +775,9 @@ function transpose(x) return x end +@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol, ::Type{Val{false}}) cholfact!(A, Val{false}) +@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol = :U) cholfact!(A) + # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index e961ef09f5a57..246f1b3c8c140 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -121,8 +121,10 @@ non_hermitian_error(f) = throw(ArgumentError("matrix is not symmetric/" * # chol!. Destructive methods for computing Cholesky factor of real symmetric or Hermitian # matrix -chol!(A::Hermitian) = _chol!(A.data, UpperTriangular) -chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = _chol!(A.data, UpperTriangular) +chol!(A::Hermitian) = + _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) +chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = + _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) function chol!(A::StridedMatrix) ishermitian(A) || non_hermitian_error("chol!") return _chol!(A, UpperTriangular) @@ -135,14 +137,22 @@ end function chol(A::Hermitian) T = promote_type(typeof(chol(one(eltype(A)))), Float32) AA = similar(A, T, size(A)) - copy!(AA, A.data) - chol!(Hermitian(AA)) + if A.uplo == 'U' + copy!(AA, A.data) + else + Base.ccopy!(AA, A.data) + end + chol!(Hermitian(AA, :U)) end function chol{T<:Real,S<:AbstractMatrix}(A::Symmetric{T,S}) TT = promote_type(typeof(chol(one(T))), Float32) AA = similar(A, TT, size(A)) - copy!(AA, A.data) - chol!(Hermitian(AA)) + if A.uplo == 'U' + copy!(AA, A.data) + else + Base.ccopy!(AA, A.data) + end + chol!(Hermitian(AA, :U)) end ## for StridedMatrices, check that matrix is symmetric/Hermitian @@ -170,15 +180,15 @@ chol(x::Number, args...) = _chol!(x, nothing) # cholfact!. Destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -function cholfact!(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) - if uplo == :U +function cholfact!(A::Hermitian, ::Type{Val{false}}) + if A.uplo == 'U' Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') end end -function cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) - if uplo == :U +function cholfact!{T<:Real,S}(A::Symmetric{T,S}, ::Type{Val{false}}) + if A.uplo == 'U' Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') @@ -187,7 +197,7 @@ end ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky + cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the factorisation @@ -196,37 +206,36 @@ integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo, Val{false}) + return cholfact!(Hermitian(A, uplo), Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact!(A::Hermitian, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) -cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) +cholfact!(A::Hermitian) = cholfact!(A, Val{false}) +cholfact!{T<:Real,S}(A::Symmetric{T,S}) = cholfact!(A, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact!(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo) + return cholfact!(Hermitian(A, uplo)) end ## With pivoting ### BLAS/LAPACK element types function cholfact!{T<:BlasReal,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, - uplo::Symbol, ::Type{Val{true}}; tol = 0.0) - uplochar = char_uplo(uplo) - AA, piv, rank, info = LAPACK.pstrf!(uplochar, A.data, tol) - return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, uplochar, piv, rank, tol, info) + ::Type{Val{true}}; tol = 0.0) + AA, piv, rank, info = LAPACK.pstrf!(A.uplo, A.data, tol) + return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, A.uplo, piv, rank, tol, info) end ### Non BLAS/LAPACK element types (generic). Since generic fallback for pivoted Cholesky ### is not implemented yet we throw an error -cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, ::Type{Val{true}}; +cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = throw(ArgumentError("generic pivoted Cholesky factorization is not implemented yet")) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the @@ -235,26 +244,25 @@ e.g. for integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo, Val{true}; tol = tol) + return cholfact!(Hermitian(A, uplo), Val{true}; tol = tol) end - - # cholfact. Non-destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) +cholfact(A::Hermitian, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, uplo::Symbol, Val{false}) -> Cholesky + cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky Compute the Cholesky factorization of a dense symmetric positive definite matrix `A` -and return a `Cholesky` factorization. -The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `Cholesky` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` +`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, +the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `Cholesky` objects: `size`, `\\`, `inv`, `det`. @@ -262,36 +270,35 @@ A `PosDefException` exception is thrown in case the matrix is not positive defin """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo, Val{false}) + return cholfact(Hermitian(A, uplo), Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact(A::Hermitian, uplo::Symbol = :U) = cholfact(A, uplo, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol = :U) = - cholfact(A, uplo, Val{false}) +cholfact(A::Hermitian) = cholfact(A, Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = cholfact(A, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo) + return cholfact(Hermitian(A, uplo)) end ## With pivoting -cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) = +cholfact(A::Hermitian, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - uplo, Val{true}; tol = tol) -cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, - ::Type{Val{true}}; tol = 0.0) = + Val{true}; tol = tol) +cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - uplo, Val{true}; tol = tol) + Val{true}; tol = tol) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix `A` -and return a `CholeskyPivoted` factorization. -The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `CholeskyPivoted` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` +`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, +the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `PivotedCholesky` objects: `size`, `\\`, `inv`, `det`, and `rank`. @@ -300,7 +307,7 @@ For negative values, the tolerance is the machine precision. """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo, Val{true}; tol = tol) + return cholfact(Hermitian(A, uplo), Val{true}; tol = tol) end ## Number diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 0a06293817286..ff1a4808cd8d5 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -308,17 +308,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the square root of a non-negative number ``x``\ . -.. function:: cholfact(A, uplo::Symbol, Val{false}) -> Cholesky +.. function:: cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky .. Docstring generated from Julia source - Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. + Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. -.. function:: cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source - Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. + Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. .. function:: cholfact(A; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor @@ -344,13 +344,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. -.. function:: cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky +.. function:: cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky .. Docstring generated from Julia source The same as ``cholfact``\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. An ``InexactError`` exception is thrown if the factorisation produces a number not representable by the element type of ``A``\ , e.g. for integer types. -.. function:: cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index 00e4696b4f12b..1c61f12060e5f 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -78,7 +78,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test tril(lapd.factors) ≈ capd[:L] if eltya <: Real capds = cholfact(apds) - lapds = cholfact(apds, :L) + lapds = cholfact(Symmetric(apds.data, :L)) ls = lapds[:L] @test ls*ls' ≈ apd @test triu(capds.factors) ≈ lapds[:U] From b17b8bba8400277450bfff1f84a2e72effb0f308 Mon Sep 17 00:00:00 2001 From: Art <wildart@users.noreply.github.com> Date: Thu, 11 Aug 2016 22:08:02 -0400 Subject: [PATCH 0906/1117] disable openssl in curl build when build agains mbedtls [fix #17910] (#17955) --- deps/curl.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/curl.mk b/deps/curl.mk index 12295cde6ad08..0e228d956feaa 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -16,7 +16,7 @@ $(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VE $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure mkdir -p $(dir $@) cd $(dir $@) && \ - $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --with-mbedtls=$(build_shlibdir)/.. CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --without-ssl --with-mbedtls=$(build_prefix) CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" touch -c $@ $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) From 5d2e42b2531226c8fada82e699922f95dbce8165 Mon Sep 17 00:00:00 2001 From: xorJane <xorJane@users.noreply.github.com> Date: Thu, 11 Aug 2016 19:09:00 -0700 Subject: [PATCH 0907/1117] Move documentation inline for error types declared in base/base.jl. (#17959) --- base/base.jl | 59 ++++++++++++++++++++++++++++++- base/docs/helpdb/Base.jl | 75 ---------------------------------------- 2 files changed, 58 insertions(+), 76 deletions(-) diff --git a/base/base.jl b/base/base.jl index 910db8d8a6fec..9fa8d77cb4a15 100644 --- a/base/base.jl +++ b/base/base.jl @@ -1,5 +1,10 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +""" + SystemError(prefix::AbstractString, [errno::Int32]) + +A system call failed with an error code (in the `errno` global variable). +""" type SystemError <: Exception prefix::AbstractString errnum::Int32 @@ -9,10 +14,22 @@ type SystemError <: Exception SystemError(p::AbstractString) = new(p, Libc.errno()) end +""" + ParseError(msg) + +The expression passed to the `parse` function could not be interpreted as a valid Julia +expression. +""" type ParseError <: Exception msg::AbstractString end +""" + ArgumentError(msg) + +The parameters to a function call do not match a valid signature. Argument `msg` is a +descriptive error string. +""" type ArgumentError <: Exception msg::AbstractString end @@ -21,25 +38,53 @@ end # var::Symbol #end +""" + KeyError(key) + +An indexing operation into an `Associative` (`Dict`) or `Set` like object tried to access or +delete a non-existent element. +""" type KeyError <: Exception key end +""" + MethodError(f, args) + +A method with the required type signature does not exist in the given generic function. +Alternatively, there is no unique most-specific method. +""" type MethodError <: Exception f args end +""" + EOFError() + +No more data was available to read from a file or stream. +""" type EOFError <: Exception end +""" + DimensionMismatch([msg]) + +The objects called do not have matching dimensionality. Optional argument `msg` is a +descriptive error string. +""" type DimensionMismatch <: Exception msg::AbstractString end DimensionMismatch() = DimensionMismatch("") +""" + AssertionError([msg]) + +The asserted condition did not evaluate to `true`. +Optional argument `msg` is a descriptive error string. +""" type AssertionError <: Exception msg::AbstractString - AssertionError() = new("") AssertionError(msg) = new(msg) end @@ -48,12 +93,24 @@ end #Subtypes should put the exception in an 'error' field abstract WrappedException <: Exception +""" + LoadError(file::AbstractString, line::Int, error) + +An error occurred while `include`ing, `require`ing, or `using` a file. The error specifics +should be available in the `.error` field. +""" type LoadError <: WrappedException file::AbstractString line::Int error end +""" + InitError(mod::Symbol, error) + +An error occurred when running a module's `__init__` function. The actual error thrown is +available in the `.error` field. +""" type InitError <: WrappedException mod::Symbol error diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index b7748cff42198..9c6b91207fa98 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2145,13 +2145,6 @@ Remove each element of `iterable` from set `s` in-place. """ setdiff! -""" - EOFError() - -No more data was available to read from a file or stream. -""" -EOFError - """ isascii(c::Union{Char,AbstractString}) -> Bool @@ -2953,12 +2946,6 @@ handle properly. """ OutOfMemoryError -""" - SystemError(prefix::AbstractString, [errno::Int32]) - -A system call failed with an error code (in the `errno` global variable). -""" -SystemError """ binomial(n,k) @@ -2997,14 +2984,6 @@ detailed system information is shown as well. """ versioninfo -""" - DimensionMismatch([msg]) - -The objects called do not have matching dimensionality. Optional argument `msg` is a -descriptive error string. -""" -DimensionMismatch - """ sort!(v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) @@ -3575,13 +3554,6 @@ cannot be used with empty collections (see `reduce(op, itr)`). """ foldr(op, itr) -""" - ParseError(msg) - -The expression passed to the `parse` function could not be interpreted as a valid Julia expression. -""" -ParseError - """ delete!(collection, key) @@ -4434,22 +4406,6 @@ Get the file name part of a path. """ basename -""" - ArgumentError(msg) - -The parameters to a function call do not match a valid signature. Argument `msg` is a -descriptive error string. -""" -ArgumentError - -""" - KeyError(key) - -An indexing operation into an `Associative` (`Dict`) or `Set` like object tried to access or -delete a non-existent element. -""" -KeyError - """ isdiag(A) -> Bool @@ -5306,13 +5262,6 @@ the array length. If the array length is excessive, the excess portion is filled """ digits! -""" - MethodError(f, args) - -A method with the required type signature does not exist in the given generic function. Alternatively, there is no unique most-specific method. -""" -MethodError - """ cat(dims, A...) @@ -6605,22 +6554,6 @@ Compute the phase angle in radians of a complex number `z`. """ angle -""" - LoadError(file::AbstractString, line::Int, error) - -An error occurred while `include`ing, `require`ing, or `using` a file. The error specifics -should be available in the `.error` field. -""" -LoadError - -""" - InitError(mod::Symbol, error) - -An error occurred when running a module's `__init__` function. The actual error thrown is -available in the `.error` field. -""" -InitError - """ copy!(dest, src) @@ -7497,14 +7430,6 @@ Integer division was attempted with a denominator value of 0. """ DivideError -""" - AssertionError([msg]) - -The asserted condition did not evaluate to `true`. -Optional argument `msg` is a descriptive error string. -""" -AssertionError - """ Ac_ldiv_Bc(A, B) From 49271f43670ec42be89c8853bdf8dc8c4641766b Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer+github@college.harvard.edu> Date: Fri, 12 Aug 2016 02:12:35 +0000 Subject: [PATCH 0908/1117] Fix typo in libgit2 credentials callback (#17975) --- base/libgit2/callbacks.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 91a0b8d4ca42b..fd2580da27ee7 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -121,8 +121,8 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, end passdef end - (creds.user != username) || (creds.pass != userpass) || - (creds.prvkey != privatekey) || (creds.pubkey != publickey) && reset!(creds) + ((creds.user != username) || (creds.pass != passphrase) || + (creds.prvkey != privatekey) || (creds.pubkey != publickey)) && reset!(creds) creds.user = username # save credentials creds.prvkey = privatekey # save credentials From 2b2d4a4ff7d291997cd1770274a1c9a1149990ec Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 11 Aug 2016 23:17:25 +0800 Subject: [PATCH 0909/1117] Fix MSVC compilation Also update out-of-date comment about codegen steps --- src/codegen.cpp | 16 +++++++--------- src/llvm-gcroot.cpp | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 259a66fbacdee..69c8ea00f1586 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4315,9 +4315,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func // step 7. set up GC frame allocate_gc_frame(b0, &ctx); - // step 8. allocate space for exception handler contexts - - // step 9. allocate local variables slots + // step 8. allocate local variables slots // must be in the first basic block for the llvm mem2reg pass to work // get pointers for locals stored in the gc frame array (argTemp) @@ -4381,7 +4379,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func maybe_alloc_arrayvar(i, &ctx); } - // step 10. move args into local variables + // step 9. move args into local variables Function::arg_iterator AI = f->arg_begin(); if (ctx.sret) AI++; // skip sret slot @@ -4484,7 +4482,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } } - // step 11. allocate rest argument if necessary + // step 10. allocate rest argument if necessary if (va && ctx.vaSlot != -1) { jl_varinfo_t &vi = ctx.slots[ctx.vaSlot]; if (!vi.escapes) { @@ -4522,7 +4520,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } } - // step 12. associate labels with basic blocks to resolve forward jumps + // step 11. associate labels with basic blocks to resolve forward jumps BasicBlock *prev=NULL; for(i=0; i < stmtslen; i++) { jl_value_t *ex = jl_array_ptr_ref(stmts,i); @@ -4542,7 +4540,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } } - // step 13. compile body statements + // step 12. compile body statements if (ctx.debug_enabled) // set initial line number builder.SetCurrentDebugLocation(topdebugloc); @@ -4720,7 +4718,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } } - // step 14, Apply LLVM level inlining + // step 13, Apply LLVM level inlining for(std::vector<CallInst*>::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) { Function *inlinef = (*it)->getCalledFunction(); assert(inlinef->getParent()); @@ -4735,7 +4733,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } } - // step 15. Perform any delayed instantiations + // step 14. Perform any delayed instantiations if (ctx.debug_enabled) { ctx.dbuilder->finalize(); } diff --git a/src/llvm-gcroot.cpp b/src/llvm-gcroot.cpp index cf1a545e20dc1..3802d94d49f5f 100644 --- a/src/llvm-gcroot.cpp +++ b/src/llvm-gcroot.cpp @@ -21,6 +21,8 @@ #include <vector> #include <queue> #include <set> +#include <functional> +#include <utility> #include "julia.h" @@ -191,6 +193,28 @@ struct HandlerData { std::unique_ptr<std::vector<CallInst*>> parent_vec; CallInst *parent{nullptr}; bool processed{false}; +#ifdef _COMPILER_MICROSOFT_ + // MSVC 2013 seems to call the copy version instead of the move verion + // without this, which triggers compilation error since `std::unique_ptr` + // is not copyable. + HandlerData() = default; + HandlerData(HandlerData &&other) + : HandlerData() + { + operator=(std::move(other)); + } + HandlerData(const HandlerData&) = delete; + HandlerData &operator=(HandlerData &&other) + { + std::swap(leaves, other.leaves); + std::swap(nested, other.nested); + std::swap(parent_vec, other.parent_vec); + std::swap(parent, other.parent); + std::swap(processed, other.processed); + return *this; + } + HandlerData &operator=(const HandlerData&) = delete; +#endif }; void JuliaGCAllocator::lowerHandlers() From 9b2f3217e2b01f8d048da0c76f241a762ab98b1a Mon Sep 17 00:00:00 2001 From: Stefan Karpinski <stefan@karpinski.org> Date: Fri, 12 Aug 2016 00:43:28 -0400 Subject: [PATCH 0910/1117] more manual eta reduction (companion to 9cead4f7874a2aa9b1a8d6adc5) (#17982) --- test/arrayops.jl | 4 ++-- test/functional.jl | 4 ++-- test/offsetarray.jl | 4 ++-- test/parallel_exec.jl | 2 +- test/reduce.jl | 4 ++-- test/reflection.jl | 2 +- test/runtests.jl | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/arrayops.jl b/test/arrayops.jl index 5280484c95faf..2df9c599ced99 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1556,8 +1556,8 @@ copy!(S, A) @test cumsum(A, 1) == cumsum(B, 1) == cumsum(S, 1) @test cumsum(A, 2) == cumsum(B, 2) == cumsum(S, 2) -@test mapslices(v->sort(v), A, 1) == mapslices(v->sort(v), B, 1) == mapslices(v->sort(v), S, 1) -@test mapslices(v->sort(v), A, 2) == mapslices(v->sort(v), B, 2) == mapslices(v->sort(v), S, 2) +@test mapslices(sort, A, 1) == mapslices(sort, B, 1) == mapslices(sort, S, 1) +@test mapslices(sort, A, 2) == mapslices(sort, B, 2) == mapslices(sort, S, 2) @test flipdim(A, 1) == flipdim(B, 1) == flipdim(S, 2) @test flipdim(A, 2) == flipdim(B, 2) == flipdim(S, 2) diff --git a/test/functional.jl b/test/functional.jl index 5ed8f3ce51db8..1c9a68d6430ab 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -10,8 +10,8 @@ let a = [1.0, 2.0] @test isequal(a, sin([1.0, 2.0])) end # map -- ranges.jl -@test isequal(map(i->sqrt(i), 1:5), [sqrt(i) for i in 1:5]) -@test isequal(map(i->sqrt(i), 2:6), [sqrt(i) for i in 2:6]) +@test isequal(map(sqrt, 1:5), [sqrt(i) for i in 1:5]) +@test isequal(map(sqrt, 2:6), [sqrt(i) for i in 2:6]) # map on ranges should evaluate first value only once (#4453) let io=IOBuffer(3) diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 2136e4dd580e0..7731895896f91 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -308,8 +308,8 @@ v = OffsetArray(rand(8), (-2,)) @test sort(A, 1) == OffsetArray(sort(parent(A), 1), A.offsets) @test sort(A, 2) == OffsetArray(sort(parent(A), 2), A.offsets) -@test mapslices(v->sort(v), A, 1) == OffsetArray(mapslices(v->sort(v), parent(A), 1), A.offsets) -@test mapslices(v->sort(v), A, 2) == OffsetArray(mapslices(v->sort(v), parent(A), 2), A.offsets) +@test mapslices(sort, A, 1) == OffsetArray(mapslices(sort, parent(A), 1), A.offsets) +@test mapslices(sort, A, 2) == OffsetArray(mapslices(sort, parent(A), 2), A.offsets) @test rotl90(A) == OffsetArray(rotl90(parent(A)), A.offsets[[2,1]]) @test rotr90(A) == OffsetArray(rotr90(parent(A)), A.offsets[[2,1]]) diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index a5be3fa85915d..37f58388b55c0 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -991,7 +991,7 @@ end # issue #15406 v15406 = remotecall_wait(() -> 1, id_other) fetch(v15406) -remotecall_wait(t -> fetch(t), id_other, v15406) +remotecall_wait(fetch, id_other, v15406) # Test various forms of remotecall* invocations diff --git a/test/reduce.jl b/test/reduce.jl index 4edf431e14966..a86b602dd2b86 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -160,8 +160,8 @@ prod2(itr) = invoke(prod, Tuple{Any}, itr) @test maxabs([1, -2, 3, -4]) == 4 @test minabs([-1, 2, -3, 4]) == 1 -@test maximum(x->abs2(x), 3:7) == 49 -@test minimum(x->abs2(x), 3:7) == 9 +@test maximum(abs2, 3:7) == 49 +@test minimum(abs2, 3:7) == 9 @test maximum(Int16[1]) === Int16(1) @test maximum(collect(Int16(1):Int16(100))) === Int16(100) diff --git a/test/reflection.jl b/test/reflection.jl index cd8e7d702245e..35bd8ddc831c3 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -43,7 +43,7 @@ test_code_reflections(test_bin_reflection, code_native) mktemp() do f, io OLDSTDOUT = STDOUT redirect_stdout(io) - @test try @code_native map(y->abs(y), rand(3)); true; catch false; end + @test try @code_native map(abs, rand(3)); true; catch false; end redirect_stdout(OLDSTDOUT) end diff --git a/test/runtests.jl b/test/runtests.jl index 4d91b652b1aac..90505c41d2400 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -41,7 +41,7 @@ cd(dirname(@__FILE__)) do test = shift!(tests) local resp try - resp = remotecall_fetch(t -> runtests(t), p, test) + resp = remotecall_fetch(runtests, p, test) catch e resp = e end From 7a463891002202c88028fac1f502b4c3cee73639 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Fri, 12 Aug 2016 00:40:50 -0700 Subject: [PATCH 0911/1117] Fix atahn to atanh (#17977) * Fix atahn to atanh * More tests --- base/special/trig.jl | 2 +- test/math.jl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/base/special/trig.jl b/base/special/trig.jl index 2ad94211c590f..4a4e0b94170a7 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -321,7 +321,7 @@ end for (tfa, tfainv, hfa, hfainv, fn) in ((:asec, :acos, :asech, :acosh, "secant"), (:acsc, :asin, :acsch, :asinh, "cosecant"), - (:acot, :atan, :acoth, :atahn, "cotangent")) + (:acot, :atan, :acoth, :atanh, "cotangent")) tname = string(tfa) hname = string(hfa) @eval begin diff --git a/test/math.jl b/test/math.jl index 2394bd460e4ea..26a744b3211f0 100644 --- a/test/math.jl +++ b/test/math.jl @@ -192,6 +192,16 @@ end @test exp2(Float16(2.)) ≈ exp2(2.) @test log(e) == 1 +#test abstractarray trig fxns +TAA = rand(2,2) +TAA = (TAA + TAA.')/2. +STAA = Symmetric(TAA) +@test full(atanh(STAA)) == atanh(TAA) +@test full(asinh(STAA)) == asinh(TAA) +@test full(acosh(STAA+Symmetric(ones(TAA)))) == acosh(TAA+ones(TAA)) +@test full(acsch(STAA+Symmetric(ones(TAA)))) == acsch(TAA+ones(TAA)) +@test full(acoth(STAA+Symmetric(ones(TAA)))) == acoth(TAA+ones(TAA)) + # check exp2(::Integer) matches exp2(::Float) for ii in -2048:2048 expected = exp2(float(ii)) From 47d192efe090af24133baba8d949d932df2fffa2 Mon Sep 17 00:00:00 2001 From: ranjanan <benditlikeranjan@gmail.com> Date: Fri, 12 Aug 2016 19:23:06 +0530 Subject: [PATCH 0912/1117] Write up test for #15592 parse("[1] [2]") should throw a ParseError --- test/parse.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/parse.jl b/test/parse.jl index f33e55273118e..3c625c192dc8d 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -662,3 +662,8 @@ end # issue #17701 @test expand(:(i==3 && i+=1)) == Expr(:error, "invalid assignment location \"==(i,3)&&i\"") + +# PR #15592 +let str = "[1] [2]" + @test_throws ParseError parse(str) +end From c850e6f90c6167ff1bed7bce244bb4c5b2be9e0a Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 13 Aug 2016 04:32:31 -0500 Subject: [PATCH 0913/1117] Fix missing type parameter in _array_for This fixes a performance regression in the lucompletepiv benchmark triggered by generalizing _array_for to work with indices. This call, first introduced in b363cc70ee6f09bb4472944f8690283793c6052a, likely always triggered dynamic method dispatch. The generalization to indices seemed to make that worse. --- base/array.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index 4eba2f531ad78..8cbdbdeebff37 100644 --- a/base/array.jl +++ b/base/array.jl @@ -288,8 +288,8 @@ else _default_eltype(itr::ANY) = Any end -_array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = similar(Array{T}, indices(itr)) +_array_for{T}(::Type{T}, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) +_array_for{T}(::Type{T}, itr, ::HasShape) = similar(Array{T}, indices(itr)) function collect(itr::Generator) isz = iteratorsize(itr.iter) From 631708f954d19a0642a4f9d555bc2722536ee2a8 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 13 Aug 2016 06:19:17 -0500 Subject: [PATCH 0914/1117] Provide a more useful error message for calling map on a Dict with a non-Pair return type --- base/dict.jl | 3 +++ test/dict.jl | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/base/dict.jl b/base/dict.jl index d66b93377c7f8..69c41ece8d8cc 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -848,3 +848,6 @@ function similar(t::ImmutableDict) end return t end + +_similar_for{P<:Pair}(c::Dict, ::Type{P}, itr, isz) = similar(c, P) +_similar_for(c::Associative, T, itr, isz) = throw(ArgumentError("for Associatives, similar requires an element type of Pair;\n if calling map, consider a comprehension instead")) diff --git a/test/dict.jl b/test/dict.jl index 45dcbbe998c8d..0af6b4d3807d7 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -388,6 +388,11 @@ d = Dict('a'=>1, 'b'=>1, 'c'=> 3) @test [d[k] for k in keys(d)] == [d[k] for k in eachindex(d)] == [v for (k, v) in d] == [d[x[1]] for (i, x) in enumerate(d)] +# generators, similar +d = Dict(:a=>"a") +@test @inferred(map(identity, d)) == d +@test @inferred(map(p->p.first=>p.second[1], d)) == Dict(:a=>'a') +@test_throws ArgumentError map(p->p.second, d) # Issue 12451 @test_throws ArgumentError Dict(0) From f3292799c6a47f612a7b1004ec354a51b9792a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Sat, 13 Aug 2016 13:17:43 +0200 Subject: [PATCH 0915/1117] Move env.jl test to separate test file --- test/choosetests.jl | 2 +- test/env.jl | 32 ++++++++++++++++++++++++++++++++ test/sysinfo.jl | 31 ------------------------------- 3 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 test/env.jl diff --git a/test/choosetests.jl b/test/choosetests.jl index d02ed1dcf3cf0..f15e61a139d6c 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -26,7 +26,7 @@ function choosetests(choices = []) "priorityqueue", "file", "read", "mmap", "version", "resolve", "pollfd", "mpfr", "broadcast", "complex", "socket", "floatapprox", "datafmt", "reflection", "regex", "float16", - "combinatorics", "sysinfo", "rounding", "ranges", "mod2pi", + "combinatorics", "sysinfo", "env", "rounding", "ranges", "mod2pi", "euler", "show", "lineedit", "replcompletions", "repl", "replutil", "sets", "test", "goto", "llvmcall", "grisu", "nullable", "meta", "stacktraces", "profile", "libgit2", "docs", diff --git a/test/env.jl b/test/env.jl new file mode 100644 index 0000000000000..592c378cf65ee --- /dev/null +++ b/test/env.jl @@ -0,0 +1,32 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +@test !("f=a=k=e=n=a=m=e" ∈ keys(ENV)) + +# issue #10994 +@test_throws ArgumentError ENV["bad\0name"] = "ok" +@test_throws ArgumentError ENV["okname"] = "bad\0val" +@test_throws ArgumentError Sys.set_process_title("bad\0title") + +withenv("bad"=>"dog") do + @test_throws ArgumentError ENV["bad\0cat"] +end + +# issue #11170 +withenv("TEST"=>"nonempty") do + @test ENV["TEST"] == "nonempty" +end +withenv("TEST"=>"") do + @test ENV["TEST"] == "" +end + +let c = collect(ENV) + @test isa(c, Vector) + @test length(ENV) == length(c) + @test isempty(ENV) || first(ENV) in c +end + +# test for non-existent keys +key = randstring(25) +@test !haskey(ENV,key) +@test_throws KeyError ENV[key] +@test get(ENV,key,"default") == "default" diff --git a/test/sysinfo.jl b/test/sysinfo.jl index 645c34efce765..261935ac715f1 100644 --- a/test/sysinfo.jl +++ b/test/sysinfo.jl @@ -6,34 +6,3 @@ sprint(Base.Sys.cpu_summary) @test Base.Sys.uptime() > 0 Base.Sys.loadavg() - -@test !("f=a=k=e=n=a=m=e" ∈ keys(ENV)) - -# issue #10994 -@test_throws ArgumentError ENV["bad\0name"] = "ok" -@test_throws ArgumentError ENV["okname"] = "bad\0val" -@test_throws ArgumentError Sys.set_process_title("bad\0title") - -withenv("bad"=>"dog") do - @test_throws ArgumentError ENV["bad\0cat"] -end - -# issue #11170 -withenv("TEST"=>"nonempty") do - @test ENV["TEST"] == "nonempty" -end -withenv("TEST"=>"") do - @test ENV["TEST"] == "" -end - -let c = collect(ENV) - @test isa(c, Vector) - @test length(ENV) == length(c) - @test isempty(ENV) || first(ENV) in c -end - -# test for non-existent keys -key = randstring(25) -@test !haskey(ENV,key) -@test_throws KeyError ENV[key] -@test get(ENV,key,"default") == "default" From 395c137416772dca20ea2e9bafa083d9c24e7c67 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 13 Aug 2016 08:00:04 -0500 Subject: [PATCH 0916/1117] Silence unneeded output in backtrace test --- test/backtrace.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/backtrace.jl b/test/backtrace.jl index bdc93b000e0d9..5a73200d79d07 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -121,7 +121,8 @@ end @test_broken hasbt2 function btmacro() - @time backtrace() + ret = @timed backtrace() + ret[1] end lkup = map(StackTraces.lookup, btmacro()) hasme = hasbtmacro = false From 9e04023449311cf4a3866529f66354d8e9accbd4 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 13 Aug 2016 08:03:55 -0500 Subject: [PATCH 0917/1117] Force inlining on indices(A, d) --- base/abstractarray.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index a4b6cbe6bda0b..452a7f1643d61 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -51,7 +51,11 @@ size{N}(x, d1::Integer, d2::Integer, dx::Vararg{Integer, N}) = (size(x, d1), siz Returns the valid range of indices for array `A` along dimension `d`. """ -indices{T,N}(A::AbstractArray{T,N}, d) = d <= N ? indices(A)[d] : OneTo(1) +function indices{T,N}(A::AbstractArray{T,N}, d) + @_inline_meta + d <= N ? indices(A)[d] : OneTo(1) +end + """ indices(A) From 223f311b40767e29012e84a61f5a73769a9ee6f9 Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Thu, 4 Aug 2016 23:35:28 +0200 Subject: [PATCH 0918/1117] Fix `@threadcall` argument passing. Fixes #17819. Also document not being allowed to call back into Julia. --- base/threadcall.jl | 1 + src/ccalltest.c | 10 ++++++++++ test/ccall.jl | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/base/threadcall.jl b/base/threadcall.jl index be3e99f324cb8..da877d9707109 100644 --- a/base/threadcall.jl +++ b/base/threadcall.jl @@ -75,6 +75,7 @@ function do_threadcall(wrapper::Function, rettype::Type, argtypes::Vector, argva y = cconvert(T, x) push!(roots, y) unsafe_store!(convert(Ptr{T}, ptr), unsafe_convert(T, y)) + ptr += sizeof(T) end # create return buffer diff --git a/src/ccalltest.c b/src/ccalltest.c index e01aafa300fb2..ccd7e1cd57fde 100644 --- a/src/ccalltest.c +++ b/src/ccalltest.c @@ -21,6 +21,7 @@ int verbose = 1; int c_int = 0; + ////////////////////////////////// // Test for proper argument register truncation @@ -520,6 +521,7 @@ JL_DLLEXPORT void finalizer_cptr(void* v) set_c_int(-1); } + ////////////////////////////////// // Turn off verbose for automated tests, leave on for debugging @@ -527,6 +529,10 @@ JL_DLLEXPORT void set_verbose(int level) { verbose = level; } + +////////////////////////////////// +// Other tests + JL_DLLEXPORT void *test_echo_p(void *p) { return p; } @@ -723,3 +729,7 @@ JL_DLLEXPORT float32x4_t test_ppc64_vec2(int64_t d1, float32x4_t a, float32x4_t } #endif + +JL_DLLEXPORT int threadcall_args(int a, int b) { + return a + b; +} diff --git a/test/ccall.jl b/test/ccall.jl index 7f0792a87fdd5..56a6b1bf3f5be 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -589,6 +589,10 @@ threadcall_test_func(x) = @test threadcall_test_func(3) == 1 @test threadcall_test_func(259) == 1 +# issue 17819 +# NOTE: can't use cfunction or reuse ccalltest Struct methods, as those call into the runtime +@test @threadcall((:threadcall_args, libccalltest), Cint, (Cint, Cint), 1, 2) == 3 + let n=3 tids = Culong[] @sync for i in 1:10^n From 074c5247e98bf151d48ba7a1dbdc4725384c99fd Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sat, 13 Aug 2016 12:10:46 -0400 Subject: [PATCH 0919/1117] stop curl from rebuilding incessantly --- deps/curl.mk | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deps/curl.mk b/deps/curl.mk index 0e228d956feaa..b8bf9ccbd8fec 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -1,6 +1,6 @@ ## CURL ## -CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/.libs/libcurl.$(SHLIB_EXT) +CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/lib/.libs/libcurl.$(SHLIB_EXT) CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT) ifneq ($(OS),WINNT) @@ -9,18 +9,22 @@ endif $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2 + $(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET) $(JLCHECKSUM) $< cd $(dir $<) && $(TAR) jxf $(notdir $<) touch -c $@ + $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --without-ssl --with-mbedtls=$(build_prefix) CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" touch -c $@ + $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) touch -c $@ + $(BUILDDIR)/curl-$(CURL_VER)/checked: $(CURL_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) ifneq ($(OS),WINNT) @@ -28,6 +32,7 @@ ifneq ($(OS),WINNT) endif endif echo 1 > $@ + $(CURL_OBJ_TARGET): $(CURL_SRC_TARGET) $(call make-install,curl-$(CURL_VER),$(LIBTOOL_CCLD)) $(INSTALL_NAME_CMD)libcurl.$(SHLIB_EXT) $@ From c731d5535f0f566f9019871d14f3440278fdf9a5 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 11 Aug 2016 04:52:25 -0700 Subject: [PATCH 0920/1117] Add cygpath_w call so `make -C test/perf` works in cygwin-cross --- test/perf/Makefile | 6 +----- test/perf/shootout/fasta.jl | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/perf/Makefile b/test/perf/Makefile index 0ff723aa70f7b..adc07a2425e64 100644 --- a/test/perf/Makefile +++ b/test/perf/Makefile @@ -9,11 +9,7 @@ all: micro kernel cat shootout blas lapack simd sort spell sparse micro kernel cat shootout blas lapack simd sort spell sparse: @$(MAKE) $(QUIET_MAKE) -C $(SRCDIR)/shootout -ifneq ($(OS),WINNT) - @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/$@/perf.jl | perl -nle '@_=split/,/; printf "%-18s %8.3f %8.3f %8.3f %8.3f\n", $$_[1], $$_[2], $$_[3], $$_[4], $$_[5]' -else - @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/$@/perf.jl 2> /dev/null -endif + @$(call spawn,$(JULIA_EXECUTABLE)) $(call cygpath_w,$(SRCDIR)/$@/perf.jl) | perl -nle '@_=split/,/; printf "%-18s %8.3f %8.3f %8.3f %8.3f\n", $$_[1], $$_[2], $$_[3], $$_[4], $$_[5]' codespeed: @$(MAKE) $(QUIET_MAKE) -C $(SRCDIR)/shootout diff --git a/test/perf/shootout/fasta.jl b/test/perf/shootout/fasta.jl index 13bbec2c22875..3746a15e7e8ae 100644 --- a/test/perf/shootout/fasta.jl +++ b/test/perf/shootout/fasta.jl @@ -55,7 +55,7 @@ end rng_state = 42.0 function fasta(n=25000000) - repeat_fasta(alu, 2n) - random_fasta(iub1, iub2, 3n) - random_fasta(homosapiens1, homosapiens2, 5n) + repeat_fasta(alu, 2n) + random_fasta(iub1, iub2, 3n) + random_fasta(homosapiens1, homosapiens2, 5n) end From f9be671aa9aed614e9db87d8396c8053a6189b5f Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 11 Aug 2016 11:38:24 -0700 Subject: [PATCH 0921/1117] fix deprecated syntax "global rng_state::Float64" --- test/perf/shootout/fasta.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/shootout/fasta.jl b/test/perf/shootout/fasta.jl index 3746a15e7e8ae..fe8129d1d74c7 100644 --- a/test/perf/shootout/fasta.jl +++ b/test/perf/shootout/fasta.jl @@ -22,7 +22,7 @@ const IA = 3877.0 const IC = 29573.0 function gen_random() - global rng_state::Float64 = ((rng_state::Float64 * IA + IC) % IM) / IM + global rng_state = ((rng_state::Float64 * IA + IC) % IM) / IM end function repeat_fasta(src, n) k = length(src) From 0de84af91113914173b6ee29fceeaed286d636f3 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 13 Aug 2016 18:46:35 +0800 Subject: [PATCH 0922/1117] A trick to fix continue/break/return handling in for loop testset. Fix #17908 --- base/test.jl | 22 ++++++++++++++--- test/test.jl | 69 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/base/test.jl b/base/test.jl index 7d23115354bbd..0b2009ba72465 100644 --- a/base/test.jl +++ b/base/test.jl @@ -720,6 +720,13 @@ function testset_forloop(args, testloop) # wrapped in the outer loop provided by the user tests = testloop.args[2] blk = quote + # Trick to handle `break` and `continue` in the test code before + # they can be handled properly by `finally` lowering. + if !first_iteration + pop_testset() + push!(arr, finish(ts)) + end + first_iteration = false ts = $(testsettype)($desc; $options...) push_testset(ts) try @@ -729,13 +736,20 @@ function testset_forloop(args, testloop) # error in this test set record(ts, Error(:nontest_error, :(), err, catch_backtrace())) end - pop_testset() - finish(ts) end quote arr = Array{Any,1}(0) - $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), - :(push!(arr, $blk)))) + first_iteration = true + local ts + try + $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), blk)) + finally + # Handle `return` in test body + if !first_iteration + pop_testset() + push!(arr, finish(ts)) + end + end arr end end diff --git a/test/test.jl b/test/test.jl index d168a38eeb859..7e2e8b70823af 100644 --- a/test/test.jl +++ b/test/test.jl @@ -183,6 +183,50 @@ end @test typeof(tss[1]) == Base.Test.DefaultTestSet @test typeof(tss[1].results[1]) == Base.Test.Pass +# Issue #17908 (return) +testset_depth17908 = Test.get_testset_depth() +@testset for i in 1:3 + i > 1 && return + @test i == 1 +end +# The return aborts the control flow so the expression above doesn't return a +# value. The only thing we can test is whether the testset is properly popped. +# Do not use `@test` since the issue this is testing will swallow the error. +@assert testset_depth17908 == Test.get_testset_depth() + +# Issue #17462 and Issue #17908 (break, continue) +testset_depth17462 = Test.get_testset_depth() +counter_17462_pre = 0 +counter_17462_post = 0 +tss17462 = @testset for x in [1,2,3,4] + counter_17462_pre += 1 + if x == 1 + @test counter_17462_pre == x + continue + @test false + elseif x == 3 + @test counter_17462_pre == x + break + @test false + elseif x == 4 + @test false + else + @test counter_17462_pre == x + @test x == 2 + @test counter_17462_post == 0 + end + counter_17462_post += 1 +end +# Do not use `@test` since the issue this is testing will swallow the error. +# Same for the `@assert` in the for loop below +@assert testset_depth17462 == Test.get_testset_depth() +@assert length(tss17462) == 3 +for ts17462 in tss17462 + @assert isa(ts17462, Base.Test.DefaultTestSet) +end +@test counter_17462_pre == 3 +@test counter_17462_post == 1 + # now we're done running tests with DefaultTestSet so we can go back to STDOUT redirect_stdout(OLD_STDOUT) @@ -320,28 +364,3 @@ end @test @inferred(inferrable_kwtest(1; y=1)) == 2 @test @inferred(uninferrable_kwtest(1)) == 3 @test_throws ErrorException @inferred(uninferrable_kwtest(1; y=2)) == 2 - -# Issue #17462 -counter_17462_pre = 0 -counter_17462_post = 0 -@testset for x in [1,2,3,4] - counter_17462_pre += 1 - if x == 1 - @test counter_17462_pre == x - continue - @test false - elseif x == 3 - @test counter_17462_pre == x - break - @test false - elseif x == 4 - @test false - else - @test counter_17462_pre == x - @test x == 2 - @test counter_17462_post == 0 - end - counter_17462_post += 1 -end -@test counter_17462_pre == 3 -@test counter_17462_post == 1 From e25106b2a4f63d2c6996664dccc5e385ec5dbe0d Mon Sep 17 00:00:00 2001 From: albap <albapompeo@gmail.com> Date: Fri, 12 Aug 2016 18:45:13 -0400 Subject: [PATCH 0923/1117] Remove nonstandard header sys/sysctl.h when unneeded This header prevents me from building Julia on musl libc. ``` /home/alba/julia/src/sys.c:16:24: fatal error: sys/sysctl.h: No such file or directory compilation terminated. Makefile:94: recipe for target sys.o failed make[1]: *** [sys.o] Error 1 ``` --- src/sys.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/sys.c b/src/sys.c index 0723b3dc60bf1..a312dd20e6a33 100644 --- a/src/sys.c +++ b/src/sys.c @@ -10,19 +10,23 @@ #include <stdlib.h> #include <string.h> #include <assert.h> +#include <errno.h> +#include <signal.h> +#include <fcntl.h> + #ifdef _OS_WINDOWS_ #include <psapi.h> #else +#include <unistd.h> +#if !defined(_SC_NPROCESSORS_ONLN) || defined(_OS_FREEBSD_) || defined(_OS_DARWIN_) +// try secondary location for _SC_NPROCESSORS_ONLN, or for HW_AVAILCPU on BSDs #include <sys/sysctl.h> +#endif #include <sys/wait.h> #include <sys/ptrace.h> -#include <unistd.h> #include <sys/mman.h> #include <dlfcn.h> #endif -#include <errno.h> -#include <signal.h> -#include <fcntl.h> #ifndef _OS_WINDOWS_ // for getrusage @@ -355,7 +359,10 @@ JL_DLLEXPORT int jl_cpu_cores(void) } return count; #elif defined(_SC_NPROCESSORS_ONLN) - return sysconf(_SC_NPROCESSORS_ONLN); + long count = sysconf(_SC_NPROCESSORS_ONLN); + if (count < 1) + return 1; + return count; #elif defined(_OS_WINDOWS_) //Try to get WIN7 API method GAPC gapc = (GAPC) jl_dlsym_e( @@ -372,6 +379,7 @@ JL_DLLEXPORT int jl_cpu_cores(void) return info.dwNumberOfProcessors; } #else +#warning "cpu core detection not defined for this platform" return 1; #endif } From e7730927cf4460897896b8e926a62116549654a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kenta=20Sato=20=28=E4=BD=90=E8=97=A4=20=E5=BB=BA=E5=A4=AA?= =?UTF-8?q?=29?= <bicycle1885@gmail.com> Date: Sun, 14 Aug 2016 12:30:58 +0900 Subject: [PATCH 0924/1117] fix a typo in Base.split (#18008) * fix a typo in Base.split * fixup! fix a typo in Base.split --- base/docs/helpdb/Base.jl | 2 +- doc/stdlib/strings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 9c6b91207fa98..d510c91a7b7b2 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1226,7 +1226,7 @@ Return an array of substrings by splitting the given string on occurrences of th character delimiters, which may be specified in any of the formats allowed by `search`'s second argument (i.e. a single character, collection of characters, string, or regular expression). If `chars` is omitted, it defaults to the set of all space characters, and -`keep` is taken to be `false`. The two keyword arguments are optional: they are are a +`keep` is taken to be `false`. The two keyword arguments are optional: they are a maximum size for the result and a flag determining whether empty fields should be kept in the result. """ diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index ff929e55114c2..bfb8999981544 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -259,7 +259,7 @@ .. Docstring generated from Julia source - Return an array of substrings by splitting the given string on occurrences of the given character delimiters, which may be specified in any of the formats allowed by ``search``\ 's second argument (i.e. a single character, collection of characters, string, or regular expression). If ``chars`` is omitted, it defaults to the set of all space characters, and ``keep`` is taken to be ``false``\ . The two keyword arguments are optional: they are are a maximum size for the result and a flag determining whether empty fields should be kept in the result. + Return an array of substrings by splitting the given string on occurrences of the given character delimiters, which may be specified in any of the formats allowed by ``search``\ 's second argument (i.e. a single character, collection of characters, string, or regular expression). If ``chars`` is omitted, it defaults to the set of all space characters, and ``keep`` is taken to be ``false``\ . The two keyword arguments are optional: they are a maximum size for the result and a flag determining whether empty fields should be kept in the result. .. function:: rsplit(string, [chars]; limit=0, keep=true) From 5ee8a7a5f0d783e8cd94dab99c1b7aaff8659813 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 13 Aug 2016 21:24:45 -0700 Subject: [PATCH 0925/1117] Fix a row-indexing bug with sparse matrices that have non-Int indices, introduced by #13612 and which has caused `make -C test/perf` to fail for the last 9 months searchsortedfirst does not have methods for general Integer indices --- base/sparse/sparsevector.jl | 4 ++-- test/sparsedir/sparse.jl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index cd002a2574bdb..9a8990d5c47e4 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -426,8 +426,8 @@ function Base.getindex{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, i::Integer, J::Abstract @inbounds for j = 1:nJ col = J[j] rowI = i - ptrA = colptrA[col] - stopA = colptrA[col+1]-1 + ptrA = Int(colptrA[col]) + stopA = Int(colptrA[col+1]-1) if ptrA <= stopA if rowvalA[ptrA] <= rowI ptrA = searchsortedfirst(rowvalA, rowI, ptrA, stopA, Base.Order.Forward) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index e6d4e9767e2b1..4cfc2b248097d 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -1588,3 +1588,8 @@ end # Test temporary fix for issue #16548 in PR #16979. Brittle. Expect to remove with `\` revisions. @test which(\, (SparseMatrixCSC, AbstractVecOrMat)).module == Base.SparseArrays + +# Row indexing a SparseMatrixCSC with non-Int integer type +let A = sparse(UInt32[1,2,3], UInt32[1,2,3], [1.0,2.0,3.0]) + @test A[1,1:3] == A[1,:] == [1,0,0] +end From d67aefac721d5c878e7bcd9a7dc876bc57871ce6 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Sun, 14 Aug 2016 11:54:11 +0200 Subject: [PATCH 0926/1117] specialize on type in getindex for types --- base/array.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base/array.jl b/base/array.jl index 8cbdbdeebff37..da83d1faa0b80 100644 --- a/base/array.jl +++ b/base/array.jl @@ -126,7 +126,7 @@ similar{N}(a::Array, T::Type, dims::Dims{N}) = Array{T,N}(dims) similar{T,N}(a::Array{T}, dims::Dims{N}) = Array{T,N}(dims) # T[x...] constructs Array{T,1} -function getindex(T::Type, vals...) +function getindex{T}(::Type{T}, vals...) a = Array{T,1}(length(vals)) @inbounds for i = 1:length(vals) a[i] = vals[i] @@ -134,10 +134,10 @@ function getindex(T::Type, vals...) return a end -getindex(T::Type) = Array{T,1}(0) -getindex(T::Type, x) = (a = Array{T,1}(1); @inbounds a[1] = x; a) -getindex(T::Type, x, y) = (a = Array{T,1}(2); @inbounds (a[1] = x; a[2] = y); a) -getindex(T::Type, x, y, z) = (a = Array{T,1}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a) +getindex{T}(::Type{T}) = (@_inline_meta; Array{T,1}(0)) +getindex{T}(::Type{T}, x) = (@_inline_meta; a = Array{T,1}(1); @inbounds a[1] = x; a) +getindex{T}(::Type{T}, x, y) = (@_inline_meta; a = Array{T,1}(2); @inbounds (a[1] = x; a[2] = y); a) +getindex{T}(::Type{T}, x, y, z) = (@_inline_meta; a = Array{T,1}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a) function getindex(::Type{Any}, vals::ANY...) a = Array{Any,1}(length(vals)) From c231a39ea3bfd8c82ac2d1053cbc5f3ca53d31e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Sat, 13 Aug 2016 13:20:09 +0200 Subject: [PATCH 0927/1117] Fix #17956 and add test. The bug was introduced in 103db50aa6c67d8e732e8e2bd7f5743bae17e369 --- base/env.jl | 2 +- test/env.jl | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/base/env.jl b/base/env.jl index fc3fe87e78c70..3c77619400b82 100644 --- a/base/env.jl +++ b/base/env.jl @@ -94,7 +94,7 @@ end function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) pos = block[1] blk = block[2] - len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + 1 buf = Array{UInt16}(len) unsafe_copy!(pointer(buf), pos, len) env = transcode(String, buf) diff --git a/test/env.jl b/test/env.jl index 592c378cf65ee..23e8710398cab 100644 --- a/test/env.jl +++ b/test/env.jl @@ -30,3 +30,36 @@ key = randstring(25) @test !haskey(ENV,key) @test_throws KeyError ENV[key] @test get(ENV,key,"default") == "default" + +# Test for #17956 +@test length(ENV) > 1 +k1, k2 = "__test__", "__test1__" +withenv(k1=>k1, k2=>k2) do + b_k1, b_k2 = false, false + for (k, v) in ENV + if k==k1 + b_k1=true + elseif k==k2 + b_k2=true + end + end + @test b_k1 && b_k2 + io = IOBuffer() + show(io, ENV) + s = takebuf_string(io) + @test contains(s, "$k1=$k1") + @test contains(s, "$k2=$k2") + + @test pop!(ENV, k1) == k1 + @test !haskey(ENV, k1) + ENV[k1] = k1 + @test pop!(ENV, k1) == k1 + @test pop!(ENV, k1, "not_there") == "not_there" + + ENV[k1] = k1 + @test delete!(ENV, k1) == ENV + @test !haskey(ENV, k1) +end + +# Test for #10853 +@test withenv(Dict{Any,Any}()...) do; true; end From 60f19813aca3210b7c4a04ec5b4e4b181a7ecd9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Sat, 13 Aug 2016 13:46:06 +0200 Subject: [PATCH 0928/1117] Deprecate `delete!(ENV, k ,def)` as it is inherently type unstable and no other associative type implement the method. --- base/deprecated.jl | 5 +++++ base/env.jl | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 7dffd61acc9ba..6b3360200814d 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -785,4 +785,9 @@ const _oldstyle_array_vcat_ = false @deprecate write(x) write(STDOUT::IO, x) +function delete!(::EnvHash, k::AbstractString, def) + depwarn("`delete!(ENV, k, def)` should be replaced with `pop!(ENV, k, def)`. Be aware that `pop!` returns `k` or `def`, while `delete!` returns `ENV` or `def`.", :delete!) + haskey(ENV,k) ? delete!(ENV,k) : def +end + # End deprecations scheduled for 0.6 diff --git a/base/env.jl b/base/env.jl index 3c77619400b82..893191d015678 100644 --- a/base/env.jl +++ b/base/env.jl @@ -78,7 +78,6 @@ in(k::AbstractString, ::KeyIterator{EnvHash}) = _hasenv(k) pop!(::EnvHash, k::AbstractString) = (v = ENV[k]; _unsetenv(k); v) pop!(::EnvHash, k::AbstractString, def) = haskey(ENV,k) ? pop!(ENV,k) : def delete!(::EnvHash, k::AbstractString) = (_unsetenv(k); ENV) -delete!(::EnvHash, k::AbstractString, def) = haskey(ENV,k) ? delete!(ENV,k) : def setindex!(::EnvHash, v, k::AbstractString) = _setenv(k,string(v)) push!(::EnvHash, k::AbstractString, v) = setindex!(ENV, v, k) From f5b3004494b085b19c1ecac85140d31d75579d32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Sat, 13 Aug 2016 13:50:12 +0200 Subject: [PATCH 0929/1117] move doc for `withenv` --- base/docs/helpdb/Base.jl | 11 ----------- base/env.jl | 10 +++++++++- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index d510c91a7b7b2..f02519fb3f317 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2127,17 +2127,6 @@ Largest integer less than or equal to `x/y`. """ fld -""" - withenv(f::Function, kv::Pair...) - -Execute `f()` in an environment that is temporarily modified (not replaced as in `setenv`) -by zero or more `"var"=>val` arguments `kv`. `withenv` is generally used via the -`withenv(kv...) do ... end` syntax. A value of `nothing` can be used to temporarily unset an -environment variable (if it is set). When `withenv` returns, the original environment has -been restored. -""" -withenv - """ setdiff!(s, iterable) diff --git a/base/env.jl b/base/env.jl index 893191d015678..351ef42680d0b 100644 --- a/base/env.jl +++ b/base/env.jl @@ -138,7 +138,15 @@ function show(io::IO, ::EnvHash) end end -# temporarily set and then restore an environment value +""" + withenv(f::Function, kv::Pair...) + +Execute `f()` in an environment that is temporarily modified (not replaced as in `setenv`) +by zero or more `"var"=>val` arguments `kv`. `withenv` is generally used via the +`withenv(kv...) do ... end` syntax. A value of `nothing` can be used to temporarily unset an +environment variable (if it is set). When `withenv` returns, the original environment has +been restored. +""" function withenv{T<:AbstractString}(f::Function, keyvals::Pair{T}...) old = Dict{T,Any}() for (key,val) in keyvals From fc622dd141aa87945af7fb35d1b537dd4a062cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= <dhoegh91@gmail.com> Date: Sun, 14 Aug 2016 12:51:18 +0200 Subject: [PATCH 0930/1117] Formatting: indent the OS specific blocks. --- base/env.jl | 152 +++++++++++++++++++++++++--------------------------- 1 file changed, 74 insertions(+), 78 deletions(-) diff --git a/base/env.jl b/base/env.jl index 351ef42680d0b..78e2e0515145a 100644 --- a/base/env.jl +++ b/base/env.jl @@ -1,61 +1,59 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license if is_windows() -const ERROR_ENVVAR_NOT_FOUND = UInt32(203) + const ERROR_ENVVAR_NOT_FOUND = UInt32(203) -_getenvlen(var::Vector{UInt16}) = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,C_NULL,0) -_hasenv(s::Vector{UInt16}) = _getenvlen(s) != 0 || Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND -_hasenv(s::AbstractString) = _hasenv(cwstring(s)) + _getenvlen(var::Vector{UInt16}) = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,C_NULL,0) + _hasenv(s::Vector{UInt16}) = _getenvlen(s) != 0 || Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND + _hasenv(s::AbstractString) = _hasenv(cwstring(s)) -function access_env(onError::Function, str::AbstractString) - var = cwstring(str) - len = _getenvlen(var) - if len == 0 - return Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND ? "" : onError(str) + function access_env(onError::Function, str::AbstractString) + var = cwstring(str) + len = _getenvlen(var) + if len == 0 + return Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND ? "" : onError(str) + end + val = zeros(UInt16,len) + ret = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,val,len) + if (ret == 0 && len != 1) || ret != len-1 || val[end] != 0 + error(string("getenv: ", str, ' ', len, "-1 != ", ret, ": ", Libc.FormatMessage())) + end + pop!(val) # NUL + return transcode(String, val) end - val = zeros(UInt16,len) - ret = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,val,len) - if (ret == 0 && len != 1) || ret != len-1 || val[end] != 0 - error(string("getenv: ", str, ' ', len, "-1 != ", ret, ": ", Libc.FormatMessage())) + + function _setenv(svar::AbstractString, sval::AbstractString, overwrite::Bool=true) + var = cwstring(svar) + val = cwstring(sval) + if overwrite || !_hasenv(var) + ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,val) + systemerror(:setenv, ret == 0) + end end - pop!(val) # NUL - return transcode(String, val) -end -function _setenv(svar::AbstractString, sval::AbstractString, overwrite::Bool=true) - var = cwstring(svar) - val = cwstring(sval) - if overwrite || !_hasenv(var) - ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,val) + function _unsetenv(svar::AbstractString) + var = cwstring(svar) + ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,C_NULL) systemerror(:setenv, ret == 0) end -end - -function _unsetenv(svar::AbstractString) - var = cwstring(svar) - ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,C_NULL) - systemerror(:setenv, ret == 0) -end - else # !windows -_getenv(var::AbstractString) = ccall(:getenv, Cstring, (Cstring,), var) -_hasenv(s::AbstractString) = _getenv(s) != C_NULL - -function access_env(onError::Function, var::AbstractString) - val = _getenv(var) - val == C_NULL ? onError(var) : unsafe_string(val) -end + _getenv(var::AbstractString) = ccall(:getenv, Cstring, (Cstring,), var) + _hasenv(s::AbstractString) = _getenv(s) != C_NULL -function _setenv(var::AbstractString, val::AbstractString, overwrite::Bool=true) - ret = ccall(:setenv, Int32, (Cstring,Cstring,Int32), var, val, overwrite) - systemerror(:setenv, ret != 0) -end + function access_env(onError::Function, var::AbstractString) + val = _getenv(var) + val == C_NULL ? onError(var) : unsafe_string(val) + end -function _unsetenv(var::AbstractString) - ret = ccall(:unsetenv, Int32, (Cstring,), var) - systemerror(:unsetenv, ret != 0) -end + function _setenv(var::AbstractString, val::AbstractString, overwrite::Bool=true) + ret = ccall(:setenv, Int32, (Cstring,Cstring,Int32), var, val, overwrite) + systemerror(:setenv, ret != 0) + end + function _unsetenv(var::AbstractString) + ret = ccall(:unsetenv, Int32, (Cstring,), var) + systemerror(:unsetenv, ret != 0) + end end # os test ## ENV: hash interface ## @@ -82,45 +80,43 @@ setindex!(::EnvHash, v, k::AbstractString) = _setenv(k,string(v)) push!(::EnvHash, k::AbstractString, v) = setindex!(ENV, v, k) if is_windows() -start(hash::EnvHash) = (pos = ccall(:GetEnvironmentStringsW,stdcall,Ptr{UInt16},()); (pos,pos)) -function done(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) - if unsafe_load(block[1]) == 0 - ccall(:FreeEnvironmentStringsW, stdcall, Int32, (Ptr{UInt16},), block[2]) - return true + start(hash::EnvHash) = (pos = ccall(:GetEnvironmentStringsW,stdcall,Ptr{UInt16},()); (pos,pos)) + function done(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) + if unsafe_load(block[1]) == 0 + ccall(:FreeEnvironmentStringsW, stdcall, Int32, (Ptr{UInt16},), block[2]) + return true + end + return false end - return false -end -function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) - pos = block[1] - blk = block[2] - len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + 1 - buf = Array{UInt16}(len) - unsafe_copy!(pointer(buf), pos, len) - env = transcode(String, buf) - m = match(r"^(=?[^=]+)=(.*)$"s, env) - if m === nothing - error("malformed environment entry: $env") + function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) + pos = block[1] + blk = block[2] + len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + 1 + buf = Array{UInt16}(len) + unsafe_copy!(pointer(buf), pos, len) + env = transcode(String, buf) + m = match(r"^(=?[^=]+)=(.*)$"s, env) + if m === nothing + error("malformed environment entry: $env") + end + return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+len*2, blk)) end - return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+len*2, blk)) -end - else # !windows -start(::EnvHash) = 0 -done(::EnvHash, i) = (ccall(:jl_environ, Any, (Int32,), i) === nothing) + start(::EnvHash) = 0 + done(::EnvHash, i) = (ccall(:jl_environ, Any, (Int32,), i) === nothing) -function next(::EnvHash, i) - env = ccall(:jl_environ, Any, (Int32,), i) - if env === nothing - throw(BoundsError()) - end - env = env::String - m = match(r"^(.*?)=(.*)$"s, env) - if m === nothing - error("malformed environment entry: $env") + function next(::EnvHash, i) + env = ccall(:jl_environ, Any, (Int32,), i) + if env === nothing + throw(BoundsError()) + end + env = env::String + m = match(r"^(.*?)=(.*)$"s, env) + if m === nothing + error("malformed environment entry: $env") + end + return (Pair{String,String}(m.captures[1], m.captures[2]), i+1) end - return (Pair{String,String}(m.captures[1], m.captures[2]), i+1) -end - end # os-test #TODO: Make these more efficent From 0b7b53581906a1adc5559c2d6f02b46c7ae67244 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sun, 14 Aug 2016 18:52:56 +0800 Subject: [PATCH 0931/1117] Lock libmap and use atomic load/store on library handle/symbol GVs Fix #18020 --- src/ccall.cpp | 41 +++++++++++++++++++++++++++++++++++------ src/runtime_ccall.cpp | 16 +++++++++++----- test/threads.jl | 12 ++++++++++++ 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 9303786cc0b14..f85e30081eb03 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -142,10 +142,21 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, // *llvmgv = jl_load_and_lookup(f_lib, f_name, libptrgv); // } // return (*llvmgv) + BasicBlock *enter_bb = builder.GetInsertBlock(); BasicBlock *dlsym_lookup = BasicBlock::Create(jl_LLVMContext, "dlsym"); BasicBlock *ccall_bb = BasicBlock::Create(jl_LLVMContext, "ccall"); Constant *initnul = ConstantPointerNull::get((PointerType*)T_pvoidfunc); - builder.CreateCondBr(builder.CreateICmpNE(builder.CreateLoad(llvmgv), initnul), ccall_bb, dlsym_lookup); + LoadInst *llvmf_orig = builder.CreateAlignedLoad(llvmgv, sizeof(void*)); + // This in principle needs a consume ordering so that load from + // this pointer sees valid value. However, this is not supported by + // LLVM (or agreed on in the C/C++ standard FWIW) and should be + // almost impossible to happen on every platform we support since this + // ordering is enforced by the hardware and LLVM has to speculate a + // invalid load from the `cglobal` but doesn't depend on the `cglobal` + // value for this to happen. + // llvmf_orig->setAtomic(AtomicOrdering::Consume); + builder.CreateCondBr(builder.CreateICmpNE(llvmf_orig, initnul), + ccall_bb, dlsym_lookup); assert(f->getParent() != NULL); f->getBasicBlockList().push_back(dlsym_lookup); @@ -162,13 +173,20 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, #else Value *llvmf = builder.CreateCall3(prepare_call(jldlsym_func), libname, stringConstPtr(f_name), libptrgv); #endif - builder.CreateStore(llvmf, llvmgv); + auto store = builder.CreateAlignedStore(llvmf, llvmgv, sizeof(void*)); +# ifdef LLVM39 + store->setAtomic(AtomicOrdering::Release); +# else + store->setAtomic(Release); +# endif builder.CreateBr(ccall_bb); f->getBasicBlockList().push_back(ccall_bb); builder.SetInsertPoint(ccall_bb); - llvmf = builder.CreateLoad(llvmgv); - return builder.CreatePointerCast(llvmf,funcptype); + PHINode *p = builder.CreatePHI(T_pvoidfunc, 2); + p->addIncoming(llvmf_orig, enter_bb); + p->addIncoming(llvmf, dlsym_lookup); + return builder.CreatePointerCast(p, funcptype); } static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, @@ -247,7 +265,12 @@ static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, builder.SetInsertPoint(b0); Value *ptr = runtime_sym_lookup(funcptype, f_lib, f_name, plt, libptrgv, llvmgv, runtime_lib); - builder.CreateStore(builder.CreateBitCast(ptr, T_pvoidfunc), got); + auto store = builder.CreateAlignedStore(builder.CreateBitCast(ptr, T_pvoidfunc), got, sizeof(void*)); +# ifdef LLVM39 + store->setAtomic(AtomicOrdering::Release); +# else + store->setAtomic(Release); +# endif SmallVector<Value*, 16> args; for (Function::arg_iterator arg = plt->arg_begin(), arg_e = plt->arg_end(); arg != arg_e; ++arg) args.push_back(&*arg); @@ -292,7 +315,13 @@ static Value *emit_plt(FunctionType *functype, const AttributeSet &attrs, assert(!LM.m); got = prepare_global(slot); } - return builder.CreateBitCast(builder.CreateLoad(got), funcptype); + LoadInst *got_val = builder.CreateAlignedLoad(got, sizeof(void*)); + // See comment in `runtime_sym_lookup` above. This in principle needs a + // consume ordering too. This is even less likely to cause issue though + // since the only thing we do to this loaded pointer is to call it + // immediately. + // got_val->setAtomic(AtomicOrdering::Consume); + return builder.CreateBitCast(got_val, funcptype); } // --- ABI Implementations --- diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 0ef02ee4b9ece..ca86501e223d8 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -133,7 +133,7 @@ extern "C" void jl_init_runtime_ccall(void) // map from user-specified lib names to handles static std::map<std::string, void*> libMap; - +static jl_mutex_t libmap_lock; extern "C" void *jl_get_library(const char *f_lib) { @@ -146,21 +146,27 @@ void *jl_get_library(const char *f_lib) #endif if (f_lib == NULL) return jl_RTLD_DEFAULT_handle; - hnd = libMap[f_lib]; + JL_LOCK_NOGC(&libmap_lock); + // This is the only operation we do on the map, which doesn't invalidate + // any references or iterators. + void **map_slot = &libMap[f_lib]; + JL_UNLOCK_NOGC(&libmap_lock); + hnd = jl_atomic_load_acquire(map_slot); if (hnd != NULL) return hnd; + // We might run this concurrently on two threads but it doesn't matter. hnd = jl_load_dynamic_library(f_lib, JL_RTLD_DEFAULT); if (hnd != NULL) - libMap[f_lib] = hnd; + jl_atomic_store_release(map_slot, hnd); return hnd; } extern "C" JL_DLLEXPORT void *jl_load_and_lookup(const char *f_lib, const char *f_name, void **hnd) { - void *handle = *hnd; + void *handle = jl_atomic_load_acquire(hnd); if (!handle) - *hnd = handle = jl_get_library(f_lib); + jl_atomic_store_release(hnd, (handle = jl_get_library(f_lib))); return jl_dlsym(handle, f_name); } diff --git a/test/threads.jl b/test/threads.jl index 5b4d99d917712..4e21bc5da8740 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -385,3 +385,15 @@ if ccall(:jl_threading_enabled, Cint, ()) == 0 else @test_throws ErrorException cglobal(:jl_tls_states) end + +# Thread safety of `jl_load_and_lookup`. +function test_load_and_lookup_18020(n) + @threads for i in 1:n + try + ccall(:jl_load_and_lookup, + Ptr{Void}, (Cstring, Cstring, Ref{Ptr{Void}}), + "$i", :f, C_NULL) + end + end +end +test_load_and_lookup_18020(10000) From fb42b5869c52e9cb5175391a3ad0319bbcf752d5 Mon Sep 17 00:00:00 2001 From: WooKyoung Noh <wookay.noh@gmail.com> Date: Mon, 15 Aug 2016 21:08:36 +0900 Subject: [PATCH 0932/1117] Remove RNG from exports (#18023) --- base/random.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/random.jl b/base/random.jl index c6cf2bfdeb2c0..03e9b94c5d4f3 100644 --- a/base/random.jl +++ b/base/random.jl @@ -15,7 +15,7 @@ export srand, randsubseq,randsubseq!, shuffle,shuffle!, randperm, randcycle, - AbstractRNG, RNG, MersenneTwister, RandomDevice, + AbstractRNG, MersenneTwister, RandomDevice, GLOBAL_RNG, randjump From 6b1b10c8c1884d2f8499e1ce2ee7c9eaae35875f Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sat, 13 Aug 2016 13:09:41 -0400 Subject: [PATCH 0933/1117] fix effect_free computation over getfield fix #18015 --- base/inference.jl | 183 +++++++++++++++++++++------------------------- test/inference.jl | 18 ++++- 2 files changed, 102 insertions(+), 99 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 779e99d16efe4..869b30aa0f3e2 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1106,7 +1106,7 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState) if isa(e,QuoteNode) return abstract_eval_constant((e::QuoteNode).value) elseif isa(e,SSAValue) - return abstract_eval_ssavalue(e::SSAValue, sv) + return abstract_eval_ssavalue(e::SSAValue, sv.linfo) elseif isa(e,Slot) return vtypes[e.id].typ elseif isa(e,Symbol) @@ -1190,8 +1190,8 @@ function abstract_eval_global(M::Module, s::Symbol) return Any end -function abstract_eval_ssavalue(s::SSAValue, sv::InferenceState) - typ = sv.linfo.ssavaluetypes[s.id+1] +function abstract_eval_ssavalue(s::SSAValue, linfo::LambdaInfo) + typ = linfo.ssavaluetypes[s.id + 1] if typ === NF return Bottom end @@ -1921,7 +1921,7 @@ function finish(me::InferenceState) if !ispure && length(me.linfo.code) < 10 ispure = true for stmt in me.linfo.code - if !statement_effect_free(stmt, me) + if !statement_effect_free(stmt, me.linfo) ispure = false; break end end @@ -2177,20 +2177,21 @@ function occurs_more(e::ANY, pred, n) return 0 end -function exprtype(x::ANY, sv::InferenceState) - if isa(x,Expr) +function exprtype(x::ANY, linfo::LambdaInfo) + if isa(x, Expr) return (x::Expr).typ - elseif isa(x,SlotNumber) - return sv.linfo.slottypes[x.id] - elseif isa(x,TypedSlot) + elseif isa(x, SlotNumber) + return linfo.slottypes[x.id] + elseif isa(x, TypedSlot) return (x::Slot).typ - elseif isa(x,SSAValue) - return abstract_eval_ssavalue(x::SSAValue, sv) - elseif isa(x,Symbol) - return abstract_eval_global(sv.mod, x::Symbol) - elseif isa(x,QuoteNode) + elseif isa(x, SSAValue) + return abstract_eval_ssavalue(x::SSAValue, linfo) + elseif isa(x, Symbol) + mod = isdefined(linfo, :def) ? linfo.def.module : current_module() + return abstract_eval_global(mod, x::Symbol) + elseif isa(x, QuoteNode) return abstract_eval_constant((x::QuoteNode).value) - elseif isa(x,GlobalRef) + elseif isa(x, GlobalRef) return abstract_eval_global(x.mod, (x::GlobalRef).name) else return abstract_eval_constant(x) @@ -2221,28 +2222,28 @@ function is_pure_builtin(f::ANY) return false end -function statement_effect_free(e::ANY, sv) - if isa(e,Expr) +function statement_effect_free(e::ANY, linfo::LambdaInfo) + if isa(e, Expr) if e.head === :(=) - return !isa(e.args[1],GlobalRef) && effect_free(e.args[2], sv, false) + return !isa(e.args[1], GlobalRef) && effect_free(e.args[2], linfo, false) elseif e.head === :gotoifnot - return effect_free(e.args[1], sv, false) + return effect_free(e.args[1], linfo, false) end - elseif isa(e,LabelNode) || isa(e,GotoNode) + elseif isa(e, LabelNode) || isa(e, GotoNode) return true end - return effect_free(e, sv, false) + return effect_free(e, linfo, false) end # detect some important side-effect-free calls (allow_volatile=true) # and some affect-free calls (allow_volatile=false) -- affect_free means the call # cannot be affected by previous calls, except assignment nodes -function effect_free(e::ANY, sv, allow_volatile::Bool) - if isa(e,GlobalRef) +function effect_free(e::ANY, linfo::LambdaInfo, allow_volatile::Bool) + if isa(e, GlobalRef) return (isdefined(e.mod, e.name) && (allow_volatile || isconst(e.mod, e.name))) - elseif isa(e,Symbol) + elseif isa(e, Symbol) return allow_volatile - elseif isa(e,Expr) + elseif isa(e, Expr) e = e::Expr head = e.head if head === :static_parameter || head === :meta || head === :line || @@ -2251,32 +2252,17 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) end ea = e.args if head === :call && !isa(e.args[1], SSAValue) && !isa(e.args[1], Slot) - if is_known_call_p(e, is_pure_builtin, sv) + if is_known_call_p(e, is_pure_builtin, linfo) if !allow_volatile - if is_known_call(e, arrayref, sv) || is_known_call(e, arraylen, sv) + if is_known_call(e, arrayref, linfo) || is_known_call(e, arraylen, linfo) return false - elseif is_known_call(e, getfield, sv) - # arguments must be immutable to ensure e is affect_free - first = true - for a in ea - if first # first "arg" is the function name - first = false - continue - end - if isa(a,Symbol) - return false - end - if isa(a,SSAValue) - typ = widenconst(exprtype(a,sv)) - if !isa(typ,DataType) || typ.mutable - return false - end - end - if !effect_free(a,sv,allow_volatile) - return false - end + elseif is_known_call(e, getfield, linfo) + # first argument must be immutable to ensure e is affect_free + a = ea[2] + typ = widenconst(exprtype(a, linfo)) + if !isa(typ, DataType) || typ.mutable || typ.abstract + return false end - return true end end # fall-through @@ -2286,7 +2272,7 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) elseif head === :new if !allow_volatile a = ea[1] - typ = widenconst(exprtype(a,sv)) + typ = widenconst(exprtype(a, linfo)) if !isType(typ) || !isa((typ::Type).parameters[1],DataType) || ((typ::Type).parameters[1]::DataType).mutable return false end @@ -2300,11 +2286,11 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) return false end for a in ea - if !effect_free(a,sv,allow_volatile) + if !effect_free(a, linfo, allow_volatile) return false end end - elseif isa(e,LabelNode) || isa(e,GotoNode) + elseif isa(e, LabelNode) || isa(e, GotoNode) return false end return true @@ -2313,12 +2299,12 @@ end #### post-inference optimizations #### -function inline_as_constant(val::ANY, argexprs, sv) +function inline_as_constant(val::ANY, argexprs, linfo::LambdaInfo) # check if any arguments aren't effect_free and need to be kept around stmts = Any[] for i = 1:length(argexprs) arg = argexprs[i] - if !effect_free(arg, sv, false) + if !effect_free(arg, linfo, false) push!(stmts, arg) end end @@ -2365,7 +2351,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference istopfunction(topmod, f, :typejoin) || istopfunction(topmod, f, :promote_type)) # XXX: compute effect_free for the actual arguments - if length(argexprs) < 2 || effect_free(argexprs[2], sv, true) + if length(argexprs) < 2 || effect_free(argexprs[2], enclosing, true) return (e.typ.parameters[1],()) else return (e.typ.parameters[1], Any[argexprs[2]]) @@ -2373,11 +2359,11 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end end if istopfunction(topmod, f, :isbits) && length(atypes)==2 && isType(atypes[2]) && - effect_free(argexprs[2],sv,true) && isleaftype(atypes[2].parameters[1]) + effect_free(argexprs[2], enclosing, true) && isleaftype(atypes[2].parameters[1]) return (isbits(atypes[2].parameters[1]),()) end if is(f, Core.kwfunc) && length(argexprs) == 2 && isa(e.typ, Const) - if effect_free(argexprs[2], sv, true) + if effect_free(argexprs[2], enclosing, true) return (e.typ.val, ()) else return (e.typ.val, Any[argexprs[2]]) @@ -2407,7 +2393,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference local ti = atypes[i] if arg_hoisted || isa(ti, Union) aei = ex.args[i] - if !effect_free(aei, sv, false) + if !effect_free(aei, enclosing, false) arg_hoisted = true newvar = newvar!(sv, ti) insert!(stmts, 1, Expr(:(=), newvar, aei)) @@ -2516,11 +2502,11 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference (isType(e.typ) || isa(e.typ,Const)) if isType(e.typ) if !has_typevars(e.typ.parameters[1]) - return inline_as_constant(e.typ.parameters[1], argexprs, sv) + return inline_as_constant(e.typ.parameters[1], argexprs, enclosing) end else assert(isa(e.typ,Const)) - return inline_as_constant(e.typ.val, argexprs, sv) + return inline_as_constant(e.typ.val, argexprs, enclosing) end end @@ -2535,7 +2521,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end if linfo !== nothing && linfo.jlcall_api == 2 # in this case function can be inlined to a constant - return inline_as_constant(linfo.constval, argexprs, sv) + return inline_as_constant(linfo.constval, argexprs, enclosing) elseif linfo !== nothing && !linfo.inlineable return invoke_NF() elseif linfo === nothing || linfo.code === nothing @@ -2601,7 +2587,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference for i=na:-1:1 # stmts_free needs to be calculated in reverse-argument order #args_i = args[i] aei = argexprs[i] - aeitype = argtype = widenconst(exprtype(aei,sv)) + aeitype = argtype = widenconst(exprtype(aei, enclosing)) # ok for argument to occur more than once if the actual argument # is a symbol or constant, or is not affected by previous statements @@ -2613,17 +2599,17 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference if occ < 6 occ += occurs_more(b, x->(isa(x,Slot)&&x.id==i), 6) end - # TODO: passing `sv` here is wrong since it refers to the enclosing function - if occ > 0 && affect_free && !effect_free(b, sv, true) #TODO: we could short-circuit this test better by memoizing effect_free(b) in the for loop over i + if occ > 0 && affect_free && !effect_free(b, linfo, true) + #TODO: we might be able to short-circuit this test better by memoizing effect_free(b) in the for loop over i affect_free = false end if occ > 5 && !affect_free break end end - free = effect_free(aei,sv,true) + free = effect_free(aei, enclosing, true) if ((occ==0 && is(aeitype,Bottom)) || (occ > 1 && !inline_worthy(aei, occ*2000)) || - (affect_free && !free) || (!affect_free && !effect_free(aei,sv,false))) + (affect_free && !free) || (!affect_free && !effect_free(aei, enclosing, false))) if occ != 0 vnew = newvar!(sv, aeitype) argexprs[i] = vnew @@ -2825,8 +2811,8 @@ end function mk_tuplecall(args, sv::InferenceState) e = Expr(:call, top_tuple, args...) - e.typ = tuple_tfunc(Tuple{Any[widenconst(exprtype(x,sv)) for x in args]...}) - e + e.typ = tuple_tfunc(Tuple{Any[widenconst(exprtype(x, sv.linfo)) for x in args]...}) + return e end function inlining_pass!(linfo::LambdaInfo, sv::InferenceState) @@ -2865,11 +2851,11 @@ function inlining_pass(e::Expr, sv, linfo) # don't inline first (global) arguments of ccall, as this needs to be evaluated # by the interpreter and inlining might put in something it can't handle, # like another ccall (or try to move the variables out into the function) - if is_known_call(e, Core.Intrinsics.ccall, sv) + if is_known_call(e, Core.Intrinsics.ccall, linfo) # 4 is rewritten to 2 below to handle the callee. i0 = 4 isccall = true - elseif is_known_call(e, Core.Intrinsics.llvmcall, sv) + elseif is_known_call(e, Core.Intrinsics.llvmcall, linfo) i0 = 5 isccall = false else @@ -2895,8 +2881,8 @@ function inlining_pass(e::Expr, sv, linfo) end res = inlining_pass(ei, sv, linfo) res1 = res[1] - if has_stmts && !effect_free(res1, sv, false) - restype = exprtype(res1,sv) + if has_stmts && !effect_free(res1, linfo, false) + restype = exprtype(res1, linfo) vnew = newvar!(sv, restype) argloc[i] = vnew unshift!(stmts, Expr(:(=), vnew, res1)) @@ -2909,7 +2895,7 @@ function inlining_pass(e::Expr, sv, linfo) prepend!(stmts,res2) if !has_stmts for stmt in res2 - if !effect_free(stmt, sv, true) + if !effect_free(stmt, linfo, true) has_stmts = true end end @@ -2930,7 +2916,7 @@ function inlining_pass(e::Expr, sv, linfo) end end - ft = exprtype(arg1, sv) + ft = exprtype(arg1, linfo) if isa(ft, Const) f = ft.val else @@ -2948,7 +2934,7 @@ function inlining_pass(e::Expr, sv, linfo) a2 = e.args[3] if isa(a2, Symbol) || isa(a2, Slot) || isa(a2, SSAValue) - ta2 = exprtype(a2, sv) + ta2 = exprtype(a2, linfo) if isa(ta2, Const) a2 = ta2.val end @@ -2960,7 +2946,7 @@ function inlining_pass(e::Expr, sv, linfo) a1 = e.args[2] basenumtype = Union{corenumtype, Main.Base.Complex64, Main.Base.Complex128, Main.Base.Rational} if isa(a1, basenumtype) || ((isa(a1, Symbol) || isa(a1, Slot) || isa(a1, SSAValue)) && - exprtype(a1, sv) ⊑ basenumtype) + exprtype(a1, linfo) ⊑ basenumtype) if square e.args = Any[GlobalRef(Main.Base,:*), a1, a1] res = inlining_pass(e, sv, linfo) @@ -2984,7 +2970,7 @@ function inlining_pass(e::Expr, sv, linfo) ata = Vector{Any}(length(e.args)) ata[1] = ft for i = 2:length(e.args) - a = exprtype(e.args[i], sv) + a = exprtype(e.args[i], linfo) (a === Bottom || isvarargtype(a)) && return (e, stmts) ata[i] = a end @@ -2999,7 +2985,7 @@ function inlining_pass(e::Expr, sv, linfo) if !is(res,NF) # iteratively inline apply(f, tuple(...), tuple(...), ...) in order # to simplify long vararg lists as in multi-arg + - if isa(res,Expr) && is_known_call(res, _apply, sv) + if isa(res,Expr) && is_known_call(res, _apply, linfo) e = res::Expr f = _apply; ft = abstract_eval_constant(f) else @@ -3007,18 +2993,19 @@ function inlining_pass(e::Expr, sv, linfo) end end - if is(f,_apply) + if is(f, _apply) na = length(e.args) newargs = Vector{Any}(na-2) for i = 3:na aarg = e.args[i] - t = widenconst(exprtype(aarg,sv)) - if isa(aarg,Expr) && (is_known_call(aarg, tuple, sv) || is_known_call(aarg, svec, sv)) + t = widenconst(exprtype(aarg, linfo)) + if isa(aarg,Expr) && (is_known_call(aarg, tuple, linfo) || is_known_call(aarg, svec, linfo)) # apply(f,tuple(x,y,...)) => f(x,y,...) newargs[i-2] = aarg.args[2:end] elseif isa(aarg, Tuple) newargs[i-2] = Any[ QuoteNode(x) for x in aarg ] - elseif isa(t,DataType) && t.name===Tuple.name && !isvatuple(t) && effect_free(aarg,sv,true) && length(t.parameters) <= MAX_TUPLE_SPLAT + elseif isa(t, DataType) && t.name === Tuple.name && !isvatuple(t) && + effect_free(aarg, linfo, true) && length(t.parameters) <= MAX_TUPLE_SPLAT # apply(f,t::(x,y)) => f(t[1],t[2]) tp = t.parameters newargs[i-2] = Any[ mk_getfield(aarg,j,tp[j]) for j=1:length(tp) ] @@ -3030,7 +3017,7 @@ function inlining_pass(e::Expr, sv, linfo) e.args = [Any[e.args[2]]; newargs...] # now try to inline the simplified call - ft = exprtype(e.args[1], sv) + ft = exprtype(e.args[1], linfo) if isa(ft,Const) f = ft.val else @@ -3053,23 +3040,23 @@ function add_slot!(linfo::LambdaInfo, typ, is_sa, name=compiler_temp_sym) push!(linfo.slotnames, name) push!(linfo.slottypes, typ) push!(linfo.slotflags, Slot_Assigned + is_sa * Slot_AssignedOnce) - SlotNumber(id) + return SlotNumber(id) end -function is_known_call(e::Expr, func, sv) +function is_known_call(e::Expr, func::ANY, linfo::LambdaInfo) if e.head !== :call return false end - f = exprtype(e.args[1], sv) - return isa(f,Const) && f.val === func + f = exprtype(e.args[1], linfo) + return isa(f, Const) && f.val === func end -function is_known_call_p(e::Expr, pred, sv) +function is_known_call_p(e::Expr, pred::ANY, linfo::LambdaInfo) if e.head !== :call return false end - f = exprtype(e.args[1], sv) - return isa(f,Const) && pred(f.val) + f = exprtype(e.args[1], linfo) + return isa(f, Const) && pred(f.val) end function delete_var!(linfo, id, T) @@ -3172,7 +3159,7 @@ function occurs_outside_getfield(linfo::LambdaInfo, e::ANY, sym::ANY, end if isa(e,Expr) e = e::Expr - if is_known_call(e, getfield, sv) && symequal(e.args[2],sym) + if is_known_call(e, getfield, linfo) && symequal(e.args[2],sym) idx = e.args[3] if isa(idx,QuoteNode) && (idx.value in field_names) return false @@ -3228,7 +3215,7 @@ function _getfield_elim_pass!(e::Expr, sv) for i = 1:length(e.args) e.args[i] = _getfield_elim_pass!(e.args[i], sv) end - if is_known_call(e, getfield, sv) && length(e.args)==3 && + if is_known_call(e, getfield, sv.linfo) && length(e.args)==3 && (isa(e.args[3],Int) || isa(e.args[3],QuoteNode)) e1 = e.args[2] j = e.args[3] @@ -3243,7 +3230,7 @@ function _getfield_elim_pass!(e::Expr, sv) ok = true for k = 2:length(e1.args) k == j+1 && continue - if !effect_free(e1.args[k], sv, true) + if !effect_free(e1.args[k], sv.linfo, true) ok = false; break end end @@ -3273,10 +3260,10 @@ _getfield_elim_pass!(e::ANY, sv) = e # getfield(..., 1 <= x <= n) or getfield(..., x in f) on the result function is_allocation(e :: ANY, sv::InferenceState) isa(e, Expr) || return false - if is_known_call(e, tuple, sv) + if is_known_call(e, tuple, sv.linfo) return (length(e.args)-1,()) elseif e.head === :new - typ = widenconst(exprtype(e, sv)) + typ = widenconst(exprtype(e, sv.linfo)) if isleaftype(typ) @assert(isa(typ,DataType)) nf = length(e.args)-1 @@ -3305,7 +3292,7 @@ function gotoifnot_elim_pass!(linfo::LambdaInfo, sv::InferenceState) expr = expr::Expr expr.head === :gotoifnot || continue cond = expr.args[1] - condt = exprtype(cond, sv) + condt = exprtype(cond, linfo) isa(condt, Const) || continue val = (condt::Const).val # Codegen should emit an unreachable if val is not a Bool so @@ -3382,7 +3369,7 @@ function alloc_elim_pass!(linfo::LambdaInfo, sv::InferenceState) isa(tupelt,QuoteNode) || isa(tupelt, SSAValue)) vals[j] = tupelt else - elty = exprtype(tupelt,sv) + elty = exprtype(tupelt, linfo) if is_ssa tmpv = newvar!(sv, elty) else @@ -3437,7 +3424,7 @@ end function replace_getfield!(linfo::LambdaInfo, e::Expr, tupname, vals, field_names, sv) for i = 1:length(e.args) a = e.args[i] - if isa(a,Expr) && is_known_call(a, getfield, sv) && + if isa(a,Expr) && is_known_call(a, getfield, linfo) && symequal(a.args[2],tupname) idx = if isa(a.args[3], Int) a.args[3] @@ -3460,7 +3447,7 @@ function replace_getfield!(linfo::LambdaInfo, e::Expr, tupname, vals, field_name end elseif isa(val,SSAValue) val = val::SSAValue - typ = exprtype(val, sv) + typ = exprtype(val, linfo) if a.typ ⊑ typ && !(typ ⊑ a.typ) sv.linfo.ssavaluetypes[val.id+1] = a.typ end diff --git a/test/inference.jl b/test/inference.jl index 1d99d6539bc14..f1f4fbb9b1457 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -68,7 +68,7 @@ end abstract Outer5906{T} immutable Inner5906{T} - a:: T + a:: T end immutable Empty5906{T} <: Outer5906{T} @@ -306,3 +306,19 @@ let T = Array{Tuple{Vararg{Float64,TypeVar(:dim)}},1}, @test Base.return_types(f16530b, (Symbol,)) == Any[TTlim] end @test f16530a(:d) == Vector + + +# issue #18015 +type Triple18015 + a::Int + b::Int + c::Int +end +a18015(tri) = tri.a +b18015(tri) = tri.b +c18015(tri) = tri.c +setabc18015!(tri, a, b, c) = (tri.a = a; tri.b = b; tri.c = c) +let tri = Triple18015(1, 2, 3) + setabc18015!(tri, b18015(tri), c18015(tri), a18015(tri)) + @test tri.a === 2 && tri.b === 3 && tri.c === 1 +end From edfe0b1e702efd24e77ebbc5dfe1d0746daa90f1 Mon Sep 17 00:00:00 2001 From: Anders Claesson <anders.claesson@gmail.com> Date: Mon, 15 Aug 2016 15:15:02 +0000 Subject: [PATCH 0934/1117] Fix a trivial typo in devdocs --- doc/devdocs/object.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/devdocs/object.rst b/doc/devdocs/object.rst index 38c31075ddc51..99137ea5be094 100644 --- a/doc/devdocs/object.rst +++ b/doc/devdocs/object.rst @@ -43,7 +43,7 @@ the values can also be extracted directly as an array access:: As an example, a "boxed" :c:type:`uint16_t` is stored as follows:: struct { - oqaque metadata; + opaque metadata; struct { uint16_t data; -- 2 bytes } jl_value_t; From 3ca0f9c50688a3be59de7a737bedc7a58b953635 Mon Sep 17 00:00:00 2001 From: Twan Koolen <koolen.twan@gmail.com> Date: Mon, 15 Aug 2016 13:08:54 -0400 Subject: [PATCH 0935/1117] Make Colon immutable. Fixes 18034. --- base/essentials.jl | 2 +- test/subarray.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/base/essentials.jl b/base/essentials.jl index fed6c191ea4de..30b8d83148f46 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -202,7 +202,7 @@ function isassigned(v::SimpleVector, i::Int) end # index colon -type Colon +immutable Colon end const (:) = Colon() diff --git a/test/subarray.jl b/test/subarray.jl index 0ca8b0b05c858..2cfc8d772955d 100644 --- a/test/subarray.jl +++ b/test/subarray.jl @@ -484,3 +484,15 @@ u = (1,2:3) let size=(x,y)-> error("should not happen") @test X[1:end,2,2] == @view X[1:end,2,2] end + +# issue #18034 +# ensure that it is possible to create an isbits, LinearFast view of an immutable Array +let + immutable ImmutableTestArray{T, N} <: Base.DenseArray{T, N} + end + Base.size(::Union{ImmutableTestArray, Type{ImmutableTestArray}}) = (0, 0) + Base.linearindexing(::Union{ImmutableTestArray, Type{ImmutableTestArray}}) = Base.LinearFast() + a = ImmutableTestArray{Float64, 2}() + @test Base.linearindexing(view(a, :, :)) == Base.LinearFast() + @test isbits(view(a, :, :)) +end From 04994f97cbc3962d7f4339fe10548c0cc02fe5b3 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 15 Aug 2016 15:37:31 -0500 Subject: [PATCH 0936/1117] Make `find` indices-aware --- base/array.jl | 8 ++++++-- test/offsetarray.jl | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index da83d1faa0b80..2e0f5b4feb98b 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1077,15 +1077,18 @@ function find(testf::Function, A) # use a dynamic-length array to store the indexes, then copy to a non-padded # array for the return tmpI = Array{Int,1}(0) + inds = _index_remapper(A) for (i,a) = enumerate(A) if testf(a) - push!(tmpI, i) + push!(tmpI, inds[i]) end end I = Array{Int,1}(length(tmpI)) copy!(I, tmpI) return I end +_index_remapper(A::AbstractArray) = linearindices(A) +_index_remapper(iter) = Colon() # safe for objects that don't implement length """ find(A) @@ -1110,9 +1113,10 @@ function find(A) nnzA = countnz(A) I = Vector{Int}(nnzA) count = 1 + inds = _index_remapper(A) for (i,a) in enumerate(A) if a != 0 - I[count] = i + I[count] = inds[i] count += 1 end end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 7731895896f91..a8ce0835895ee 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -281,6 +281,11 @@ I,J,N = findnz(z) @test I == [-1] @test J == [0] @test N == [2] +h = OffsetArray([-1,1,-2,2,0], (-3,)) +@test find(h) == [-2:1;] +@test find(x->x>0, h) == [-1,1] +@test find(x->x<0, h) == [-2,0] +@test find(x->x==0, h) == [2] v = OffsetArray([1,1e100,1,-1e100], (-3,))*1000 v2 = OffsetArray([1,-1e100,1,1e100], (5,))*1000 From 12a01dc826e6ac6b13653c14f39df5b0227e7bb9 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 15 Aug 2016 15:44:51 -0500 Subject: [PATCH 0937/1117] Fixes and more tests for indexing with non-1 logical arrays --- base/multidimensional.jl | 2 +- test/offsetarray.jl | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index fe4f37a6d60fa..0a2b9489cec51 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -318,7 +318,7 @@ function _unsafe_getindex(::LinearFast, src::AbstractArray, I::AbstractArray{Boo D = eachindex(dest) Ds = start(D) - s = 0 + s = first(linearindices(src))-1 for i in eachindex(I) s += 1 @inbounds if I[i] diff --git a/test/offsetarray.jl b/test/offsetarray.jl index a8ce0835895ee..ce88abbb329e7 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -7,6 +7,7 @@ let # Basics v0 = rand(4) v = OffsetArray(v0, (-3,)) +h = OffsetArray([-1,1,-2,2,0], (-3,)) @test indices(v) == (-2:1,) @test_throws ErrorException size(v) @test_throws ErrorException size(v, 1) @@ -48,6 +49,16 @@ S = OffsetArray(view(A0, 1:2, 1:2), (-1,2)) # LinearSlow @test eachindex(A) == 1:4 @test eachindex(S) == CartesianRange((0:1,3:4)) +# logical indexing +@test A[A .> 2] == [3,4] +@test_throws BoundsError h[trues(2)] +@test_throws BoundsError h[trues(5)] +@test h[OffsetArray(trues(5), (-3,))] == parent(h) +@test h[OffsetArray([true,false,false,true,true], (-3,))] == parent(h)[[1,4,5]] +@test A[OffsetArray([true false; false true], A.offsets)] == [1,4] +@test A[OffsetArray([true true; false true], A.offsets)] == [1,3,4] +@test_throws BoundsError A[[true true; false true]] + # view S = view(A, :, 3) @test S == OffsetArray([1,2], (A.offsets[1],)) @@ -172,9 +183,6 @@ v = view(A0, i1, 1) v = view(A0, 1:1, i1) @test indices(v) === (Base.OneTo(1), -4:-3) -# logical indexing -@test A[A .> 2] == [3,4] - # copy! and fill! a = OffsetArray{Int}((-3:-1,)) fill!(a, -1) @@ -281,7 +289,6 @@ I,J,N = findnz(z) @test I == [-1] @test J == [0] @test N == [2] -h = OffsetArray([-1,1,-2,2,0], (-3,)) @test find(h) == [-2:1;] @test find(x->x>0, h) == [-1,1] @test find(x->x<0, h) == [-2,0] From b92d580c0c4eb38f99c07d208c073662baf9bed5 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Thu, 11 Aug 2016 01:01:21 -0700 Subject: [PATCH 0938/1117] add support for split debuginfo files There seem to be a few variations on the tools and arguments available for creating split debuginfo files. I used the following: objcopy --only-keep-debug libjulia.so libjulia.so.debug strip -g libjulia.so objcopy --add-gnu-debuglink=libjulia.so.debug libjulia.so --- src/debuginfo.cpp | 553 +++++++++++++++++++++++++++++----------------- 1 file changed, 352 insertions(+), 201 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index b61abe00e817c..c02eda00de75b 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -748,6 +748,8 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, #ifdef _OS_DARWIN_ #include <mach-o/dyld.h> +#else +#define LC_UUID 0 #endif #ifndef _OS_WINDOWS_ #include <dlfcn.h> @@ -761,11 +763,10 @@ typedef struct { typedef std::map<uint64_t, objfileentry_t, revcomp> obfiletype; static obfiletype objfilemap; -#ifdef _OS_DARWIN_ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) { # ifdef LLVM37 - for (auto Load : obj->load_commands ()) { + for (auto Load : obj->load_commands()) # else # ifdef LLVM35 uint32_t LoadCommandCount = obj->getHeader().ncmds; @@ -773,8 +774,9 @@ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) uint32_t LoadCommandCount = obj->getHeader().NumLoadCommands; # endif llvm::object::MachOObjectFile::LoadCommandInfo Load = obj->getFirstLoadCommandInfo(); - for (unsigned I = 0; ; ++I) { + for (unsigned I = 0; ; ++I) # endif + { if ( # ifdef LLVM35 Load.C.cmd == LC_UUID @@ -782,7 +784,7 @@ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) Load.C.Type == LC_UUID # endif ) { - memcpy(uuid,((MachO::uuid_command*)Load.Ptr)->uuid,16); + memcpy(uuid, ((const MachO::uuid_command*)Load.Ptr)->uuid, 16); return true; } # ifndef LLVM37 @@ -796,6 +798,115 @@ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) } return false; } + +#ifdef LLVM36 +struct debug_link_info { + StringRef filename; + uint32_t crc32; +}; +static debug_link_info getDebuglink(const object::ObjectFile &Obj) +{ + debug_link_info info = {}; + for (const object::SectionRef &Section: Obj.sections()) { + StringRef sName; + if (!Section.getName(sName) && sName == ".gnu_debuglink") { + StringRef Contents; + if (!Section.getContents(Contents)) { + size_t length = Contents.find('\0'); + info.filename = Contents.substr(0, length); + info.crc32 = *(const uint32_t*)Contents.substr(LLT_ALIGN(length + 1, 4), 4).data(); + break; + } + } + } + return info; +} +/* + * crc function from http://svnweb.freebsd.org/base/head/sys/libkern/crc32.c (and lldb) + * + * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or + * code or tables extracted from it, as desired without restriction. + */ +static uint32_t +calc_gnu_debuglink_crc32(const void *buf, size_t size) +{ + static const uint32_t g_crc32_tab[] = + { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d + }; + const uint8_t *p = (const uint8_t *)buf; + uint32_t crc; + + crc = ~0U; + while (size--) + crc = g_crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); + return crc ^ ~0U; +} +static ErrorOr<object::OwningBinary<object::ObjectFile>> openDebugInfo(StringRef debuginfopath, const debug_link_info &info) +{ + auto SplitFile = MemoryBuffer::getFile(debuginfopath); + if (std::error_code EC = SplitFile.getError()) + return EC; + + uint32_t crc32 = calc_gnu_debuglink_crc32( + SplitFile.get()->getBufferStart(), + SplitFile.get()->getBufferSize()); + if (crc32 != info.crc32) + return object::object_error::arch_not_found; + + auto error_splitobj = object::ObjectFile::createObjectFile( + SplitFile.get().get()->getMemBufferRef(), + sys::fs::file_magic::unknown); + if (std::error_code EC = error_splitobj.getError()) + return EC; + + // successfully validated and loaded split debug info file + return object::OwningBinary<object::ObjectFile>( + std::move(error_splitobj.get()), + std::move(SplitFile.get())); +} #endif static uint64_t jl_sysimage_base; @@ -817,55 +928,57 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, *context = NULL; *slide = 0; *section_slide = 0; + // GOAL: Determine containing Library -// Assigning fname, fbase, msize +// Assigning fname, fbase #ifdef _OS_WINDOWS_ IMAGEHLP_MODULE64 ModuleInfo; - bool isvalid; ModuleInfo.SizeOfStruct = sizeof(IMAGEHLP_MODULE64); jl_in_stackwalk = 1; - isvalid = SymGetModuleInfo64(GetCurrentProcess(), (DWORD64)pointer, &ModuleInfo); + bool isvalid = SymGetModuleInfo64(GetCurrentProcess(), (DWORD64)pointer, &ModuleInfo); jl_in_stackwalk = 0; - if (isvalid) { - char *fname = ModuleInfo.LoadedImageName; - if (!fname[0]) // empirically, LoadedImageName might be missing - fname = ModuleInfo.ImageName; - DWORD64 fbase = ModuleInfo.BaseOfImage; - bool insysimage = (fbase == jl_sysimage_base); - if (isSysImg) - *isSysImg = insysimage; - if (onlySysImg && !insysimage) { - return false; - } - static char frame_info_func[ - sizeof(SYMBOL_INFO) + - MAX_SYM_NAME * sizeof(TCHAR)]; - DWORD64 dwDisplacement64 = 0; - DWORD64 dwAddress = pointer; - PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)frame_info_func; - pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); - pSymbol->MaxNameLen = MAX_SYM_NAME; - jl_in_stackwalk = 1; - if (SymFromAddr(GetCurrentProcess(), dwAddress, &dwDisplacement64, - pSymbol)) { - // SymFromAddr returned success - // errors are ignored, but are hopefully patched up by - // using llvm to read the object (below) - if (name) - jl_copy_str(name, pSymbol->Name); - if (saddr) - *saddr = (void*)(uintptr_t)pSymbol->Address; - } - else if (saddr) { - *saddr = NULL; - } + if (!isvalid) return false; + + StringRef fname = ModuleInfo.LoadedImageName; + if (fname.empty()) // empirically, LoadedImageName might be missing + fname = ModuleInfo.ImageName; + DWORD64 fbase = ModuleInfo.BaseOfImage; + bool insysimage = (fbase == jl_sysimage_base); + if (isSysImg) + *isSysImg = insysimage; + if (onlySysImg && !insysimage) { + return false; + } + static char frame_info_func[ + sizeof(SYMBOL_INFO) + + MAX_SYM_NAME * sizeof(TCHAR)]; + DWORD64 dwDisplacement64 = 0; + DWORD64 dwAddress = pointer; + PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)frame_info_func; + pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); + pSymbol->MaxNameLen = MAX_SYM_NAME; + jl_in_stackwalk = 1; + if (SymFromAddr(GetCurrentProcess(), dwAddress, &dwDisplacement64, + pSymbol)) { + // SymFromAddr returned success + // errors are ignored, but are hopefully patched up by + // using llvm to read the object (below) + if (name) + jl_copy_str(name, pSymbol->Name); + if (saddr) + *saddr = (void*)(uintptr_t)pSymbol->Address; + } + else if (saddr) { + *saddr = NULL; + } - // If we didn't find the filename before in the debug - // info, use the dll name - if (filename && !*filename) - jl_copy_str(filename, fname); + // If we didn't find the filename before in the debug + // info, use the dll name + if (filename && !*filename) + jl_copy_str(filename, fname.data()); + + jl_in_stackwalk = 0; - jl_in_stackwalk = 0; #else // ifdef _OS_WINDOWS_ Dl_info dlinfo; int dladdr_success; @@ -876,201 +989,239 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, #else dladdr_success = dladdr((void*)pointer, &dlinfo) != 0; #endif + if (!dladdr_success || !dlinfo.dli_fname) + return false; - if (dladdr_success && dlinfo.dli_fname) { #ifdef __GLIBC__ - // dlinfo.dli_fbase is not the right value for the main executable on linux - fbase = (uintptr_t)extra_info->l_addr; + // dlinfo.dli_fbase is not the right value for the main executable on linux + fbase = (uintptr_t)extra_info->l_addr; #else - fbase = (uintptr_t)dlinfo.dli_fbase; -#endif - const char *fname; - if (saddr) - *saddr = dlinfo.dli_saddr; + fbase = (uintptr_t)dlinfo.dli_fbase; +#endif + StringRef fname; + if (saddr) + *saddr = dlinfo.dli_saddr; + bool insysimage = (fbase == jl_sysimage_base); + if (isSysImg) + *isSysImg = insysimage; + if (onlySysImg && !insysimage) { + return false; + } + // In case we fail with the debug info lookup, we at least still + // have the function name, even if we don't have line numbers + if (name) + jl_copy_str(name, dlinfo.dli_sname); + if (filename) + jl_copy_str(filename, dlinfo.dli_fname); + fname = dlinfo.dli_fname; +#endif // ifdef _OS_WINDOWS_ + + int isdarwin = 0, islinux = 0, iswindows = 0; #if defined(_OS_DARWIN_) - size_t msize = (size_t)(((uint64_t)-1)-fbase); + isdarwin = 1; +#elif defined(_OS_LINUX_) || defined(_OS_FREEBSD_) + islinux = 1; +#elif defined(_OS_WINDOWS_) + iswindows = 1; +#endif + +#ifndef LLVM35 + if (iswindows) { + return true; + } #endif - bool insysimage = (fbase == jl_sysimage_base); - if (isSysImg) - *isSysImg = insysimage; - if (onlySysImg && !insysimage) { - return false; - } - // In case we fail with the debug info lookup, we at least still - // have the function name, even if we don't have line numbers - if (name) - jl_copy_str(name, dlinfo.dli_sname); - if (filename) - jl_copy_str(filename, dlinfo.dli_fname); - fname = dlinfo.dli_fname; -#endif // ifdef _OS_WINDOWS_ // GOAL: Read debuginfo from file -#if !defined(_OS_WINDOWS_) || defined(LLVM35) - // TODO: need read/write lock here for objfilemap synchronization - obfiletype::iterator it = objfilemap.find(fbase); - if (it != objfilemap.end()) { - // Return cached value - *obj = it->second.obj; - *context = it->second.ctx; - *slide = it->second.slide; - *section_slide = it->second.section_slide; - } - else { + // TODO: need read/write lock here for objfilemap synchronization + obfiletype::iterator it = objfilemap.find(fbase); + if (it != objfilemap.end()) { + // Return cached value + *obj = it->second.obj; + *context = it->second.ctx; + *slide = it->second.slide; + *section_slide = it->second.section_slide; + return true; + } + // GOAL: Assign errorobj -#if defined(_OS_DARWIN_) + StringRef objpath; + std::string debuginfopath; + uint8_t uuid[16], uuid2[16]; + if (isdarwin) { + size_t msize = (size_t)(((uint64_t)-1) - fbase); #ifdef LLVM36 - std::unique_ptr<MemoryBuffer> membuf = MemoryBuffer::getMemBuffer( - StringRef((const char *)fbase, msize), "", false); - auto origerrorobj = llvm::object::ObjectFile::createObjectFile( - membuf->getMemBufferRef(), sys::fs::file_magic::unknown); -#elif defined(LLVM35) - MemoryBuffer *membuf = MemoryBuffer::getMemBuffer( + std::unique_ptr<MemoryBuffer> membuf = MemoryBuffer::getMemBuffer( StringRef((const char *)fbase, msize), "", false); - std::unique_ptr<MemoryBuffer> buf(membuf); - auto origerrorobj = llvm::object::ObjectFile::createObjectFile( - buf, sys::fs::file_magic::unknown); -#else - MemoryBuffer *membuf = MemoryBuffer::getMemBuffer( - StringRef((const char *)fbase, msize), "", false); - llvm::object::ObjectFile *origerrorobj = llvm::object::ObjectFile::createObjectFile( - membuf); -#endif - if (!origerrorobj) { - objfileentry_t entry = {*obj,*context,*slide,*section_slide}; - objfilemap[fbase] = entry; - return true; - } - -#ifdef LLVM36 - *obj = (llvm::object::MachOObjectFile *)origerrorobj.get().release(); + auto origerrorobj = llvm::object::ObjectFile::createObjectFile( + membuf->getMemBufferRef(), sys::fs::file_magic::unknown); #elif defined(LLVM35) - *obj = (llvm::object::MachOObjectFile *)origerrorobj.get(); + MemoryBuffer *membuf = MemoryBuffer::getMemBuffer( + StringRef((const char *)fbase, msize), "", false); + std::unique_ptr<MemoryBuffer> buf(membuf); + auto origerrorobj = llvm::object::ObjectFile::createObjectFile( + buf, sys::fs::file_magic::unknown); #else - *obj = (llvm::object::MachOObjectFile *)origerrorobj; + MemoryBuffer *membuf = MemoryBuffer::getMemBuffer( + StringRef((const char *)fbase, msize), "", false); + std::unique_ptr<llvm::object::ObjectFile> origerrorobj(llvm::object::ObjectFile::createObjectFile( + membuf)); #endif - llvm::object::MachOObjectFile *morigobj = (llvm::object::MachOObjectFile *)*obj; - - - // First find the uuid of the object file (we'll use this to make sure we find the - // correct debug symbol file). - uint8_t uuid[16], uuid2[16]; - if (!getObjUUID(morigobj,uuid)) { - objfileentry_t entry = {*obj,*context,*slide,*section_slide}; - objfilemap[fbase] = entry; - return true; - } + if (!origerrorobj) { + objfileentry_t entry = {}; + objfilemap[fbase] = entry; + return true; + } - // On OS X debug symbols are not contained in the dynamic library and that's why - // we can't have nice things (easily). For now we only support .dSYM files in the same directory - // as the shared library. In the future we may use DBGCopyFullDSYMURLForUUID from CoreFoundation to make - // use of spotlight to find the .dSYM file. - char dsympath[PATH_MAX]; - strlcpy(dsympath, fname, sizeof(dsympath)); - strlcat(dsympath, ".dSYM/Contents/Resources/DWARF/", sizeof(dsympath)); - strlcat(dsympath, strrchr(fname,'/')+1, sizeof(dsympath)); -#ifdef LLVM35 - auto errorobj = llvm::object::ObjectFile::createObjectFile(dsympath); + llvm::object::MachOObjectFile *morigobj = (llvm::object::MachOObjectFile*) +#ifdef LLVM36 + origerrorobj.get().get(); #else - llvm::object::ObjectFile *errorobj = llvm::object::ObjectFile::createObjectFile(dsympath); + origerrorobj.get(); #endif -#else // ifndef _OS_DARWIN_ + // First find the uuid of the object file (we'll use this to make sure we find the + // correct debug symbol file). + if (!getObjUUID(morigobj, uuid)) { + objfileentry_t entry = {}; + objfilemap[fbase] = entry; + return true; + } - // On Linux systems we need to mmap another copy because of the permissions on the mmap'ed shared library. - // On Windows we need to mmap another copy since reading the in-memory copy seems to return object_error:unexpected_eof + // On OS X debug symbols are not contained in the dynamic library. + // For now we only support .dSYM files in the same directory + // as the shared library. In the future we may use DBGCopyFullDSYMURLForUUID from CoreFoundation to make + // use of spotlight to find the .dSYM file. + size_t sep = fname.rfind('/'); + debuginfopath = fname; + debuginfopath += ".dSYM/Contents/Resources/DWARF/"; + debuginfopath += fname.substr(sep + 1); + objpath = debuginfopath; + } + else { + // On Linux systems we need to mmap another copy because of the permissions on the mmap'ed shared library. + // On Windows we need to mmap another copy since reading the in-memory copy seems to return object_error:unexpected_eof + objpath = fname; + } #ifdef LLVM35 - auto errorobj = llvm::object::ObjectFile::createObjectFile(fname); + auto errorobj = llvm::object::ObjectFile::createObjectFile(objpath); #else - llvm::object::ObjectFile *errorobj = llvm::object::ObjectFile::createObjectFile(fname); + std::unique_ptr<llvm::object::ObjectFile> errorobj(llvm::object::ObjectFile::createObjectFile(objpath)); #endif -#endif // ifdef _OS_DARWIN_ // GOAL: Assign *obj, *context, *slide (if above succeeded) - if (errorobj) { + if (errorobj) { #ifdef LLVM36 - auto binary = errorobj.get().takeBinary(); - *obj = binary.first.release(); - binary.second.release(); -#elif defined(LLVM35) - *obj = errorobj.get(); + auto *debugobj = errorobj->getBinary(); #else - *obj = errorobj; + auto *debugobj = errorobj.get(); #endif -#ifdef _OS_DARWIN_ - if (getObjUUID((llvm::object::MachOObjectFile *)*obj,uuid2) && - memcmp(uuid,uuid2,sizeof(uuid)) == 0) { -#endif -#ifdef LLVM37 - *context = new DWARFContextInMemory(**obj); -#elif defined(LLVM36) - *context = DIContext::getDWARFContext(**obj); -#else - *context = DIContext::getDWARFContext(const_cast<object::ObjectFile*>(*obj)); -#endif - *slide = -(int64_t)fbase; -#ifdef _OS_DARWIN_ - } - else { - // If we're here the, the dsym does not match the dylib. Use the original - // object instead. For consistency (and to make sure we get a sensible size - // for the memory buffer), we also use a fresh copy mapped from - // the file rather than reusing the one in memory. We may want to revisit - // that in the future (ideally, once we support fewer LLVM versions). - errorobj = llvm::object::ObjectFile::createObjectFile(fname); - assert(errorobj); + + if (islinux) { #ifdef LLVM36 - auto binary = errorobj.get().takeBinary(); - *obj = binary.first.release(); - binary.second.release(); -#elif defined(LLVM35) - *obj = errorobj.get(); -#else - *obj = errorobj; -#endif - delete morigobj; + // if the file has a .gnu_debuglink section, + // try to load its companion file instead + // in the expected locations + // for now, we don't support the build-id method + debug_link_info info = getDebuglink(*debugobj); + if (!info.filename.empty()) { + size_t sep = fname.rfind('/'); + ErrorOr<object::OwningBinary<object::ObjectFile>> DebugInfo(std::errc::no_such_file_or_directory); + if (fname.substr(sep + 1) != info.filename) { + debuginfopath = fname.substr(0, sep + 1); + debuginfopath += info.filename; + DebugInfo = openDebugInfo(debuginfopath, info); } -#endif -#if defined(_OS_WINDOWS_) - assert((*obj)->isCOFF()); - const llvm::object::COFFObjectFile *coffobj = (const llvm::object::COFFObjectFile *)*obj; - const llvm::object::pe32plus_header *pe32plus; - coffobj->getPE32PlusHeader(pe32plus); - if (pe32plus != NULL) { - *slide = pe32plus->ImageBase - fbase; - *section_slide = -(int64_t)pe32plus->ImageBase; + if (DebugInfo.getError()) { + debuginfopath = fname.substr(0, sep + 1); + debuginfopath += ".debug/"; + debuginfopath += info.filename; + DebugInfo = openDebugInfo(debuginfopath, info); + } + if (DebugInfo.getError()) { + debuginfopath = "/usr/lib/debug/"; + debuginfopath += fname.substr(0, sep + 1); + debuginfopath += info.filename; + DebugInfo = openDebugInfo(debuginfopath, info); + } + if (DebugInfo.getError()) { + // no split debug info found + // should we warn the user? } else { - const llvm::object::pe32_header *pe32; - coffobj->getPE32Header(pe32); - if (pe32 == NULL) { - *obj = NULL; - *context = NULL; - *slide = 0; - } - else { - *slide = pe32->ImageBase - fbase; - *section_slide = -(int64_t)pe32->ImageBase; - } + errorobj = std::move(DebugInfo); + debugobj = errorobj->getBinary(); } + } #endif + } + + if (isdarwin) { + // verify the UUID matches + if (!getObjUUID((llvm::object::MachOObjectFile*)debugobj, uuid2) || + memcmp(uuid, uuid2, sizeof(uuid)) != 0) { + objfileentry_t entry = {}; + objfilemap[fbase] = entry; + return true; + } + } + + if (iswindows) { +#ifdef LLVM35 + assert(debugobj->isCOFF()); + const llvm::object::COFFObjectFile *coffobj = (const llvm::object::COFFObjectFile*)debugobj; + const llvm::object::pe32plus_header *pe32plus; + coffobj->getPE32PlusHeader(pe32plus); + if (pe32plus != NULL) { + *slide = pe32plus->ImageBase - fbase; + *section_slide = -(int64_t)pe32plus->ImageBase; } -#ifdef LLVM39 else { - // TODO: report the error instead of silently consuming it? - // jl_error might run into the same error again... - consumeError(errorobj.takeError()); + const llvm::object::pe32_header *pe32; + coffobj->getPE32Header(pe32); + if (pe32 == NULL) { + objfileentry_t entry = {}; + objfilemap[fbase] = entry; + return true; + } + else { + *slide = pe32->ImageBase - fbase; + *section_slide = -(int64_t)pe32->ImageBase; + } } #endif - - // update cache - objfileentry_t entry = {*obj,*context,*slide,*section_slide}; - objfilemap[fbase] = entry; } + else { + *slide = -(int64_t)fbase; + } + +#ifdef LLVM37 + *context = new DWARFContextInMemory(*debugobj); +#elif defined(LLVM36) + *context = DIContext::getDWARFContext(*debugobj); +#else + *context = DIContext::getDWARFContext(debugobj); +#endif + *obj = debugobj; +#ifdef LLVM36 + auto binary = errorobj->takeBinary(); + binary.first.release(); + binary.second.release(); +#else + errorobj.release(); #endif - return true; } - return false; +#ifdef LLVM39 + else { + // TODO: report the error instead of silently consuming it? + // jl_error might run into the same error again... + consumeError(errorobj.takeError()); + } +#endif + + // update cache + objfileentry_t entry = {*obj, *context, *slide, *section_slide}; + objfilemap[fbase] = entry; + return true; } // *name and *filename should be either NULL or malloc'd pointer From cd62eb1e8b1e5349b802deed9ba4190c3cbfa0b0 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 16 Aug 2016 00:29:05 -0400 Subject: [PATCH 0939/1117] fall back to dynamic dispatch in union-splitting optimization This should help #17932. If a function returns an unanticipated type that wasn't part of the `Union` seen by inference, do dynamic dispatch instead of erroring. --- base/inference.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 869b30aa0f3e2..29efbf8237d34 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -9,7 +9,8 @@ const MAX_TUPLETYPE_LEN = 15 const MAX_TUPLE_DEPTH = 4 const MAX_TUPLE_SPLAT = 16 -const MAX_UNION_SPLITTING = 6 +const MAX_UNION_SPLITTING = 4 +const UNION_SPLIT_MISMATCH_ERROR = false # alloc_elim_pass! relies on `Slot_AssignedOnce | Slot_UsedUndef` being # SSA. This should be true now but can break if we start to track conditional @@ -2429,7 +2430,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference all = false end end - if all + if UNION_SPLIT_MISMATCH_ERROR && all error_label === nothing && (error_label = genlabel(sv)) push!(stmts, GotoNode(error_label.label)) else From acc744ec68c2bb46897bb204c7a3f739bedb349e Mon Sep 17 00:00:00 2001 From: Mus M <mus-m@outlook.com> Date: Tue, 16 Aug 2016 01:33:45 -0400 Subject: [PATCH 0940/1117] Update modules.rst adding type to show for v0.5 (#18018) Update to add type annotation otherwise we get ``` ERROR: MethodError: no method matching display(::MyType) ``` --- doc/manual/modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index aab14db34fb52..72241c2ea00a7 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -36,7 +36,7 @@ not meant to be run, but is shown for illustrative purposes:: bar(x) = 2x foo(a::MyType) = bar(a.x) + 1 - show(io, a::MyType) = print(io, "MyType $(a.x)") + show(io::IO, a::MyType) = print(io, "MyType $(a.x)") end Note that the style is not to indent the body of the module, since From 7a036e904b2ac7a29afdf796339cf0ec19965f2d Mon Sep 17 00:00:00 2001 From: Michele Zaffalon <michele.zaffalon@gmail.com> Date: Tue, 16 Aug 2016 07:34:28 +0200 Subject: [PATCH 0941/1117] added example of array type conversion (#17663) * added example of array type conversion ... because it seems to bite less experienced users like myself. (Stolen from Terry Seaward's email, https://groups.google.com/d/msg/julia-users/wzaJw_FfNlA/qcuZ4FggEL4J) * added example of array type conversion --- doc/manual/conversion-and-promotion.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 61969179addc5..11651893358c3 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -90,6 +90,16 @@ action: julia> typeof(ans) Float64 + julia> a = Any[1 2 3; 4 5 6] + 2x3 Array{Any,2}: + 1 2 3 + 4 5 6 + + julia> convert(Array{Float64}, a) + 2x3 Array{Float64,2}: + 1.0 2.0 3.0 + 4.0 5.0 6.0 + Conversion isn't always possible, in which case a no method error is thrown indicating that ``convert`` doesn't know how to perform the requested conversion: From bc3b2a47fd92a0d95969e71c96c91b0cb614d322 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 15 Aug 2016 13:00:55 -0700 Subject: [PATCH 0942/1117] test libccalltest.so with split debug info --- Make.inc | 1 + base/stacktraces.jl | 4 ++-- src/Makefile | 23 ++++++++++++++++++++++- src/debuginfo.cpp | 22 ++++++++++++++++++---- src/stackwalk.c | 10 +++++++--- test/stacktraces.jl | 10 ++++++++++ 6 files changed, 60 insertions(+), 10 deletions(-) diff --git a/Make.inc b/Make.inc index 568a94cb22cf7..003f50db78376 100644 --- a/Make.inc +++ b/Make.inc @@ -460,6 +460,7 @@ endif #ARCH LD := link endif #USEMSVC RANLIB := $(CROSS_COMPILE)ranlib +OBJCOPY := $(CROSS_COMPILE)objcopy # file extensions ifeq ($(OS), WINNT) diff --git a/base/stacktraces.jl b/base/stacktraces.jl index b4be771e44628..55a7d901225ad 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -56,7 +56,7 @@ immutable StackFrame # this type should be kept platform-agnostic so that profil "true if the code is from an inlined frame" inlined::Bool "representation of the pointer to the execution context as returned by `backtrace`" - pointer::Int64 # Large enough to be read losslessly on 32- and 64-bit machines. + pointer::UInt64 # Large enough to be read losslessly on 32- and 64-bit machines. end StackFrame(func, file, line) = StackFrame(func, file, line, Nullable{LambdaInfo}(), false, false, 0) @@ -123,7 +123,7 @@ inlined at that point, innermost function first. """ function lookup(pointer::Ptr{Void}) infos = ccall(:jl_lookup_code_address, Any, (Ptr{Void}, Cint), pointer - 1, false) - isempty(infos) && return [StackFrame(empty_sym, empty_sym, -1, Nullable{LambdaInfo}(), true, false, convert(Int64, pointer))] + isempty(infos) && return [StackFrame(empty_sym, empty_sym, -1, Nullable{LambdaInfo}(), true, false, convert(UInt64, pointer))] res = Array{StackFrame}(length(infos)) for i in 1:length(infos) info = infos[i] diff --git a/src/Makefile b/src/Makefile index 1f85e9169348b..ab2e599ae2370 100644 --- a/src/Makefile +++ b/src/Makefile @@ -124,8 +124,26 @@ $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(SRCDIR)/llvm-version.h $(HEADERS) $(LLV @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG_HOST) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) libccalltest: $(build_shlibdir)/libccalltest.$(SHLIB_EXT) + +ifeq ($(OS), Linux) +JULIA_SPLITDEBUG := 1 +else +JULIA_SPLITDEBUG := 0 +endif $(build_shlibdir)/libccalltest.$(SHLIB_EXT): $(SRCDIR)/ccalltest.c - @$(call PRINT_CC, $(CC) $(CFLAGS) $(CPPFLAGS) $(DEBUGFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS)) + @$(call PRINT_CC, $(CC) $(CFLAGS) $(CPPFLAGS) $(DEBUGFLAGS) -O3 $< $(fPIC) -shared -o $@.tmp $(LDFLAGS)) +ifeq ($(JULIA_SPLITDEBUG),1) + @# Create split debug info file for libccalltest stacktraces test + @# packagers should disable this by setting JULIA_SPLITDEBUG=0 if this is already done by your build system + $(OBJCOPY) --only-keep-debug $@.tmp $@.debug + $(OBJCOPY) --strip-debug $@.tmp + $(OBJCOPY) --add-gnu-debuglink=$@.debug $@.tmp +endif + @## clang should have made the dSYM split-debug directory, + @## but we are intentionally not going to give it the correct name + @## because we want to test the non-default debug configuration + @#rm -r $@.dSYM && mv $@.tmp.dSYM $@.dSYM + mv $@.tmp $@ julia_flisp.boot.inc.phony: $(BUILDDIR)/julia_flisp.boot.inc @@ -221,9 +239,11 @@ ifneq ($(OS), WINNT) @ln -sf libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia-debug.$(SHLIB_EXT) endif $(DSYMUTIL) $@ + $(BUILDDIR)/libjulia-debug.a: $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a rm -f $@ @$(call PRINT_LINK, ar -rcs $@ $(DOBJS)) + libjulia-debug: $(build_shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT): $(SRCDIR)/julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV) @@ -234,6 +254,7 @@ ifneq ($(OS), WINNT) @ln -sf libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia.$(SHLIB_EXT) endif $(DSYMUTIL) $@ + $(BUILDDIR)/libjulia.a: julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a rm -f $@ @$(call PRINT_LINK, ar -rcs $@ $(OBJS)) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index c02eda00de75b..c7e598fc5de4e 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -666,10 +666,19 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, // This function is not allowed to reference any TLS variables // since it can be called from an unmanaged thread on OSX. if (!context) { - if (demangle && (*frames)[0].func_name != NULL) { - char *oldname = (*frames)[0].func_name; - (*frames)[0].func_name = jl_demangle(oldname); - free(oldname); + if (demangle) { + if ((*frames)[0].func_name != NULL) { + char *oldname = (*frames)[0].func_name; + (*frames)[0].func_name = jl_demangle(oldname); + free(oldname); + } + else { + // We do this to hide the jlcall wrappers when getting julia backtraces, + // but it is still good to have them for regular lookup of C frames. + // Technically not true, but we don't want them + // in julia backtraces, so close enough + (*frames)[0].fromC = 1; + } } return 1; } @@ -686,6 +695,9 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, int fromC = (*frames)[0].fromC; int n_frames = inlineInfo.getNumberOfFrames(); + if (n_frames == 0) + // no line number info available in the context, return without the context + return lookup_pointer(NULL, frames, pointer, demangle, noInline); if (noInline) n_frames = 1; if (n_frames > 1) { @@ -921,6 +933,7 @@ extern "C" void jl_register_fptrs(uint64_t sysimage_base, void **fptrs, jl_lambd sysimg_fvars_n = n; } +extern "C" void jl_refresh_dbg_module_list(void); bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, llvm::DIContext **context, int64_t *slide, int64_t *section_slide, bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) { @@ -934,6 +947,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, #ifdef _OS_WINDOWS_ IMAGEHLP_MODULE64 ModuleInfo; ModuleInfo.SizeOfStruct = sizeof(IMAGEHLP_MODULE64); + jl_refresh_dbg_module_list(); jl_in_stackwalk = 1; bool isvalid = SymGetModuleInfo64(GetCurrentProcess(), (DWORD64)pointer, &ModuleInfo); jl_in_stackwalk = 0; diff --git a/src/stackwalk.c b/src/stackwalk.c index e67308ea01b25..f45b840a0c24c 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -194,17 +194,21 @@ static DWORD64 WINAPI JuliaGetModuleBase64( #endif } +// Might be called from unmanaged thread. int needsSymRefreshModuleList; BOOL (WINAPI *hSymRefreshModuleList)(HANDLE); -static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) +void jl_refresh_dbg_module_list(void) { - // Might be called from unmanaged thread. if (needsSymRefreshModuleList && hSymRefreshModuleList != 0 && !jl_in_stackwalk) { jl_in_stackwalk = 1; hSymRefreshModuleList(GetCurrentProcess()); jl_in_stackwalk = 0; needsSymRefreshModuleList = 0; } +} +static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) +{ + jl_refresh_dbg_module_list(); #if !defined(_CPU_X86_64_) if (jl_in_stackwalk) { return 0; @@ -372,7 +376,7 @@ JL_DLLEXPORT jl_value_t *jl_lookup_code_address(void *ip, int skipC) jl_svecset(r, 3, frame.linfo != NULL ? (jl_value_t*)frame.linfo : jl_nothing); jl_svecset(r, 4, jl_box_bool(frame.fromC)); jl_svecset(r, 5, jl_box_bool(frame.inlined)); - jl_svecset(r, 6, jl_box_long((intptr_t)ip)); + jl_svecset(r, 6, jl_box_voidpointer(ip)); } free(frames); JL_GC_POP(); diff --git a/test/stacktraces.jl b/test/stacktraces.jl index 2f36384a11112..2ba19b597b3de 100644 --- a/test/stacktraces.jl +++ b/test/stacktraces.jl @@ -109,3 +109,13 @@ let li = typeof(getfield).name.mt.cache.func::LambdaInfo, repr = string(sf) @test repr == " in getfield(...) at b:3" end + +let ctestptr = cglobal((:ctest, "libccalltest")), + ctest = StackTraces.lookup(ctestptr + 1) + + @test length(ctest) == 1 + @test ctest[1].func === :ctest + @test isnull(ctest[1].linfo) + @test ctest[1].from_c + @test ctest[1].pointer === UInt64(ctestptr) +end From 186905314a189efef70134b60695843470baf01e Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 15 Aug 2016 22:51:55 -0400 Subject: [PATCH 0943/1117] fix "improved" correctness of fieldtype the getfield_tfunc was missing a test for whether the fields were all equivalent before concluding that the result type was exact fixes the fix #17953 fix #18037 --- base/inference.jl | 9 +++++---- test/inference.jl | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 29efbf8237d34..196d98705b91c 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -479,7 +479,7 @@ function getfield_tfunc(s0::ANY, name) # in the current type system typ = limit_type_depth(R, 0, true, filter!(x->isa(x,TypeVar), Any[s.parameters...])) - return typ, isleaftype(s) && typeseq(typ, R) + return typ, isleaftype(s) && isa(R, Type) && typeof(R) === typeof(typ) && typeseq(R, typ) end end end @@ -502,14 +502,15 @@ function getfield_tfunc(s0::ANY, name) elseif length(s.types) == 1 && isempty(s.parameters) return s.types[1], true else - R = reduce(tmerge, Bottom, map(unwrapva,s.types)) #=Union{s.types...}=# + R = reduce(tmerge, Bottom, map(unwrapva, s.types)) #=Union{s.types...}=# + alleq = isa(R, Type) && typeof(R) === typeof(s.types[1]) && typeseq(R, s.types[1]) # do the same limiting as the known-symbol case to preserve type-monotonicity if isempty(s.parameters) - return R, typeseq(R, s.types[1]) + return R, alleq else typ = limit_type_depth(R, 0, true, filter!(x->isa(x,TypeVar), Any[s.parameters...])) - return typ, isleaftype(s) && typeseq(typ, R) + return typ, alleq && isleaftype(s) && typeof(R) === typeof(typ) && typeseq(R, typ) end end end diff --git a/test/inference.jl b/test/inference.jl index f1f4fbb9b1457..a324716b2511b 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -307,6 +307,21 @@ let T = Array{Tuple{Vararg{Float64,TypeVar(:dim)}},1}, end @test f16530a(:d) == Vector +let T1 = Tuple{Int, Float64}, + T2 = Tuple{Int, Float32}, + T = Tuple{T1, T2} + + global f18037 + f18037() = fieldtype(T, 1) + f18037(i) = fieldtype(T, i) + + @test f18037() === T1 + @test f18037(1) === T1 + @test f18037(2) === T2 + + @test Base.return_types(f18037, ()) == Any[Type{T1}] + @test Base.return_types(f18037, (Int,)) == Any[Type{TypeVar(:T, Tuple{Int, AbstractFloat})}] +end # issue #18015 type Triple18015 From 7d1533a9a60ff556aaef9eba65508623c701f762 Mon Sep 17 00:00:00 2001 From: Simon Byrne <simonbyrne@gmail.com> Date: Tue, 16 Aug 2016 12:06:08 +0100 Subject: [PATCH 0944/1117] Document known problem with setrounding. Fixes the doc problem mentioned in #17926, but not the underlying problem. --- base/rounding.jl | 25 +++++++++++++++++-- .../integers-and-floating-point-numbers.rst | 8 ++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/base/rounding.jl b/base/rounding.jl index 43fe9565440e2..4571419584c05 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -100,12 +100,15 @@ end setrounding(T, mode) Set the rounding mode of floating point type `T`, controlling the rounding of basic -arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/`](:func:`/`) -and [`sqrt`](:func:`sqrt`)) and type conversion. +arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), +[`/`](:func:`/`) and [`sqrt`](:func:`sqrt`)) and type conversion. Other numerical +functions may give incorrect or invalid values when using rounding modes other than the +default `RoundNearest`. Note that this may affect other types, for instance changing the rounding mode of `Float64` will change the rounding mode of `Float32`. See [`RoundingMode`](:obj:`RoundingMode`) for available modes. + """ setrounding(T::Type, mode) @@ -117,6 +120,8 @@ arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/` and [`sqrt`](:func:`sqrt`)) and type conversion. See [`RoundingMode`](:obj:`RoundingMode`) for available modes. + +**Warning**: This feature is still experimental, and may give unexpected or incorrect values. """ :rounding @@ -138,6 +143,22 @@ equivalent to: setrounding(T, old) See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. + +**Warning**: This feature is still experimental, and may give unexpected or incorrect values. A known problem is the interaction with compiler optimisations, e.g. + + julia> setrounding(Float64,RoundDown) do + 1.1 + 0.1 + end + 1.2000000000000002 + +Here the compiler is *constant folding*, that is evaluating a known constant expression at compile time, however the rounding mode is only changed at runtime, so this is not reflected in the function result. This can be avoided by moving constants outside the expression, e.g. + + julia> x = 1.1; y = 0.1; + + julia> setrounding(Float64,RoundDown) do + x + y + end + 1.2 """ function setrounding{T}(f::Function, ::Type{T}, rounding::RoundingMode) old_rounding_raw = rounding_raw(T) diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index 36279871cc0f5..8ffdf7e745d08 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -507,12 +507,16 @@ rounded to an appropriate representable value, however, if wanted, the manner in which this rounding is done can be changed according to the rounding modes presented in the `IEEE 754 standard <https://en.wikipedia.org/wiki/IEEE_754-2008>`_:: +.. doctest:: + + + julia> x = 1.1; y = 0.1; - julia> 1.1 + 0.1 + julia> x + y 1.2000000000000002 julia> setrounding(Float64,RoundDown) do - 1.1 + 0.1 + x + y end 1.2 From ff741bfe0ec0bd80063be2e685b587b37a7fd530 Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Tue, 16 Aug 2016 11:07:04 -0400 Subject: [PATCH 0945/1117] Clear the IRBuilder's insertion point after emitting a function. The function might get finalized, invalidating the IP. However, in some cases this invalid IP may get saved and restored, accessing the invalid IP while doing so. Example code path accessing an invalid IP: -> jl_cfunction_object (nested_compile=true, but doesn't change IP) -> gen_cfun_wrapper -> jl_compile_linfo (saves and restores invalid IP) --- src/codegen.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index 69c8ea00f1586..377700d5530e2 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4718,6 +4718,8 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } } + builder.ClearInsertionPoint(); + // step 13, Apply LLVM level inlining for(std::vector<CallInst*>::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) { Function *inlinef = (*it)->getCalledFunction(); From aa7e3caf17aeb1d8744e7623b175aece35c57f24 Mon Sep 17 00:00:00 2001 From: Simon Byrne <simonbyrne@gmail.com> Date: Tue, 16 Aug 2016 16:30:46 +0100 Subject: [PATCH 0946/1117] update docs, use new markdown syntax --- base/rounding.jl | 27 ++++++++++++++------------- doc/stdlib/numbers.rst | 28 +++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/base/rounding.jl b/base/rounding.jl index 4571419584c05..76e3490b78bc7 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -109,6 +109,8 @@ Note that this may affect other types, for instance changing the rounding mode o will change the rounding mode of `Float32`. See [`RoundingMode`](:obj:`RoundingMode`) for available modes. +!!! warning + This feature is still experimental, and may give unexpected or incorrect values. """ setrounding(T::Type, mode) @@ -120,8 +122,6 @@ arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/` and [`sqrt`](:func:`sqrt`)) and type conversion. See [`RoundingMode`](:obj:`RoundingMode`) for available modes. - -**Warning**: This feature is still experimental, and may give unexpected or incorrect values. """ :rounding @@ -144,21 +144,22 @@ equivalent to: See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. -**Warning**: This feature is still experimental, and may give unexpected or incorrect values. A known problem is the interaction with compiler optimisations, e.g. +!!! warning + This feature is still experimental, and may give unexpected or incorrect values. A known problem is the interaction with compiler optimisations, e.g. - julia> setrounding(Float64,RoundDown) do - 1.1 + 0.1 - end - 1.2000000000000002 + julia> setrounding(Float64,RoundDown) do + 1.1 + 0.1 + end + 1.2000000000000002 -Here the compiler is *constant folding*, that is evaluating a known constant expression at compile time, however the rounding mode is only changed at runtime, so this is not reflected in the function result. This can be avoided by moving constants outside the expression, e.g. + Here the compiler is *constant folding*, that is evaluating a known constant expression at compile time, however the rounding mode is only changed at runtime, so this is not reflected in the function result. This can be avoided by moving constants outside the expression, e.g. - julia> x = 1.1; y = 0.1; + julia> x = 1.1; y = 0.1; - julia> setrounding(Float64,RoundDown) do - x + y - end - 1.2 + julia> setrounding(Float64,RoundDown) do + x + y + end + 1.2 """ function setrounding{T}(f::Function, ::Type{T}, rounding::RoundingMode) old_rounding_raw = rounding_raw(T) diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index ae1d47fa34c5f..1d89e90c5db58 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -371,10 +371,14 @@ General Number Functions and Constants .. Docstring generated from Julia source - Set the rounding mode of floating point type ``T``\ , controlling the rounding of basic arithmetic functions (:func:`+`\ , :func:`-`\ , :func:`*`\ , :func:`/` and :func:`sqrt`\ ) and type conversion. + Set the rounding mode of floating point type ``T``\ , controlling the rounding of basic arithmetic functions (:func:`+`\ , :func:`-`\ , :func:`*`\ , :func:`/` and :func:`sqrt`\ ) and type conversion. Other numerical functions may give incorrect or invalid values when using rounding modes other than the default ``RoundNearest``\ . Note that this may affect other types, for instance changing the rounding mode of ``Float64`` will change the rounding mode of ``Float32``\ . See :obj:`RoundingMode` for available modes. + .. warning:: + This feature is still experimental, and may give unexpected or incorrect values. + + .. function:: setrounding(f::Function, T, mode) .. Docstring generated from Julia source @@ -390,6 +394,28 @@ General Number Functions and Constants See :obj:`RoundingMode` for available rounding modes. + .. warning:: + This feature is still experimental, and may give unexpected or incorrect values. A known problem is the interaction with compiler optimisations, e.g. + + .. code-block:: julia + + julia> setrounding(Float64,RoundDown) do + 1.1 + 0.1 + end + 1.2000000000000002 + + Here the compiler is *constant folding*, that is evaluating a known constant expression at compile time, however the rounding mode is only changed at runtime, so this is not reflected in the function result. This can be avoided by moving constants outside the expression, e.g. + + .. code-block:: julia + + julia> x = 1.1; y = 0.1; + + julia> setrounding(Float64,RoundDown) do + x + y + end + 1.2 + + .. function:: get_zero_subnormals() -> Bool .. Docstring generated from Julia source From c45ad1abe8965041b273d660b7eaecf771b4bca6 Mon Sep 17 00:00:00 2001 From: Simon Byrne <simonbyrne@gmail.com> Date: Tue, 16 Aug 2016 16:46:13 +0100 Subject: [PATCH 0947/1117] wrap help text --- base/rounding.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/base/rounding.jl b/base/rounding.jl index 76e3490b78bc7..cbb740af82889 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -110,6 +110,7 @@ will change the rounding mode of `Float32`. See [`RoundingMode`](:obj:`RoundingM available modes. !!! warning + This feature is still experimental, and may give unexpected or incorrect values. """ setrounding(T::Type, mode) @@ -145,14 +146,19 @@ equivalent to: See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. !!! warning - This feature is still experimental, and may give unexpected or incorrect values. A known problem is the interaction with compiler optimisations, e.g. + + This feature is still experimental, and may give unexpected or incorrect values. A + known problem is the interaction with compiler optimisations, e.g. julia> setrounding(Float64,RoundDown) do 1.1 + 0.1 end 1.2000000000000002 - Here the compiler is *constant folding*, that is evaluating a known constant expression at compile time, however the rounding mode is only changed at runtime, so this is not reflected in the function result. This can be avoided by moving constants outside the expression, e.g. + Here the compiler is *constant folding*, that is evaluating a known constant + expression at compile time, however the rounding mode is only changed at runtime, so + this is not reflected in the function result. This can be avoided by moving constants + outside the expression, e.g. julia> x = 1.1; y = 0.1; From 4053f3a84759a2f4e2211229e8ef8b19bca97dd4 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Tue, 16 Aug 2016 11:23:27 -0500 Subject: [PATCH 0948/1117] Fix bug with operator precendence --- base/libgit2/callbacks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index fd2580da27ee7..cbab587516e82 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -160,7 +160,7 @@ function authenticate_userpass(creds::UserPasswordCredentials, libgit2credptr::P urlusername : username) userpass = prompt("Password for '$schema$username@$host'", password=true) end - (creds.user != username) || (creds.pass != userpass) && reset!(creds) + ((creds.user != username) || (creds.pass != userpass)) && reset!(creds) creds.user = username # save credentials creds.pass = userpass # save credentials From 81b9e9ff7410ba4fa2fcfd6c5bc711103e334be9 Mon Sep 17 00:00:00 2001 From: Pranav Thulasiram Bhat <pranavtbhat@gmail.com> Date: Tue, 16 Aug 2016 22:52:19 +0530 Subject: [PATCH 0949/1117] Define find and findnz for SparseVector (#18049) --- base/sparse/sparsevector.jl | 51 +++++++++++++++++++++++++++++++++- test/sparsedir/sparsevector.jl | 7 +++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index 9a8990d5c47e4..e77cef2776cd9 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -2,7 +2,7 @@ ### Common definitions -import Base: scalarmax, scalarmin, sort +import Base: scalarmax, scalarmin, sort, find, findnz ### The SparseVector @@ -560,6 +560,55 @@ function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector) SparseVector(n, rowvalB, nzvalB) end +function find{Tv,Ti}(x::SparseVector{Tv,Ti}) + numnz = nnz(x) + I = Array(Ti, numnz) + + nzind = x.nzind + nzval = x.nzval + + count = 1 + @inbounds for i = 1 : numnz + if nzval[i] != 0 + I[count] = nzind[i] + count += 1 + end + end + + count -= 1 + if numnz != count + deleteat!(I, (count+1):numnz) + end + + return I +end + +function findnz{Tv,Ti}(x::SparseVector{Tv,Ti}) + numnz = nnz(x) + + I = Array(Ti, numnz) + V = Array(Tv, numnz) + + nzind = x.nzind + nzval = x.nzval + + count = 1 + @inbounds for i = 1 : numnz + if nzval[i] != 0 + I[count] = nzind[i] + V[count] = nzval[i] + count += 1 + end + end + + count -= 1 + if numnz != count + deleteat!(I, (count+1):numnz) + deleteat!(V, (count+1):numnz) + end + + return (I, V) +end ### Generic functions operating on AbstractSparseVector diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index b9079c95807be..fc91b9ba3f11d 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -225,6 +225,13 @@ let x = SparseVector(10, [2, 7, 9], [2.0, 7.0, 9.0]) @test Base.SparseArrays.dropstored!(x, 5) == SparseVector(10, [7, 9], [7.0, 9.0]) end +# find and findnz tests +@test find(spv_x1) == find(x1_full) +@test findnz(spv_x1) == (find(x1_full), filter(x->x!=0, x1_full)) +let xc = SparseVector(8, [2, 3, 5], [1.25, 0, -0.75]), fc = full(xc) + @test find(xc) == find(fc) + @test findnz(xc) == ([2, 5], [1.25, -0.75]) +end ### Array manipulation From ae4b42f400f146f3a004b7ea324d05af13e4516a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 16 Aug 2016 13:33:01 -0400 Subject: [PATCH 0950/1117] assume `getfield` is non-volatile if inferred to be constant fixes regression caused by #18017 --- base/inference.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index 196d98705b91c..8c423674a7b99 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2258,7 +2258,7 @@ function effect_free(e::ANY, linfo::LambdaInfo, allow_volatile::Bool) if !allow_volatile if is_known_call(e, arrayref, linfo) || is_known_call(e, arraylen, linfo) return false - elseif is_known_call(e, getfield, linfo) + elseif is_known_call(e, getfield, linfo) && !isa(exprtype(e,linfo), Const) # first argument must be immutable to ensure e is affect_free a = ea[2] typ = widenconst(exprtype(a, linfo)) From 3b55c99883e1f066e212c2aac92e2afec0c01e16 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 16 Aug 2016 13:39:59 -0400 Subject: [PATCH 0951/1117] fix #18051, allocation in sparse vector getindex Caused by the return type of `ord()` being unknown. --- base/sort.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/sort.jl b/base/sort.jl index d388e55f6647c..91e752202c57b 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -184,6 +184,7 @@ for s in [:searchsortedfirst, :searchsortedlast, :searchsorted] $s(v::AbstractVector, x; lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) = $s(v,x,ord(lt,by,rev,order)) + $s(v::AbstractVector, x) = $s(v, x, Forward) end end From d6470bcb9fbb236fbf3cafa5879af19ae206ccf5 Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Tue, 16 Aug 2016 20:27:57 +0200 Subject: [PATCH 0952/1117] Add test for #18054. Note that this test requires a memory sanitizer (ASAN, valgrind) to detect failure. --- test/core.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/core.jl b/test/core.jl index ee4605ff60704..b3dd90d9bba49 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4469,3 +4469,10 @@ let k(x) = (k = x; k) @test k(1) == 1 end + +# PR #18054: compilation of cfunction leaves IRBuilder in bad state, +# causing heap-use-after-free when compiling f18054 +function f18054() + return Cint(0) +end +cfunction(f18054, Cint, ()) From f815ddda0a1880e94ffcce2d81b7128f03fe4a06 Mon Sep 17 00:00:00 2001 From: Mus M <mus-m@outlook.com> Date: Tue, 16 Aug 2016 14:35:17 -0400 Subject: [PATCH 0953/1117] Fix for consistency. --- doc/manual/interfaces.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 22b1e808c946f..fc8fbf349f122 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -66,7 +66,7 @@ A simple example is an iterable sequence of square numbers with a defined length end Base.start(::Squares) = 1 Base.next(S::Squares, state) = (state*state, state+1) - Base.done(S::Squares, s) = s > S.count; + Base.done(S::Squares, state) = state > S.count; Base.eltype(::Type{Squares}) = Int # Note that this is defined for the type Base.length(S::Squares) = S.count; From f1aa31af6718fe3d1ab060833b920ab3725f1f8e Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Tue, 16 Aug 2016 15:24:10 -0400 Subject: [PATCH 0954/1117] fix #10633, remove redundant definitions for `map(Integer, a)` etc. --- base/abstractarray.jl | 4 ---- test/functional.jl | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 452a7f1643d61..ebf99e5806245 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -713,10 +713,6 @@ of_indices(x, y) = similar(dims->y, oftype(indices(x), indices(y))) full(x::AbstractArray) = x -map(::Type{Integer}, a::Array) = map!(Integer, similar(a,typeof(Integer(one(eltype(a))))), a) -map(::Type{Signed}, a::Array) = map!(Signed, similar(a,typeof(Signed(one(eltype(a))))), a) -map(::Type{Unsigned}, a::Array) = map!(Unsigned, similar(a,typeof(Unsigned(one(eltype(a))))), a) - ## range conversions ## map{T<:Real}(::Type{T}, r::StepRange) = T(r.start):T(r.step):T(last(r)) diff --git a/test/functional.jl b/test/functional.jl index 1c9a68d6430ab..7846491f05153 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -32,6 +32,10 @@ end # maps of strings (character arrays) -- string.jl @test map((c)->Char(c+1), "abcDEF") == "bcdEFG" +# issue #10633 +@test isa(map(Integer, Any[1, 2]), Vector{Int}) +@test isa(map(Integer, Any[]), Vector{Integer}) + # filter -- array.jl @test isequal(filter(x->(x>1), [0 1 2 3 2 1 0]), [2, 3, 2]) # TODO: @test_throws isequal(filter(x->x+1, [0 1 2 3 2 1 0]), [2, 3, 2]) From a7acb3f90de018eb224ce3ed73a824e8d404f6fe Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Tue, 16 Aug 2016 14:34:46 -0500 Subject: [PATCH 0955/1117] Improve URL regex --- base/libgit2/callbacks.jl | 10 ++++------ base/libgit2/utils.jl | 8 +++++++- test/libgit2.jl | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index cbab587516e82..13f2f88e168de 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -209,12 +209,10 @@ function credentials_callback(libgit2credptr::Ptr{Ptr{Void}}, url_ptr::Cstring, url = unsafe_string(url_ptr) # parse url for schema and host - urlparts = match(urlmatcher, url) - schema = urlparts.captures[1] - urlusername = urlparts.captures[4] - urlusername = urlusername === nothing ? "" : String(urlusername) - host = urlparts.captures[5] - schema = schema === nothing ? "" : schema*"://" + urlparts = match(URL_REGEX, url) + schema = urlparts[:scheme] === nothing ? "" : urlparts[:scheme] * "://" + urlusername = urlparts[:user] === nothing ? "" : urlparts[:user] + host = urlparts[:host] # get credentials object from payload pointer @assert payload_ptr != C_NULL diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index ba2149dbda576..ab8928687e3c7 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -1,6 +1,12 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -const urlmatcher = r"^(http[s]?|git|ssh)?(:\/\/)?((\w+)@)?([A-Za-z0-9\-\.]+)(:[0-9]+)?(.*)$" +const URL_REGEX = r""" +^(?:(?<scheme>https?|git|ssh)\:\/\/)? +(?:(?<user>.*?)(?:\:(?<password>.*?))?@)? +(?<host>[A-Za-z0-9\-\.]+) +(?:\:(?<port>\d+)?)? +(?<path>.*?)$ +"""x function version() major = Ref{Cint}(0) diff --git a/test/libgit2.jl b/test/libgit2.jl index a0466c21c7e15..96a183c3d6e27 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -76,6 +76,39 @@ const LIBGIT2_MIN_VER = v"0.23.0" @test sig3.email == sig.email #end +#@testset "URL parsing" begin + # Use all named group + m = match(LibGit2.URL_REGEX, "https://user:pass@hostname.com:80/path/repo.git") + @test m[:scheme] == "https" + @test m[:user] == "user" + @test m[:password] == "pass" + @test m[:host] == "hostname.com" + @test m[:port] == "80" + @test m[:path] == "/path/repo.git" + + # Realistic example HTTP example + m = match(LibGit2.URL_REGEX, "https://github.com/JuliaLang/Example.jl.git") + @test m[:scheme] == "https" + @test m[:user] == nothing + @test m[:password] == nothing + @test m[:host] == "github.com" + @test m[:port] == nothing + @test m[:path] == "/JuliaLang/Example.jl.git" + + # Realistic example SSH example + m = match(LibGit2.URL_REGEX, "git@github.com:JuliaLang/Example.jl.git") + @test m[:scheme] == nothing + @test m[:user] == "git" + @test m[:password] == nothing + @test m[:host] == "github.com" + @test m[:port] == nothing + @test m[:path] == "JuliaLang/Example.jl.git" + + # Make sure that a username can contain special characters + m = match(LibGit2.URL_REGEX, "user-name@hostname.com") + @test m[:user] == "user-name" +#end + mktempdir() do dir # test parameters repo_url = "https://github.com/JuliaLang/Example.jl" From 9ca347bafd7642ff554fa3be9c4525a774c874ba Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Tue, 16 Aug 2016 14:49:11 -0500 Subject: [PATCH 0956/1117] Some indices generalizations for linalg/generic (#18032) --- base/linalg/generic.jl | 56 +++++++++++++++++++++--------------------- base/linalg/linalg.jl | 4 ++- test/offsetarray.jl | 4 +++ 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 3451999961ba7..261bed72b8c39 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -19,8 +19,8 @@ function generic_scale!(s::Number, X::AbstractArray) end function generic_scale!(C::AbstractArray, X::AbstractArray, s::Number) - if length(C) != length(X) - throw(DimensionMismatch("first array has length $(length(C)) which does not match the length of the second, $(length(X)).")) + if _length(C) != _length(X) + throw(DimensionMismatch("first array has length $(_length(C)) which does not match the length of the second, $(_length(X)).")) end for (IC, IX) in zip(eachindex(C), eachindex(X)) @inbounds C[IC] = X[IX]*s @@ -29,9 +29,9 @@ function generic_scale!(C::AbstractArray, X::AbstractArray, s::Number) end function generic_scale!(C::AbstractArray, s::Number, X::AbstractArray) - if length(C) != length(X) - throw(DimensionMismatch("first array has length $(length(C)) which does not -match the length of the second, $(length(X)).")) + if _length(C) != _length(X) + throw(DimensionMismatch("first array has length $(_length(C)) which does not +match the length of the second, $(_length(X)).")) end for (IC, IX) in zip(eachindex(C), eachindex(X)) @inbounds C[IC] = s*X[IX] @@ -126,7 +126,7 @@ function generic_vecnorm2(x) s = start(x) (v, s) = next(x, s) T = typeof(maxabs) - if isfinite(length(x)*maxabs*maxabs) && maxabs*maxabs != 0 # Scaling not necessary + if isfinite(_length(x)*maxabs*maxabs) && maxabs*maxabs != 0 # Scaling not necessary sum::promote_type(Float64, T) = norm_sqr(v) while !done(x, s) (v, s) = next(x, s) @@ -156,7 +156,7 @@ function generic_vecnormp(x, p) T = typeof(float(norm(v))) end spp::promote_type(Float64, T) = p - if -1 <= p <= 1 || (isfinite(length(x)*maxabs^spp) && maxabs^spp != 0) # scaling not necessary + if -1 <= p <= 1 || (isfinite(_length(x)*maxabs^spp) && maxabs^spp != 0) # scaling not necessary sum::promote_type(Float64, T) = norm(v)^spp while !done(x, s) (v, s) = next(x, s) @@ -253,14 +253,14 @@ end @inline norm(x::Number, p::Real=2) = vecnorm(x, p) function vecdot(x::AbstractArray, y::AbstractArray) - lx = length(x) - if lx != length(y) - throw(DimensionMismatch("first array has length $(lx) which does not match the length of the second, $(length(y)).")) + lx = _length(x) + if lx != _length(y) + throw(DimensionMismatch("first array has length $(lx) which does not match the length of the second, $(_length(y)).")) end if lx == 0 return dot(zero(eltype(x)), zero(eltype(y))) end - s = zero(dot(x[1], y[1])) + s = zero(dot(first(x), first(y))) for (Ix, Iy) in zip(eachindex(x), eachindex(y)) @inbounds s += dot(x[Ix], y[Iy]) end @@ -378,11 +378,11 @@ condskeel(A::AbstractMatrix, x::AbstractVector, p::Real=Inf) = norm(abs(inv(A))* condskeel{T<:Integer}(A::AbstractMatrix{T}, x::AbstractVector, p::Real=Inf) = norm(abs(inv(float(A)))*abs(A)*abs(x), p) function issymmetric(A::AbstractMatrix) - m, n = size(A) - if m != n + indsm, indsn = indices(A) + if indsm != indsn return false end - for i = 1:(n-1), j = (i+1):n + for i = first(indsn):last(indsn)-1, j = (i+1):last(indsn) if A[i,j] != transpose(A[j,i]) return false end @@ -393,11 +393,11 @@ end issymmetric(x::Number) = true function ishermitian(A::AbstractMatrix) - m, n = size(A) - if m != n + indsm, indsn = indices(A) + if indsm != indsn return false end - for i = 1:n, j = i:n + for i = indsn, j = i:last(indsn) if A[i,j] != ctranspose(A[j,i]) return false end @@ -497,26 +497,26 @@ end # BLAS-like in-place y = x*α+y function (see also the version in blas.jl # for BlasFloat Arrays) function axpy!(α, x::AbstractArray, y::AbstractArray) - n = length(x) - if n != length(y) - throw(DimensionMismatch("x has length $n, but y has length $(length(y))")) + n = _length(x) + if n != _length(y) + throw(DimensionMismatch("x has length $n, but y has length $(_length(y))")) end - for i = 1:n - @inbounds y[i] += x[i]*α + for (IY, IX) in zip(eachindex(y), eachindex(x)) + @inbounds y[IY] += x[IX]*α end y end function axpy!{Ti<:Integer,Tj<:Integer}(α, x::AbstractArray, rx::AbstractArray{Ti}, y::AbstractArray, ry::AbstractArray{Tj}) - if length(rx) != length(ry) - throw(DimensionMismatch("rx has length $(length(rx)), but ry has length $(length(ry))")) - elseif minimum(rx) < 1 || maximum(rx) > length(x) + if _length(rx) != _length(ry) + throw(DimensionMismatch("rx has length $(_length(rx)), but ry has length $(_length(ry))")) + elseif !checkindex(Bool, linearindices(x), rx) throw(BoundsError(x, rx)) - elseif minimum(ry) < 1 || maximum(ry) > length(y) + elseif !checkindex(Bool, linearindices(y), ry) throw(BoundsError(y, ry)) end - for i = 1:length(rx) - @inbounds y[ry[i]] += x[rx[i]]*α + for (IY, IX) in zip(eachindex(ry), eachindex(rx)) + @inbounds y[ry[IY]] += x[rx[IX]]*α end y end diff --git a/base/linalg/linalg.jl b/base/linalg/linalg.jl index dc36d649fae31..556020776082a 100644 --- a/base/linalg/linalg.jl +++ b/base/linalg/linalg.jl @@ -10,7 +10,9 @@ import Base: USE_BLAS64, abs, big, ceil, conj, convert, copy, copy!, copy_transp imag, inv, isapprox, kron, ndims, parent, power_by_squaring, print_matrix, promote_rule, real, round, setindex!, show, similar, size, transpose, transpose!, trunc -using Base: promote_op +using Base: promote_op, _length +# We use `_length` because of non-1 indices; releases after julia 0.5 +# can go back to `length`. `_length(A)` is equivalent to `length(linearindices(A))`. export # Modules diff --git a/test/offsetarray.jl b/test/offsetarray.jl index ce88abbb329e7..6d99433d03d61 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -294,6 +294,10 @@ I,J,N = findnz(z) @test find(x->x<0, h) == [-2,0] @test find(x->x==0, h) == [2] +@test_approx_eq vecnorm(v) vecnorm(parent(v)) +@test_approx_eq vecnorm(A) vecnorm(parent(A)) +@test_approx_eq vecdot(v, v) vecdot(v0, v0) + v = OffsetArray([1,1e100,1,-1e100], (-3,))*1000 v2 = OffsetArray([1,-1e100,1,1e100], (5,))*1000 @test isa(v, OffsetArray) From 1a77b8ca11a317ceb4c54b820a545cfbf24ee565 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Tue, 16 Aug 2016 15:13:07 -0500 Subject: [PATCH 0957/1117] Correct invalid SSH URL An SSH URL with a path uses a colon between the host and the path: git@github.com:JuliaLang/Example.jl (correct) git@github.com/JuliaLang/Example.jl (incorrect) --- test/libgit2-online.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/libgit2-online.jl b/test/libgit2-online.jl index 436003eb37160..61b683f087138 100644 --- a/test/libgit2-online.jl +++ b/test/libgit2-online.jl @@ -7,13 +7,12 @@ ######### # init & clone mktempdir() do dir - repo_url = "github.com/JuliaLang/Example.jl" - https_prefix = "https://" - ssh_prefix = "git@" + repo_url_https = "https://github.com/JuliaLang/Example.jl" + repo_url_git = "git@github.com:JuliaLang/Example.jl" #@testset "Cloning repository" begin #@testset "with 'https' protocol" begin repo_path = joinpath(dir, "Example1") - repo = LibGit2.clone(https_prefix*repo_url, repo_path) + repo = LibGit2.clone(repo_url_https, repo_path) try @test isdir(repo_path) @test isdir(joinpath(repo_path, ".git")) @@ -27,7 +26,7 @@ mktempdir() do dir repo_path = joinpath(dir, "Example2") # credentials are required because github tries to authenticate on unknown repo cred = LibGit2.UserPasswordCredentials("","") # empty credentials cause authentication error - LibGit2.clone(https_prefix*repo_url*randstring(10), repo_path, payload=Nullable(cred)) + LibGit2.clone(repo_url_https*randstring(10), repo_path, payload=Nullable(cred)) error("unexpected") catch ex @test isa(ex, LibGit2.Error.GitError) @@ -37,12 +36,13 @@ mktempdir() do dir #TODO: remove or condition on libgit2 features this test when ssh protocol will be supported #@testset "with 'ssh' protocol (by default is not supported)" begin + repo_path = joinpath(dir, "Example3") + repo = LibGit2.clone(repo_url_git, repo_path) try - repo_path = joinpath(dir, "Example3") - @test_throws LibGit2.Error.GitError LibGit2.clone(ssh_prefix*repo_url, repo_path) - catch ex - # but we cloned succesfully, so check that repo was created - ex.fail == 1 && @test isdir(joinpath(path, ".git")) + @test isdir(repo_path) + @test isdir(joinpath(repo_path, ".git")) + finally + finalize(repo) end #end #end From 14579d23e2bc57e4a138e83bebeaac6e58f73686 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 17 Aug 2016 00:06:21 -0400 Subject: [PATCH 0958/1117] move recheck_tuple_intersection to thread-local --- src/jltypes.c | 78 +++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index a0e4409b8745a..9caf8b720f0a8 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -392,10 +392,14 @@ static void extend(jl_value_t *var, jl_value_t *val, cenv_t *soln) } static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var); + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var); static jl_value_t *intersect_union(jl_uniontype_t *a, jl_value_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var) { int eq0 = eqc->n, co0 = penv->n; size_t i, l = jl_svec_len(a->types); @@ -411,11 +415,11 @@ static jl_value_t *intersect_union(jl_uniontype_t *a, jl_value_t *b, for(i=0; i < l; i++) { int eq_l = eqc->n, co_l = penv->n; jl_value_t *ti = jl_type_intersect(jl_svecref(a->types,i), b, - penv, eqc, var); + penv, eqc, recheck_tuple_intersection, var); if (ti == (jl_value_t*)jl_bottom_type) { eqc->n = eq0; penv->n = co0; ti = jl_type_intersect(jl_svecref(a->types,i), b, - penv, eqc, var); + penv, eqc, recheck_tuple_intersection, var); if (ti != (jl_value_t*)jl_bottom_type) { // tvar conflict among union elements; keep the conflicting // constraints rolled back @@ -553,10 +557,10 @@ to be returned, with one exception illustrated by: where typeintersect(B,C) == Bottom. */ -int recheck_tuple_intersection = 0; // "flag" above - static jl_value_t *intersect_tuple(jl_datatype_t *a, jl_datatype_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, // "flag" above + variance_t var) { jl_svec_t *ap = a->parameters, *bp = b->parameters; size_t alenr = jl_svec_len(ap), blenr = jl_svec_len(bp); @@ -590,7 +594,7 @@ static jl_value_t *intersect_tuple(jl_datatype_t *a, jl_datatype_t *b, // Do we need to store "at least N" constraints in penv? // Formerly, typeintersect(Tuple{A,Vararg{B}}, NTuple{N,C}) did that if (akind == JL_VARARG_BOUND || bkind == JL_VARARG_BOUND) - recheck_tuple_intersection = 1; + *recheck_tuple_intersection = 1; } if (bottom) return (jl_value_t*) jl_bottom_type; if (n == 0) return jl_typeof(jl_emptytuple); @@ -626,17 +630,17 @@ static jl_value_t *intersect_tuple(jl_datatype_t *a, jl_datatype_t *b, bi++; } assert(ae!=NULL && be!=NULL); - ce = jl_type_intersect(ae,be,penv,eqc,var); + ce = jl_type_intersect(ae, be, penv, eqc, recheck_tuple_intersection, var); if (ce == (jl_value_t*)jl_bottom_type) { if (var!=invariant && aseq && bseq) { // (X∩Y)==∅ → (X...)∩(Y...) == () // We don't need to set bindings here because - // recheck_tuple_intersection=1 + // *recheck_tuple_intersection = 1 if (n == 1) { JL_GC_POP(); return (jl_value_t*)jl_typeof(jl_emptytuple); } - jl_svec_set_len_unsafe(tc,jl_svec_len(tc)-1); + jl_svec_set_len_unsafe(tc, jl_svec_len(tc) - 1); goto done_intersect_tuple; } JL_GC_POP(); @@ -653,7 +657,9 @@ static jl_value_t *intersect_tuple(jl_datatype_t *a, jl_datatype_t *b, } static jl_value_t *intersect_tag(jl_datatype_t *a, jl_datatype_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var) { assert(a->name == b->name); assert(jl_svec_len(a->parameters) == jl_svec_len(b->parameters)); @@ -678,7 +684,7 @@ static jl_value_t *intersect_tag(jl_datatype_t *a, jl_datatype_t *b, return (jl_value_t*)jl_bottom_type; } } - ti = jl_type_intersect(ap,bp,penv,eqc,invariant); + ti = jl_type_intersect(ap, bp, penv, eqc, recheck_tuple_intersection, invariant); if (bp == (jl_value_t*)jl_bottom_type && !((jl_tvar_t*)ap)->bound) { // "Union{}" as a type parameter @@ -687,7 +693,7 @@ static jl_value_t *intersect_tag(jl_datatype_t *a, jl_datatype_t *b, } } else if (jl_is_typevar(bp)) { - ti = jl_type_intersect(ap,bp,penv,eqc,invariant); + ti = jl_type_intersect(ap, bp, penv, eqc, recheck_tuple_intersection, invariant); if (ap == (jl_value_t*)jl_bottom_type && !((jl_tvar_t*)bp)->bound) { // "Union{}" as a type parameter @@ -701,7 +707,7 @@ static jl_value_t *intersect_tag(jl_datatype_t *a, jl_datatype_t *b, if (tva || tvb) { if (jl_subtype_invariant(ap,bp,0) || jl_subtype_invariant(bp,ap,0)) { - ti = jl_type_intersect(ap,bp,penv,eqc,invariant); + ti = jl_type_intersect(ap, bp, penv, eqc, recheck_tuple_intersection, invariant); } else { ti = (jl_value_t*)jl_bottom_type; @@ -786,7 +792,9 @@ static int match_intersection_mode = 0; static jl_value_t *meet_tvars(jl_tvar_t *a, jl_tvar_t *b); static jl_value_t *intersect_typevar(jl_tvar_t *a, jl_value_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var) { jl_value_t *both=NULL; jl_tvar_t *new_b=NULL; @@ -828,7 +836,7 @@ static jl_value_t *intersect_typevar(jl_tvar_t *a, jl_value_t *b, } } else { - b = jl_type_intersect(a->ub, b, penv, eqc, covariant); + b = jl_type_intersect(a->ub, b, penv, eqc, recheck_tuple_intersection, covariant); if (b == jl_bottom_type) { JL_GC_POP(); return b; @@ -924,7 +932,7 @@ static jl_value_t *intersect_typevar(jl_tvar_t *a, jl_value_t *b, return (jl_value_t*)a; } -static jl_value_t *approxify_type(jl_datatype_t *dt, jl_svec_t *pp) +static jl_value_t *approxify_type(jl_datatype_t *dt, jl_svec_t *pp, int *recheck_tuple_intersection) { size_t i, l = jl_svec_len(dt->parameters); jl_svec_t *p = jl_alloc_svec(l); @@ -942,7 +950,9 @@ static jl_value_t *approxify_type(jl_datatype_t *dt, jl_svec_t *pp) } static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var) { if (jl_is_typector(a)) a = (jl_value_t*)((jl_typector_t*)a)->body; @@ -953,13 +963,13 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, if (var == covariant && !((jl_tvar_t*)a)->bound) a = ((jl_tvar_t*)a)->ub; else if (a != jl_ANY_flag) - return intersect_typevar((jl_tvar_t*)a, b, penv, eqc, var); + return intersect_typevar((jl_tvar_t*)a, b, penv, eqc, recheck_tuple_intersection, var); } if (jl_is_typevar(b)) { if (var == covariant && !((jl_tvar_t*)b)->bound) b = ((jl_tvar_t*)b)->ub; else if (b != jl_ANY_flag) - return intersect_typevar((jl_tvar_t*)b, a, penv, eqc, var); + return intersect_typevar((jl_tvar_t*)b, a, penv, eqc, recheck_tuple_intersection, var); } if (a == (jl_value_t*)jl_bottom_type || b == (jl_value_t*)jl_bottom_type) return (jl_value_t*)jl_bottom_type; @@ -971,19 +981,19 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, } // union if (jl_is_uniontype(a)) - return intersect_union((jl_uniontype_t*)a, b, penv, eqc, var); + return intersect_union((jl_uniontype_t*)a, b, penv, eqc, recheck_tuple_intersection, var); if (jl_is_uniontype(b)) - return intersect_union((jl_uniontype_t*)b, a, penv, eqc, var); + return intersect_union((jl_uniontype_t*)b, a, penv, eqc, recheck_tuple_intersection, var); if (a == (jl_value_t*)jl_any_type || a == jl_ANY_flag) return b; if (b == (jl_value_t*)jl_any_type || b == jl_ANY_flag) return a; // tuple if (jl_is_tuple_type(a)) { if (jl_is_tuple_type(b)) { - return intersect_tuple((jl_datatype_t*)a, (jl_datatype_t*)b, penv,eqc,var); + return intersect_tuple((jl_datatype_t*)a, (jl_datatype_t*)b, penv, eqc, recheck_tuple_intersection, var); } } if (jl_is_tuple_type(b)) { - return jl_type_intersect(b, a, penv,eqc,var); + return jl_type_intersect(b, a, penv, eqc, recheck_tuple_intersection, var); } // tag if (!jl_is_datatype(a) || !jl_is_datatype(b)) @@ -991,7 +1001,7 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, jl_datatype_t *tta = (jl_datatype_t*)a; jl_datatype_t *ttb = (jl_datatype_t*)b; if (tta->name == ttb->name) - return (jl_value_t*)intersect_tag(tta, ttb, penv, eqc, var); + return (jl_value_t*)intersect_tag(tta, ttb, penv, eqc, recheck_tuple_intersection, var); jl_datatype_t *super = NULL; jl_datatype_t *sub = NULL; jl_value_t *env = NULL; @@ -1055,10 +1065,10 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, if (var == covariant && sub == (jl_datatype_t*)sub->name->primary && jl_has_typevars_from((jl_value_t*)sub->super, ((jl_datatype_t*)sub->name->primary)->parameters)) - env = approxify_type((jl_datatype_t*)sub->super, ((jl_datatype_t*)sub->name->primary)->parameters); + env = approxify_type((jl_datatype_t*)sub->super, ((jl_datatype_t*)sub->name->primary)->parameters, recheck_tuple_intersection); else env = (jl_value_t*)sub->super; - super = (jl_datatype_t*)jl_type_intersect((jl_value_t*)env, (jl_value_t*)super, penv, eqc, var); + super = (jl_datatype_t*)jl_type_intersect((jl_value_t*)env, (jl_value_t*)super, penv, eqc, recheck_tuple_intersection, var); if ((jl_value_t*)super == jl_bottom_type) { JL_GC_POP(); @@ -1128,7 +1138,7 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, for(int e=0; e < jl_svec_len(env); e+=2) { if (jl_svecref(env, e) == tp) { elt = jl_type_intersect(elt, jl_svecref(env, e+1), - penv, eqc, invariant); + penv, eqc, recheck_tuple_intersection, invariant); // note: elt might be Union{} if "Union{}" was the type parameter break; } @@ -1490,14 +1500,14 @@ jl_value_t *jl_type_intersection_matching(jl_value_t *a, jl_value_t *b, jl_value_t **pti = &rts[0]; jl_value_t **extraroot = &rts[1]; - recheck_tuple_intersection = 0; + int recheck_tuple_intersection = 0; JL_TRY { // This is kind of awful, but an inner call to instantiate_type // might fail due to a mismatched type parameter. The problem is // that we allow Range{T} to exist, even though the declaration of // Range specifies Range{T<:Real}. Therefore intersection cannot see // that some parameter values actually don't match. - *pti = jl_type_intersect(a, b, &env, &eqc, covariant); + *pti = jl_type_intersect(a, b, &env, &eqc, &recheck_tuple_intersection, covariant); } JL_CATCH { *pti = (jl_value_t*)jl_bottom_type; @@ -1511,8 +1521,8 @@ jl_value_t *jl_type_intersection_matching(jl_value_t *a, jl_value_t *b, int e; if (recheck_tuple_intersection) { - for(e=0; e < eqc.n; e+=2) { - jl_value_t *val = eqc.data[e+1]; + for (e = 0; e < eqc.n; e += 2) { + jl_value_t *val = eqc.data[e + 1]; if (jl_is_long(val)) break; } @@ -1526,7 +1536,7 @@ jl_value_t *jl_type_intersection_matching(jl_value_t *a, jl_value_t *b, to find all other constraints on N first, then do intersection again with that knowledge. */ - *pti = jl_type_intersect(a, b, &env, &eqc, covariant); + *pti = jl_type_intersect(a, b, &env, &eqc, &recheck_tuple_intersection, covariant); if (*pti == (jl_value_t*)jl_bottom_type) { JL_GC_POP(); return *pti; From 83028fe3f31e55670b180c21ff6cc04a59ece22c Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 17 Aug 2016 00:42:44 -0400 Subject: [PATCH 0959/1117] ensure in_typeinf_loop is only read by the thread with the typeinf lock --- base/inference.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index 8c423674a7b99..01c0e3eb10dbc 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1470,7 +1470,11 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr end end - if caller === nothing && in_typeinf_loop + ccall(:jl_typeinf_begin, Void, ()) + thread_in_typeinf_loop = in_typeinf_loop::Bool + ccall(:jl_typeinf_end, Void, ()) + + if caller === nothing && thread_in_typeinf_loop # if the caller needed the ast, but we are already in the typeinf loop # then just return early -- we can't fulfill this request # if the client was inlining, then this means we decided not to try to infer this From db612608ba6c8afb7cf0503de2eaa74ad87e8d15 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Wed, 17 Aug 2016 13:18:49 +0200 Subject: [PATCH 0960/1117] do not catch everything in isassigned --- base/abstractarray.jl | 9 ++++++--- test/abstractarray.jl | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index ebf99e5806245..252a7b271316d 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -181,12 +181,15 @@ function _strides{M,T,N}(out::NTuple{M}, A::AbstractArray{T,N}) end function isassigned(a::AbstractArray, i::Int...) - # TODO try a[i...] true - catch - false + catch e + if isa(e, BoundsError) || isa(e, UndefRefError) + return false + else + rethrow(e) + end end end diff --git a/test/abstractarray.jl b/test/abstractarray.jl index b3ff1b2f18c77..73a4b0a32ce8a 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -408,6 +408,13 @@ function test_primitives{T}(::Type{T}, shape, ::Type{TestAbstractArray}) @test convert(Array, X) == X end +let + type TestThrowNoGetindex{T} <: AbstractVector{T} end + Base.length(::TestThrowNoGetindex) = 2 + Base.size(::TestThrowNoGetindex) = (2,) + @test_throws ErrorException isassigned(TestThrowNoGetindex{Float64}(), 1) +end + function test_in_bounds(::Type{TestAbstractArray}) n = rand(2:5) sz = rand(2:5, n) From da5e010ecc6c4db4f2b02b4d3a358581c4d8de6a Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 17 Aug 2016 04:35:13 -0700 Subject: [PATCH 0961/1117] Reinstate downloading old versions of winrpm gcc dll's (#18059) This mostly reverts #17906 and puts #15521 back in place, but from a slightly more permanent download location now. Unfortunately using the latest opensuse copy of the gcc dll's is causing issues when a cygwin-built Julia tries to load an opensuse-built libzmq.dll. Using these slightly old gcc 5 versions seems to fix it. I suspect the difference is which libstdc++ ABI is being used. GCC 5 has both old and new available, but many distros had it using the old ABI by default for compatibility. GCC 6 (which opensuse is using now) is more likely to be using the new ABI by default. --- Makefile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index fd2d8edb109cc..c10090e4b73c2 100644 --- a/Makefile +++ b/Makefile @@ -471,7 +471,7 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) + cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) ifeq ($(USE_GPL_LIBS), 1) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ cp busybox.exe $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) @@ -600,7 +600,14 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686)) $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920.exe && \ 7z x -y 7z920.exe 7z.exe 7z.dll && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.2 \ - "mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \ + "mingw32-libexpat1 mingw32-zlib1" && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libgfortran3-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libquadmath0-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libstdc++6-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libgcc_s_sjlj1-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libssp0-5.3.0-1.1.noarch.rpm && \ + for i in *.rpm; do 7z x -y $$i; done && \ + for i in *.cpio; do 7z x -y $$i; done && \ cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . else ifeq ($(ARCH),x86_64) cd $(JULIAHOME)/dist-extras && \ @@ -609,7 +616,14 @@ else ifeq ($(ARCH),x86_64) mv _7z.dll 7z.dll && \ mv _7z.exe 7z.exe && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.2 \ - "mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \ + "mingw64-libexpat1 mingw64-zlib1" && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libgfortran3-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libquadmath0-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libstdc++6-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libgcc_s_seh1-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libssp0-5.3.0-1.1.noarch.rpm && \ + for i in *.rpm; do 7z x -y $$i; done && \ + for i in *.cpio; do 7z x -y $$i; done && \ cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . else $(error no win-extras target for ARCH=$(ARCH)) From 40ca2367b0349c7ddc727c3422cc61a9e03738b7 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 17 Aug 2016 13:24:57 -0500 Subject: [PATCH 0962/1117] Removed SSH test from libgit2-online Test was only working for environments which had GitHub SSH keys setup. --- test/libgit2-online.jl | 19 +++---------------- test/libgit2.jl | 1 - 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/test/libgit2-online.jl b/test/libgit2-online.jl index 61b683f087138..dd02d6c42d37f 100644 --- a/test/libgit2-online.jl +++ b/test/libgit2-online.jl @@ -7,12 +7,11 @@ ######### # init & clone mktempdir() do dir - repo_url_https = "https://github.com/JuliaLang/Example.jl" - repo_url_git = "git@github.com:JuliaLang/Example.jl" + repo_url = "https://github.com/JuliaLang/Example.jl" #@testset "Cloning repository" begin #@testset "with 'https' protocol" begin repo_path = joinpath(dir, "Example1") - repo = LibGit2.clone(repo_url_https, repo_path) + repo = LibGit2.clone(repo_url, repo_path) try @test isdir(repo_path) @test isdir(joinpath(repo_path, ".git")) @@ -26,25 +25,13 @@ mktempdir() do dir repo_path = joinpath(dir, "Example2") # credentials are required because github tries to authenticate on unknown repo cred = LibGit2.UserPasswordCredentials("","") # empty credentials cause authentication error - LibGit2.clone(repo_url_https*randstring(10), repo_path, payload=Nullable(cred)) + LibGit2.clone(repo_url*randstring(10), repo_path, payload=Nullable(cred)) error("unexpected") catch ex @test isa(ex, LibGit2.Error.GitError) @test ex.code == LibGit2.Error.EAUTH end #end - - #TODO: remove or condition on libgit2 features this test when ssh protocol will be supported - #@testset "with 'ssh' protocol (by default is not supported)" begin - repo_path = joinpath(dir, "Example3") - repo = LibGit2.clone(repo_url_git, repo_path) - try - @test isdir(repo_path) - @test isdir(joinpath(repo_path, ".git")) - finally - finalize(repo) - end - #end #end end diff --git a/test/libgit2.jl b/test/libgit2.jl index 96a183c3d6e27..ed07633b47b48 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -112,7 +112,6 @@ const LIBGIT2_MIN_VER = v"0.23.0" mktempdir() do dir # test parameters repo_url = "https://github.com/JuliaLang/Example.jl" - ssh_prefix = "git@" cache_repo = joinpath(dir, "Example") test_repo = joinpath(dir, "Example.Test") test_sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time(), 0), 0) From 2bfbdcc6194f6f8e79258d14a5ef3a0d45a2a2b9 Mon Sep 17 00:00:00 2001 From: Curtis Vogt <curtis.vogt@gmail.com> Date: Wed, 17 Aug 2016 13:40:45 -0500 Subject: [PATCH 0963/1117] Update URL parsing tests Added a new explicit SSH protocol test which generated other changes. --- test/libgit2.jl | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/test/libgit2.jl b/test/libgit2.jl index ed07633b47b48..d2a8fbc91b48d 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -77,16 +77,34 @@ const LIBGIT2_MIN_VER = v"0.23.0" #end #@testset "URL parsing" begin - # Use all named group - m = match(LibGit2.URL_REGEX, "https://user:pass@hostname.com:80/path/repo.git") + # HTTPS URL + m = match(LibGit2.URL_REGEX, "https://user:pass@server.com:80/org/project.git") @test m[:scheme] == "https" @test m[:user] == "user" @test m[:password] == "pass" - @test m[:host] == "hostname.com" + @test m[:host] == "server.com" @test m[:port] == "80" - @test m[:path] == "/path/repo.git" + @test m[:path] == "/org/project.git" - # Realistic example HTTP example + # SSH URL + m = match(LibGit2.URL_REGEX, "ssh://user:pass@server:22/project.git") + @test m[:scheme] == "ssh" + @test m[:user] == "user" + @test m[:password] == "pass" + @test m[:host] == "server" + @test m[:port] == "22" + @test m[:path] == "/project.git" + + # SSH URL using scp-like syntax + m = match(LibGit2.URL_REGEX, "user@server:project.git") + @test m[:scheme] == nothing + @test m[:user] == "user" + @test m[:password] == nothing + @test m[:host] == "server" + @test m[:port] == nothing + @test m[:path] == "project.git" + + # Realistic example from GitHub using HTTPS m = match(LibGit2.URL_REGEX, "https://github.com/JuliaLang/Example.jl.git") @test m[:scheme] == "https" @test m[:user] == nothing @@ -95,7 +113,7 @@ const LIBGIT2_MIN_VER = v"0.23.0" @test m[:port] == nothing @test m[:path] == "/JuliaLang/Example.jl.git" - # Realistic example SSH example + # Realistic example from GitHub using SSH m = match(LibGit2.URL_REGEX, "git@github.com:JuliaLang/Example.jl.git") @test m[:scheme] == nothing @test m[:user] == "git" @@ -104,7 +122,7 @@ const LIBGIT2_MIN_VER = v"0.23.0" @test m[:port] == nothing @test m[:path] == "JuliaLang/Example.jl.git" - # Make sure that a username can contain special characters + # Make sure usernames can contain special characters m = match(LibGit2.URL_REGEX, "user-name@hostname.com") @test m[:user] == "user-name" #end From 7a2499dcac4013ab0167723228984c2e974b4a1a Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Wed, 17 Aug 2016 13:00:19 -0700 Subject: [PATCH 0964/1117] Add missing tests for diagonal and uniformscaling (#17921) * Add missing tests for diagonal and uniformscaling * Change A_mul_B to * * Moved things out of nested loop that don't need to be there --- test/linalg/diagonal.jl | 183 ++++++++++++++++++---------------- test/linalg/uniformscaling.jl | 2 + 2 files changed, 100 insertions(+), 85 deletions(-) diff --git a/test/linalg/diagonal.jl b/test/linalg/diagonal.jl index be856257cef9d..6d87b0f97b3cb 100644 --- a/test/linalg/diagonal.jl +++ b/test/linalg/diagonal.jl @@ -36,6 +36,13 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test D[1,1] == d[1] @test D[1,2] == 0 + @test issymmetric(D) + @test istriu(D) + @test istril(D) + if elty <: Real + @test ishermitian(D) + end + debug && println("Simple unary functions") for op in (-,) @test op(D)==op(DM) @@ -69,6 +76,10 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) debug && println("Linear solve") @test_approx_eq_eps D*v DM*v n*eps(relty)*(elty<:Complex ? 2:1) @test_approx_eq_eps D*U DM*U n^2*eps(relty)*(elty<:Complex ? 2:1) + + @test U.'*D ≈ U.'*full(D) + @test U'*D ≈ U'*full(D) + if relty != BigFloat @test_approx_eq_eps D\v DM\v 2n^2*eps(relty)*(elty<:Complex ? 2:1) @test_approx_eq_eps D\U DM\U 2n^3*eps(relty)*(elty<:Complex ? 2:1) @@ -95,98 +106,100 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) b = view(rand(elty,n+1),collect(1:n+1)) @test_throws DimensionMismatch A_ldiv_B!(D,b) end + end + end + debug && println("Binary operations") + d = convert(Vector{elty}, randn(n)) + D2 = Diagonal(d) + DM2= diagm(d) + for op in (+, -, *) + @test full(op(D, D2)) ≈ op(DM, DM2) + end + # binary ops with plain numbers + a = rand() + @test full(a*D) ≈ a*DM + @test full(D*a) ≈ DM*a + @test full(D/a) ≈ DM/a + if relty <: BlasFloat + b = rand(elty,n,n) + b = sparse(b) + @test A_mul_B!(copy(D), copy(b)) ≈ full(D)*full(b) + @test At_mul_B!(copy(D), copy(b)) ≈ full(D).'*full(b) + @test Ac_mul_B!(copy(D), copy(b)) ≈ full(D)'*full(b) + end - debug && println("Binary operations") - d = convert(Vector{elty}, randn(n)) - D2 = Diagonal(d) - DM2= diagm(d) - for op in (+, -, *) - @test full(op(D, D2)) ≈ op(DM, DM2) - end - # binary ops with plain numbers - a = rand() - @test full(a*D) ≈ a*DM - @test full(D*a) ≈ DM*a - @test full(D/a) ≈ DM/a - if relty <: BlasFloat - b = rand(elty,n,n) - b = sparse(b) - @test A_mul_B!(copy(D), copy(b)) ≈ full(D)*full(b) - @test At_mul_B!(copy(D), copy(b)) ≈ full(D).'*full(b) - @test Ac_mul_B!(copy(D), copy(b)) ≈ full(D)'*full(b) - end + #a few missing mults + bd = Bidiagonal(D2) + @test D*D2.' ≈ full(D)*full(D2).' + @test D2*D.' ≈ full(D2)*full(D).' + @test D2*D' ≈ full(D2)*full(D)' - @test U.'*D ≈ U.'*full(D) - @test U'*D ≈ U'*full(D) + #division of two Diagonals + @test D/D2 ≈ Diagonal(D.diag./D2.diag) + @test D\D2 ≈ Diagonal(D2.diag./D.diag) + # test triu/tril + @test istriu(D) + @test istril(D) + @test triu(D,1) == zeros(D) + @test triu(D,0) == D + @test triu(D,-1) == D + @test tril(D,1) == D + @test tril(D,-1) == zeros(D) + @test tril(D,0) == D + @test_throws ArgumentError tril(D,n+1) + @test_throws ArgumentError triu(D,n+1) - #division of two Diagonals - @test D/D2 ≈ Diagonal(D.diag./D2.diag) - @test D\D2 ≈ Diagonal(D2.diag./D.diag) - # test triu/tril - @test istriu(D) - @test istril(D) - @test triu(D,1) == zeros(D) - @test triu(D,0) == D - @test triu(D,-1) == D - @test tril(D,1) == D - @test tril(D,-1) == zeros(D) - @test tril(D,0) == D - @test_throws ArgumentError tril(D,n+1) - @test_throws ArgumentError triu(D,n+1) - - # factorize - @test factorize(D) == D - - debug && println("Eigensystem") - eigD = eigfact(D) - @test Diagonal(eigD[:values]) ≈ D - @test eigD[:vectors] == eye(D) - - debug && println("ldiv") - v = rand(n + 1) - @test_throws DimensionMismatch D\v - v = rand(n) - @test D\v ≈ DM\v - V = rand(n + 1, n) - @test_throws DimensionMismatch D\V - V = rand(n, n) - @test D\V ≈ DM\V - - debug && println("conj and transpose") - @test transpose(D) == D - if elty <: BlasComplex - @test full(conj(D)) ≈ conj(DM) - @test ctranspose(D) == conj(D) - end + # factorize + @test factorize(D) == D - #logdet - if relty <: Real - ld=convert(Vector{relty},rand(n)) - @test logdet(Diagonal(ld)) ≈ logdet(diagm(ld)) - end + debug && println("Eigensystem") + eigD = eigfact(D) + @test Diagonal(eigD[:values]) ≈ D + @test eigD[:vectors] == eye(D) - #similar - @test isa(similar(D), Diagonal{elty}) - @test isa(similar(D, Int), Diagonal{Int}) - @test isa(similar(D, (3,2)), Matrix{elty}) - @test isa(similar(D, Int, (3,2)), Matrix{Int}) - - #10036 - @test issymmetric(D2) - @test ishermitian(D2) - if elty <: Complex - dc = d + im*convert(Vector{elty}, ones(n)) - D3 = Diagonal(dc) - @test issymmetric(D3) - @test !ishermitian(D3) - end + debug && println("ldiv") + v = rand(n + 1) + @test_throws DimensionMismatch D\v + v = rand(n) + @test D\v ≈ DM\v + V = rand(n + 1, n) + @test_throws DimensionMismatch D\V + V = rand(n, n) + @test D\V ≈ DM\V - U, s, V = svd(D) - @test (U*Diagonal(s))*V' ≈ D - @test svdvals(D) == s - @test svdfact(D)[:V] == V - end + debug && println("conj and transpose") + @test transpose(D) == D + if elty <: BlasComplex + @test full(conj(D)) ≈ conj(DM) + @test ctranspose(D) == conj(D) end + + #logdet + if relty <: Real + ld=convert(Vector{relty},rand(n)) + @test logdet(Diagonal(ld)) ≈ logdet(diagm(ld)) + end + + #similar + @test isa(similar(D), Diagonal{elty}) + @test isa(similar(D, Int), Diagonal{Int}) + @test isa(similar(D, (3,2)), Matrix{elty}) + @test isa(similar(D, Int, (3,2)), Matrix{Int}) + + #10036 + @test issymmetric(D2) + @test ishermitian(D2) + if elty <: Complex + dc = d + im*convert(Vector{elty}, ones(n)) + D3 = Diagonal(dc) + @test issymmetric(D3) + @test !ishermitian(D3) + end + + U, s, V = svd(D) + @test (U*Diagonal(s))*V' ≈ D + @test svdvals(D) == s + @test svdfact(D)[:V] == V end D = Diagonal(Matrix{Float64}[randn(3,3), randn(2,2)]) diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index 4d1c3e20d0ad8..f20dd0315cab8 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -8,8 +8,10 @@ srand(123) @test I[1,1] == 1 # getindex @test I[1,2] == 0 # getindex @test I === I' # transpose +@test ndims(I) == 2 @test one(UniformScaling{Float32}) == UniformScaling(one(Float32)) @test zero(UniformScaling{Float32}) == UniformScaling(zero(Float32)) +@test eltype(one(UniformScaling{Float32})) == Float32 @test zero(UniformScaling(rand(Complex128))) == zero(UniformScaling{Complex128}) @test one(UniformScaling(rand(Complex128))) == one(UniformScaling{Complex128}) @test eltype(one(UniformScaling(rand(Complex128)))) == Complex128 From 7479c56acbd632c57c2336376202084ac1ceb05a Mon Sep 17 00:00:00 2001 From: Katie Hyatt <kshyatt@physics.ucsb.edu> Date: Wed, 17 Aug 2016 13:38:07 -0700 Subject: [PATCH 0965/1117] Add tests for flipdim, unary ops --- test/abstractarray.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 73a4b0a32ce8a..34b1f21c6e3d8 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -716,3 +716,16 @@ let @test !issparse(m1) @test !issparse(m2) end + +#isinteger and isreal +@test isinteger(Diagonal(rand(1:5,5))) +@test isreal(Diagonal(rand(5))) + +#unary ops +let A = Diagonal(rand(1:5,5)) + @test +(A) == A + @test *(A) == A +end + +#flipdim on empty +@test flipdim(Diagonal([]),1) == Diagonal([]) From 302964ffec95140e1593682c9a5944f7cbd6eb02 Mon Sep 17 00:00:00 2001 From: Katie Hyatt <kshyatt@physics.ucsb.edu> Date: Wed, 17 Aug 2016 13:38:17 -0700 Subject: [PATCH 0966/1117] Add tests for iteration methods on numbers --- test/numbers.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/numbers.jl b/test/numbers.jl index 8d7c680a0a709..4baf2c79b2d76 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2770,11 +2770,17 @@ testmi(map(UInt32, 0:1000), map(UInt32, 1:100)) testmi(typemax(UInt32)-UInt32(1000):typemax(UInt32), map(UInt32, 1:100)) @test ndims(1) == 0 +@test ndims(Integer) == 0 @test size(1,1) == 1 @test_throws BoundsError size(1,-1) @test indices(1) == () @test indices(1,1) == 1:1 @test_throws BoundsError indices(1,-1) +@test isinteger(Integer(2)) == true +@test size(1) == () +@test length(1) == 1 +@test endof(1) == 1 +@test eltype(Integer) == Integer # issue #15920 @test Rational(0, 1) / Complex(3, 2) == 0 From b2396291019fff6e9628e27b2080a68c13fbabbf Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 17 Aug 2016 19:26:41 -0400 Subject: [PATCH 0967/1117] split static-eval to its individual users most users of static-eval didn't need most of it's complexity and didn't actually want its deprecation warning behavior this also will make it easier to delete this function as the users of it get fixed not to need it --- src/alloc.c | 40 +++++++++++++++++++-------- src/anticodegen.c | 6 ---- src/cgutils.cpp | 15 +++++----- src/codegen.cpp | 56 ++++++++++++++++---------------------- src/julia_internal.h | 2 -- src/toplevel.c | 65 ++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 122 insertions(+), 62 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 42c6931c20cbe..79da428f63028 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -264,6 +264,7 @@ JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) return jv; } + extern jl_value_t *jl_builtin_getfield; jl_value_t *jl_resolve_globals(jl_value_t *expr, jl_lambda_info_t *lam) { @@ -280,20 +281,37 @@ jl_value_t *jl_resolve_globals(jl_value_t *expr, jl_lambda_info_t *lam) e->head == boundscheck_sym || e->head == simdloop_sym) { } else { - if (e->head == call_sym && jl_expr_nargs(e) == 3 && jl_is_quotenode(jl_exprarg(e,2)) && + if (e->head == call_sym && jl_expr_nargs(e) == 3 && jl_is_quotenode(jl_exprarg(e, 2)) && lam->def->module != NULL) { // replace getfield(module_expr, :sym) with GlobalRef - jl_value_t *s = jl_fieldref(jl_exprarg(e,2),0); - jl_value_t *fe = jl_exprarg(e,0); + jl_value_t *s = jl_fieldref(jl_exprarg(e, 2), 0); + jl_value_t *fe = jl_exprarg(e, 0); if (jl_is_symbol(s) && jl_is_globalref(fe)) { - jl_value_t *f = jl_static_eval(fe, NULL, lam->def->module, lam, 0, 0); + jl_binding_t *b = jl_get_binding(jl_globalref_mod(fe), jl_globalref_name(fe)); + jl_value_t *f = NULL; + if (b && b->constp) { + f = b->value; + } if (f == jl_builtin_getfield) { - jl_value_t *me = jl_exprarg(e,1); - if (jl_is_globalref(me) || - (jl_is_symbol(me) && jl_binding_resolved_p(lam->def->module, (jl_sym_t*)me))) { - jl_value_t *m = jl_static_eval(me, NULL, lam->def->module, lam, 0, 0); - if (m && jl_is_module(m)) - return jl_module_globalref((jl_module_t*)m, (jl_sym_t*)s); + jl_value_t *me = jl_exprarg(e, 1); + jl_module_t *me_mod = NULL; + jl_sym_t *me_sym = NULL; + if (jl_is_globalref(me)) { + me_mod = jl_globalref_mod(me); + me_sym = jl_globalref_name(me); + } + else if (jl_is_symbol(me) && jl_binding_resolved_p(lam->def->module, (jl_sym_t*)me)) { + me_mod = lam->def->module; + me_sym = (jl_sym_t*)me; + } + if (me_mod && me_sym) { + jl_binding_t *b = jl_get_binding(me_mod, me_sym); + if (b && b->constp) { + jl_value_t *m = b->value; + if (m && jl_is_module(m)) { + return jl_module_globalref((jl_module_t*)m, (jl_sym_t*)s); + } + } } } } @@ -303,7 +321,7 @@ jl_value_t *jl_resolve_globals(jl_value_t *expr, jl_lambda_info_t *lam) e->head == bitstype_sym || e->head == module_sym) i++; for(; i < jl_array_len(e->args); i++) { - jl_exprargset(e, i, jl_resolve_globals(jl_exprarg(e,i), lam)); + jl_exprargset(e, i, jl_resolve_globals(jl_exprarg(e, i), lam)); } } } diff --git a/src/anticodegen.c b/src/anticodegen.c index 4ee97cb5ad86e..948c2a81b08c3 100644 --- a/src/anticodegen.c +++ b/src/anticodegen.c @@ -36,12 +36,6 @@ int jl_getFunctionInfo(jl_frame_t **frames, uintptr_t pointer, int skipC, int no return 0; } -jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, - jl_lambda_info_t *li, int sparams, int allow_alloc) -{ - return NULL; -} - void jl_register_fptrs(uint64_t sysimage_base, void **fptrs, jl_lambda_info_t **linfos, size_t n) { (void)sysimage_base; (void)fptrs; (void)linfos; (void)n; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 9ad239345595d..5336f5ad95f88 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -895,9 +895,6 @@ static Value *julia_bool(Value *cond) // --- get the inferred type of an AST node --- -static jl_value_t *static_eval(jl_value_t *ex, jl_codectx_t *ctx, bool sparams=true, - bool allow_alloc=true); - static inline jl_module_t *topmod(jl_codectx_t *ctx) { return jl_base_relative_to(ctx->module); @@ -945,11 +942,13 @@ static jl_value_t *expr_type(jl_value_t *e, jl_codectx_t *ctx) goto type_of_constant; } if (jl_is_globalref(e)) { - jl_value_t *v = static_eval(e, ctx); - if (v == NULL) - return (jl_value_t*)jl_any_type; - e = v; - goto type_of_constant; + jl_sym_t *s = (jl_sym_t*)jl_globalref_name(e); + jl_binding_t *b = jl_get_binding(jl_globalref_mod(e), s); + if (b && b->constp) { + e = b->value; + goto type_of_constant; + } + return (jl_value_t*)jl_any_type; } if (jl_is_symbol(e)) { jl_binding_t *b = jl_get_binding(ctx->module, (jl_sym_t*)e); diff --git a/src/codegen.cpp b/src/codegen.cpp index 377700d5530e2..fbb0bf1a5731c 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1499,10 +1499,6 @@ extern "C" void jl_write_malloc_log(void) write_log_data(mallocData, ".mem"); } -// --- code gen for intrinsic functions --- - -#include "intrinsics.cpp" - // --- constant determination --- static void show_source_loc(JL_STREAM *out, jl_codectx_t *ctx) @@ -1522,16 +1518,14 @@ static void cg_bdw(jl_binding_t *b, jl_codectx_t *ctx) } } + // try to statically evaluate, NULL if not possible -extern "C" -jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, - jl_lambda_info_t *linfo, int sparams, int allow_alloc) +static jl_value_t *static_eval(jl_value_t *ex, jl_codectx_t *ctx, int sparams=true, int allow_alloc=true) { - jl_codectx_t *ctx = (jl_codectx_t*)ctx_; if (jl_is_symbol(ex)) { jl_sym_t *sym = (jl_sym_t*)ex; - if (jl_is_const(mod, sym)) - return jl_get_global(mod, sym); + if (jl_is_const(ctx->module, sym)) + return jl_get_global(ctx->module, sym); return NULL; } if (jl_is_slot(ex)) @@ -1539,36 +1533,34 @@ jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, if (jl_is_ssavalue(ex)) { ssize_t idx = ((jl_ssavalue_t*)ex)->id; assert(idx >= 0); - if (ctx != NULL && ctx->ssavalue_assigned.at(idx)) { + if (ctx->ssavalue_assigned.at(idx)) { return ctx->SAvalues.at(idx).constant; } return NULL; } if (jl_is_quotenode(ex)) - return jl_fieldref(ex,0); + return jl_fieldref(ex, 0); if (jl_is_lambda_info(ex)) return NULL; jl_module_t *m = NULL; jl_sym_t *s = NULL; if (jl_is_globalref(ex)) { - s = (jl_sym_t*)jl_globalref_name(ex); - if (s && jl_is_symbol(s)) { - jl_binding_t *b = jl_get_binding(jl_globalref_mod(ex), s); - if (b && b->constp) { - if (b->deprecated) cg_bdw(b, ctx); - return b->value; - } + s = jl_globalref_name(ex); + jl_binding_t *b = jl_get_binding(jl_globalref_mod(ex), s); + if (b && b->constp) { + if (b->deprecated) cg_bdw(b, ctx); + return b->value; } return NULL; } if (jl_is_expr(ex)) { jl_expr_t *e = (jl_expr_t*)ex; if (e->head == call_sym) { - jl_value_t *f = jl_static_eval(jl_exprarg(e,0),ctx,mod,linfo,sparams,allow_alloc); + jl_value_t *f = static_eval(jl_exprarg(e, 0), ctx, sparams, allow_alloc); if (f) { if (jl_array_dim0(e->args) == 3 && f==jl_builtin_getfield) { - m = (jl_module_t*)jl_static_eval(jl_exprarg(e,1),ctx,mod,linfo,sparams,allow_alloc); - s = (jl_sym_t*)jl_static_eval(jl_exprarg(e,2),ctx,mod,linfo,sparams,allow_alloc); + m = (jl_module_t*)static_eval(jl_exprarg(e, 1), ctx, sparams, allow_alloc); + s = (jl_sym_t*)static_eval(jl_exprarg(e, 2), ctx, sparams, allow_alloc); if (m && jl_is_module(m) && s && jl_is_symbol(s)) { jl_binding_t *b = jl_get_binding(m, s); if (b && b->constp) { @@ -1586,7 +1578,7 @@ jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, jl_value_t **v; JL_GC_PUSHARGS(v, n); for (i = 0; i < n; i++) { - v[i] = jl_static_eval(jl_exprarg(e,i+1),ctx,mod,linfo,sparams,allow_alloc); + v[i] = static_eval(jl_exprarg(e, i+1), ctx, sparams, allow_alloc); if (v[i] == NULL) { JL_GC_POP(); return NULL; @@ -1608,9 +1600,9 @@ jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, } } else if (e->head == static_parameter_sym) { - size_t idx = jl_unbox_long(jl_exprarg(e,0)); - if (linfo && idx <= jl_svec_len(linfo->sparam_vals)) { - jl_value_t *e = jl_svecref(linfo->sparam_vals, idx - 1); + size_t idx = jl_unbox_long(jl_exprarg(e, 0)); + if (idx <= jl_svec_len(ctx->linfo->sparam_vals)) { + jl_value_t *e = jl_svecref(ctx->linfo->sparam_vals, idx - 1); if (jl_is_typevar(e)) return NULL; return e; @@ -1621,15 +1613,9 @@ jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, return ex; } -static jl_value_t *static_eval(jl_value_t *ex, jl_codectx_t *ctx, bool sparams, - bool allow_alloc) -{ - return jl_static_eval(ex, ctx, ctx->module, ctx->linfo, sparams, allow_alloc); -} - static bool is_constant(jl_value_t *ex, jl_codectx_t *ctx, bool sparams=true) { - return static_eval(ex,ctx,sparams) != NULL; + return static_eval(ex, ctx, sparams) != NULL; } static bool slot_eq(jl_value_t *e, int sl) @@ -1637,6 +1623,10 @@ static bool slot_eq(jl_value_t *e, int sl) return jl_is_slot(e) && jl_slot_number(e)-1 == sl; } +// --- code gen for intrinsic functions --- + +#include "intrinsics.cpp" + // --- find volatile variables --- // assigned in a try block and used outside that try block diff --git a/src/julia_internal.h b/src/julia_internal.h index 6eec5c076d43a..fc9514260c02f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -305,8 +305,6 @@ jl_value_t *jl_parse_eval_all(const char *fname, const char *content, size_t contentlen); jl_value_t *jl_interpret_toplevel_thunk(jl_lambda_info_t *lam); jl_value_t *jl_interpret_toplevel_expr(jl_value_t *e); -jl_value_t *jl_static_eval(jl_value_t *ex, void *ctx_, jl_module_t *mod, - jl_lambda_info_t *li, int sparams, int allow_alloc); int jl_is_toplevel_only_expr(jl_value_t *e); jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr); diff --git a/src/toplevel.c b/src/toplevel.c index a402f06d27f9c..742d84779fbe2 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -263,6 +263,67 @@ JL_DLLEXPORT jl_module_t *jl_base_relative_to(jl_module_t *m) return jl_top_module; } +// try to statically evaluate, NULL if not possible +// remove this once jl_has_intrinsics is deleted +extern jl_value_t *jl_builtin_getfield; +static jl_value_t *jl_static_eval(jl_value_t *ex, jl_module_t *mod, + jl_lambda_info_t *linfo, int sparams) +{ + if (jl_is_symbol(ex)) { + jl_sym_t *sym = (jl_sym_t*)ex; + if (jl_is_const(mod, sym)) + return jl_get_global(mod, sym); + return NULL; + } + if (jl_is_slot(ex)) + return NULL; + if (jl_is_ssavalue(ex)) + return NULL; + if (jl_is_quotenode(ex)) + return jl_fieldref(ex, 0); + if (jl_is_lambda_info(ex)) + return NULL; + jl_module_t *m = NULL; + jl_sym_t *s = NULL; + if (jl_is_globalref(ex)) { + jl_binding_t *b = jl_get_binding(jl_globalref_mod(ex), jl_globalref_name(ex)); + if (b && b->constp) { + return b->value; + } + return NULL; + } + if (jl_is_expr(ex)) { + jl_expr_t *e = (jl_expr_t*)ex; + if (e->head == call_sym) { + jl_value_t *f = jl_static_eval(jl_exprarg(e, 0), mod, linfo, sparams); + if (f) { + if (jl_array_dim0(e->args) == 3 && f==jl_builtin_getfield) { + m = (jl_module_t*)jl_static_eval(jl_exprarg(e, 1), mod, linfo, sparams); + s = (jl_sym_t*)jl_static_eval(jl_exprarg(e, 2), mod, linfo, sparams); + if (m && jl_is_module(m) && s && jl_is_symbol(s)) { + jl_binding_t *b = jl_get_binding(m, s); + if (b && b->constp) { + return b->value; + } + } + } + } + } + else if (e->head == static_parameter_sym) { + size_t idx = jl_unbox_long(jl_exprarg(e, 0)); + if (linfo && idx <= jl_svec_len(linfo->sparam_vals)) { + jl_value_t *e = jl_svecref(linfo->sparam_vals, idx - 1); + if (jl_is_typevar(e)) + return NULL; + return e; + } + } + return NULL; + } + return ex; +} + + int jl_has_intrinsics(jl_lambda_info_t *li, jl_value_t *v, jl_module_t *m) { if (!jl_is_expr(v)) return 0; @@ -273,13 +334,13 @@ int jl_has_intrinsics(jl_lambda_info_t *li, jl_value_t *v, jl_module_t *m) return 0; jl_value_t *e0 = jl_exprarg(e, 0); if (e->head == call_sym) { - jl_value_t *sv = jl_static_eval(e0, NULL, m, li, li != NULL, 0); + jl_value_t *sv = jl_static_eval(e0, m, li, li != NULL); if (sv && jl_typeis(sv, jl_intrinsic_type)) return 1; } if (0 && e->head == assign_sym && jl_is_ssavalue(e0)) { // code branch needed for *very-linear-mode*, but not desirable otherwise jl_value_t *e1 = jl_exprarg(e, 1); - jl_value_t *sv = jl_static_eval(e1, NULL, m, li, li != NULL, 0); + jl_value_t *sv = jl_static_eval(e1, m, li, li != NULL); if (sv && jl_typeis(sv, jl_intrinsic_type)) return 1; } From 017aa1eb9fdb81920358f4e5f773ea259b88c60b Mon Sep 17 00:00:00 2001 From: mcprentiss <mcprentiss@gmail.com> Date: Wed, 17 Aug 2016 22:15:45 -0500 Subject: [PATCH 0968/1117] updates to README.md (#18093) This link has moved. --- test/perf/shootout/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/shootout/README b/test/perf/shootout/README index 948bb7eeb447c..ec8c91df1749c 100644 --- a/test/perf/shootout/README +++ b/test/perf/shootout/README @@ -1,6 +1,6 @@ This directory contains the Julia version of the "The Computer Language Benchmarks Game": -http://shootout.alioth.debian.org/ +https://benchmarksgame.alioth.debian.org/ The source code for all the benchmarks are available there: http://alioth.debian.org/scm/viewvc.php/shootout/bench/?root=shootout From 0f2cc750c70c7dd57462d1a32d8a706f6eefce98 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 16 Aug 2016 19:48:03 +0000 Subject: [PATCH 0969/1117] improve FreeBSD / generic posix support --- Make.inc | 5 +++++ README.md | 16 +++++++++++++--- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + deps/libgit2.mk | 7 ++++++- deps/libuv.version | 2 +- src/cgmemmgr.cpp | 2 +- src/flisp/Makefile | 4 ++-- src/gc-pages.c | 4 ++++ src/gc.c | 8 ++++++-- src/julia_threads.h | 2 +- src/safepoint.c | 2 +- src/signals-unix.c | 17 +++++++++++++++-- src/task.c | 2 +- 16 files changed, 58 insertions(+), 17 deletions(-) delete mode 100644 deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 delete mode 100644 deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 create mode 100644 deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/md5 create mode 100644 deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/sha512 diff --git a/Make.inc b/Make.inc index 003f50db78376..b8b04c9332313 100644 --- a/Make.inc +++ b/Make.inc @@ -340,6 +340,11 @@ endif STDLIBCPP_FLAG := +ifeq ($(OS), FreeBSD) +USEGCC := 0 +USECLANG := 1 +endif + ifeq ($(OS), Darwin) DARWINVER := $(shell uname -r | cut -b 1-2) DARWINVER_GTE13 := $(shell expr `uname -r | cut -b 1-2` \>= 13) diff --git a/README.md b/README.md index d893f1b32c036..aee51ca376a81 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ developers may find the notes in [CONTRIBUTING](https://github.com/JuliaLang/jul - **FreeBSD** - **Windows** -All systems are supported with both x86/64 (64-bit) and x86 (32-bit) architectures. Experimental and early support for [ARM](https://github.com/JuliaLang/julia/blob/master/README.arm.md) is available too. +All systems are supported with both x86/64 (64-bit) and x86 (32-bit) architectures. Experimental and early support for [ARM](https://github.com/JuliaLang/julia/blob/master/README.arm.md), AARCH64, and POWER (little-endian) is available too. <a name="Source-Download-and-Compilation"/> ## Source Download and Compilation @@ -235,12 +235,22 @@ When building Julia, or its dependencies, libraries installed by third party pac ### FreeBSD -On *FreeBSD Release 9.0*, install the `gcc47`, `git`, and `gmake` packages/ports, and compile Julia with the command: +On *FreeBSD Release 11.0*, install the gfortran, git, cmake, and gmake packages/ports (`pkg install gcc6 gmake git cmake`), and compile Julia with the command: - $ gmake FC=gfortran47 + $ echo 'FC=gfortran6' >> Make.user + $ gmake You must use the `gmake` command on FreeBSD instead of `make`. +Note that Julia is community-supported and we have little control over our upstream dependencies, you may still run into issues with its dependencies and YMMV. Current known issues includes: + + - The x86 arch doesn't support threading due to lack of compiler runtime library support (set JULIA_THREADS=0). + - libunwind needs a small patch to its tests to compile. + - OpenBLAS patches in pkg haven't been upstreamed. + - gfortran can't link binaries. Set `FFLAGS=-Wl,-rpath,/usr/local/lib/gcc6` to workaround this (upstream bug submitted to FreeBSD pkg maintainers). + - System libraries installed by pkg are not on the compiler path by default. You may need to add `LDFLAGS=/usr/local/lib` and `CPPFLAGS=/usr/local/include` to your environment or Make.user file to build successfully. + + ### Windows In order to build Julia on Windows, see [README.windows](https://github.com/JuliaLang/julia/blob/master/README.windows.md). diff --git a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 deleted file mode 100644 index e42a20ffd5a35..0000000000000 --- a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -cc07a8ef026fa42eeaddf7b6e074096e diff --git a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 deleted file mode 100644 index bec2d8d44203f..0000000000000 --- a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -e128cec9548ff2f52a78743203d3f06dceef3cf7cbeb3a7a5f7453b132576833f82688eed52cf74c035f57828deac079cd845ad47c8cd6197a8e0bebf3586fc2 diff --git a/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/md5 b/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/md5 new file mode 100644 index 0000000000000..89795f7987bbc --- /dev/null +++ b/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/md5 @@ -0,0 +1 @@ +4c91d4c9161555c911630b0a70ddec03 diff --git a/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/sha512 b/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/sha512 new file mode 100644 index 0000000000000..faf62292a62af --- /dev/null +++ b/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/sha512 @@ -0,0 +1 @@ +c53513a5aea84405bf302b084a23f24f54148aac90a2bd666219ce14879723baab959942934d0d801a4572fffd07e60a7d574ade8d7eb57b6da8216063c20a48 diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 224d30f9304ec..10a3a59d3a913 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -43,9 +43,14 @@ ifeq ($(OS),Linux) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied endif ifneq ($(OS),WINNT) +ifeq ($(USE_SYSTEM_CURL), 0) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(CURL_OBJ_TARGET) endif -$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied $(LIBSSH2_OBJ_TARGET) +endif +ifeq ($(USE_SYSTEM_LIBSSH2), 0) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(LIBSSH2_OBJ_TARGET) +endif +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) diff --git a/deps/libuv.version b/deps/libuv.version index 478ebca66589a..4ba4b36712aaf 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv1.9.0 -LIBUV_SHA1=28743d6091531340cfe316de2b2d385fe1778ff5 +LIBUV_SHA1=8d5131b6c1595920dd30644cd1435b4f344b46c8 diff --git a/src/cgmemmgr.cpp b/src/cgmemmgr.cpp index 5c3b187ecab25..a85dfa57a227b 100644 --- a/src/cgmemmgr.cpp +++ b/src/cgmemmgr.cpp @@ -47,7 +47,7 @@ static void *map_anon_page(size_t size) mem = (char*)LLT_ALIGN(uintptr_t(mem), jl_page_size); #else // _OS_WINDOWS_ void *mem = mmap(nullptr, size, PROT_READ | PROT_WRITE, - MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); assert(mem != MAP_FAILED && "Cannot allocate RW memory"); #endif // _OS_WINDOWS_ return mem; diff --git a/src/flisp/Makefile b/src/flisp/Makefile index 5d7cf6fc51ccd..5de2aff8a4888 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -77,10 +77,10 @@ CCLD := $(LD) endif $(BUILDDIR)/$(EXENAME)-debug: $(DOBJS) $(LIBFILES_debug) $(BUILDDIR)/$(LIBTARGET)-debug.a $(BUILDDIR)/flmain.dbg.obj | $(BUILDDIR)/flisp.boot - @$(call PRINT_LINK, $(CCLD) $(DEBUGFLAGS) $(DOBJS) $(BUILDDIR)/flmain.dbg.obj -o $@ $(BUILDDIR)/$(LIBTARGET)-debug.a $(LIBFILES_debug) $(LIBS) $(OSLIBS)) + @$(call PRINT_LINK, $(CCLD) $(DEBUGFLAGS) $(LDFLAGS) $(DOBJS) $(BUILDDIR)/flmain.dbg.obj -o $@ $(BUILDDIR)/$(LIBTARGET)-debug.a $(LIBFILES_debug) $(LIBS) $(OSLIBS)) $(BUILDDIR)/$(EXENAME): $(OBJS) $(LIBFILES_release) $(BUILDDIR)/$(LIBTARGET).a $(BUILDDIR)/flmain.o | $(BUILDDIR)/flisp.boot - @$(call PRINT_LINK, $(CCLD) $(SHIPFLAGS) $(OBJS) $(BUILDDIR)/flmain.o -o $@ $(BUILDDIR)/$(LIBTARGET).a $(LIBFILES_release) $(LIBS) $(OSLIBS)) + @$(call PRINT_LINK, $(CCLD) $(SHIPFLAGS) $(LDFLAGS) $(OBJS) $(BUILDDIR)/flmain.o -o $@ $(BUILDDIR)/$(LIBTARGET).a $(LIBFILES_release) $(LIBS) $(OSLIBS)) ifneq ($(BUILDDIR),.) $(BUILDDIR)/flisp.boot: flisp.boot diff --git a/src/gc-pages.c b/src/gc-pages.c index ead16010cee15..fbe2b27450fe3 100644 --- a/src/gc-pages.c +++ b/src/gc-pages.c @@ -37,6 +37,10 @@ void jl_gc_init_page(void) #endif } +#ifndef MAP_NORESERVE // not defined in POSIX, FreeBSD, etc. +#define MAP_NORESERVE (0) +#endif + // Try to allocate a memory block for a region with `pg_cnt` pages. // Return `NULL` if allocation failed. Result is aligned to `GC_PAGE_SZ`. static char *jl_gc_try_alloc_region(int pg_cnt) diff --git a/src/gc.c b/src/gc.c index c95a9fbc62d59..44aa2d2133b91 100644 --- a/src/gc.c +++ b/src/gc.c @@ -618,7 +618,11 @@ JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) bigval_t *v = (bigval_t*)malloc_cache_align(allocsz); if (v == NULL) jl_throw(jl_memory_exception); +#ifdef JULIA_ENABLE_THREADING jl_atomic_fetch_add(&gc_num.allocd, allocsz); +#else + gc_num.allocd += allocsz; +#endif gc_num.bigalloc++; #ifdef MEMDEBUG memset(v, 0xee, allocsz); @@ -1602,7 +1606,7 @@ static void post_mark(arraylist_t *list) } // collector entry point and control -static volatile uint64_t jl_gc_disable_counter = 0; +static volatile uint32_t jl_gc_disable_counter = 0; JL_DLLEXPORT int jl_gc_enable(int on) { @@ -2094,7 +2098,7 @@ void *jl_gc_perm_alloc_nolock(size_t sz) pool = (void*)LLT_ALIGN((uintptr_t)pool, JL_SMALL_BYTE_ALIGNMENT); #else void *pool = mmap(0, GC_PERM_POOL_SIZE, PROT_READ | PROT_WRITE, - MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (__unlikely(pool == MAP_FAILED)) return NULL; #endif diff --git a/src/julia_threads.h b/src/julia_threads.h index e18eb7283e1d4..5302fa88c31d6 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -159,7 +159,7 @@ static inline unsigned long JL_CONST_FUNC jl_thread_self(void) * 2. (most importantly) we need interoperability between code written * in different languages. * The current c++ standard (c++14) does not allow using c11 atomic - * functions or types and there's currently no grantee that the two + * functions or types and there's currently no guarantee that the two * types are compatible (although most of them probably are). * We also need to access these atomic variables from the LLVM JIT code * which is very hard unless the layout of the object is fully diff --git a/src/safepoint.c b/src/safepoint.c index 6644f32b63c61..2874266ebe4b5 100644 --- a/src/safepoint.c +++ b/src/safepoint.c @@ -93,7 +93,7 @@ void jl_safepoint_init(void) char *addr = (char*)VirtualAlloc(NULL, pgsz * 3, MEM_COMMIT, PAGE_READONLY); #else char *addr = (char*)mmap(0, pgsz * 3, PROT_READ, - MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) addr = NULL; #endif diff --git a/src/signals-unix.c b/src/signals-unix.c index 9bf9bdb01f734..f9cea3354e1cb 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -84,12 +84,24 @@ static void jl_call_in_ctx(jl_ptls_t ptls, void (*fptr)(void), void *_ctx) *(void**)rsp = NULL; ctx->uc_mcontext.gregs[REG_RSP] = rsp; ctx->uc_mcontext.gregs[REG_RIP] = (uintptr_t)fptr; +#elif defined(_OS_FREEBSD_) && defined(_CPU_X86_64_) + ucontext_t *ctx = (ucontext_t*)_ctx; + rsp -= sizeof(void*); + *(void**)rsp = NULL; + ctx->uc_mcontext.mc_rsp = rsp; + ctx->uc_mcontext.mc_rip = (uintptr_t)fptr; #elif defined(_OS_LINUX_) && defined(_CPU_X86_) ucontext_t *ctx = (ucontext_t*)_ctx; rsp -= sizeof(void*); *(void**)rsp = NULL; ctx->uc_mcontext.gregs[REG_ESP] = rsp; ctx->uc_mcontext.gregs[REG_EIP] = (uintptr_t)fptr; +#elif defined(_OS_FREEBSD_) && defined(_CPU_X86_) + ucontext_t *ctx = (ucontext_t*)_ctx; + rsp -= sizeof(void*); + *(void**)rsp = NULL; + ctx->uc_mcontext.mc_esp = rsp; + ctx->uc_mcontext.mc_eip = (uintptr_t)fptr; #elif defined(_OS_LINUX_) && defined(_CPU_AARCH64_) ucontext_t *ctx = (ucontext_t*)_ctx; ctx->uc_mcontext.sp = rsp; @@ -112,7 +124,8 @@ static void jl_call_in_ctx(jl_ptls_t ptls, void (*fptr)(void), void *_ctx) ctx->uc_mcontext64->__ss.__rsp = rsp; ctx->uc_mcontext64->__ss.__rip = (uintptr_t)fptr; #else - // TODO Add support for FreeBSD and PowerPC(64)? +#warning "julia: throw-in-context not supported on this platform" + // TODO Add support for PowerPC(64)? fptr(); #endif } @@ -436,7 +449,7 @@ static void *alloc_sigstack(size_t size) // Add one guard page to catch stack overflow in the signal handler size = LLT_ALIGN(size, pagesz) + pagesz; void *stackbuff = mmap(0, size, PROT_READ | PROT_WRITE, - MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (stackbuff == MAP_FAILED) jl_errorf("fatal error allocating signal stack: mmap: %s", strerror(errno)); diff --git a/src/task.c b/src/task.c index 105e80d34d579..81b9b564211ce 100644 --- a/src/task.c +++ b/src/task.c @@ -359,7 +359,7 @@ static void ctx_switch(jl_ptls_t ptls, jl_task_t *t, jl_jmp_buf *where) " push %%ebp;\n" // instead of ESP " jmp %P1;\n" // call `start_task` with fake stack frame " ud2" - : : "r" (stackbase), ""(&start_task) : "memory" ); + : : "r" (stackbase), "X"(&start_task) : "memory" ); #elif defined(_CPU_AARCH64_) asm(" mov sp, %0;\n" " mov x29, xzr;\n" // Clear link register (x29) and frame pointer From df5f4d9319062bdfedd3668a532d86ca2ba492bd Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Wed, 17 Aug 2016 22:21:49 -0700 Subject: [PATCH 0970/1117] Added test for complex on SharedArrays (#18097) --- test/parallel_exec.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index 37f58388b55c0..3d0b48d730fe4 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -473,6 +473,18 @@ for (x,i) in enumerate(d) @test x == i end +# complex +sd = SharedArray(Int,10) +se = SharedArray(Int,10) +@sync @parallel for i=1:10 + sd[i] = i + se[i] = i +end +sc = complex(sd,se) +for (x,i) in enumerate(sc) + @test i == complex(x,x) +end + # Once finalized accessing remote references and shared arrays should result in exceptions. function finalize_and_test(r) finalize(r) From a38743a3425bdacc5fbe4612a5ffb4a69b66f933 Mon Sep 17 00:00:00 2001 From: Elliot Saba <staticfloat@gmail.com> Date: Wed, 17 Aug 2016 23:43:57 -0700 Subject: [PATCH 0971/1117] Set LIB_INSTALL_DIR for cmake-based projects (#18047) * Set LIB_INSTALL_DIR for cmake-based projects This fixes `MULTIARCH_INSTALL=1` installations that place libraries not into `<prefix>/lib` but into `<prefix>/lib/<host-triplet>`, like Debian and Ubuntu. Without this change, `libgit2`, `libssh2` and `mbedtls` would place libraries into `<prefix>/lib`, silently failing until the Julia bootstrap process attempted to dlopen one of them. * Apply `LIB_INSTALL_DIR` to `CMAKE_COMMON` --- deps/Makefile | 2 +- deps/libssh2.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 37b4defd90884..1b799d922aaee 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -46,7 +46,7 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXX CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS) CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS) -CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) +CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) -DLIB_INSTALL_DIR=$(build_shlibdir) ifneq ($(VERBOSE), 0) CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON endif diff --git a/deps/libssh2.mk b/deps/libssh2.mk index be13f58c8653e..eb2282a882e12 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -8,7 +8,7 @@ LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT) LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT) LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_BUILD_TYPE=Release ifeq ($(OS),WINNT) LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF From ec6ee2cb40b5bac243099bae21ceb7e8b0c07b3f Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 18 Aug 2016 04:16:26 -0700 Subject: [PATCH 0972/1117] readme formatting [ci skip] --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index aee51ca376a81..bf89ab33271bf 100644 --- a/README.md +++ b/README.md @@ -242,13 +242,13 @@ On *FreeBSD Release 11.0*, install the gfortran, git, cmake, and gmake packages/ You must use the `gmake` command on FreeBSD instead of `make`. -Note that Julia is community-supported and we have little control over our upstream dependencies, you may still run into issues with its dependencies and YMMV. Current known issues includes: +Note that Julia is community-supported and we have little control over our upstream dependencies, you may still run into issues with dependencies and YMMV. Current known issues include: - - The x86 arch doesn't support threading due to lack of compiler runtime library support (set JULIA_THREADS=0). + - The x86 arch doesn't support threading due to lack of compiler runtime library support (set `JULIA_THREADS=0`). - libunwind needs a small patch to its tests to compile. - OpenBLAS patches in pkg haven't been upstreamed. - - gfortran can't link binaries. Set `FFLAGS=-Wl,-rpath,/usr/local/lib/gcc6` to workaround this (upstream bug submitted to FreeBSD pkg maintainers). - - System libraries installed by pkg are not on the compiler path by default. You may need to add `LDFLAGS=/usr/local/lib` and `CPPFLAGS=/usr/local/include` to your environment or Make.user file to build successfully. + - gfortran can't link binaries. Set `FFLAGS=-Wl,-rpath,/usr/local/lib/gcc6` to work around this (upstream bug submitted to FreeBSD pkg maintainers). + - System libraries installed by pkg are not on the compiler path by default. You may need to add `LDFLAGS=/usr/local/lib` and `CPPFLAGS=/usr/local/include` to your environment or `Make.user` file to build successfully. ### Windows From 2c741746b6d9beff47c59aa9ec55fa731a0e0bb5 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 18 Aug 2016 20:08:17 +0800 Subject: [PATCH 0973/1117] Fix compiler warnings on 32bits --- src/llvm-ptls.cpp | 1 + src/support/hashing.h | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 91d588243fcf5..f9567a3a5b007 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -147,6 +147,7 @@ void LowerPTLS::runOnFunction(LLVMContext &ctx, Module &M, Function *F, Value *tls = nullptr; assert(0 && "Cannot emit thread pointer for this architecture."); # endif + (void)T_pint8; ptlsStates->replaceAllUsesWith(tls); ptlsStates->eraseFromParent(); } diff --git a/src/support/hashing.h b/src/support/hashing.h index 1be2a9e7b5ec0..5f9b41d6f700d 100644 --- a/src/support/hashing.h +++ b/src/support/hashing.h @@ -3,6 +3,9 @@ #ifndef HASHING_H #define HASHING_H +#include "utils.h" +#include "dtypes.h" + #ifdef __cplusplus extern "C" { #endif @@ -22,10 +25,17 @@ JL_DLLEXPORT uint32_t memhash32(const char *buf, size_t n); JL_DLLEXPORT uint32_t memhash32_seed(const char *buf, size_t n, uint32_t seed); #ifdef _P64 -#define bitmix(a,b) int64hash((a)^bswap_64(b)) +STATIC_INLINE uint64_t bitmix(uint64_t a, uint64_t b) +{ + return int64hash(a^bswap_64(b)); +} #else -#define bitmix(a,b) int64to32hash((((uint64_t)a)<<32)|((uint64_t)b)) +STATIC_INLINE uint32_t bitmix(uint32_t a, uint32_t b) +{ + return int64to32hash((((uint64_t)a) << 32) | (uint64_t)b); +} #endif +#define bitmix(a, b) (bitmix)((uintptr_t)(a), (uintptr_t)(b)) #ifdef __cplusplus } From e03ad1d9450fe02ccb603a5c7c355648212c7f76 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 18 Aug 2016 10:39:39 +0800 Subject: [PATCH 0974/1117] Improve arch/cpu detection/selection on ARM and AArch64 * Allow `cpu_target` to specify a generic arch, matching the behavior on x86 * Detect the CPU arch version with `uname` * Require `armv6` Close #13270 (`armv5` is not supported) Fix #18042 * Remove warning about generic arch since it's not really useful Fix #17549 * Require at least the same ARM arch version and profile the C code is compiled with --- README.arm.md | 7 ++- src/codegen.cpp | 130 +++++++++++++++++++++++++++++++++++++++++++----- src/disasm.cpp | 4 +- 3 files changed, 125 insertions(+), 16 deletions(-) diff --git a/README.arm.md b/README.arm.md index 1c50538172cb9..c9c823cd68d4f 100644 --- a/README.arm.md +++ b/README.arm.md @@ -1,7 +1,12 @@ # Julia binaries for ARM [Nightly builds](https://status.julialang.org/download/linux-arm) are -available for ARM. +available for ARMv7-A. + +# Hardware requirements + +Julia requires at least `armv6` and `vfpv2` instruction sets. It's recommanded +to use at least `armv7-a`. `armv5` or soft float are not supported. # Building Julia on ARM diff --git a/src/codegen.cpp b/src/codegen.cpp index fbb0bf1a5731c..873a4558e1e7e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -87,6 +87,7 @@ #if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) # include <llvm/IR/InlineAsm.h> +# include <sys/utsname.h> #endif #if defined(USE_POLLY) #include <polly/RegisterPasses.h> @@ -5531,10 +5532,68 @@ static void init_julia_llvm_env(Module *m) addOptimizationPasses(jl_globalPM); } +static inline std::string getNativeTarget() +{ + std::string cpu = sys::getHostCPUName(); +#if defined(_CPU_ARM_) + // Try slightly harder than LLVM at determine the CPU architecture. + if (cpu == "generic") { + // This is the most reliable way I can find + // `/proc/cpuinfo` changes between kernel versions + struct utsname name; + if (uname(&name) >= 0) { + // name.machine is the elf_platform in the kernel. + if (strcmp(name.machine, "armv6l") == 0) { + return "armv6"; + } + if (strcmp(name.machine, "armv7l") == 0) { + return "armv7"; + } + if (strcmp(name.machine, "armv7ml") == 0) { + // Thumb + return "armv7-m"; + } + if (strcmp(name.machine, "armv8l") == 0 || + strcmp(name.machine, "aarch64") == 0) { + return "armv8"; + } + } + } +#endif + return cpu; +} + +#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) +// Check if the cpu name is a ARM/AArch64 arch name and return a +// string that can be used as LLVM feature name +static inline std::string checkARMArchFeature(const std::string &cpu) +{ + const char *prefix = "armv"; + size_t prefix_len = strlen(prefix); + if (cpu.size() <= prefix_len || + memcmp(cpu.data(), prefix, prefix_len) != 0 || + cpu[prefix_len] < '1' || cpu[prefix_len] > '9') + return std::string(); +#if defined(_CPU_ARM_) + // "v7" and "v8" are not available in the form of `armv*` + // in the feature list + if (cpu == "armv7") { + return "v7"; + } + else if (cpu == "armv8") { + return "v8"; + } + return cpu; +#else + return cpu.substr(3); +#endif +} +#endif + // Helper to figure out what features to set for the LLVM target // If the user specifies native (or does not specify) we default // using the API provided by LLVM -static inline SmallVector<std::string,10> getTargetFeatures() +static inline SmallVector<std::string,10> getTargetFeatures(std::string &cpu) { StringMap<bool> HostFeatures; if (!strcmp(jl_options.cpu_target,"native")) { @@ -5563,16 +5622,63 @@ static inline SmallVector<std::string,10> getTargetFeatures() #endif // Figure out if we know the cpu_target - std::string cpu = strcmp(jl_options.cpu_target,"native") ? jl_options.cpu_target : sys::getHostCPUName(); - if (cpu.empty() || cpu == "generic") { - jl_printf(JL_STDERR, "WARNING: unable to determine host cpu name.\n"); -#if defined(_CPU_ARM_) && defined(__ARM_PCS_VFP) - // Check if this is required when you have read the features directly from the processor - // This affects the platform calling convention. - // TODO: enable vfp3 for ARMv7+ (but adapt the ABI) - HostFeatures["vfp2"] = true; -#endif + cpu = (strcmp(jl_options.cpu_target,"native") ? jl_options.cpu_target : + getNativeTarget()); +#if defined(_CPU_ARM_) + // Figure out what we are compiling against from the C defines. + // This might affect ABI but is fine since + // 1. We define the C ABI explicitly. + // 2. This does not change when running the same binary on different + // machines. + // This shouldn't affect making generic binaries since that requires a + // generic C -march anyway. + HostFeatures["vfp2"] = true; + + // Arch version +#if __ARM_ARCH >= 8 + HostFeatures["v8"] = true; +#elif __ARM_ARCH >= 7 + HostFeatures["v7"] = true; +#else + // minimum requirement + HostFeatures["v6"] = true; +#endif + + // ARM profile + // Only do this on ARM and not AArch64 since LLVM aarch64 backend + // doesn't support setting profiles. + // AFAIK there's currently no 64bit R and M profile either + // (v8r and v8m are both 32bit) +#if defined(__ARM_ARCH_PROFILE) +# if __ARM_ARCH_PROFILE == 'A' + HostFeatures["aclass"] = true; +# elif __ARM_ARCH_PROFILE == 'R' + HostFeatures["rclass"] = true; +# elif __ARM_ARCH_PROFILE == 'M' + // Thumb + HostFeatures["mclass"] = true; +# endif +#endif +#endif // _CPU_ARM_ + + // On ARM and AArch64, allow using cpu_target to specify a CPU architecture + // which is specified in the feature set in LLVM. +#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) + // Supported ARM arch names on LLVM 3.8: + // armv6, armv6-m, armv6j, armv6k, armv6kz, armv6s-m, armv6t2, + // armv7, armv7-a, armv7-m, armv7-r, armv7e-m, armv7k, armv7s, + // armv8, armv8-a, armv8.1-a, armv8.2-a + // Additional ARM arch names on LLVM 3.9: + // armv8-m.base, armv8-m.main + // + // Supported AArch64 arch names on LLVM 3.8: + // armv8.1a, armv8.2a + std::string arm_arch = checkARMArchFeature(cpu); + if (!arm_arch.empty()) { + HostFeatures[arm_arch] = true; + cpu = "generic"; } +#endif SmallVector<std::string,10> attr; for (StringMap<bool>::const_iterator it = HostFeatures.begin(); it != HostFeatures.end(); it++) { @@ -5689,8 +5795,8 @@ extern "C" void jl_init_codegen(void) TheTriple.setEnvironment(Triple::ELF); #endif #endif - std::string TheCPU = strcmp(jl_options.cpu_target,"native") ? jl_options.cpu_target : sys::getHostCPUName(); - SmallVector<std::string, 10> targetFeatures = getTargetFeatures( ); + std::string TheCPU; + SmallVector<std::string, 10> targetFeatures = getTargetFeatures(TheCPU); jl_TargetMachine = eb.selectTarget( TheTriple, "", diff --git a/src/disasm.cpp b/src/disasm.cpp index df3f15604e11b..356e57add0b9b 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -370,9 +370,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, { // GC safe // Get the host information - std::string TripleName; - if (TripleName.empty()) - TripleName = sys::getDefaultTargetTriple(); + std::string TripleName = sys::getDefaultTargetTriple(); Triple TheTriple(Triple::normalize(TripleName)); std::string MCPU = sys::getHostCPUName(); From 4ce6ab46735582f70ec2687ebfade02637592fa7 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 18 Aug 2016 04:23:29 -0700 Subject: [PATCH 0975/1117] fix "recommanded" typo --- README.arm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.arm.md b/README.arm.md index c9c823cd68d4f..f31eed9d51ef3 100644 --- a/README.arm.md +++ b/README.arm.md @@ -5,7 +5,7 @@ available for ARMv7-A. # Hardware requirements -Julia requires at least `armv6` and `vfpv2` instruction sets. It's recommanded +Julia requires at least `armv6` and `vfpv2` instruction sets. It's recommended to use at least `armv7-a`. `armv5` or soft float are not supported. # Building Julia on ARM From c10148ebeefe9a5deb4b6166d004ddab43b084fc Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Tue, 16 Aug 2016 19:58:37 +0200 Subject: [PATCH 0976/1117] ASAN: provide default options. --- src/init.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/init.c b/src/init.c index a2d74fe60cc91..d4bf640a181fe 100644 --- a/src/init.c +++ b/src/init.c @@ -46,6 +46,14 @@ extern BOOL (WINAPI *hSymRefreshModuleList)(HANDLE); #include <unistd.h> #endif +#ifdef JL_ASAN_ENABLED +JL_DLLEXPORT const char* __asan_default_options() { + return "allow_user_segv_handler=1:detect_leaks=0"; + // FIXME: enable LSAN after fixing leaks & defining __lsan_default_suppressions(), + // or defining __lsan_default_options = exitcode=0 once publicly available +} +#endif + static const char system_image_path[256] = "\0" JL_SYSTEM_IMAGE_PATH; jl_options_t jl_options = { 0, // quiet @@ -628,17 +636,6 @@ void _julia_init(JL_IMAGE_SEARCH rel) } #endif - -#ifdef JL_ASAN_ENABLED - const char *asan_options = getenv("ASAN_OPTIONS"); - if (!asan_options || !(strstr(asan_options, "allow_user_segv_handler=1") || - strstr(asan_options, "handle_segv=0"))) { - jl_printf(JL_STDERR,"WARNING: ASAN overrides Julia's SIGSEGV handler; " - "disable SIGSEGV handling or allow custom handlers.\n"); - } - -#endif - jl_init_threading(); jl_gc_init(); From 152c2c9faa41d430ec8e145de9bc73b4f3c2dc3e Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Tue, 16 Aug 2016 20:19:27 +0200 Subject: [PATCH 0977/1117] ASAN/MSAN: document use of sanitizers. [ci skip] --- doc/devdocs/C.rst | 1 + doc/devdocs/sanitizers.rst | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 doc/devdocs/sanitizers.rst diff --git a/doc/devdocs/C.rst b/doc/devdocs/C.rst index 60150eb777d9b..b5866a7fa18ea 100644 --- a/doc/devdocs/C.rst +++ b/doc/devdocs/C.rst @@ -12,3 +12,4 @@ backtraces debuggingtips valgrind + sanitizers diff --git a/doc/devdocs/sanitizers.rst b/doc/devdocs/sanitizers.rst new file mode 100644 index 0000000000000..262e259825d5c --- /dev/null +++ b/doc/devdocs/sanitizers.rst @@ -0,0 +1,44 @@ +***************** +Sanitizer support +***************** + +General considerations +---------------------- + +Using Clang's sanitizers obviously require you to use Clang (``USECLANG=1``), but there's +another catch: most sanitizers require a run-time library, provided by the host compiler, +while the instrumented code generated by Julia's JIT relies on functionality from that +library. This implies that the LLVM version of your host compiler matches that of the LLVM +library used within Julia. + +An easy solution is to have an dedicated build folder for providing a matching toolchain, by +building with ``BUILD_LLVM_CLANG=1`` and overriding ``LLVM_USE_CMAKE=1`` (Autotool-based +builds are incompatible with ASAN). You can then refer to this toolchain from another build +folder by specifying ``USECLANG=1`` while overriding the ``CC`` and ``CXX`` variables. + + +Address Sanitizer (ASAN) +------------------------ + +For detecting or debugging memory bugs, you can use Clang's `address sanitizer (ASAN) +<http://clang.llvm.org/docs/AddressSanitizer.html>`_. By compiling with +``SANITIZE=1`` you enable ASAN for the Julia compiler and its generated code. In addition, +you can specify ``LLVM_SANITIZE=1`` to sanitize the LLVM library as well. Note that these +options incur a high performance and memory cost. For example, using ASAN for Julia and LLVM +makes ``testall1`` takes 8-10 times as long while using 20 times as much memory (this can +be reduced to respectively a factor of 3 and 4 by using the options described below). + +By default, Julia sets the ``allow_user_segv_handler=1`` ASAN flag, which is required for +signal delivery to work properly. You can define other options using the ``ASAN_OPTIONS`` +environment flag, in which case you'll need to repeat the default option mentioned before. +For example, memory usage can be reduced by specifying ``fast_unwind_on_malloc=0`` and +``malloc_context_size=2``, at the cost of backtrace accuracy. For now, Julia also sets +``detect_leaks=0``, but this should be removed in the future. + + +Memory Sanitizer (MSAN) +----------------------- + +For detecting use of uninitialized memory, you can use Clang's `memory sanitizer (MSAN) +<http://clang.llvm.org/docs/MemorySanitizer.html>`_ by compiling with +``SANITIZE_MEMORY=1``. From 8379703f7fe19335809d9f4dc6704c1d62b39543 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Thu, 18 Aug 2016 06:39:14 -0700 Subject: [PATCH 0978/1117] Missing tests for strings (#18089) --- test/strings/basic.jl | 5 +++++ test/strings/search.jl | 2 ++ 2 files changed, 7 insertions(+) diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 1008ce9566a17..e736da9e69890 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -7,6 +7,11 @@ let d = [0x61,0x62,0x63,0x21] end @test String("abc!") == "abc!" +@test isempty(string()) +@test eltype(GenericString) == Char +@test start("abc") == 1 +@test cmp("ab","abc") == -1 + # {starts,ends}with @test startswith("abcd", 'a') @test startswith("abcd", "a") diff --git a/test/strings/search.jl b/test/strings/search.jl index 6cc7b7233619a..8fcfe9c62d5d3 100644 --- a/test/strings/search.jl +++ b/test/strings/search.jl @@ -373,3 +373,5 @@ end # string searchindex with a two-char UTF-8 (4 byte) string literal @test rsearchindex("\U1f596\U1f596", "\U1f596\U1f596") == 1 @test rsearchindex("\U1f596\U1f596", "\U1f596\U1f596", endof("\U1f596\U1f596\U1f596")) == 1 + +@test_throws ErrorException "ab" ∈ "abc" From 8663edeb96d53eb4b74ed525294285aca9a67985 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 17 Aug 2016 15:49:55 -0400 Subject: [PATCH 0979/1117] factor out the jl_get_llvmf code more cleanly jl_get_llvmf was left in so that this can be backported to v0.5 without altering the C api (in case it was used in a package) also correct / add error handling and proper locks --- base/reflection.jl | 41 +++++++-- src/codegen.cpp | 204 +++++++++++++++++++++++++++---------------- src/gf.c | 2 +- src/julia_internal.h | 2 +- test/ambiguous.jl | 1 + 5 files changed, 163 insertions(+), 87 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index f6facf3ff6123..80d29f822cfff 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -339,26 +339,44 @@ uncompressed_ast(l::LambdaInfo) = isa(l.code,Array{UInt8,1}) ? ccall(:jl_uncompress_ast, Array{Any,1}, (Any,Any), l, l.code) : l.code # Printing code representations in IR and assembly -function _dump_function(f, t::ANY, native, wrapper, strip_ir_metadata, dump_module) - ccall(:jl_is_in_pure_context, Bool, ()) && error("native reflection cannot be used from generated functions") +function _dump_function(f::ANY, t::ANY, native::Bool, wrapper::Bool, strip_ir_metadata::Bool, dump_module::Bool) + ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions") if isa(f, Core.Builtin) throw(ArgumentError("argument is not a generic function")) end - t = tt_cons(Core.Typeof(f), to_tuple_type(t)) - llvmf = ccall(:jl_get_llvmf, Ptr{Void}, (Any, Bool, Bool), t, wrapper, native) + # get the LambdaInfo for the method match + meth = which(f, t) + t = to_tuple_type(t) + ft = isa(f, Type) ? Type{f} : typeof(f) + tt = Tuple{ft, t.parameters...} + (ti, env) = ccall(:jl_match_method, Any, (Any, Any, Any), + tt, meth.sig, meth.tvars)::SimpleVector + li = func_for_method_checked(meth, tt) + # try to infer it + (linfo, ty, inf) = Core.Inference.typeinf(li, ti, env, true) + # get the code for it + return _dump_function(linfo, native, wrapper, strip_ir_metadata, dump_module) +end +function _dump_function(linfo::LambdaInfo, native::Bool, wrapper::Bool, strip_ir_metadata::Bool, dump_module::Bool) + if native + llvmf = ccall(:jl_get_llvmf_decl, Ptr{Void}, (Any, Bool), linfo, wrapper) + else + llvmf = ccall(:jl_get_llvmf_defn, Ptr{Void}, (Any, Bool), linfo, wrapper) + end if llvmf == C_NULL - error("did not find a unique method for the specified argument types") + error("could not compile the specified method") end if native - str = ccall(:jl_dump_function_asm, Ref{String}, (Ptr{Void},Cint), llvmf, 0) + str = ccall(:jl_dump_function_asm, Ref{String}, (Ptr{Void}, Cint), llvmf, 0) else str = ccall(:jl_dump_function_ir, Ref{String}, (Ptr{Void}, Bool, Bool), llvmf, strip_ir_metadata, dump_module) end - isleaftype(t) || (str = "# WARNING: This code may not match what actually runs.\n" * str) + # TODO: use jl_is_cacheable_sig instead of isleaftype + isleaftype(linfo.specTypes) || (str = "; WARNING: This code may not match what actually runs.\n" * str) return str end @@ -450,11 +468,16 @@ function which(f::ANY, t::ANY) return first(ms) else ft = isa(f,Type) ? Type{f} : typeof(f) - m = ccall(:jl_gf_invoke_lookup, Any, (Any,), Tuple{ft, t.parameters...}) + tt = Tuple{ft, t.parameters...} + m = ccall(:jl_gf_invoke_lookup, Any, (Any,), tt) if m === nothing error("no method found for the specified argument types") end - return m.func::Method + meth = m.func::Method + if ccall(:jl_has_call_ambiguities, Int32, (Any, Any), tt, meth) != 0 + error("method match is ambiguous for the specified argument types") + end + return meth end end diff --git a/src/codegen.cpp b/src/codegen.cpp index 873a4558e1e7e..3acbae8de822a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1042,88 +1042,109 @@ void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) extern "C" JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp); extern "C" JL_DLLEXPORT -void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) +void *jl_get_llvmf_defn(jl_lambda_info_t *linfo, bool getwrapper) { - jl_lambda_info_t *linfo = NULL, *temp = NULL; - JL_GC_PUSH3(&linfo, &temp, &tt); - if (tt != NULL) { - linfo = jl_get_specialization1(tt); - if (linfo == NULL) { - linfo = jl_method_lookup_by_type( - ((jl_datatype_t*)jl_tparam0(tt))->name->mt, tt, 0, 0); - if (linfo == NULL || jl_has_call_ambiguities(tt, linfo->def)) { - JL_GC_POP(); - return NULL; - } + if (linfo->def && linfo->def->lambda_template->code == jl_nothing) { + // not a generic function + return NULL; + } + + jl_lambda_info_t *temp = NULL; + JL_GC_PUSH1(&temp); + if (linfo->code == jl_nothing && linfo->def) { + // re-infer if we've deleted the code + // first copy the linfo to avoid corrupting it and + // confusing the compiler about the + // validity of the code it already generated + temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals); + jl_type_infer(temp, 0); + if (temp->code == jl_nothing || temp->inInference) { + // something went wrong: abort! + JL_GC_POP(); + return NULL; } } - if (linfo == NULL) { - // no function found for argument tuple type - JL_GC_POP(); - return NULL; + + // Backup the info for the nested compile + JL_LOCK(&codegen_lock); + BasicBlock *old = nested_compile ? builder.GetInsertBlock() : NULL; + DebugLoc olddl = builder.getCurrentDebugLocation(); + bool last_n_c = nested_compile; + nested_compile = true; + // emit this function into a new module + jl_llvm_functions_t declarations; + std::unique_ptr<Module> m; + JL_TRY { + m = emit_function(temp ? temp : linfo, &declarations); + } + JL_CATCH { + // something failed! + nested_compile = last_n_c; + if (old != NULL) { + builder.SetInsertPoint(old); + builder.SetCurrentDebugLocation(olddl); + } + JL_UNLOCK(&codegen_lock); // Might GC + jl_rethrow_with_add("error compiling %s", jl_symbol_name(linfo->def ? linfo->def->name : anonymous_sym)); + } + // Restore the previous compile context + if (old != NULL) { + builder.SetInsertPoint(old); + builder.SetCurrentDebugLocation(olddl); } - if (linfo->def->lambda_template->code == jl_nothing) { + nested_compile = last_n_c; + + jl_globalPM->run(*m.get()); + Function *f = (llvm::Function*)declarations.functionObject; + Function *specf = (llvm::Function*)declarations.specFunctionObject; + // swap declarations for definitions and destroy declarations + if (specf) { + Function *tempf = cast<Function>(m->getNamedValue(specf->getName())); + delete specf; + specf = tempf; + } + if (f) { + Function *tempf = cast<Function>(m->getNamedValue(f->getName())); + delete f; + f = tempf; + } + // clone the name from the runtime linfo, if it exists + // to give the user a (false) sense of stability + Function *specf_decl = (Function*)linfo->functionObjectsDecls.specFunctionObject; + if (specf_decl) { + specf->setName(specf_decl->getName()); + } + Function *f_decl = (Function*)linfo->functionObjectsDecls.functionObject; + if (f_decl) { + f->setName(f_decl->getName()); + } + m.release(); // the return object `llvmf` will be the owning pointer + JL_UNLOCK(&codegen_lock); // Might GC + JL_GC_POP(); + if (getwrapper || !specf) + return f; + else + return specf; +} + + +extern "C" JL_DLLEXPORT +void *jl_get_llvmf_decl(jl_lambda_info_t *linfo, bool getwrapper) +{ + if (linfo->def && linfo->def->lambda_template->code == jl_nothing) { // not a generic function - JL_GC_POP(); return NULL; } - // make sure to compile this normally first, - // since `emit_function` doesn't handle recursive compilation correctly + // compile this normally linfo = jl_compile_for_dispatch(linfo); - if (!getdeclarations) { - if (linfo->code == jl_nothing) { - // re-infer if we've deleted the code - // first copy the linfo to avoid corrupting it and - // confusing the compiler about the - // validity of the code it already generated - temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals); - jl_type_infer(temp, 0); - if (temp->code == jl_nothing || temp->inInference) { - JL_GC_POP(); - return NULL; - } - } - // emit this function into a new module - Function *f, *specf; - jl_llvm_functions_t declarations; - std::unique_ptr<Module> m = emit_function(temp ? temp : linfo, &declarations); - jl_globalPM->run(*m.get()); - f = (llvm::Function*)declarations.functionObject; - specf = (llvm::Function*)declarations.specFunctionObject; - // swap declarations for definitions and destroy declarations - if (specf) { - Function *tempf = cast<Function>(m->getNamedValue(specf->getName())); - delete specf; - specf = tempf; - } - if (f) { - Function *tempf = cast<Function>(m->getNamedValue(f->getName())); - delete f; - f = tempf; - } - Function *specf_decl = (Function*)linfo->functionObjectsDecls.specFunctionObject; - if (specf_decl) { - specf->setName(specf_decl->getName()); - } - Function *f_decl = (Function*)linfo->functionObjectsDecls.functionObject; - if (f_decl) { - f->setName(f_decl->getName()); - } - m.release(); // the return object `llvmf` will be the owning pointer - JL_GC_POP(); - if (getwrapper || !specf) { - return f; - } - else { - return specf; - } - } - if (linfo->jlcall_api == 2) { + if (linfo->jlcall_api == 2 && linfo->def) { // normally we don't generate native code for these functions, so need an exception here - // This leaks a bit of memory to cache the native code that we'll never actually need + // This leaks a bit of memory to cache native code that we'll never actually need if (linfo->functionObjectsDecls.functionObject == NULL) { + jl_lambda_info_t *temp = NULL; + JL_GC_PUSH1(&temp); temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals); jl_type_infer(temp, 0); temp->jlcall_api = 0; @@ -1134,20 +1155,49 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) } jl_compile_linfo(temp); linfo->functionObjectsDecls = temp->functionObjectsDecls; + JL_GC_POP(); } jl_set_lambda_code_null(linfo); } - Function *llvmf; - if (!getwrapper && linfo->functionObjectsDecls.specFunctionObject != NULL) { - llvmf = (Function*)linfo->functionObjectsDecls.specFunctionObject; + + if (getwrapper || !linfo->functionObjectsDecls.specFunctionObject) + return linfo->functionObjectsDecls.functionObject; + else + return linfo->functionObjectsDecls.specFunctionObject; +} + + +extern "C" JL_DLLEXPORT +void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) +{ // DEPRECATED + jl_lambda_info_t *linfo = NULL, *temp = NULL; + JL_GC_PUSH3(&linfo, &temp, &tt); + if (tt != NULL) { + linfo = jl_get_specialization1(tt); + if (linfo == NULL) { + linfo = jl_method_lookup_by_type( + ((jl_datatype_t*)jl_tparam0(tt))->name->mt, tt, 0, 0); + if (linfo == NULL || jl_has_call_ambiguities(tt, linfo->def)) { + JL_GC_POP(); + return NULL; + } + } } - else { - llvmf = (Function*)linfo->functionObjectsDecls.functionObject; + if (linfo == NULL) { + // no function found for argument tuple type + JL_GC_POP(); + return NULL; } + void *f; + if (getdeclarations) + f = jl_get_llvmf_decl(linfo, getwrapper); + else + f = jl_get_llvmf_defn(linfo, getwrapper); JL_GC_POP(); - return llvmf; + return f; } + // print an llvm IR acquired from jl_get_llvmf // warning: this takes ownership of, and destroys, f->getParent() extern "C" JL_DLLEXPORT @@ -1160,6 +1210,7 @@ const jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump if (!llvmf || (!llvmf->isDeclaration() && !llvmf->getParent())) jl_error("jl_dump_function_ir: Expected Function* in a temporary Module"); + JL_LOCK(&codegen_lock); // Might GC if (!llvmf->getParent()) { // print the function declaration as-is llvmf->print(stream); @@ -1203,6 +1254,7 @@ const jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump } delete m; } + JL_UNLOCK(&codegen_lock); // Might GC return jl_cstr_to_string(const_cast<char*>(stream.str().c_str())); } diff --git a/src/gf.c b/src/gf.c index 2c18d7d2b228e..225ecf7569640 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1383,7 +1383,7 @@ JL_DLLEXPORT jl_value_t *jl_get_spec_lambda(jl_tupletype_t *types) return li ? li : jl_nothing; } -int jl_has_call_ambiguities(jl_tupletype_t *types, jl_method_t *m) +JL_DLLEXPORT int jl_has_call_ambiguities(jl_tupletype_t *types, jl_method_t *m) { if (m->ambig == jl_nothing) return 0; for (size_t i = 0; i < jl_array_len(m->ambig); i++) { diff --git a/src/julia_internal.h b/src/julia_internal.h index fc9514260c02f..3b643ce9ba00f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -424,7 +424,7 @@ JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); -int jl_has_call_ambiguities(jl_tupletype_t *types, jl_method_t *m); +JL_DLLEXPORT int jl_has_call_ambiguities(jl_tupletype_t *types, jl_method_t *m); uint32_t jl_module_next_counter(jl_module_t *m); void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig); diff --git a/test/ambiguous.jl b/test/ambiguous.jl index 3b2296ce0a630..37a8135f2db88 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -71,6 +71,7 @@ code_native(io, ambig, (Int, Int)) # Test that ambiguous cases fail appropriately @test precompile(ambig, (UInt8, Int)) == false cfunction(ambig, Int, (UInt8, Int)) # test for a crash (doesn't throw an error) +@test_throws ErrorException which(ambig, (UInt8, Int)) @test_throws ErrorException code_llvm(io, ambig, (UInt8, Int)) @test_throws ErrorException code_native(io, ambig, (UInt8, Int)) From a7384009a5b28207a51030b7c4a89fb00459768a Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 18 Aug 2016 05:49:20 +0000 Subject: [PATCH 0980/1117] Fix compilation on llvm 3.9 --- src/debuginfo.cpp | 79 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index c7e598fc5de4e..89e4e5d4a6859 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -29,14 +29,10 @@ #else #include <llvm/ExecutionEngine/JITMemoryManager.h> #endif -#ifdef _OS_DARWIN_ #include <llvm/Object/MachO.h> -#endif -#ifdef _OS_WINDOWS_ #include <llvm/Object/COFF.h> -# ifdef LLVM37 -# include <llvm/Object/ELFObjectFile.h> -# endif +#ifdef LLVM37 +# include <llvm/Object/ELFObjectFile.h> #endif #if defined(USE_MCJIT) && !defined(LLVM36) && defined(_OS_DARWIN_) @@ -896,23 +892,52 @@ calc_gnu_debuglink_crc32(const void *buf, size_t size) crc = g_crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); return crc ^ ~0U; } -static ErrorOr<object::OwningBinary<object::ObjectFile>> openDebugInfo(StringRef debuginfopath, const debug_link_info &info) + +template<typename T> +static inline void ignoreError(T &err) +{ +#if defined(LLVM39) && !defined(NDEBUG) + consumeError(err.takeError()); +#endif +} + +#ifdef LLVM39 +static Expected<object::OwningBinary<object::ObjectFile>> +#else +static ErrorOr<object::OwningBinary<object::ObjectFile>> +#endif +openDebugInfo(StringRef debuginfopath, const debug_link_info &info) { auto SplitFile = MemoryBuffer::getFile(debuginfopath); - if (std::error_code EC = SplitFile.getError()) + if (std::error_code EC = SplitFile.getError()) { +#ifdef LLVM39 + return errorCodeToError(EC); +#else return EC; +#endif + } uint32_t crc32 = calc_gnu_debuglink_crc32( SplitFile.get()->getBufferStart(), SplitFile.get()->getBufferSize()); - if (crc32 != info.crc32) + if (crc32 != info.crc32) { +#ifdef LLVM39 + return errorCodeToError(object::object_error::arch_not_found); +#else return object::object_error::arch_not_found; +#endif + } auto error_splitobj = object::ObjectFile::createObjectFile( SplitFile.get().get()->getMemBufferRef(), sys::fs::file_magic::unknown); - if (std::error_code EC = error_splitobj.getError()) - return EC; + if (!error_splitobj) { +#ifdef LLVM39 + return error_splitobj.takeError(); +#else + return error_splitobj.getError(); +#endif + } // successfully validated and loaded split debug info file return object::OwningBinary<object::ObjectFile>( @@ -1139,32 +1164,44 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, debug_link_info info = getDebuglink(*debugobj); if (!info.filename.empty()) { size_t sep = fname.rfind('/'); - ErrorOr<object::OwningBinary<object::ObjectFile>> DebugInfo(std::errc::no_such_file_or_directory); +#ifdef LLVM39 + Expected<object::OwningBinary<object::ObjectFile>> + DebugInfo(errorCodeToError(std::make_error_code(std::errc::no_such_file_or_directory))); + // Can't find a way to construct an empty Expected object + // that can be ignored. + ignoreError(DebugInfo); +#else + ErrorOr<object::OwningBinary<object::ObjectFile>> + DebugInfo(std::errc::no_such_file_or_directory); +#endif if (fname.substr(sep + 1) != info.filename) { debuginfopath = fname.substr(0, sep + 1); debuginfopath += info.filename; DebugInfo = openDebugInfo(debuginfopath, info); } - if (DebugInfo.getError()) { + if (!DebugInfo) { debuginfopath = fname.substr(0, sep + 1); debuginfopath += ".debug/"; debuginfopath += info.filename; + ignoreError(DebugInfo); DebugInfo = openDebugInfo(debuginfopath, info); } - if (DebugInfo.getError()) { + if (!DebugInfo) { debuginfopath = "/usr/lib/debug/"; debuginfopath += fname.substr(0, sep + 1); debuginfopath += info.filename; + ignoreError(DebugInfo); DebugInfo = openDebugInfo(debuginfopath, info); } - if (DebugInfo.getError()) { - // no split debug info found - // should we warn the user? - } - else { + if (DebugInfo) { errorobj = std::move(DebugInfo); + // Yes, we've checked, and yes LLVM want us to check again. + assert(errorobj); debugobj = errorobj->getBinary(); } + else { + ignoreError(DebugInfo); + } } #endif } @@ -1224,13 +1261,11 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, errorobj.release(); #endif } -#ifdef LLVM39 else { // TODO: report the error instead of silently consuming it? // jl_error might run into the same error again... - consumeError(errorobj.takeError()); + ignoreError(errorobj); } -#endif // update cache objfileentry_t entry = {*obj, *context, *slide, *section_slide}; From c41951a11c5ff08a1ac81bc82f250f666f4ad949 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 18 Aug 2016 12:05:33 -0400 Subject: [PATCH 0981/1117] disable llvm optimizations in JIT when generating output When saving native code we re-run optimization on the whole module as part of object file emission, so it is not needed for JIT compiled code. We are already passing -O0 for package precompilation; this extends that benefit to the system image. --- src/jitlayers.cpp | 9 ++++++++- src/timing.h | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 50fb1683a68f1..e30336385ba22 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -439,7 +439,13 @@ JuliaOJIT::JuliaOJIT(TargetMachine &TM) } ) { - addOptimizationPasses(&PM); + if (!jl_generating_output()) { + addOptimizationPasses(&PM); + } + else { + PM.add(createLowerGCFramePass(tbaa_gcframe)); + PM.add(createLowerPTLSPass(imaging_mode, tbaa_const)); + } if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) llvm_unreachable("Target does not support MC emission."); @@ -1038,6 +1044,7 @@ static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, extern "C" void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len) { + JL_TIMING(NATIVE_DUMP); assert(imaging_mode); // We don't want to use MCJIT's target machine because // it uses the large code model and we may potentially diff --git a/src/timing.h b/src/timing.h index 70a885018cdac..3fb58b537baf4 100644 --- a/src/timing.h +++ b/src/timing.h @@ -46,7 +46,8 @@ static inline uint64_t rdtscp(void) X(AST_COMPRESS), \ X(AST_UNCOMPRESS), \ X(SYSIMG_LOAD), \ - X(SYSIMG_DUMP), + X(SYSIMG_DUMP), \ + X(NATIVE_DUMP), enum jl_timing_owners { #define X(name) JL_TIMING_ ## name From 61b50b1d2c4757c908c431de2b63d475ae5d95ae Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Thu, 18 Aug 2016 09:16:24 +0530 Subject: [PATCH 0982/1117] Build curl with minimal external libraries. Link to our own libssh2. --- deps/curl.mk | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/deps/curl.mk b/deps/curl.mk index b8bf9ccbd8fec..060b41b7969a1 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -18,7 +18,14 @@ $(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VE $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure mkdir -p $(dir $@) cd $(dir $@) && \ - $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --without-ssl --with-mbedtls=$(build_prefix) CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) \ + --without-ssl --without-gnutls --without-gssapi \ + --without-libidn --without-libmetalink --without-librtmp \ + --without-nghttp2 --without-nss --without-polarssl \ + --without-spnego --disable-ares --disable-ldap \ + --disable-ldaps --without-zsh-functions-dir \ + --with-libssh2=$(build_prefix) --with-mbedtls=$(build_prefix) \ + CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" touch -c $@ $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status From a738dbd647537734945f3cddb8fc801d9b5f3687 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 18 Aug 2016 12:21:55 -0400 Subject: [PATCH 0983/1117] fix #18085, segfault on method add in loop Closures that are part of out-of-scope method adds were not lifted out of top-level loops. --- src/julia-syntax.scm | 2 +- test/core.jl | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index f32dd65d9d82c..234660ec2ce30 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2882,7 +2882,7 @@ f(x) = yt(x) (let* ((exprs (lift-toplevel (convert-lambda lam2 '|#anon| #t '()))) (top-stmts (cdr exprs)) (newlam (renumber-slots-and-labels (linearize (car exprs))))) - `(block + `(toplevel-butlast ,@top-stmts ,@sp-inits (method ,name ,(cl-convert sig fname lam namemap toplevel interp) diff --git a/test/core.jl b/test/core.jl index b3dd90d9bba49..66ba2ef72368f 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4476,3 +4476,11 @@ function f18054() return Cint(0) end cfunction(f18054, Cint, ()) + +# issue #18085 +f18085(a,x...) = (0,) +for (f,g) in ((:asin,:sin), (:acos,:cos)) + gx = eval(g) + f18085(::Type{Val{f}},x...) = map(x->2gx(x), f18085(Val{g},x...)) +end +@test f18085(Val{:asin},3) === (0.0,) From c3d06c9c90ecd905694ea7042fa20d73e0b8bbaf Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Thu, 18 Aug 2016 14:12:52 -0400 Subject: [PATCH 0984/1117] make dlopen more conservative about opening files the user didn't request (#18061) * make dlopen more conservative about opening files the user didn't request * update manual * recognize \server\foo and /foo/bar absolute paths on windows, similar to isabspath * simplify isabspath from init.c and use it in dlopen instead of duplicating --- base/libdl.jl | 6 +++++ doc/stdlib/libdl.rst | 2 ++ src/dlload.c | 60 +++++++++++++++++--------------------------- src/init.c | 10 +++----- src/julia_internal.h | 2 ++ 5 files changed, 36 insertions(+), 44 deletions(-) diff --git a/base/libdl.jl b/base/libdl.jl index bac9f95e7f353..f2edd0130f0ee 100644 --- a/base/libdl.jl +++ b/base/libdl.jl @@ -69,6 +69,12 @@ end Load a shared library, returning an opaque handle. +The extension given by the constant `dlext` (`.so`, `.dll`, or `.dylib`) +can be omitted from the `libfile` string, as it is automatically appended +if needed. If `libfile` is not an absolute path name, then the paths +in the array `DL_LOAD_PATH` are searched for `libfile`, followed by the +system load path. + The optional flags argument is a bitwise-or of zero or more of `RTLD_LOCAL`, `RTLD_GLOBAL`, `RTLD_LAZY`, `RTLD_NOW`, `RTLD_NODELETE`, `RTLD_NOLOAD`, `RTLD_DEEPBIND`, and `RTLD_FIRST`. These are converted to the corresponding flags of the POSIX (and/or GNU libc and/or MacOS) diff --git a/doc/stdlib/libdl.rst b/doc/stdlib/libdl.rst index 6da4dffbc60f7..2f41c87bd11ad 100644 --- a/doc/stdlib/libdl.rst +++ b/doc/stdlib/libdl.rst @@ -14,6 +14,8 @@ The names in :mod:`Base.Libdl` are not exported and need to be called e.g. as `` Load a shared library, returning an opaque handle. + The extension given by the constant ``dlext`` (``.so``\ , ``.dll``\ , or ``.dylib``\ ) can be omitted from the ``libfile`` string, as it is automatically appended if needed. If ``libfile`` is not an absolute path name, then the paths in the array ``DL_LOAD_PATH`` are searched for ``libfile``\ , followed by the system load path. + The optional flags argument is a bitwise-or of zero or more of ``RTLD_LOCAL``\ , ``RTLD_GLOBAL``\ , ``RTLD_LAZY``\ , ``RTLD_NOW``\ , ``RTLD_NODELETE``\ , ``RTLD_NOLOAD``\ , ``RTLD_DEEPBIND``\ , and ``RTLD_FIRST``\ . These are converted to the corresponding flags of the POSIX (and/or GNU libc and/or MacOS) dlopen command, if possible, or are ignored if the specified functionality is not available on the current platform. The default flags are platform specific. On MacOS the default ``dlopen`` flags are ``RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL`` while on other platforms the defaults are ``RTLD_LAZY|RTLD_DEEPBIND|RTLD_LOCAL``\ . An important usage of these flags is to specify non default behavior for when the dynamic library loader binds library references to exported symbols and if the bound references are put into process local or global scope. For instance ``RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL`` allows the library's symbols to be available for usage in other shared libraries, addressing situations where there are dependencies between shared libraries. .. function:: dlopen_e(libfile::AbstractString [, flags::Integer]) diff --git a/src/dlload.c b/src/dlload.c index 1966c92803f58..414e594da97fd 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -21,17 +21,15 @@ extern "C" { #endif -// The empty extension at the beginning and the end is a trick to change -// the order of the loop. #if defined(__APPLE__) -static char const *const extensions[] = { "", ".dylib", "" }; +static char const *const extensions[] = { "", ".dylib" }; #elif defined(_OS_WINDOWS_) -static char const *const extensions[] = { "", ".dll", "" }; +static char const *const extensions[] = { "", ".dll" }; extern int needsSymRefreshModuleList; #else -static char const *const extensions[] = { "", ".so", "" }; +static char const *const extensions[] = { "", ".so" }; #endif -#define N_EXTENSIONS (sizeof(extensions) / sizeof(char*) - 1) +#define N_EXTENSIONS (sizeof(extensions) / sizeof(char*)) static int endswith_extension(const char *path) { @@ -118,11 +116,10 @@ static void *jl_load_dynamic_library_(const char *modname, unsigned flags, int t int i; uv_stat_t stbuf; void *handle; - // This determines if we try the no-extension name first or last - // We want to make sure the last one we try has higher chance of being - // a real file since the error reported will otherwise be a unhelpful - // file not found error due to the extra or missing extension name. - int hasext = endswith_extension(modname); + int abspath; + // number of extensions to try — if modname already ends with the + // standard extension, then we don't try adding additional extensions + int n_extensions = endswith_extension(modname) ? 1 : N_EXTENSIONS; /* this branch returns handle of libjulia @@ -139,26 +136,15 @@ static void *jl_load_dynamic_library_(const char *modname, unsigned flags, int t #endif goto done; } - /* - this branch shortcuts absolute paths - */ -#ifdef _OS_WINDOWS_ - else if (modname[1] == ':') { -#else - else if (modname[0] == '/') { -#endif - handle = jl_dlopen(modname, flags); - if (handle) - goto done; - // bail out and show the error if file actually exists - if (jl_stat(modname, (char*)&stbuf) == 0) - goto notfound; - } + + abspath = isabspath(modname); + /* this branch permutes all base paths in DL_LOAD_PATH with all extensions - note: skip when !jl_base_module to avoid UndefVarError(:DL_LOAD_PATH) + note: skip when !jl_base_module to avoid UndefVarError(:DL_LOAD_PATH), + and also skip for absolute paths */ - else if (jl_base_module != NULL) { + if (!abspath && jl_base_module != NULL) { jl_array_t *DL_LOAD_PATH = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("DL_LOAD_PATH")); if (DL_LOAD_PATH != NULL) { size_t j; @@ -167,9 +153,8 @@ static void *jl_load_dynamic_library_(const char *modname, unsigned flags, int t size_t len = strlen(dl_path); if (len == 0) continue; - for (i=0; i < N_EXTENSIONS; i++) { - // Do the no-ext one last if hasext == 1 - const char *ext = extensions[i + hasext]; + for (i=0; i < n_extensions; i++) { + const char *ext = extensions[i]; path[0] = '\0'; if (dl_path[len-1] == PATHSEPSTRING[0]) snprintf(path, PATHBUF, "%s%s%s", dl_path, modname, ext); @@ -187,9 +172,8 @@ static void *jl_load_dynamic_library_(const char *modname, unsigned flags, int t } // now fall back and look in default library paths, for all extensions - for(i=0; i < N_EXTENSIONS; i++) { - // Do the no-ext one last if hasext == 1 - const char *ext = extensions[i + hasext]; + for(i=0; i < n_extensions; i++) { + const char *ext = extensions[i]; path[0] = '\0'; snprintf(path, PATHBUF, "%s%s", modname, ext); handle = jl_dlopen(path, flags); @@ -199,9 +183,11 @@ static void *jl_load_dynamic_library_(const char *modname, unsigned flags, int t #if defined(__linux__) || defined(__FreeBSD__) // check map of versioned libs from "libX" to full soname "libX.so.ver" - handle = jl_dlopen_soname(modname, strlen(modname), flags); - if (handle) - goto done; + if (!abspath && n_extensions > 1) { // soname map only works for libX + handle = jl_dlopen_soname(modname, strlen(modname), flags); + if (handle) + goto done; + } #endif notfound: diff --git a/src/init.c b/src/init.c index d4bf640a181fe..4538e99b2b47f 100644 --- a/src/init.c +++ b/src/init.c @@ -438,13 +438,9 @@ int isabspath(const char *in) if (c0 == '/' || c0 == '\\') { return 1; // absolute path relative to %CD% (current drive), or UNC } - else { - int s = strlen(in); - if (s > 2) { - char c1 = in[1]; - char c2 = in[2]; - if (c1 == ':' && (c2 == '/' || c2 == '\\')) return 1; // absolute path - } + else if (c0 && in[1] == ':') { + char c2 = in[2]; + return c2 == '/' || c2 == '\\'; // absolute path with drive name } #else if (in[0] == '/') return 1; // absolute path diff --git a/src/julia_internal.h b/src/julia_internal.h index 3b643ce9ba00f..d4e0f32e51058 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -763,6 +763,8 @@ STATIC_INLINE void *jl_get_frame_addr(void) JL_DLLEXPORT jl_array_t *jl_array_cconvert_cstring(jl_array_t *a); +int isabspath(const char *in); + #ifdef __cplusplus } #endif From 99e7c7f623bd44eae88db2373f502cd879bcf7c3 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 18 Aug 2016 16:57:37 -0400 Subject: [PATCH 0985/1117] fix more of the #18017 regression --- base/inference.jl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 01c0e3eb10dbc..2649514a32bd1 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2262,12 +2262,15 @@ function effect_free(e::ANY, linfo::LambdaInfo, allow_volatile::Bool) if !allow_volatile if is_known_call(e, arrayref, linfo) || is_known_call(e, arraylen, linfo) return false - elseif is_known_call(e, getfield, linfo) && !isa(exprtype(e,linfo), Const) - # first argument must be immutable to ensure e is affect_free - a = ea[2] - typ = widenconst(exprtype(a, linfo)) - if !isa(typ, DataType) || typ.mutable || typ.abstract - return false + elseif is_known_call(e, getfield, linfo) + et = exprtype(e,linfo) + if !isa(et,Const) && !(isType(et) && isleaftype(et)) + # first argument must be immutable to ensure e is affect_free + a = ea[2] + typ = widenconst(exprtype(a, linfo)) + if !isa(typ, DataType) || typ.mutable || typ.abstract + return false + end end end end From 6d2441e699025ba78b3eff46ba6e726fe453701f Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Thu, 18 Aug 2016 18:24:33 -0400 Subject: [PATCH 0986/1117] Add istriu, istril, issymmetric, and ishermitian methods for UniformScaling (#17950) --- base/linalg/uniformscaling.jl | 5 +++++ test/linalg/uniformscaling.jl | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/base/linalg/uniformscaling.jl b/base/linalg/uniformscaling.jl index 227e41ac5dc0c..91098a08da173 100644 --- a/base/linalg/uniformscaling.jl +++ b/base/linalg/uniformscaling.jl @@ -29,6 +29,11 @@ one{T}(J::UniformScaling{T}) = one(UniformScaling{T}) zero{T}(::Type{UniformScaling{T}}) = UniformScaling(zero(T)) zero{T}(J::UniformScaling{T}) = zero(UniformScaling{T}) +istriu(::UniformScaling) = true +istril(::UniformScaling) = true +issymmetric(::UniformScaling) = true +ishermitian(J::UniformScaling) = isreal(J.λ) + (+)(J1::UniformScaling, J2::UniformScaling) = UniformScaling(J1.λ+J2.λ) (+){T}(B::BitArray{2},J::UniformScaling{T}) = Array(B) + J (+)(J::UniformScaling, B::BitArray{2}) = J + Array(B) diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index f20dd0315cab8..477306c6fc6ba 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -16,6 +16,12 @@ srand(123) @test one(UniformScaling(rand(Complex128))) == one(UniformScaling{Complex128}) @test eltype(one(UniformScaling(rand(Complex128)))) == Complex128 @test -one(UniformScaling(2)) == UniformScaling(-1) +@test istriu(I) +@test istril(I) +@test issymmetric(I) +@test issymmetric(UniformScaling(complex(1.0,1.0))) +@test ishermitian(I) +@test !ishermitian(UniformScaling(complex(1.0,1.0))) α = randn() @test α .* UniformScaling(1.0) == UniformScaling(1.0) .* α From 4931889b29a47e1619d24aab94c586cfdb7c8396 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Thu, 18 Aug 2016 19:25:16 -0400 Subject: [PATCH 0987/1117] fix #18129, optimize some more cases of captured variables --- src/julia-syntax.scm | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 234660ec2ce30..b971fc16526c7 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2732,25 +2732,27 @@ f(x) = yt(x) (any vinfo:sa vi)) (let* ((leading (filter (lambda (x) (and (pair? x) - (or (and (eq? (car x) 'method) - (length> x 2)) - (eq? (car x) '=)))) + (let ((cx (car x))) + (or (and (eq? cx 'method) (length> x 2)) + (eq? cx '=) + (eq? cx 'call))))) (take-statements-while (lambda (e) (or (atom? e) (memq (car e) '(quote top core line inert local unnecessary meta inbounds boundscheck simdloop implicit-global global globalref - const newvar = null method)))) + const newvar = null method call)))) (lam:body lam)))) - (unused (map cadr leading)) + (unused (map cadr (filter (lambda (x) (memq (car x) '(method =))) + leading))) (def (table))) ;; TODO: reorder leading statements to put assignments where the RHS is ;; `simple-atom?` at the top. (for-each (lambda (e) (set! unused (filter (lambda (v) (not (expr-uses-var e v))) unused)) - (if (memq (cadr e) unused) + (if (and (memq (car e) '(method =)) (memq (cadr e) unused)) (put! def (cadr e) #t))) leading) (for-each (lambda (v) From cf852e77d5d035215204e1da81e9e99e6ef550b8 Mon Sep 17 00:00:00 2001 From: Katie Hyatt <kshyatt@physics.ucsb.edu> Date: Thu, 18 Aug 2016 18:04:47 -0700 Subject: [PATCH 0988/1117] Missing tests for intfuncs --- test/intfuncs.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/intfuncs.jl b/test/intfuncs.jl index 7a6c84c54670e..822bad6805a07 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -2,6 +2,7 @@ # Int32 and Int64 take different code paths -- test both for T in (Int32, Int64) + @test gcd(T(3)) === T(3) @test gcd(T(3), T(5)) === T(1) @test gcd(T(3), T(15)) === T(3) @test gcd(T(0), T(15)) === T(15) @@ -16,6 +17,7 @@ for T in (Int32, Int64) @test gcd(typemin(T), T(1)) === T(1) @test_throws OverflowError gcd(typemin(T), typemin(T)) + @test lcm(T(2)) === T(2) @test lcm(T(2), T(3)) === T(6) @test lcm(T(4), T(6)) === T(12) @test lcm(T(3), T(0)) === T(0) @@ -86,6 +88,8 @@ let n = rand(Int) @test ndigits(n) == ndigits(big(n)) == ndigits(n, 10) end +@test bin('3') == "110011" +@test bin('3',7) == "0110011" @test bin(3) == "11" @test bin(3, 2) == "11" @test bin(3, 3) == "011" @@ -103,8 +107,11 @@ end @test base(2, 5, 7) == "0000101" +@test bits(Int16(3)) == "0000000000000011" +@test bits('3') == "00000000000000000000000000110011" @test bits(1035) == (Int == Int32 ? "00000000000000000000010000001011" : "0000000000000000000000000000000000000000000000000000010000001011") +@test bits(Int128(3)) == "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011" @test digits(4, 2) == [0, 0, 1] @test digits(5, 3) == [2, 1] @@ -116,6 +123,9 @@ end @test count_zeros(Int64(1)) == 63 +@test factorial(3) == 6 +@test_throws DomainError factorial(-3) + @test isqrt(4) == 2 @test isqrt(5) == 2 # issue #4884 From 2b5fbcf1bfa889aa1b4ace0c6d5e3fb30e5fd536 Mon Sep 17 00:00:00 2001 From: Katie Hyatt <kshyatt@physics.ucsb.edu> Date: Thu, 18 Aug 2016 19:39:52 -0700 Subject: [PATCH 0989/1117] Tests for missing abstract array utils --- test/abstractarray.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 34b1f21c6e3d8..db26e0099f896 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -729,3 +729,8 @@ end #flipdim on empty @test flipdim(Diagonal([]),1) == Diagonal([]) + +# ndims and friends +@test ndims(Diagonal(rand(1:5,5))) == 2 +@test ndims(Diagonal{Float64}) == 2 +@test Base.elsize(Diagonal(rand(1:5,5))) == sizeof(Int) From 66c501de350fb3ecb20f8e511a5cb6424ba02844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Biernat?= <pawel.biernat+github@gmail.com> Date: Fri, 19 Aug 2016 11:05:36 +0200 Subject: [PATCH 0990/1117] rational powers (fixes #18114) (#18118) Fixes precision problem for rational powers of BigFloats (#18114) --- base/rational.jl | 4 ++-- test/numbers.jl | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/base/rational.jl b/base/rational.jl index 067b8b570deac..6ff823701088d 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -375,8 +375,8 @@ function ^(x::Rational, n::Integer) end ^(x::Number, y::Rational) = x^(y.num/y.den) -^{T<:AbstractFloat}(x::T, y::Rational) = x^(convert(T, y.num / y.den)) -^{T<:AbstractFloat}(x::Complex{T}, y::Rational) = x^(convert(T, y.num / y.den)) +^{T<:AbstractFloat}(x::T, y::Rational) = x^convert(T,y) +^{T<:AbstractFloat}(x::Complex{T}, y::Rational) = x^convert(T,y) ^{T<:Rational}(z::Complex{T}, n::Bool) = n ? z : one(z) # to resolve ambiguity function ^{T<:Rational}(z::Complex{T}, n::Integer) diff --git a/test/numbers.jl b/test/numbers.jl index 4baf2c79b2d76..2094122fa4f53 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2137,7 +2137,8 @@ rationalize(nextfloat(0.0)) == 0//1 # rational-exponent promotion rules (issue #3155): @test 2.0f0^(1//3) == 2.0f0^(1.0f0/3) @test 2^(1//3) == 2^(1/3) - +# no loss of precision for rational powers (issue #18114) +@test BigFloat(2)^(BigFloat(1)/BigFloat(3)) == BigFloat(2)^(1//3) # large shift amounts @test Int32(-1)>>31 == -1 From 02a11ee402c0638dc6eb51af8cc1114532f255bc Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Fri, 19 Aug 2016 10:32:29 +0800 Subject: [PATCH 0991/1117] Use unique identifier to avoid incorrect ditype reuse --- src/cgutils.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 5336f5ad95f88..a0776a6b38e70 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -96,7 +96,18 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed jl_datatype_t *jdt = (jl_datatype_t*)jt; if (jdt->ditype != NULL) { #ifdef LLVM37 - return (llvm::DIType*)jdt->ditype; + DIType* t = (DIType*)jdt->ditype; +#ifndef LLVM39 + // On LLVM 3.7 and 3.8, DICompositeType with a unique name + // are ref'd by their unique name and needs to be explicitly + // retained in order to be used in the module. + if (auto *Composite = dyn_cast<DICompositeType>(t)) { + if (Composite->getRawIdentifier()) { + dbuilder->retainType(Composite); + } + } +#endif + return t; #else return DIType((llvm::MDNode*)jdt->ditype); #endif @@ -130,17 +141,22 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed else if (jl_is_structtype(jt)) { jl_datatype_t *jst = (jl_datatype_t*)jt; size_t ntypes = jl_datatype_nfields(jst); + const char *tname = jl_symbol_name(jdt->name->name); + std::stringstream unique_name; + unique_name << tname << "_" << globalUnique++; llvm::DICompositeType *ct = dbuilder->createStructType( NULL, // Scope - jl_symbol_name(jdt->name->name), // Name + tname, // Name NULL, // File 0, // LineNumber 8 * jdt->size, // SizeInBits - 8 * jdt->layout->alignment, // AlignmentInBits + 8 * jdt->layout->alignment, // AlignInBits 0, // Flags NULL, // DerivedFrom DINodeArray(), // Elements - dwarf::DW_LANG_Julia // RuntimeLanguage + dwarf::DW_LANG_Julia, // RuntimeLanguage + nullptr, // VTableHolder + unique_name.str() // UniqueIdentifier ); jdt->ditype = ct; std::vector<llvm::Metadata*> Elements; From 8341028ca5b003b96e6f2afb74088472187e5fde Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Fri, 19 Aug 2016 05:11:45 -0700 Subject: [PATCH 0992/1117] More tests for osnames (#18121) --- test/misc.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/misc.jl b/test/misc.jl index d10b46aafa732..bf3795ffb8051 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -206,6 +206,15 @@ end @test isa(ex, ErrorException) && ex.msg == "cannot assign variables in other modules" end +@test !Base.is_unix(:Windows) +@test !Base.is_linux(:Windows) +@test Base.is_linux(:Linux) +@test Base.is_windows(:Windows) +@test Base.is_windows(:NT) +@test !Base.is_windows(:Darwin) +@test Base.is_apple(:Darwin) +@test Base.is_apple(:Apple) +@test !Base.is_apple(:Windows) @test Base.is_unix(:Darwin) @test Base.is_unix(:FreeBSD) @test_throws ArgumentError Base.is_unix(:BeOS) From dc7a22f0c3d0aac723565a3465eb26cdea2e298f Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Fri, 19 Aug 2016 05:12:33 -0700 Subject: [PATCH 0993/1117] Two missing tests for intset (#18128) --- test/intset.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/intset.jl b/test/intset.jl index 5ebd531a46e09..ff062ba16c859 100644 --- a/test/intset.jl +++ b/test/intset.jl @@ -14,6 +14,7 @@ data_out = collect(s) # eltype, similar @test is(eltype(IntSet()), Int64) +@test is(eltype(IntSet), Int64) @test isequal(similar(IntSet([1,2,3])), IntSet()) # show @@ -33,6 +34,8 @@ s = IntSet([1,2,10,20,200,300,1000,10000,10002]) @test !in(1,s) @test !in(10002,s) @test in(10000,s) +@test in(10000.0,s) +@test !in(10002.0,s) @test_throws ArgumentError first(IntSet()) @test_throws ArgumentError last(IntSet()) t = copy(s) From 81554655f35ba77c8bde69074d511b6e2ae6fdf5 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Fri, 19 Aug 2016 07:19:58 -0500 Subject: [PATCH 0994/1117] Make similar faster and safer (#18107) * Make similar faster and safer By converting the eltype argument into a ::Type{T} parameter, we avoid runtime method lookup. Perhaps more importantly, the introduction of NeedsShaping makes `similar` safer, by preventing any possibility of an infinite recursion. * Test that similar throws a MethodError for unsupported dims types --- base/abstractarray.jl | 15 ++++++++------- test/offsetarray.jl | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 252a7b271316d..fbecc43efa4f8 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -9,6 +9,7 @@ typealias RangeIndex Union{Int, Range{Int}, AbstractUnitRange{Int}, Colon} typealias DimOrInd Union{Integer, AbstractUnitRange} typealias IntOrInd Union{Int, AbstractUnitRange} typealias DimsOrInds{N} NTuple{N,DimOrInd} +typealias NeedsShaping Union{Tuple{Integer,Vararg{Integer}}, Tuple{OneTo,Vararg{OneTo}}} macro _inline_pure_meta() Expr(:meta, :inline, :pure) @@ -420,14 +421,14 @@ different element type it will create a regular `Array` instead: 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 """ -similar{T}(a::AbstractArray{T}) = similar(a, T) -similar( a::AbstractArray, T::Type) = similar(a, T, to_shape(indices(a))) -similar{T}(a::AbstractArray{T}, dims::Tuple) = similar(a, T, to_shape(dims)) -similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, to_shape(dims)) -similar( a::AbstractArray, T::Type, dims::DimOrInd...) = similar(a, T, to_shape(dims)) -similar( a::AbstractArray, T::Type, dims) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray{T}) = similar(a, T) +similar{T}(a::AbstractArray, ::Type{T}) = similar(a, T, to_shape(indices(a))) +similar{T}(a::AbstractArray{T}, dims::Tuple) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray, ::Type{T}, dims::DimOrInd...) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray, ::Type{T}, dims::NeedsShaping) = similar(a, T, to_shape(dims)) # similar creates an Array by default -similar{N}(a::AbstractArray, T::Type, dims::Dims{N}) = Array{T,N}(dims) +similar{T,N}(a::AbstractArray, ::Type{T}, dims::Dims{N}) = Array{T,N}(dims) to_shape(::Tuple{}) = () to_shape(dims::Dims) = dims diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 6d99433d03d61..854712b721d02 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -369,3 +369,20 @@ for s = -5:5 end end # let + +# Check that similar throws a MethodError rather than a +# StackOverflowError if no appropriate method has been defined +# (#18107) +module SimilarUR + using Base.Test + immutable MyURange <: AbstractUnitRange{Int} + start::Int + stop::Int + end + ur = MyURange(1,3) + a = Array{Int}(2) + @test_throws MethodError similar(a, ur) + @test_throws MethodError similar(a, Float64, ur) + @test_throws MethodError similar(a, Float64, (ur,)) + @test_throws MethodError similar(a, (2.0,3.0)) +end From 706946a7d90884501ccbecb5e125e87acb7902ec Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Fri, 19 Aug 2016 14:56:23 +0200 Subject: [PATCH 0995/1117] Fix incorrect code block in rounding modes docs (#18137) The `::` syntax appearing before the `doctest::` block was causing the doctest not be rendered correctly. --- doc/manual/integers-and-floating-point-numbers.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index 8ffdf7e745d08..bd92ee746dacd 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -505,11 +505,10 @@ Rounding modes If a number doesn't have an exact floating-point representation, it must be rounded to an appropriate representable value, however, if wanted, the manner in which this rounding is done can be changed according to the rounding modes -presented in the `IEEE 754 standard <https://en.wikipedia.org/wiki/IEEE_754-2008>`_:: +presented in the `IEEE 754 standard <https://en.wikipedia.org/wiki/IEEE_754-2008>`_. .. doctest:: - julia> x = 1.1; y = 0.1; julia> x + y From 766d1de67f379fab7eaa0c27f792eb52789d979e Mon Sep 17 00:00:00 2001 From: David Beach <dbeach24@users.noreply.github.com> Date: Thu, 18 Aug 2016 14:15:22 -0600 Subject: [PATCH 0996/1117] Bugfix #18073 (#18105) fix #18073: `any([])` and `all([])` (#18105) (cherry picked from commit d259be55130f9d695262c3e528697c451cbb4e9e) --- base/reduce.jl | 4 ++-- test/reduce.jl | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/base/reduce.jl b/base/reduce.jl index fe8b7e3320da4..6ba2d8e457544 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -457,7 +457,7 @@ any(f::Predicate, itr) = mapreduce_sc_impl(f, |, itr) any(f::typeof(identity), itr) = eltype(itr) <: Bool ? mapreduce_sc_impl(f, |, itr) : - reduce(or_bool_only, itr) + reduce(or_bool_only, false, itr) """ all(p, itr) -> Bool @@ -474,7 +474,7 @@ all(f::Predicate, itr) = mapreduce_sc_impl(f, &, itr) all(f::typeof(identity), itr) = eltype(itr) <: Bool ? mapreduce_sc_impl(f, &, itr) : - reduce(and_bool_only, itr) + reduce(and_bool_only, true, itr) ## in & contains diff --git a/test/reduce.jl b/test/reduce.jl index a86b602dd2b86..1ca4e4f75867f 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -173,6 +173,7 @@ prod2(itr) = invoke(prod, Tuple{Any}, itr) # any & all +@test any([]) == false @test any(Bool[]) == false @test any([true]) == true @test any([false, false]) == false @@ -183,6 +184,7 @@ prod2(itr) = invoke(prod, Tuple{Any}, itr) @test any([true, false, true]) == true @test any([false, false, false]) == false +@test all([]) == true @test all(Bool[]) == true @test all([true]) == true @test all([false, false]) == false @@ -193,11 +195,13 @@ prod2(itr) = invoke(prod, Tuple{Any}, itr) @test all([true, false, true]) == false @test all([false, false, false]) == false +@test any(x->x>0, []) == false @test any(x->x>0, Int[]) == false @test any(x->x>0, [-3]) == false @test any(x->x>0, [4]) == true @test any(x->x>0, [-3, 4, 5]) == true +@test all(x->x>0, []) == true @test all(x->x>0, Int[]) == true @test all(x->x>0, [-3]) == false @test all(x->x>0, [4]) == true From 68b636ef33ae5c76f136438d238e576e83f1b1a8 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Fri, 19 Aug 2016 09:50:56 -0400 Subject: [PATCH 0997/1117] Remove +-(UniformScaling, Number) to avoid ambiguous behavior as pointed (#17607) out in #17083 --- base/deprecated.jl | 5 +++++ base/linalg/uniformscaling.jl | 4 ---- test/linalg/uniformscaling.jl | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 6b3360200814d..0bac394c53cef 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -790,4 +790,9 @@ function delete!(::EnvHash, k::AbstractString, def) haskey(ENV,k) ? delete!(ENV,k) : def end +@deprecate (+)(J::UniformScaling, x::Number) J.λ + x +@deprecate (+)(x::Number, J::UniformScaling) x + J.λ +@deprecate (-)(J::UniformScaling, x::Number) J.λ - x +@deprecate (-)(x::Number, J::UniformScaling) x - J.λ + # End deprecations scheduled for 0.6 diff --git a/base/linalg/uniformscaling.jl b/base/linalg/uniformscaling.jl index 91098a08da173..f2fce351438a8 100644 --- a/base/linalg/uniformscaling.jl +++ b/base/linalg/uniformscaling.jl @@ -38,15 +38,11 @@ ishermitian(J::UniformScaling) = isreal(J.λ) (+){T}(B::BitArray{2},J::UniformScaling{T}) = Array(B) + J (+)(J::UniformScaling, B::BitArray{2}) = J + Array(B) (+)(J::UniformScaling, A::AbstractMatrix) = A + J -(+)(J::UniformScaling, x::Number) = J.λ + x -(+)(x::Number, J::UniformScaling) = x + J.λ (-)(J::UniformScaling) = UniformScaling(-J.λ) (-)(J1::UniformScaling, J2::UniformScaling) = UniformScaling(J1.λ-J2.λ) (-)(B::BitArray{2}, J::UniformScaling) = Array(B) - J (-)(J::UniformScaling, B::BitArray{2}) = J - Array(B) -(-)(J::UniformScaling, x::Number) = J.λ - x -(-)(x::Number, J::UniformScaling) = x - J.λ for (t1, t2) in ((:UnitUpperTriangular, :UpperTriangular), (:UnitLowerTriangular, :LowerTriangular)) diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index 477306c6fc6ba..e82245c941b2b 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -26,8 +26,6 @@ srand(123) α = randn() @test α .* UniformScaling(1.0) == UniformScaling(1.0) .* α @test UniformScaling(α)./α == UniformScaling(1.0) -@test α + UniformScaling(1.0) == UniformScaling(1.0) + α -@test α - UniformScaling(1.0) == -(UniformScaling(1.0) - α) @test copy(UniformScaling(one(Float64))) == UniformScaling(one(Float64)) @test sprint(show,UniformScaling(one(Float32))) == "UniformScaling{Float32}\n1.0*I" From af2b7e488229c8f009445b1cb2751d0a028527cc Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Fri, 19 Aug 2016 09:53:20 -0400 Subject: [PATCH 0998/1117] Expose MEMDEBUG, and use it to fix the #14173 test --- src/intrinsics.cpp | 8 ++++++++ test/misc.jl | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 444a76b853f7d..1d9844e20e944 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -50,6 +50,14 @@ JL_DLLEXPORT uint32_t jl_get_LLVM_VERSION(void) ; } +extern "C" JL_DLLEXPORT int8_t jl_is_memdebug() { +#ifdef MEMDEBUG + return true; +#else + return false; +#endif +} + /* low-level intrinsics design: TODO: fix description below functions like add_int expect unboxed values of matching bit-length. diff --git a/test/misc.jl b/test/misc.jl index bf3795ffb8051..e91c3ca135b8c 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -230,7 +230,8 @@ module Tmp14173 A = randn(2000, 2000) end whos(IOBuffer(), Tmp14173) # warm up -@test @allocated(whos(IOBuffer(), Tmp14173)) < 10000 +const MEMDEBUG = ccall(:jl_is_memdebug, Bool, ()) +@test @allocated(whos(IOBuffer(), Tmp14173)) < (MEMDEBUG ? 20000 : 8000) ## test conversion from UTF-8 to UTF-16 (for Windows APIs) From 516c3a1b42ae4e2b3406f888130508d3ceb41c1f Mon Sep 17 00:00:00 2001 From: Matthias Reisinger <matthias.j.reisinger@gmail.com> Date: Tue, 16 Aug 2016 15:39:36 +0200 Subject: [PATCH 0999/1117] Simplify bounds checks for multi-dimensional array accesses This simplifies the array bounds check code that is emitted for multi-dimensional array accesses that use "regular" indexing, i.e. accesses of the form `A[i1,i2,...,iN]` to some `N`-dimensional array `A`. For example, with this change, the access `A[i,j,k]` to an array with the three dimensions `m`, `n` and `o` now leads to bounds checks that correspond to the following pseudo code: ``` if (i >= m) out_of_bounds_error(); else if (j >= n) out_of_bounds_error(); else if (k >= o) out_of_bounds_error(); ``` So far, the following more complicated bounds checks would have been emitted: ``` if (i >= m) out_of_bounds_error(); else if (j >= n) out_of_bounds_error(); else if (((k * n + j) * m + i) >= m * n * o) out_of_bounds_error(); ``` Julia also allows one-dimensional and "partial" linear indexing (see #14770), i.e. the number of indices used to access an array does not have to match the actual number of dimensions of the accessed array. For this case we still have use this old scheme. One motivation for this change was the following: expressions like `((k * n + j) * m + i)` are non-affine and Polly would not be able to analyze them. This change therefore also facilitates Polly's bounds check elimination logic, which would hoist such checks out of loops or may remove them entirely where possible. --- src/cgutils.cpp | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index a0776a6b38e70..e198f36d53f93 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1262,6 +1262,14 @@ static void assign_arrayvar(jl_arrayvar_t &av, const jl_cgval_t &ainfo, jl_codec builder.CreateStore(emit_arraysize(ainfo, i+1, ctx), av.sizes[i]); } +// Returns the size of the array represented by `tinfo` for the given dimension `dim` if +// `dim` is a valid dimension, otherwise returns constant one. +static Value *emit_arraysize_for_unsafe_dim(const jl_cgval_t &tinfo, jl_value_t *ex, size_t dim, + size_t nd, jl_codectx_t *ctx) +{ + return dim > nd ? ConstantInt::get(T_size, 1) : emit_arraysize(tinfo, ex, dim, ctx); +} + static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, size_t nd, jl_value_t **args, size_t nidxs, jl_codectx_t *ctx) { @@ -1282,12 +1290,12 @@ static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, size_ for(size_t k=0; k < nidxs; k++) { idxs[k] = emit_unbox(T_size, emit_expr(args[k], ctx), NULL); } + Value *ii; for(size_t k=0; k < nidxs; k++) { - Value *ii = builder.CreateSub(idxs[k], ConstantInt::get(T_size, 1)); + ii = builder.CreateSub(idxs[k], ConstantInt::get(T_size, 1)); i = builder.CreateAdd(i, builder.CreateMul(ii, stride)); if (k < nidxs-1) { - Value *d = - k >= nd ? ConstantInt::get(T_size, 1) : emit_arraysize(ainfo, ex, k+1, ctx); + Value *d = emit_arraysize_for_unsafe_dim(ainfo, ex, k+1, nd, ctx); #if CHECK_BOUNDS==1 if (bc) { BasicBlock *okBB = BasicBlock::Create(jl_LLVMContext, "ib"); @@ -1302,9 +1310,21 @@ static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, size_ } #if CHECK_BOUNDS==1 if (bc) { - Value *alen = emit_arraylen(ainfo, ex, ctx); - // if !(i < alen) goto error - builder.CreateCondBr(builder.CreateICmpULT(i, alen), endBB, failBB); + // We have already emitted a bounds check for each index except for + // the last one which we therefore have to do here. + bool linear_indexing = nidxs < nd; + if (linear_indexing) { + // Compare the linearized index `i` against the linearized size of + // the accessed array, i.e. `if !(i < alen) goto error`. + Value *alen = emit_arraylen(ainfo, ex, ctx); + builder.CreateCondBr(builder.CreateICmpULT(i, alen), endBB, failBB); + } else { + // Compare the last index of the access against the last dimension of + // the accessed array, i.e. `if !(last_index < last_dimension) goto error`. + Value *last_index = ii; + Value *last_dimension = emit_arraysize_for_unsafe_dim(ainfo, ex, nidxs, nd, ctx); + builder.CreateCondBr(builder.CreateICmpULT(last_index, last_dimension), endBB, failBB); + } ctx->f->getBasicBlockList().push_back(failBB); builder.SetInsertPoint(failBB); From 9f2384dd7adda3524dc92ef46f77f01b7d9a2043 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Fri, 19 Aug 2016 14:26:07 -0400 Subject: [PATCH 1000/1117] replace branch in bit shift operators, helps #18135 --- base/int.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/int.jl b/base/int.jl index 79cfb0ddafb0b..6b244499f4b0b 100644 --- a/base/int.jl +++ b/base/int.jl @@ -182,11 +182,11 @@ trailing_ones(x::Integer) = trailing_zeros(~x) # note: this early during bootstrap, `>=` is not yet available # note: we only define Int shift counts here; the generic case is handled later >>(x::BitInteger, y::Int) = - 0 <= y ? x >> unsigned(y) : x << unsigned(-y) + select_value(0 <= y, x >> unsigned(y), x << unsigned(-y)) <<(x::BitInteger, y::Int) = - 0 <= y ? x << unsigned(y) : x >> unsigned(-y) + select_value(0 <= y, x << unsigned(y), x >> unsigned(-y)) >>>(x::BitInteger, y::Int) = - 0 <= y ? x >>> unsigned(y) : x << unsigned(-y) + select_value(0 <= y, x >>> unsigned(y), x << unsigned(-y)) ## integer conversions ## From 19ddfdc6e48b069073359c426c23c38cbc31aaa0 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Fri, 19 Aug 2016 14:32:12 -0400 Subject: [PATCH 1001/1117] make sure .= uses Base.identity, not a local identity (#18122) --- src/julia-syntax.scm | 2 +- test/broadcast.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index b971fc16526c7..57574c575ef76 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1706,7 +1706,7 @@ (expand-forms `(call (top broadcast!) ,(from-lambda (cadr e)) ,lhs-view ,@(caddr e)))) (if (null? lhs) (expand-forms e) - (expand-forms `(call (top broadcast!) identity ,lhs-view ,e)))))) + (expand-forms `(call (top broadcast!) (top identity) ,lhs-view ,e)))))) ;; table mapping expression head to a function expanding that form (define expand-table diff --git a/test/broadcast.jl b/test/broadcast.jl index 7cc1ceffa500f..dfbbeed1c8950 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -284,6 +284,10 @@ let d = Dict(:foo => [1,3,7], (3,4) => [5,9]) d[3,4] .-= 1 @test d[3,4] == [4,8] end +let identity = error, x = [1,2,3] + x .= 1 # make sure it goes to broadcast!(Base.identity, ...), not identity + @test x == [1,1,1] +end # PR 16988 @test Base.promote_op(+, Bool) === Int From 00a63f082b634386ab28289043236550ed5b3ff3 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sun, 7 Aug 2016 05:19:47 -0700 Subject: [PATCH 1002/1117] Make rpath settings a bit more consistent probably addresses #18106 --- deps/arpack.mk | 7 +++---- deps/blas.mk | 7 +++---- deps/curl.mk | 2 -- deps/openspecfun.mk | 2 +- deps/pcre.mk | 4 +--- deps/suitesparse.mk | 4 +--- 6 files changed, 9 insertions(+), 17 deletions(-) diff --git a/deps/arpack.mk b/deps/arpack.mk index 91a5c72f05b90..7d65c10af7860 100644 --- a/deps/arpack.mk +++ b/deps/arpack.mk @@ -32,10 +32,9 @@ ARPACK_OBJ_TARGET := $(build_shlibdir)/libarpack.$(SHLIB_EXT) ARPACK_MFLAGS := F77="$(FC)" MPIF77="$(FC)" ARPACK_FFLAGS += $(FFLAGS) $(JFFLAGS) -ARPACK_FLAGS := --with-blas="$(LIBBLAS)" --with-lapack="$(LIBLAPACK)" --disable-mpi --enable-shared FFLAGS="$(ARPACK_FFLAGS)" CFLAGS="$(CFLAGS) $(ARPACK_CFLAGS)" -ifneq ($(OS),WINNT) -ARPACK_FLAGS += LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'" -endif +ARPACK_FLAGS := --with-blas="$(LIBBLAS)" --with-lapack="$(LIBLAPACK)" \ + --disable-mpi --enable-shared FFLAGS="$(ARPACK_FFLAGS)" \ + CFLAGS="$(CFLAGS) $(ARPACK_CFLAGS)" LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)" # ARPACK-NG upstream keeps changing their download filenames $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz: | $(SRCDIR)/srccache diff --git a/deps/blas.mk b/deps/blas.mk index fcd6cd8074bb0..809c4c7297932 100644 --- a/deps/blas.mk +++ b/deps/blas.mk @@ -200,10 +200,9 @@ LAPACK_OBJ_TARGET := LAPACK_OBJ_SOURCE := endif -LAPACK_MFLAGS := NOOPT="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS) -O0" OPTS="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS)" FORTRAN="$(FC)" LOADER="$(FC)" -ifneq ($(OS),WINNT) -LAPACK_MFLAGS += BLASLIB="-Wl,-rpath,'$(build_libdir)' $(LIBBLAS)" -endif +LAPACK_MFLAGS := NOOPT="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS) -O0" \ + OPTS="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS)" FORTRAN="$(FC)" \ + LOADER="$(FC)" BLASLIB="$(RPATH_ESCAPED_ORIGIN) $(LIBBLAS)" $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.netlib.org/lapack/$(notdir $@) diff --git a/deps/curl.mk b/deps/curl.mk index 060b41b7969a1..a9536f9a1b7ef 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -3,9 +3,7 @@ CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/lib/.libs/libcurl.$(SHLIB_EXT) CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT) -ifneq ($(OS),WINNT) CURL_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) -endif $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2 diff --git a/deps/openspecfun.mk b/deps/openspecfun.mk index 6e3ccce12d02a..f6c5dfead968f 100644 --- a/deps/openspecfun.mk +++ b/deps/openspecfun.mk @@ -13,7 +13,7 @@ OPENSPECFUN_OBJ_TARGET := $(build_shlibdir)/libopenspecfun.$(SHLIB_EXT) OPENSPECFUN_OBJ_SOURCE := $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/libopenspecfun.$(SHLIB_EXT) OPENSPECFUN_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" \ USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" \ - CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" LDFLAGS="$(RPATH_ESCAPED_ORIGIN)" + CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)" ifeq ($(USE_SYSTEM_LIBM),0) OPENSPECFUN_FLAGS += USE_OPENLIBM=1 diff --git a/deps/pcre.mk b/deps/pcre.mk index 5ae612452e763..b251f0c8aeae1 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -5,9 +5,7 @@ PCRE_OBJ_TARGET := $(build_shlibdir)/libpcre2-8.$(SHLIB_EXT) # Force optimization for PCRE flags (Issue #11668) PCRE_CFLAGS := -O3 -ifneq ($(OS),WINNT) -PCRE_LDFLAGS := "-Wl,-rpath,'$(build_libdir)'" -endif +PCRE_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-$(PCRE_VER).tar.bz2 diff --git a/deps/suitesparse.mk b/deps/suitesparse.mk index 975572e762466..87a07adfe898d 100644 --- a/deps/suitesparse.mk +++ b/deps/suitesparse.mk @@ -22,9 +22,7 @@ ifneq ($(OS), WINNT) SUITE_SPARSE_LIB += -lrt endif endif -ifneq ($(OS), WINNT) -SUITE_SPARSE_LIB += -Wl,-rpath,'$(build_libdir)' -endif +SUITE_SPARSE_LIB += $(RPATH_ESCAPED_ORIGIN) SUITESPARSE_MFLAGS := CC="$(CC)" CXX="$(CXX)" F77="$(FC)" AR="$(AR)" RANLIB="$(RANLIB)" BLAS="$(LIBBLAS)" LAPACK="$(LIBLAPACK)" \ INSTALL_LIB="$(build_libdir)" INSTALL_INCLUDE="$(build_includedir)" LIB="$(SUITE_SPARSE_LIB)" \ UMFPACK_CONFIG="$(UMFPACK_CONFIG)" CHOLMOD_CONFIG="$(CHOLMOD_CONFIG)" SPQR_CONFIG="$(SPQR_CONFIG)" From fac68ce4be6576044348480aba0b1c5eef395fb1 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 29 Jul 2016 01:10:07 -0700 Subject: [PATCH 1003/1117] Delete some files from libssh2 and curl that keep changing on travis causing the cache to get re-uploaded --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 94edf04dbd460..306958eb46563 100644 --- a/.travis.yml +++ b/.travis.yml @@ -111,6 +111,10 @@ script: - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN && /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg - - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia && rm -rf julia/deps/build/julia-env + - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia && + rm -rf julia/deps/build/julia-env && + rm -rf julia/deps/build/libssh2-*/CMakeFiles/Makefile2 && + rm -rf julia/deps/build/curl-*/config.log && + rm -rf julia/deps/build/curl-*/libtool # uncomment the following if failures are suspected to be due to the out-of-memory killer # - dmesg From b3afbc72f0f8fe6d004b86dc2151722ef7a34c10 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 19 Aug 2016 03:42:15 -0700 Subject: [PATCH 1004/1117] Install all 3 mbedtls dlls on windows and build tests --- deps/mbedtls.mk | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 150a39f2a1196..08b04c58568d2 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -16,7 +16,7 @@ MBEDTLS_OPTS := $(CMAKE_COMMON) -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ ifeq ($(OS),WINNT) MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=OFF ifeq ($(BUILD_OS),WINNT) -MBEDTLS_OPTS += -G"MSYS Makefiles" -DENABLE_TESTING=OFF +MBEDTLS_OPTS += -G"MSYS Makefiles" endif else MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=ON @@ -56,8 +56,10 @@ endif echo 1 > $@ $(MBEDTLS_OBJ_TARGET): $(MBEDTLS_OBJ_SOURCE) | $(build_shlibdir) -ifeq ($(OS), WINNT) +ifeq ($(BUILD_OS), WINNT) cp $^ $(build_shlibdir) + cp $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedx509.$(SHLIB_EXT) $(build_shlibdir) + cp $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedtls.$(SHLIB_EXT) $(build_shlibdir) else $(call make-install,mbedtls-$(MBEDTLS_VER),) endif From 7dba10ff46906e03534c4e0c1308dd84d0ea73ae Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 19 Aug 2016 04:58:04 -0700 Subject: [PATCH 1005/1117] disable libpsl in curl build should fix clean build of master indent line continuations --- deps/curl.mk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deps/curl.mk b/deps/curl.mk index a9536f9a1b7ef..73e3610167542 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -17,13 +17,13 @@ $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/ mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) \ - --without-ssl --without-gnutls --without-gssapi \ - --without-libidn --without-libmetalink --without-librtmp \ - --without-nghttp2 --without-nss --without-polarssl \ - --without-spnego --disable-ares --disable-ldap \ - --disable-ldaps --without-zsh-functions-dir \ - --with-libssh2=$(build_prefix) --with-mbedtls=$(build_prefix) \ - CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + --without-ssl --without-gnutls --without-gssapi \ + --without-libidn --without-libmetalink --without-librtmp \ + --without-nghttp2 --without-nss --without-polarssl \ + --without-spnego --without-libpsl --disable-ares \ + --disable-ldap --disable-ldaps --without-zsh-functions-dir \ + --with-libssh2=$(build_prefix) --with-mbedtls=$(build_prefix) \ + CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" touch -c $@ $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status From 943986d2a9b0592b1800dcc13e51f435aaf7d33b Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 19 Aug 2016 05:56:49 -0700 Subject: [PATCH 1006/1117] disable using zlib in mbedtls, libssh2, and libcurl since it requires additional dev headers fix #18101 --- deps/curl.mk | 2 +- deps/libssh2.mk | 2 +- deps/mbedtls.mk | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/deps/curl.mk b/deps/curl.mk index 73e3610167542..48251bc1cc204 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -17,7 +17,7 @@ $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/ mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) \ - --without-ssl --without-gnutls --without-gssapi \ + --without-ssl --without-gnutls --without-gssapi --without-zlib \ --without-libidn --without-libmetalink --without-librtmp \ --without-nghttp2 --without-nss --without-polarssl \ --without-spnego --without-libpsl --disable-ares \ diff --git a/deps/libssh2.mk b/deps/libssh2.mk index eb2282a882e12..d733afec2a7c5 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -16,7 +16,7 @@ ifeq ($(BUILD_OS),WINNT) LIBSSH2_OPTS += -G"MSYS Makefiles" endif else -LIBSSH2_OPTS += -DCRYPTO_BACKEND=mbedTLS -DENABLE_ZLIB_COMPRESSION=ON +LIBSSH2_OPTS += -DCRYPTO_BACKEND=mbedTLS -DENABLE_ZLIB_COMPRESSION=OFF endif ifeq ($(OS),Linux) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 08b04c58568d2..4134fd1520cb8 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -13,14 +13,10 @@ MBEDTLS_OBJ_TARGET := $(build_shlibdir)/libmbedcrypto.$(SHLIB_EXT) MBEDTLS_OPTS := $(CMAKE_COMMON) -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release -ifeq ($(OS),WINNT) MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=OFF ifeq ($(BUILD_OS),WINNT) MBEDTLS_OPTS += -G"MSYS Makefiles" endif -else -MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=ON -endif ifeq ($(OS),Linux) MBEDTLS_OPTS += -DCMAKE_INSTALL_RPATH="\$$ORIGIN" From b8b49f86b4b66c442dc8706fcc51e1016f68630c Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 19 Aug 2016 20:21:55 -0700 Subject: [PATCH 1007/1117] Cleanup: replace misleading STAGE#_DEPS variables with DEP_LIBS --- .travis.yml | 2 +- contrib/windows/msys_build.sh | 9 ++---- deps/Makefile | 54 ++++++++++++++++------------------- 3 files changed, 29 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 306958eb46563..1f7ff08e6711f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ before_install: brew install -v staticfloat/juliadeps/libgfortran; brew install -v --only-dependencies --HEAD julia; BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm37-julia)/bin/llvm-config-3.7.1 LLVM_SIZE=$(brew --prefix llvm37-julia)/bin/llvm-size-3.7.1"; - BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1 STAGE2_DEPS=utf8proc"; + BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1"; BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas"; for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND; do export BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1"; diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index ae343ae1dc85b..7ada885dbf90e 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -178,10 +178,7 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user # libuv since its static lib is no longer included in the binaries # openlibm since we need it as a static library to work properly # utf8proc since its headers are not in the binary download -echo 'override STAGE1_DEPS = libuv' >> Make.user -echo 'override STAGE2_DEPS = utf8proc' >> Make.user -echo 'override STAGE3_DEPS = ' >> Make.user -echo 'override STAGE4_DEPS = ' >> Make.user +echo 'override DEP_LIBS = libuv utf8proc' >> Make.user if [ -n "$USEMSVC" ]; then # Openlibm doesn't build well with MSVC right now @@ -193,14 +190,14 @@ if [ -n "$USEMSVC" ]; then make -C deps install-libuv install-utf8proc cp usr/lib/uv.lib usr/lib/libuv.a echo 'override CC += -TP' >> Make.user - echo 'override STAGE1_DEPS += dsfmt' >> Make.user + echo 'override DEP_LIBS += dsfmt' >> Make.user # Create a modified version of compile for wrapping link sed -e 's/-link//' -e 's/cl/link/g' -e 's/ -Fe/ -OUT:/' \ -e 's|$dir/$lib|$dir/lib$lib|g' deps/srccache/libuv/compile > linkld chmod +x linkld else - echo 'override STAGE1_DEPS += openlibm' >> Make.user + echo 'override DEP_LIBS += openlibm' >> Make.user make check-whitespace make VERBOSE=1 -C base version_git.jl.phony echo 'NO_GIT = 1' >> Make.user diff --git a/deps/Makefile b/deps/Makefile index 1b799d922aaee..55ed71fc3cb3a 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -88,31 +88,29 @@ MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_depsbindir) libd # prevent installing libs into usr/lib64 on opensuse unexport CONFIG_SITE -STAGE1_DEPS := -STAGE2_DEPS := ifeq ($(USE_GPL_LIBS), 1) -STAGE3_DEPS := suitesparse-wrapper +DEP_LIBS := suitesparse-wrapper else -STAGE3_DEPS := +DEP_LIBS := endif ifeq ($(USE_SYSTEM_LIBUV), 0) -STAGE1_DEPS += libuv +DEP_LIBS += libuv endif ifeq ($(USE_SYSTEM_LIBUNWIND), 0) ifeq ($(OS), Linux) -STAGE1_DEPS += unwind +DEP_LIBS += unwind else ifeq ($(OS), FreeBSD) -STAGE1_DEPS += unwind +DEP_LIBS += unwind else ifeq ($(OS), Darwin) -STAGE1_DEPS += osxunwind +DEP_LIBS += osxunwind endif endif ifeq ($(OS), Linux) ifeq ($(USE_SYSTEM_PATCHELF), 0) -STAGE1_DEPS += patchelf +DEP_LIBS += patchelf PATCHELF:=$(build_depsbindir)/patchelf else PATCHELF:=patchelf @@ -124,79 +122,79 @@ PATCHELF_BIN := $(CUSTOM_LD_LIBRARY_PATH) $(PATCHELF) ifeq ($(USE_SYSTEM_OPENLIBM), 0) ifeq ($(USE_SYSTEM_LIBM), 0) -STAGE1_DEPS += openlibm +DEP_LIBS += openlibm endif endif ifeq ($(USE_SYSTEM_OPENSPECFUN), 0) -STAGE1_DEPS += openspecfun +DEP_LIBS += openspecfun endif ifeq ($(USE_SYSTEM_DSFMT), 0) -STAGE1_DEPS += dsfmt +DEP_LIBS += dsfmt endif ifeq ($(USE_SYSTEM_LLVM), 0) -STAGE1_DEPS += llvm +DEP_LIBS += llvm endif ifeq ($(USE_SYSTEM_PCRE), 0) -STAGE1_DEPS += pcre +DEP_LIBS += pcre endif ifeq ($(USE_SYSTEM_BLAS), 0) -STAGE1_DEPS += openblas +DEP_LIBS += openblas ifeq ($(USE_BLAS64), 1) ifeq ($(OS), Darwin) -STAGE1_DEPS += objconv +DEP_LIBS += objconv endif endif endif ifeq ($(USE_GPL_LIBS), 1) ifeq ($(USE_SYSTEM_FFTW), 0) -STAGE1_DEPS += fftw +DEP_LIBS += fftw endif endif ifeq ($(USE_SYSTEM_GMP), 0) -STAGE1_DEPS += gmp +DEP_LIBS += gmp endif ifeq ($(USE_SYSTEM_MBEDTLS), 0) -STAGE1_DEPS += mbedtls +DEP_LIBS += mbedtls endif ifeq ($(USE_SYSTEM_LIBSSH2), 0) -STAGE2_DEPS += libssh2 +DEP_LIBS += libssh2 endif ifneq ($(OS), WINNT) ifeq ($(USE_SYSTEM_CURL), 0) -STAGE3_DEPS += curl +DEP_LIBS += curl endif endif ifeq ($(USE_SYSTEM_LIBGIT2), 0) -STAGE4_DEPS += libgit2 +DEP_LIBS += libgit2 endif ifeq ($(USE_SYSTEM_MPFR), 0) -STAGE2_DEPS += mpfr +DEP_LIBS += mpfr endif ifeq ($(USE_SYSTEM_ARPACK), 0) -STAGE2_DEPS += arpack +DEP_LIBS += arpack endif ifeq ($(USE_GPL_LIBS), 1) ifeq ($(USE_SYSTEM_SUITESPARSE), 0) -STAGE2_DEPS += suitesparse +DEP_LIBS += suitesparse endif endif ifeq ($(USE_SYSTEM_UTF8PROC), 0) -STAGE2_DEPS += utf8proc +DEP_LIBS += utf8proc endif # Only compile standalone LAPACK if we are not using OpenBLAS. @@ -205,7 +203,7 @@ endif # build LAPACK as the vendor LAPACK may be too old (eg. Apple vecLib) ifeq ($(USE_SYSTEM_BLAS), 1) ifeq ($(USE_SYSTEM_LAPACK), 0) -STAGE2_DEPS += lapack +DEP_LIBS += lapack endif endif @@ -217,8 +215,6 @@ endif ## Common build target prefixes -DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) $(STAGE4_DEPS) - default: install | $(build_prefix) get: $(addprefix get-, $(DEP_LIBS)) configure: $(addprefix configure-, $(DEP_LIBS)) From 57e0afcf4a29a23f1f08bcfc668c58050ab1b019 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 20 Aug 2016 00:11:33 -0400 Subject: [PATCH 1008/1117] Improve handling of curl, mbedtls, libssh2 USE_SYSTEM fix #17884 --- deps/Makefile | 4 ++-- deps/curl.mk | 8 +++++++- deps/libssh2.mk | 7 +++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 55ed71fc3cb3a..5467e02782fee 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -161,6 +161,7 @@ ifeq ($(USE_SYSTEM_GMP), 0) DEP_LIBS += gmp endif +ifeq ($(USE_SYSTEM_LIBGIT2), 0) ifeq ($(USE_SYSTEM_MBEDTLS), 0) DEP_LIBS += mbedtls endif @@ -175,9 +176,8 @@ DEP_LIBS += curl endif endif -ifeq ($(USE_SYSTEM_LIBGIT2), 0) DEP_LIBS += libgit2 -endif +endif # USE_SYSTEM_LIBGIT2 ifeq ($(USE_SYSTEM_MPFR), 0) DEP_LIBS += mpfr diff --git a/deps/curl.mk b/deps/curl.mk index 48251bc1cc204..893e15aa859ef 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -8,11 +8,17 @@ CURL_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2 -$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET) +$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(JLCHECKSUM) $< cd $(dir $<) && $(TAR) jxf $(notdir $<) touch -c $@ +ifeq ($(USE_SYSTEM_MBEDTLS), 0) +$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(MBEDTLS_OBJ_TARGET) +endif +ifeq ($(USE_SYSTEM_LIBSSH2), 0) +$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(LIBSSH2_OBJ_TARGET) +endif $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure mkdir -p $(dir $@) cd $(dir $@) && \ diff --git a/deps/libssh2.mk b/deps/libssh2.mk index d733afec2a7c5..48f111dfcc5e9 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -8,7 +8,7 @@ LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT) LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT) LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \ - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=Release ifeq ($(OS),WINNT) LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF @@ -30,7 +30,10 @@ $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-encryptedpem.patch-applied: | $(SR cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-encryptedpem.patch echo 1 > $@ -$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-encryptedpem.patch-applied $(MBEDTLS_OBJ_TARGET) +ifeq ($(USE_SYSTEM_MBEDTLS), 0) +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(MBEDTLS_OBJ_TARGET) +endif +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-encryptedpem.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBSSH2_OPTS) From 93b2cdd269d3aa62e2ed62b7ff86bbd117b85ac5 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 20 Aug 2016 00:23:37 -0400 Subject: [PATCH 1009/1117] Set CMAKE_INSTALL_LIBDIR and CMAKE_INSTALL_BINDIR in CMAKE_COMMON, so libssh2 installs without an x86_64-linux-gnu multiarch prefix --- deps/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deps/Makefile b/deps/Makefile index 5467e02782fee..5e59c43fb7e15 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -46,7 +46,9 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXX CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS) CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS) -CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) -DLIB_INSTALL_DIR=$(build_shlibdir) +CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) +CMAKE_COMMON += -DCMAKE_INSTALL_LIBDIR=$(build_libdir) -DCMAKE_INSTALL_BINDIR=$(build_bindir) +CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir) ifneq ($(VERBOSE), 0) CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON endif From ac6bfb54a3643e0157f5a1f8d95aa5dfe0bc835c Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 19 Aug 2016 21:29:18 -0700 Subject: [PATCH 1010/1117] Disable building static mbedtls library since we don't need it Go back to just copying the mbedtls dll even in cygwin-cross, otherwise it puts .dll.a files that we don't need alongside the dlls (should probably fix this upstream) --- deps/mbedtls.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 4134fd1520cb8..f9a340f83c9c1 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -11,7 +11,7 @@ MBEDTLS_OBJ_SOURCE := $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedcrypto.$ MBEDTLS_OBJ_TARGET := $(build_shlibdir)/libmbedcrypto.$(SHLIB_EXT) MBEDTLS_OPTS := $(CMAKE_COMMON) -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ - -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release + -DUSE_STATIC_MBEDTLS_LIBRARY=OFF -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=OFF ifeq ($(BUILD_OS),WINNT) @@ -52,7 +52,7 @@ endif echo 1 > $@ $(MBEDTLS_OBJ_TARGET): $(MBEDTLS_OBJ_SOURCE) | $(build_shlibdir) -ifeq ($(BUILD_OS), WINNT) +ifeq ($(OS), WINNT) cp $^ $(build_shlibdir) cp $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedx509.$(SHLIB_EXT) $(build_shlibdir) cp $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedtls.$(SHLIB_EXT) $(build_shlibdir) From 917940f445f7c82b55b248ea62677c6129c1e269 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Fri, 19 Aug 2016 22:25:21 -0700 Subject: [PATCH 1011/1117] Also just copy libgit2 dll for its install target since otherwise it puts pkg-config files in LIB_INSTALL_DIR which we don't need - if mbedtls separates runtime from library destinations then we could separate BIN_INSTALL_DIR from LIB_INSTALL_DIR or convince libgit2 to use the standard GNUInstallDirs names --- deps/libgit2.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 10a3a59d3a913..f28efaa72852d 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -67,7 +67,7 @@ endif echo 1 > $@ $(LIBGIT2_OBJ_TARGET): $(LIBGIT2_OBJ_SOURCE) | $(build_shlibdir) -ifeq ($(BUILD_OS),WINNT) +ifeq ($(OS),WINNT) cp $< $@ else $(call make-install,$(LIBGIT2_SRC_DIR),) From 5f4530af61bcd70c69ae788cb95f0dcea7341c9e Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Sat, 20 Aug 2016 00:06:30 -0700 Subject: [PATCH 1012/1117] Fix indent in sparsevector.jl and remove some unnecessary blank lines [ci skip] --- base/sparse/sparsevector.jl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index e77cef2776cd9..781df3775fef6 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -603,8 +603,8 @@ function findnz{Tv,Ti}(x::SparseVector{Tv,Ti}) count -= 1 if numnz != count - deleteat!(I, (count+1):numnz) - deleteat!(V, (count+1):numnz) + deleteat!(I, (count+1):numnz) + deleteat!(V, (count+1):numnz) end return (I, V) @@ -982,7 +982,6 @@ function _binarymap{Tx,Ty}(f::Function, x::AbstractSparseVector{Tx}, y::AbstractSparseVector{Ty}, mode::Int) - 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) R = typeof(f(zero(Tx), zero(Ty))) n = length(x) @@ -1124,7 +1123,6 @@ function _binarymap{Tx,Ty}(f::Function, x::AbstractVector{Tx}, y::AbstractSparseVector{Ty}, mode::Int) - 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) R = typeof(f(zero(Tx), zero(Ty))) n = length(x) @@ -1167,7 +1165,6 @@ function _binarymap{Tx,Ty}(f::Function, x::AbstractSparseVector{Tx}, y::AbstractVector{Ty}, mode::Int) - 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) R = typeof(f(zero(Tx), zero(Ty))) n = length(x) From dc7eaab9eff067a1e882f1f586d0a48b63b50473 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 20 Aug 2016 02:44:42 -0500 Subject: [PATCH 1013/1117] Try fixing LLVM 3.3 compilation (#18138) --- src/debuginfo.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 89e4e5d4a6859..6d0ef5b01660c 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -893,14 +893,6 @@ calc_gnu_debuglink_crc32(const void *buf, size_t size) return crc ^ ~0U; } -template<typename T> -static inline void ignoreError(T &err) -{ -#if defined(LLVM39) && !defined(NDEBUG) - consumeError(err.takeError()); -#endif -} - #ifdef LLVM39 static Expected<object::OwningBinary<object::ObjectFile>> #else @@ -958,6 +950,14 @@ extern "C" void jl_register_fptrs(uint64_t sysimage_base, void **fptrs, jl_lambd sysimg_fvars_n = n; } +template<typename T> +static inline void ignoreError(T &err) +{ +#if defined(LLVM39) && !defined(NDEBUG) + consumeError(err.takeError()); +#endif +} + extern "C" void jl_refresh_dbg_module_list(void); bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, llvm::DIContext **context, int64_t *slide, int64_t *section_slide, bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) From 496ad1d63ffe9b365a450490fc18cb19291ea5d1 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman <benditlikeranjan@gmail.com> Date: Sat, 20 Aug 2016 13:33:40 +0530 Subject: [PATCH 1014/1117] Add test for 15913 and 15896 (#17991) * Add test for 15913 and 15896 * Change name of test --- test/parse.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/parse.jl b/test/parse.jl index 3c625c192dc8d..50a6b61f437ae 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -667,3 +667,6 @@ end let str = "[1] [2]" @test_throws ParseError parse(str) end + +# issue 15896 and PR 15913 +@test_throws ErrorException eval(:(macro test15896(d; y=0) end)) From f238e5b2b5bae7bca29effff3e4cf9cb3d51502f Mon Sep 17 00:00:00 2001 From: Tracy Wadleigh <tracy.wadleigh@gmail.com> Date: Sat, 20 Aug 2016 01:15:27 -0700 Subject: [PATCH 1015/1117] Fixes #18141, eliminating null in value when iterating over `ENV`. (#18144) * Eliminate trailing null from iterator over `ENV`. Fixes #18141. * Add failing test for #18141. --- base/env.jl | 4 ++-- test/env.jl | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/base/env.jl b/base/env.jl index 78e2e0515145a..6482d7737c56e 100644 --- a/base/env.jl +++ b/base/env.jl @@ -91,7 +91,7 @@ if is_windows() function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) pos = block[1] blk = block[2] - len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + 1 + len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) buf = Array{UInt16}(len) unsafe_copy!(pointer(buf), pos, len) env = transcode(String, buf) @@ -99,7 +99,7 @@ if is_windows() if m === nothing error("malformed environment entry: $env") end - return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+len*2, blk)) + return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+(len+1)*2, blk)) end else # !windows start(::EnvHash) = 0 diff --git a/test/env.jl b/test/env.jl index 23e8710398cab..030fe3cf9b590 100644 --- a/test/env.jl +++ b/test/env.jl @@ -63,3 +63,10 @@ end # Test for #10853 @test withenv(Dict{Any,Any}()...) do; true; end + +# Test for #18141 +for (k, v) in ENV + if length(v) > 0 + @test v[end] != '\0' + end +end From a75b5e1c51701f8a64564481b2dcb91e3803d18a Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 20 Aug 2016 11:08:00 -0500 Subject: [PATCH 1016/1117] reshape(::Array, Val{N}) always returns an Array --- base/reshapedarray.jl | 23 +++++++++++++++-------- test/arrayops.jl | 8 ++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index 0f82bcca73d64..4bdfbf5ef87a3 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -51,20 +51,27 @@ end @pure rdims{N}(out::Tuple, inds::Tuple{}, ::Type{Val{N}}) = rdims((out..., OneTo(1)), (), Val{N}) @pure rdims{N}(out::Tuple, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = rdims((out..., first(inds)), tail(inds), Val{N}) -function _reshape(parent::AbstractArray, dims::Dims) - n = _length(parent) - prod(dims) == n || throw(DimensionMismatch("parent has $n elements, which is incompatible with size $dims")) - __reshape((parent, linearindexing(parent)), dims) -end -_reshape(R::ReshapedArray, dims::Dims) = _reshape(R.parent, dims) +# _reshape on Array returns an Array +_reshape(parent::Vector, dims::Dims{1}) = parent +_reshape(parent::Array, dims::Dims{1}) = reshape(parent, dims) +_reshape(parent::Array, dims::Dims) = reshape(parent, dims) # When reshaping Vector->Vector, don't wrap with a ReshapedArray -_reshape{T}(v::ReshapedArray{T,1}, dims::Tuple{Int}) = _reshape(v.parent, dims) -function _reshape(v::AbstractVector, dims::Tuple{Int}) +function _reshape(v::AbstractVector, dims::Dims{1}) len = dims[1] len == length(v) || throw(DimensionMismatch("parent has $(length(v)) elements, which is incompatible with length $len")) v end +# General reshape +function _reshape(parent::AbstractArray, dims::Dims) + n = _length(parent) + prod(dims) == n || throw(DimensionMismatch("parent has $n elements, which is incompatible with size $dims")) + __reshape((parent, linearindexing(parent)), dims) +end + +# Reshaping a ReshapedArray +_reshape{T}(v::ReshapedArray{T,1}, dims::Dims{1}) = _reshape(v.parent, dims) +_reshape(R::ReshapedArray, dims::Dims) = _reshape(R.parent, dims) function __reshape(p::Tuple{AbstractArray,LinearSlow}, dims::Dims) parent = p[1] diff --git a/test/arrayops.jl b/test/arrayops.jl index 2df9c599ced99..7b5e3f628213c 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -124,6 +124,14 @@ a = zeros(0, 5) # an empty linearslow array s = view(a, :, [2,3,5]) @test length(reshape(s, length(s))) == 0 +# reshape(a, Val{N}) +a = ones(Int,3,3) +s = view(a, 1:2, 1:2) +for N in (1,3) + @test isa(reshape(a, Val{N}), Array{Int,N}) + @test isa(reshape(s, Val{N}), Base.ReshapedArray{Int,N}) +end + @test reshape(1:5, (5,)) === 1:5 @test reshape(1:5, 5) === 1:5 From 3ce15ec27cfafc4ca807c3e546712615c86a9e8e Mon Sep 17 00:00:00 2001 From: David Beach <dbeach24@users.noreply.github.com> Date: Sat, 20 Aug 2016 10:16:10 -0600 Subject: [PATCH 1017/1117] fast rand(::Dict) and rand(::Set) (#18155) fixes #16231 --- base/random.jl | 13 +++++++++++++ test/random.jl | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/base/random.jl b/base/random.jl index 03e9b94c5d4f3..8ec17996cfe0d 100644 --- a/base/random.jl +++ b/base/random.jl @@ -292,6 +292,19 @@ function rand(r::AbstractRNG, ::Type{Char}) (c < 0xd800) ? Char(c) : Char(c+0x800) end +# random values from Dict or Set (for efficiency) +function rand(r::AbstractRNG, t::Dict) + isempty(t) && throw(ArgumentError("dict must be non-empty")) + n = length(t.slots) + while true + i = rand(r, 1:n) + Base.isslotfilled(t, i) && return (t.keys[i] => t.vals[i]) + end +end +rand(t::Dict) = rand(GLOBAL_RNG, t) +rand(r::AbstractRNG, s::Set) = rand(r, s.dict).first +rand(s::Set) = rand(GLOBAL_RNG, s) + ## Arrays of random numbers rand(r::AbstractRNG, dims::Dims) = rand(r, Float64, dims) diff --git a/test/random.jl b/test/random.jl index 1ebffa03c6e29..f988ae08f61a6 100644 --- a/test/random.jl +++ b/test/random.jl @@ -53,6 +53,12 @@ let mt = MersenneTwister() rand(coll, 2, 3) end +# rand using Dict, Set +adict = Dict(1=>2, 3=>4, 5=>6) +@test rand(adict) in adict +aset = Set(1:10) +@test rand(aset) in aset + # randn @test randn(MersenneTwister(42)) == -0.5560268761463861 A = zeros(2, 2) From 553f51a8c479ed88ead1f5508dd86de2e5a0dee6 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sat, 20 Aug 2016 12:03:03 -0500 Subject: [PATCH 1018/1117] reshape: only call to_shape when it will return Dims --- base/reshapedarray.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index 4bdfbf5ef87a3..fa00f37b0001b 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -36,8 +36,9 @@ start(R::ReshapedArrayIterator) = start(R.iter) end length(R::ReshapedArrayIterator) = length(R.iter) -reshape(parent::AbstractArray, shp::Tuple) = _reshape(parent, to_shape(shp)) reshape(parent::AbstractArray, dims::IntOrInd...) = reshape(parent, dims) +reshape(parent::AbstractArray, shp::NeedsShaping) = reshape(parent, to_shape(shp)) +reshape(parent::AbstractArray, dims::Dims) = _reshape(parent, dims) reshape{T,N}(parent::AbstractArray{T,N}, ndims::Type{Val{N}}) = parent function reshape{T,AN,N}(parent::AbstractArray{T,AN}, ndims::Type{Val{N}}) @@ -47,7 +48,10 @@ end # dimensionality N, either filling with OneTo(1) or collapsing the # product of trailing dims into the last element @pure rdims{N}(out::NTuple{N}, inds::Tuple{}, ::Type{Val{N}}) = out -@pure rdims{N}(out::NTuple{N}, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = (front(out)..., length(last(out)) * prod(map(length, inds))) +@pure function rdims{N}(out::NTuple{N}, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) + l = length(last(out)) * prod(map(length, inds)) + (front(out)..., OneTo(l)) +end @pure rdims{N}(out::Tuple, inds::Tuple{}, ::Type{Val{N}}) = rdims((out..., OneTo(1)), (), Val{N}) @pure rdims{N}(out::Tuple, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = rdims((out..., first(inds)), tail(inds), Val{N}) From f117cc9ddd4c23ec0cccd00969fc178128143cb7 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Sun, 21 Aug 2016 06:21:21 +0200 Subject: [PATCH 1019/1117] Add note about docstrings for aliases (#18157) As mentioned in https://github.com/JuliaLang/julia/pull/18041#issuecomment-241189817 docstrings attached to aliases of functions should be avoided where possible. --- doc/manual/documentation.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index a6d481f340a58..b40dedf5038a0 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -441,6 +441,32 @@ Global Variables Adds docstring ``"..."`` to the ``Binding``\ s ``a``, ``b``, and ``c``. +.. note:: + + When a ``const`` definition is only used to define an alias of another definition, such + as is the case with the function ``div`` and its alias ``÷`` in ``Base``, do not + document the alias and instead document the actual function. + + If the alias is documented and not the real definition then the docsystem (``?`` mode) + will not return the docstring attached to the alias when the real definition is + searched for. + + For example you should write + + .. code-block:: julia + + "..." + f(x) = x + 1 + const alias = f + + rather than + + .. code-block:: julia + + f(x) = x + 1 + "..." + const alias = f + .. code-block:: julia "..." From c1767e5f0621e04fba14b439e995330f7ffdb641 Mon Sep 17 00:00:00 2001 From: Keno Fischer <kfischer+github@college.harvard.edu> Date: Sun, 21 Aug 2016 22:43:49 +0000 Subject: [PATCH 1020/1117] Misc build fixes for win32 clang (#18164) --- Make.inc | 4 ++++ deps/blas.mk | 2 ++ deps/libgit2.mk | 2 ++ src/support/ENTRY.i387.h | 2 +- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index b8b04c9332313..1700797373119 100644 --- a/Make.inc +++ b/Make.inc @@ -445,7 +445,9 @@ export CCACHE_CPP2 := yes endif else #USECCACHE CC_BASE := $(shell echo $(CC) | cut -d' ' -f1) +CC_ARG := $(shell echo $(CC) | cut -d' ' -f2-) CXX_BASE := $(shell echo $(CXX) | cut -d' ' -f1) +CXX_ARG := $(shell echo $(CXX) | cut -d' ' -f2-) endif JFFLAGS := -O2 $(fPIC) @@ -683,10 +685,12 @@ endif ifeq ($(OS),WINNT) ifneq ($(ARCH),x86_64) +ifneq ($(USECLANG),1) JCFLAGS += -mincoming-stack-boundary=2 JCXXFLAGS += -mincoming-stack-boundary=2 endif endif +endif ifeq ($(USEGCC),1) ifeq ($(ISX86),1) diff --git a/deps/blas.mk b/deps/blas.mk index 809c4c7297932..752acffa1b24d 100644 --- a/deps/blas.mk +++ b/deps/blas.mk @@ -54,7 +54,9 @@ OPENBLAS_BUILD_OPTS += OSNAME=$(OS) CROSS=1 HOSTCC=$(HOSTCC) endif ifeq ($(OS),WINNT) ifneq ($(ARCH),x86_64) +ifneq ($(USECLANG),1) OPENBLAS_BUILD_OPTS += CFLAGS="$(CFLAGS) -mincoming-stack-boundary=2" +endif OPENBLAS_BUILD_OPTS += FFLAGS="$(FFLAGS) -mincoming-stack-boundary=2" endif endif diff --git a/deps/libgit2.mk b/deps/libgit2.mk index f28efaa72852d..61ad347d76312 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -11,8 +11,10 @@ LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DTHREADSAFE=ON ifeq ($(OS),WINNT) LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON ifneq ($(ARCH),x86_64) +ifneq ($(USECLANG),1) LIBGIT2_OPTS += -DCMAKE_C_FLAGS="-mincoming-stack-boundary=2" endif +endif ifeq ($(BUILD_OS),WINNT) LIBGIT2_OPTS += -G"MSYS Makefiles" else diff --git a/src/support/ENTRY.i387.h b/src/support/ENTRY.i387.h index 109ea745034d5..d80038671247a 100644 --- a/src/support/ENTRY.i387.h +++ b/src/support/ENTRY.i387.h @@ -59,7 +59,7 @@ EXT(CNAME): _START_ENTRY .globl EXT(CNAME) .section .drectve -.ascii " -export:" XSTR(CNAME) +.ascii " -export:", XSTR(CNAME) .section .text .def EXT(CNAME) .scl 2 From 45bbf0713c68beeb0ba32cf36a778805ad2c753d Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Sun, 21 Aug 2016 19:29:10 -0400 Subject: [PATCH 1021/1117] Increase MEMDEBUG allocation threshold for #14173 test --- test/misc.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/misc.jl b/test/misc.jl index e91c3ca135b8c..114c0f2b9079f 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -231,7 +231,7 @@ module Tmp14173 end whos(IOBuffer(), Tmp14173) # warm up const MEMDEBUG = ccall(:jl_is_memdebug, Bool, ()) -@test @allocated(whos(IOBuffer(), Tmp14173)) < (MEMDEBUG ? 20000 : 8000) +@test @allocated(whos(IOBuffer(), Tmp14173)) < (MEMDEBUG ? 30000 : 8000) ## test conversion from UTF-8 to UTF-16 (for Windows APIs) From 4498ead62a6610fbbb2588ec70cf78619ca311b2 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 22 Aug 2016 04:45:30 -0500 Subject: [PATCH 1022/1117] Add do-block support for redirect_std[out,err,in] Fixes #7022. In particular, these simplify testing for warnings. --- base/docs/helpdb/Base.jl | 33 ------------------- base/stream.jl | 68 +++++++++++++++++++++++++++++++++++++++ doc/stdlib/io-network.rst | 18 +++++++++++ test/show.jl | 28 +++++++++++++++- 4 files changed, 113 insertions(+), 34 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index f02519fb3f317..b1e51b084e6d2 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -3936,14 +3936,6 @@ Convert `x` from degrees to radians. """ deg2rad -""" - redirect_stdin([stream]) - -Like redirect_stdout, but for STDIN. Note that the order of the return tuple is still -(rd,wr), i.e. data to be read from STDIN, may be written to wr. -""" -redirect_stdin - """ mktemp([parent=tempdir()]) @@ -5196,13 +5188,6 @@ The result of an expression is too large for the specified type and will cause a """ OverflowError -""" - redirect_stderr([stream]) - -Like `redirect_stdout`, but for `STDERR`. -""" -redirect_stderr - """ ctranspose!(dest,src) @@ -6759,24 +6744,6 @@ General unescaping of traditional C and Unicode escape sequences. Reverse of """ unescape_string(s) -""" - redirect_stdout() - -Create a pipe to which all C and Julia level `STDOUT` output will be redirected. Returns a -tuple `(rd,wr)` representing the pipe ends. Data written to `STDOUT` may now be read from -the rd end of the pipe. The wr end is given for convenience in case the old `STDOUT` object -was cached by the user and needs to be replaced elsewhere. -""" -redirect_stdout - -""" - redirect_stdout(stream) - -Replace `STDOUT` by stream for all C and Julia level output to `STDOUT`. Note that `stream` -must be a TTY, a `Pipe` or a `TCPSocket`. -""" -redirect_stdout(stream) - """ print_with_color(color::Symbol, [io], strings...) diff --git a/base/stream.jl b/base/stream.jl index d57180fe7badb..f42af7b82c2ec 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -955,6 +955,74 @@ for (x, writable, unix_fd, c_symbol) in end end +""" + redirect_stdout() + +Create a pipe to which all C and Julia level `STDOUT` output will be redirected. Returns a +tuple `(rd,wr)` representing the pipe ends. Data written to `STDOUT` may now be read from +the rd end of the pipe. The wr end is given for convenience in case the old `STDOUT` object +was cached by the user and needs to be replaced elsewhere. +""" +redirect_stdout + +""" + redirect_stdout(stream) + +Replace `STDOUT` by stream for all C and Julia level output to `STDOUT`. Note that `stream` +must be a TTY, a `Pipe` or a `TCPSocket`. +""" +redirect_stdout(stream) + +""" + redirect_stderr([stream]) + +Like `redirect_stdout`, but for `STDERR`. +""" +redirect_stderr + +""" + redirect_stdin([stream]) + +Like redirect_stdout, but for STDIN. Note that the order of the return tuple is still +(rd,wr), i.e. data to be read from STDIN, may be written to wr. +""" +redirect_stdin + +for (F,S) in ((:redirect_stdin, :STDIN), (:redirect_stdout, :STDOUT), (:redirect_stderr, :STDERR)) + @eval function $F(f::Function, stream) + STDOLD = $S + local ret + $F(stream) + try + ret = f() + finally + $F(STDOLD) + end + ret + end +end + +""" + redirect_stdout(f::Function, stream) + +Run the function `f` while redirecting `STDOUT` to `stream`. Upon completion, `STDOUT` is restored to its prior setting. +""" +redirect_stdout(f::Function, stream) + +""" + redirect_stderr(f::Function, stream) + +Run the function `f` while redirecting `STDERR` to `stream`. Upon completion, `STDERR` is restored to its prior setting. +""" +redirect_stderr(f::Function, stream) + +""" + redirect_stdin(f::Function, stream) + +Run the function `f` while redirecting `STDIN` to `stream`. Upon completion, `STDIN` is restored to its prior setting. +""" +redirect_stdin(f::Function, stream) + mark(x::LibuvStream) = mark(x.buffer) unmark(x::LibuvStream) = unmark(x.buffer) reset(x::LibuvStream) = reset(x.buffer) diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 67a1b123c57b6..b78e2d1e22b17 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -350,18 +350,36 @@ General I/O Replace ``STDOUT`` by stream for all C and Julia level output to ``STDOUT``\ . Note that ``stream`` must be a TTY, a ``Pipe`` or a ``TCPSocket``\ . +.. function:: redirect_stdout(f::Function, stream) + + .. Docstring generated from Julia source + + Run the function ``f`` while redirecting ``STDOUT`` to ``stream``\ . Upon completion, ``STDOUT`` is restored to its prior setting. + .. function:: redirect_stderr([stream]) .. Docstring generated from Julia source Like ``redirect_stdout``\ , but for ``STDERR``\ . +.. function:: redirect_stderr(f::Function, stream) + + .. Docstring generated from Julia source + + Run the function ``f`` while redirecting ``STDERR`` to ``stream``\ . Upon completion, ``STDERR`` is restored to its prior setting. + .. function:: redirect_stdin([stream]) .. Docstring generated from Julia source Like redirect_stdout, but for STDIN. Note that the order of the return tuple is still (rd,wr), i.e. data to be read from STDIN, may be written to wr. +.. function:: redirect_stdin(f::Function, stream) + + .. Docstring generated from Julia source + + Run the function ``f`` while redirecting ``STDIN`` to ``stream``\ . Upon completion, ``STDIN`` is restored to its prior setting. + .. function:: readchomp(x) .. Docstring generated from Julia source diff --git a/test/show.jl b/test/show.jl index 2d185d76fb379..d839f7ce8933c 100644 --- a/test/show.jl +++ b/test/show.jl @@ -318,6 +318,32 @@ let oldout = STDOUT, olderr = STDERR end end +let filename = tempname() + ret = open(filename, "w") do f + redirect_stdout(f) do + println("hello") + [1,3] + end + end + @test ret == [1,3] + @test chomp(readstring(filename)) == "hello" + ret = open(filename, "w") do f + redirect_stderr(f) do + warn("hello") + [2] + end + end + @test ret == [2] + @test contains(readstring(filename), "WARNING: hello") + ret = open(filename) do f + redirect_stdin(f) do + readline() + end + end + @test contains(ret, "WARNING: hello") + rm(filename) +end + # issue #12960 type T12960 end let @@ -576,4 +602,4 @@ end # Test compact printing of homogeneous tuples @test repr(NTuple{7,Int64}) == "NTuple{7,Int64}" @test repr(Tuple{Float64, Float64, Float64, Float64}) == "NTuple{4,Float64}" -@test repr(Tuple{Float32, Float32, Float32}) == "Tuple{Float32,Float32,Float32}" \ No newline at end of file +@test repr(Tuple{Float32, Float32, Float32}) == "Tuple{Float32,Float32,Float32}" From d84ef3433e194282d3cc9caa8fcb0135a844cdac Mon Sep 17 00:00:00 2001 From: Tim Besard <tim.besard@gmail.com> Date: Mon, 22 Aug 2016 14:35:40 +0200 Subject: [PATCH 1023/1117] ASAN: disable leak detection for flisp. --- src/flisp/flmain.c | 8 ++++++++ src/init.c | 1 + 2 files changed, 9 insertions(+) diff --git a/src/flisp/flmain.c b/src/flisp/flmain.c index 401905cc7a7a8..f3861eed9e8a2 100644 --- a/src/flisp/flmain.c +++ b/src/flisp/flmain.c @@ -8,6 +8,14 @@ extern "C" { #endif +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +const char* __asan_default_options() { + return "detect_leaks=0"; +} +#endif +#endif + static value_t argv_list(fl_context_t *fl_ctx, int argc, char *argv[]) { int i; diff --git a/src/init.c b/src/init.c index 4538e99b2b47f..af07f1a3d42b4 100644 --- a/src/init.c +++ b/src/init.c @@ -51,6 +51,7 @@ JL_DLLEXPORT const char* __asan_default_options() { return "allow_user_segv_handler=1:detect_leaks=0"; // FIXME: enable LSAN after fixing leaks & defining __lsan_default_suppressions(), // or defining __lsan_default_options = exitcode=0 once publicly available + // (here and in flisp/flmain.c) } #endif From 05b0f6733ae7e4d37550ed54d0cb8358ee50d192 Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 22 Aug 2016 07:42:00 -0500 Subject: [PATCH 1024/1117] Fix fft(::Vector{Complex{Int}}) --- base/dft.jl | 1 + test/fft.jl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/base/dft.jl b/base/dft.jl index 700bbc68c2dec..f1072d2ae99e6 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -24,6 +24,7 @@ typealias FFTWFloat Union{Float32,Float64} fftwfloat(x) = _fftwfloat(float(x)) _fftwfloat{T<:FFTWFloat}(::Type{T}) = T _fftwfloat(::Type{Float16}) = Float32 +_fftwfloat{T}(::Type{Complex{T}}) = Complex{_fftwfloat(T)} _fftwfloat{T}(::Type{T}) = error("type $T not supported") _fftwfloat{T}(x::T) = _fftwfloat(T)(x) diff --git a/test/fft.jl b/test/fft.jl index b30e05ff09a5a..376d7e5e32248 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -6,6 +6,8 @@ a = rand(8) + im*rand(8) @test norm(ifft(fft(a,1),1) - a) < 1e-8 @test norm(ifft(fft(a,[1]),[1]) - a) < 1e-8 @test norm(ifft(fft(a,(1,)),(1,)) - a) < 1e-8 +a = rand(-10:10, 8) + im*rand(-10:10, 8) +@test norm(ifft(fft(a)) - a) < 1e-8 m4 = [16. 2 3 13; 5 11 10 8; From 969c277a418fbec2c4f6c6bd8c4aecd5691685d5 Mon Sep 17 00:00:00 2001 From: Matthias Reisinger <matthias.j.reisinger@gmail.com> Date: Wed, 15 Jun 2016 17:47:11 +0200 Subject: [PATCH 1025/1117] Add command line option --polly={yes|no} Passing `--polly=no` will cause `@polly` declarations to be ignored. This facilitates debugging or evaluating performance differences between using/not using Polly without having to manually remove `@polly` declarations from functions. --- base/options.jl | 1 + src/codegen.cpp | 2 +- src/init.c | 1 + src/jloptions.c | 14 ++++++++++++++ src/julia.h | 4 ++++ test/cmdlineargs.jl | 7 +++++++ 6 files changed, 28 insertions(+), 1 deletion(-) diff --git a/base/options.jl b/base/options.jl index c59bd19c5b44b..7da2224a509d5 100644 --- a/base/options.jl +++ b/base/options.jl @@ -24,6 +24,7 @@ immutable JLOptions check_bounds::Int8 depwarn::Int8 can_inline::Int8 + polly::Int8 fast_math::Int8 worker::Ptr{UInt8} handle_signals::Int8 diff --git a/src/codegen.cpp b/src/codegen.cpp index 3acbae8de822a..2789963bbb14c 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4111,7 +4111,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func #endif #ifdef USE_POLLY - if (!jl_has_meta(code, polly_sym)) { + if (!jl_has_meta(code, polly_sym) || jl_options.polly == JL_OPTIONS_POLLY_OFF) { f->addFnAttr(polly::PollySkipFnAttr); } #endif diff --git a/src/init.c b/src/init.c index af07f1a3d42b4..da8051986b48a 100644 --- a/src/init.c +++ b/src/init.c @@ -79,6 +79,7 @@ jl_options_t jl_options = { 0, // quiet JL_OPTIONS_CHECK_BOUNDS_DEFAULT, // check_bounds 1, // depwarn 1, // can_inline + JL_OPTIONS_POLLY_ON, // polly JL_OPTIONS_FAST_MATH_DEFAULT, 0, // worker JL_OPTIONS_HANDLE_SIGNALS_ON, diff --git a/src/jloptions.c b/src/jloptions.c index 3913ee7d33c45..9923d1836618b 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -46,6 +46,9 @@ static const char opts[] = " -O, --optimize={0,1,2,3} Set the optimization level (default 2 if unspecified or 3 if specified as -O)\n" " --inline={yes|no} Control whether inlining is permitted (overrides functions declared as @inline)\n" " --check-bounds={yes|no} Emit bounds checks always or never (ignoring declarations)\n" +#ifdef USE_POLLY + " --polly={yes|no} Enable or disable the polyhedral optimizer Polly (overrides @polly declaration)\n" +#endif " --math-mode={ieee,fast} Disallow or enable unsafe floating point optimizations (overrides @fastmath declaration)\n\n" // error and warning options @@ -83,6 +86,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_output_bc, opt_depwarn, opt_inline, + opt_polly, opt_math_mode, opt_worker, opt_bind_to, @@ -125,6 +129,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "output-incremental",required_argument, 0, opt_incremental }, { "depwarn", required_argument, 0, opt_depwarn }, { "inline", required_argument, 0, opt_inline }, + { "polly", required_argument, 0, opt_polly }, { "math-mode", required_argument, 0, opt_math_mode }, { "handle-signals", required_argument, 0, opt_handle_signals }, // hidden command line options @@ -398,6 +403,15 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_errorf("julia: invalid argument to --inline (%s)", optarg); } break; + case opt_polly: + if (!strcmp(optarg,"yes")) + jl_options.polly = JL_OPTIONS_POLLY_ON; + else if (!strcmp(optarg,"no")) + jl_options.polly = JL_OPTIONS_POLLY_OFF; + else { + jl_errorf("julia: invalid argument to --polly (%s)", optarg); + } + break; case opt_math_mode: if (!strcmp(optarg,"ieee")) jl_options.fast_math = JL_OPTIONS_FAST_MATH_OFF; diff --git a/src/julia.h b/src/julia.h index 5ee59917b999f..6b5c790603c83 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1668,6 +1668,7 @@ typedef struct { int8_t check_bounds; int8_t depwarn; int8_t can_inline; + int8_t polly; int8_t fast_math; const char *worker; int8_t handle_signals; @@ -1722,6 +1723,9 @@ JL_DLLEXPORT int jl_generating_output(void); #define JL_OPTIONS_DEPWARN_ON 1 #define JL_OPTIONS_DEPWARN_ERROR 2 +#define JL_OPTIONS_POLLY_ON 1 +#define JL_OPTIONS_POLLY_OFF 0 + #define JL_OPTIONS_FAST_MATH_ON 1 #define JL_OPTIONS_FAST_MATH_OFF 2 #define JL_OPTIONS_FAST_MATH_DEFAULT 0 diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index db5d6583016e2..d560c27910997 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -186,6 +186,13 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes --startup-file=no` # --inline takes yes/no as argument @test !success(`$exename --inline=false`) + # --polly + @test readchomp(`$exename -E "Bool(Base.JLOptions().polly)"`) == "true" + @test readchomp(`$exename --polly=yes -E "Bool(Base.JLOptions().polly)"`) == "true" + @test readchomp(`$exename --polly=no -E "Bool(Base.JLOptions().polly)"`) == "false" + # --polly takes yes/no as argument + @test !success(`$exename --polly=false`) + # --fast-math let JL_OPTIONS_FAST_MATH_DEFAULT = 0, JL_OPTIONS_FAST_MATH_ON = 1, From 3492a1a7d49b4a287d2c0694ca341e626d0efb6c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson <jeff.bezanson@gmail.com> Date: Mon, 22 Aug 2016 12:05:08 -0400 Subject: [PATCH 1026/1117] fix #18173, closure lowering error caused by #18130 --- src/julia-syntax.scm | 2 +- test/core.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 57574c575ef76..0d14e75fd6327 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2725,7 +2725,7 @@ f(x) = yt(x) ((eq? (car ex) 'method) (and (length> ex 2) (assq v (cadr (lam:vinfo (cadddr ex)))))) - (else #f))) + (else (expr-contains-eq v ex)))) (assert (eq? (car lam) 'lambda)) (let ((vi (car (lam:vinfo lam)))) (if (and (any vinfo:capt vi) diff --git a/test/core.jl b/test/core.jl index 66ba2ef72368f..d0295b804d87b 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4484,3 +4484,10 @@ for (f,g) in ((:asin,:sin), (:acos,:cos)) f18085(::Type{Val{f}},x...) = map(x->2gx(x), f18085(Val{g},x...)) end @test f18085(Val{:asin},3) === (0.0,) + +# issue #18173 +function f18173() + identity(()->successflag) + successflag = false +end +@test f18173() == false From 60aa5f3e2ab04ed6b907cbc963eb5d6bce637628 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 22 Aug 2016 20:08:42 -0400 Subject: [PATCH 1027/1117] Fix the part of #18164 that broke the build of cmake projects with gcc - CC_ARG and CXX_ARG are getting double processed somewhere, so cmake is trying to call the compiler as '/usr/bin/gcc gcc' add cut -s --- Make.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Make.inc b/Make.inc index 1700797373119..5f93cfe3a9a29 100644 --- a/Make.inc +++ b/Make.inc @@ -445,9 +445,9 @@ export CCACHE_CPP2 := yes endif else #USECCACHE CC_BASE := $(shell echo $(CC) | cut -d' ' -f1) -CC_ARG := $(shell echo $(CC) | cut -d' ' -f2-) +CC_ARG := $(shell echo $(CC) | cut -s -d' ' -f2-) CXX_BASE := $(shell echo $(CXX) | cut -d' ' -f1) -CXX_ARG := $(shell echo $(CXX) | cut -d' ' -f2-) +CXX_ARG := $(shell echo $(CXX) | cut -s -d' ' -f2-) endif JFFLAGS := -O2 $(fPIC) From 42d6e57be395ea79b4e24c9dca345b0fe487598a Mon Sep 17 00:00:00 2001 From: Ehsan Totoni <ehsan.totoni@intel.com> Date: Tue, 23 Aug 2016 04:58:36 -0700 Subject: [PATCH 1028/1117] ccalltest.c: ifndef Intel compiler for complex int (#18193) Complex int is GNU extension that Intel compiler does not support currently. #17145 --- src/ccalltest.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ccalltest.c b/src/ccalltest.c index ccd7e1cd57fde..02e40f4fb9c1a 100644 --- a/src/ccalltest.c +++ b/src/ccalltest.c @@ -292,6 +292,8 @@ typedef struct { complex double r89; } struct_huge4b; +// Intel compiler does not currently support complex int (which is GNU extension) +#ifndef _COMPILER_INTEL_ typedef struct { complex int r1; complex int r2; @@ -314,6 +316,7 @@ typedef struct { complex int r8; complex int r9; } struct_huge5b; +#endif // _COMPILER_INTEL_ JL_DLLEXPORT struct1 test_1(struct1 a, float b) { @@ -503,8 +506,11 @@ test_huge(3b, r1); test_huge(3c, r1); test_huge(4a, r12); test_huge(4b, r12); +// Intel compiler does not currently support complex int (which is GNU extension) +#ifndef _COMPILER_INTEL_ test_huge(5a, r1); test_huge(5b, r1); +#endif // _COMPILER_INTEL_ JL_DLLEXPORT int get_c_int(void) { From 1dd9f3a1e49ed1eb1b6421a4f0f463c40fa0c5d8 Mon Sep 17 00:00:00 2001 From: Erik Schnetter <schnetter@gmail.com> Date: Tue, 23 Aug 2016 08:00:14 -0400 Subject: [PATCH 1029/1117] Describe negative bit shift counts in NEWS.md (#18188) --- NEWS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.md b/NEWS.md index 773f693c1d33d..530777425f5b8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -195,6 +195,12 @@ This section lists changes that do not have deprecation warnings. * `map` on a dictionary now expects a function that expects and returns a `Pair`. The result is now another dictionary instead of an array ([#16622]). + * Bit shift operations (i.e. `<<`, `>>`, and `>>>`) now handle + negative shift counts differently: Negative counts are interpreted + as shifts in the opposite direction. For example, `4 >> -1 == 4 << + +1 == 8`. Previously, negative counts would implicitly overflow to + large positive counts, always yielding either `0` or `-1`. + Library improvements -------------------- From 5cddbc87879a1d0a1f6f47eeaddd0b85d099c530 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 20 Aug 2016 14:37:05 +0800 Subject: [PATCH 1030/1117] Do codegen in control flow order * Fix codegen of `SSAValue` whose assignment appears later than the user in syntactic order due to backward branches. * Disallow more expression types in value position * Use improved debug info metadata in coverage and allocation tracking too --- src/cgutils.cpp | 18 +- src/codegen.cpp | 675 ++++++++++++++++++++++++++++-------------------- test/core.jl | 12 + 3 files changed, 412 insertions(+), 293 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index e198f36d53f93..8f20befa3f61b 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -741,26 +741,12 @@ static void emit_typecheck(const jl_cgval_t &x, jl_value_t *type, const std::str builder.SetInsertPoint(passBB); } -static bool is_inbounds(jl_codectx_t *ctx) -{ - // inbounds rule is either of top two values on inbounds stack are true - bool inbounds = !ctx->inbounds.empty() && ctx->inbounds.back(); - if (ctx->inbounds.size() > 1) - inbounds |= ctx->inbounds[ctx->inbounds.size()-2]; - return inbounds; -} - -static bool is_bounds_check_block(jl_codectx_t *ctx) -{ - return !ctx->boundsCheck.empty() && ctx->boundsCheck.back(); -} - #define CHECK_BOUNDS 1 static Value *emit_bounds_check(const jl_cgval_t &ainfo, jl_value_t *ty, Value *i, Value *len, jl_codectx_t *ctx) { Value *im1 = builder.CreateSub(i, ConstantInt::get(T_size, 1)); #if CHECK_BOUNDS==1 - if ((!is_inbounds(ctx) && + if ((!ctx->is_inbounds && jl_options.check_bounds != JL_OPTIONS_CHECK_BOUNDS_OFF) || jl_options.check_bounds == JL_OPTIONS_CHECK_BOUNDS_ON) { Value *ok = builder.CreateICmpULT(im1, len); @@ -1277,7 +1263,7 @@ static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, size_ Value *i = ConstantInt::get(T_size, 0); Value *stride = ConstantInt::get(T_size, 1); #if CHECK_BOUNDS==1 - bool bc = (!is_inbounds(ctx) && + bool bc = (!ctx->is_inbounds && jl_options.check_bounds != JL_OPTIONS_CHECK_BOUNDS_OFF) || jl_options.check_bounds == JL_OPTIONS_CHECK_BOUNDS_ON; BasicBlock *failBB=NULL, *endBB=NULL; diff --git a/src/codegen.cpp b/src/codegen.cpp index 2789963bbb14c..3477bc2fe2ee2 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -506,7 +506,6 @@ typedef struct { std::vector<jl_cgval_t> SAvalues; std::vector<bool> ssavalue_assigned; std::map<int, jl_arrayvar_t> *arrayvars; - std::map<int, BasicBlock*> *labels; jl_module_t *module; jl_lambda_info_t *linfo; const char *name; @@ -519,14 +518,12 @@ typedef struct { bool vaStack; // varargs stack-allocated bool sret; int nReqArgs; - std::vector<bool> boundsCheck; - std::vector<bool> inbounds; CallInst *ptlsStates; Value *signalPage; - llvm::DIBuilder *dbuilder; bool debug_enabled; + bool is_inbounds{false}; std::vector<CallInst*> to_inline; } jl_codectx_t; @@ -3099,14 +3096,27 @@ static void emit_stmtpos(jl_value_t *expr, jl_codectx_t *ctx) } return; } - if (jl_is_expr(expr)) { - jl_sym_t *head = ((jl_expr_t*)expr)->head; - // some expression types are metadata and can be ignored in statement position - if (head == line_sym || head == meta_sym) - return; - // fall-through + if (!jl_is_expr(expr)) { + (void)emit_expr(expr, ctx); + return; + } + jl_expr_t *ex = (jl_expr_t*)expr; + jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); + jl_sym_t *head = ex->head; + if (head == line_sym || head == meta_sym || head == boundscheck_sym || + head == inbounds_sym) { + // some expression types are metadata and can be ignored + // in statement position + return; + } + else if (head == leave_sym) { + assert(jl_is_long(args[0])); + builder.CreateCall(prepare_call(jlleave_func), + ConstantInt::get(T_int32, jl_unbox_long(args[0]))); + } + else { + (void)emit_expr(expr, ctx); } - (void)emit_expr(expr, ctx); } static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) @@ -3129,33 +3139,17 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) return ctx->SAvalues.at(idx); // at this point, SAvalues[idx] actually contains the SAvalue } } + if (jl_is_globalref(expr)) { + return emit_getfield((jl_value_t*)jl_globalref_mod(expr), jl_globalref_name(expr), ctx); + } if (jl_is_labelnode(expr)) { - int labelname = jl_labelnode_label(expr); - BasicBlock *bb = (*ctx->labels)[labelname]; - assert(bb); - if (builder.GetInsertBlock()->getTerminator() == NULL) { - builder.CreateBr(bb); // all BasicBlocks must exit explicitly - } - builder.SetInsertPoint(bb); - return jl_cgval_t(); + jl_error("Labelnode in value position"); } if (jl_is_linenode(expr)) { jl_error("Linenode in value position"); } if (jl_is_gotonode(expr)) { - if (builder.GetInsertBlock()->getTerminator() == NULL) { - int labelname = jl_gotonode_label(expr); - BasicBlock *bb = (*ctx->labels)[labelname]; - assert(bb); - builder.CreateBr(bb); - BasicBlock *after = BasicBlock::Create(jl_LLVMContext, - "br", ctx->f); - builder.SetInsertPoint(after); - } - return jl_cgval_t(); - } - if (jl_is_globalref(expr)) { - return emit_getfield((jl_value_t*)jl_globalref_mod(expr), jl_globalref_name(expr), ctx); + jl_error("Gotonode in value position"); } if (!jl_is_expr(expr)) { int needroot = true; @@ -3191,19 +3185,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) // this is object-disoriented. // however, this is a good way to do it because it should *not* be easy // to add new node types. - if (head == goto_ifnot_sym) { - jl_value_t *cond = args[0]; - int labelname = jl_unbox_long(args[1]); - BasicBlock *ifso = BasicBlock::Create(jl_LLVMContext, "if", ctx->f); - BasicBlock *ifnot = (*ctx->labels)[labelname]; - assert(ifnot); - // Any branches treated as constant in type inference should be - // eliminated before running - Value *isfalse = emit_condition(cond, "if", ctx); - builder.CreateCondBr(isfalse, ifnot, ifso); - builder.SetInsertPoint(ifso); - } - else if (head == invoke_sym) { + if (head == invoke_sym) { return emit_invoke(ex, ctx); } else if (head == call_sym) { @@ -3294,82 +3276,6 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) /*isvolatile*/true), true, jl_any_type, ctx); } - else if (head == leave_sym) { - assert(jl_is_long(args[0])); - builder.CreateCall(prepare_call(jlleave_func), - ConstantInt::get(T_int32, jl_unbox_long(args[0]))); - } - else if (head == enter_sym) { - assert(jl_is_long(args[0])); - int labl = jl_unbox_long(args[0]); - CallInst *sj = builder.CreateCall(prepare_call(except_enter_func)); - // We need to mark this on the call site as well. See issue #6757 - sj->setCanReturnTwice(); - Value *isz = builder.CreateICmpEQ(sj, ConstantInt::get(T_int32,0)); - BasicBlock *tryblk = BasicBlock::Create(jl_LLVMContext, "try", - ctx->f); - BasicBlock *handlr = (*ctx->labels)[labl]; - assert(handlr); -#ifdef _OS_WINDOWS_ - BasicBlock *cond_resetstkoflw_blk = BasicBlock::Create(jl_LLVMContext, "cond_resetstkoflw", ctx->f); - BasicBlock *resetstkoflw_blk = BasicBlock::Create(jl_LLVMContext, "resetstkoflw", ctx->f); - builder.CreateCondBr(isz, tryblk, cond_resetstkoflw_blk); - builder.SetInsertPoint(cond_resetstkoflw_blk); - builder.CreateCondBr(builder.CreateICmpEQ( - literal_pointer_val(jl_stackovf_exception), - builder.CreateLoad(emit_exc_in_transit(ctx), true)), - resetstkoflw_blk, handlr); - builder.SetInsertPoint(resetstkoflw_blk); - builder.CreateCall(prepare_call(resetstkoflw_func) -# ifdef LLVM37 - , {} -# endif - ); - builder.CreateBr(handlr); -#else - builder.CreateCondBr(isz, tryblk, handlr); -#endif - builder.SetInsertPoint(tryblk); - } - else if (head == inbounds_sym) { - // manipulate inbounds stack - // note that when entering an inbounds context, we must also update - // the boundsCheck context to be false - if (jl_array_len(ex->args) > 0) { - jl_value_t *arg = args[0]; - if (arg == jl_true) { - ctx->inbounds.push_back(true); - ctx->boundsCheck.push_back(false); - } - else if (arg == jl_false) { - ctx->inbounds.push_back(false); - ctx->boundsCheck.push_back(false); - } - else { - if (!ctx->inbounds.empty()) - ctx->inbounds.pop_back(); - if (!ctx->boundsCheck.empty()) - ctx->boundsCheck.pop_back(); - } - } - return ghostValue(jl_void_type); - } - else if (head == boundscheck_sym) { - if (jl_array_len(ex->args) > 0) { - jl_value_t *arg = args[0]; - if (arg == jl_true) { - ctx->boundsCheck.push_back(true); - } - else if (arg == jl_false) { - ctx->boundsCheck.push_back(false); - } - else { - if (!ctx->boundsCheck.empty()) - ctx->boundsCheck.pop_back(); - } - } - return ghostValue(jl_void_type); - } else if (head == copyast_sym) { jl_value_t *arg = args[0]; if (jl_is_quotenode(arg)) { @@ -3387,6 +3293,21 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) jl_printf(JL_STDERR, "WARNING: could not attach metadata for @simd loop.\n"); return jl_cgval_t(); } + else if (head == goto_ifnot_sym) { + jl_error("Expr(:goto_ifnot) in value position"); + } + else if (head == leave_sym) { + jl_error("Expr(:leave) in value position"); + } + else if (head == enter_sym) { + jl_error("Expr(:enter) in value position"); + } + else if (head == inbounds_sym) { + jl_error("Expr(:inbounds) in value position"); + } + else if (head == boundscheck_sym) { + jl_error("Expr(:boundscheck) in value position"); + } else { if (!strcmp(jl_symbol_name(head), "$")) jl_error("syntax: prefix \"$\" in non-quoted expression"); @@ -3926,15 +3847,12 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func std::map<int, BasicBlock*> labels; jl_codectx_t ctx = {}; ctx.arrayvars = &arrayvars; - ctx.labels = &labels; ctx.module = lam->def ? lam->def->module : ptls->current_module; ctx.linfo = lam; ctx.name = jl_symbol_name(lam->def ? lam->def->name : anonymous_sym); ctx.funcName = ctx.name; ctx.vaSlot = -1; ctx.vaStack = false; - ctx.inbounds.push_back(false); - ctx.boundsCheck.push_back(false); ctx.spvals_ptr = NULL; // step 2. process var-info lists to see what vars need boxing @@ -4139,17 +4057,12 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func ctx.file = filename; DIBuilder dbuilder(*M); - ctx.dbuilder = &dbuilder; #ifdef LLVM37 DIFile *topfile = NULL; DISubprogram *SP = NULL; - std::vector<DILocation *> DI_loc_stack; - std::vector<DISubprogram *> DI_sp_stack; #else DIFile topfile; DISubprogram SP; - std::vector<DebugLoc> DI_loc_stack; - std::vector<DISubprogram> DI_sp_stack; #endif BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", f); @@ -4199,14 +4112,14 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func for(size_t i=0; i < jl_nparams(lam->specTypes); i++) { // assumes !va if (i < largslen && ctx.slots[i].value.isghost) continue; - ditypes.push_back(julia_type_to_di(jl_tparam(lam->specTypes,i),ctx.dbuilder,false)); + ditypes.push_back(julia_type_to_di(jl_tparam(lam->specTypes,i),&dbuilder,false)); } #ifdef LLVM38 - subrty = ctx.dbuilder->createSubroutineType(ctx.dbuilder->getOrCreateTypeArray(ditypes)); + subrty = dbuilder.createSubroutineType(dbuilder.getOrCreateTypeArray(ditypes)); #elif defined(LLVM36) - subrty = ctx.dbuilder->createSubroutineType(topfile,ctx.dbuilder->getOrCreateTypeArray(ditypes)); + subrty = dbuilder.createSubroutineType(topfile,dbuilder.getOrCreateTypeArray(ditypes)); #else - subrty = ctx.dbuilder->createSubroutineType(topfile,ctx.dbuilder->getOrCreateArray(ditypes)); + subrty = dbuilder.createSubroutineType(topfile,dbuilder.getOrCreateArray(ditypes)); #endif } @@ -4249,24 +4162,24 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func if (argname == unused_sym) continue; jl_varinfo_t &varinfo = ctx.slots[i]; #ifdef LLVM38 - varinfo.dinfo = ctx.dbuilder->createParameterVariable( + varinfo.dinfo = dbuilder.createParameterVariable( SP, // Scope (current function will be fill in later) jl_symbol_name(argname), // Variable name ctx.sret + i + 1, // Argument number (1-based) topfile, // File toplineno == -1 ? 0 : toplineno, // Line // Variable type - julia_type_to_di(varinfo.value.typ,ctx.dbuilder,false), + julia_type_to_di(varinfo.value.typ,&dbuilder,false), AlwaysPreserve, // May be deleted if optimized out 0); // Flags (TODO: Do we need any) #else - varinfo.dinfo = ctx.dbuilder->createLocalVariable( + varinfo.dinfo = dbuilder.createLocalVariable( llvm::dwarf::DW_TAG_arg_variable, // Tag SP, // Scope (current function will be fill in later) jl_symbol_name(argname), // Variable name topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) - julia_type_to_di(varinfo.value.typ, ctx.dbuilder,false), // Variable type + julia_type_to_di(varinfo.value.typ, &dbuilder,false), // Variable type AlwaysPreserve, // May be deleted if optimized out 0, // Flags (TODO: Do we need any) ctx.sret + i + 1); // Argument number (1-based) @@ -4274,23 +4187,23 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } if (va && ctx.vaSlot != -1) { #ifdef LLVM38 - ctx.slots[ctx.vaSlot].dinfo = ctx.dbuilder->createParameterVariable( + ctx.slots[ctx.vaSlot].dinfo = dbuilder.createParameterVariable( SP, // Scope (current function will be fill in later) std::string(jl_symbol_name(slot_symbol(ctx.vaSlot, &ctx))) + "...", // Variable name ctx.sret + nreq + 1, // Argument number (1-based) topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) - julia_type_to_di(ctx.slots[ctx.vaSlot].value.typ, ctx.dbuilder, false), + julia_type_to_di(ctx.slots[ctx.vaSlot].value.typ, &dbuilder, false), AlwaysPreserve, // May be deleted if optimized out 0); // Flags (TODO: Do we need any) #else - ctx.slots[ctx.vaSlot].dinfo = ctx.dbuilder->createLocalVariable( + ctx.slots[ctx.vaSlot].dinfo = dbuilder.createLocalVariable( llvm::dwarf::DW_TAG_arg_variable, // Tag SP, // Scope (current function will be fill in later) std::string(jl_symbol_name(slot_symbol(ctx.vaSlot, &ctx))) + "...", // Variable name topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) - julia_type_to_di(ctx.slots[ctx.vaSlot].value.typ, ctx.dbuilder, false), // Variable type + julia_type_to_di(ctx.slots[ctx.vaSlot].value.typ, &dbuilder, false), // Variable type AlwaysPreserve, // May be deleted if optimized out 0, // Flags (TODO: Do we need any) ctx.sret + nreq + 1); // Argument number (1-based) @@ -4302,16 +4215,16 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func if (varinfo.isArgument || s == compiler_temp_sym || s == unused_sym) continue; #ifdef LLVM38 - varinfo.dinfo = ctx.dbuilder->createAutoVariable( + varinfo.dinfo = dbuilder.createAutoVariable( #else - varinfo.dinfo = ctx.dbuilder->createLocalVariable( + varinfo.dinfo = dbuilder.createLocalVariable( llvm::dwarf::DW_TAG_auto_variable, // Tag #endif SP, // Scope (current function will be fill in later) jl_symbol_name(s), // Variable name topfile, // File toplineno == -1 ? 0 : toplineno, // Line (for now, use lineno of the function) - julia_type_to_di(varinfo.value.typ, ctx.dbuilder, false), // Variable type + julia_type_to_di(varinfo.value.typ, &dbuilder, false), // Variable type AlwaysPreserve, // May be deleted if optimized out 0 // Flags (TODO: Do we need any) #ifndef LLVM38 @@ -4383,7 +4296,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func #ifdef LLVM36 if (ctx.debug_enabled && varinfo.dinfo) { assert((Metadata*)varinfo.dinfo->getType() != jl_pvalue_dillvmt); - ctx.dbuilder->insertDeclare(lv, varinfo.dinfo, ctx.dbuilder->createExpression(), + dbuilder.insertDeclare(lv, varinfo.dinfo, dbuilder.createExpression(), #ifdef LLVM37 topdebugloc, #endif @@ -4404,14 +4317,14 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func if (ctx.debug_enabled && varinfo.dinfo) { DIExpression *expr; if ((Metadata*)varinfo.dinfo->getType() == jl_pvalue_dillvmt) { - expr = ctx.dbuilder->createExpression(); + expr = dbuilder.createExpression(); } else { SmallVector<uint64_t, 8> addr; addr.push_back(llvm::dwarf::DW_OP_deref); - expr = ctx.dbuilder->createExpression(addr); + expr = dbuilder.createExpression(addr); } - ctx.dbuilder->insertDeclare(av, varinfo.dinfo, expr, + dbuilder.insertDeclare(av, varinfo.dinfo, expr, #ifdef LLVM37 topdebugloc, #endif @@ -4470,7 +4383,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func addr.push_back((i - 1) * sizeof(void*)); if ((Metadata*)vi.dinfo->getType() != jl_pvalue_dillvmt) addr.push_back(llvm::dwarf::DW_OP_deref); - ctx.dbuilder->insertDeclare(pargArray, vi.dinfo, ctx.dbuilder->createExpression(addr), + dbuilder.insertDeclare(pargArray, vi.dinfo, dbuilder.createExpression(addr), #ifdef LLVM37 topdebugloc, #endif @@ -4502,7 +4415,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func parg = builder.CreateAlloca(theArg.V->getType(), NULL, jl_symbol_name(s)); builder.CreateStore(theArg.V, parg); } - ctx.dbuilder->insertDeclare(parg, vi.dinfo, ctx.dbuilder->createExpression(addr), + dbuilder.insertDeclare(parg, vi.dinfo, dbuilder.createExpression(addr), #ifdef LLVM37 topdebugloc, #endif @@ -4563,79 +4476,107 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } } - // step 11. associate labels with basic blocks to resolve forward jumps - BasicBlock *prev=NULL; - for(i=0; i < stmtslen; i++) { - jl_value_t *ex = jl_array_ptr_ref(stmts,i); - if (jl_is_labelnode(ex)) { - int lname = jl_labelnode_label(ex); - if (prev != NULL) { - // fuse consecutive labels - labels[lname] = prev; - } - else { - prev = BasicBlock::Create(jl_LLVMContext, "L", f); - labels[lname] = prev; - } - } - else { - prev = NULL; - } - } - - // step 12. compile body statements - if (ctx.debug_enabled) - // set initial line number - builder.SetCurrentDebugLocation(topdebugloc); - if (do_coverage) - coverageVisitLine(ctx.file, toplineno); - bool prevlabel = false; - int lno = -1; - int prevlno = -1; + // step 11. Compute properties for each statements + // This needs to be computed by iterating in the IR order + // instead of control flow order. +#ifdef LLVM37 + struct DbgState { + DebugLoc loc; + DISubprogram *sp; + StringRef file; + ssize_t line; + }; +#else + struct DbgState { + DebugLoc loc; + DISubprogram sp; + StringRef file; + ssize_t line; + }; +#endif + struct StmtProp { + DebugLoc loc; + StringRef file; + ssize_t line; + bool enabled; + bool is_inbounds; + bool loc_changed; + bool is_poploc; + }; + std::vector<StmtProp> stmtprops(stmtslen); + std::vector<DbgState> DI_stack; + std::vector<bool> boundsCheck_stack{false}; + std::vector<bool> inbounds_stack{false}; + auto is_bounds_check_block = [&] () { + return !boundsCheck_stack.empty() && boundsCheck_stack.back(); + }; + auto is_inbounds = [&] () { + // inbounds rule is either of top two values on inbounds stack are true + size_t sz = inbounds_stack.size(); + bool inbounds = sz && inbounds_stack.back(); + if (sz > 1) + inbounds |= inbounds_stack[sz - 2]; + return inbounds; + }; + StmtProp cur_prop{topdebugloc, filename, toplineno, true, false, true, false}; + // this should not be necessary but it seems that meta node can cause + // an offset between the label number and the statement number + std::map<int, int> label_map; for (i = 0; i < stmtslen; i++) { + cur_prop.loc_changed = false; + cur_prop.is_poploc = false; jl_value_t *stmt = jl_array_ptr_ref(stmts, i); - if (jl_is_linenode(stmt) || - (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == line_sym)) { - + jl_expr_t *expr = jl_is_expr(stmt) ? (jl_expr_t*)stmt : nullptr; + if (jl_is_labelnode(stmt)) { + int lname = jl_labelnode_label(stmt); + if (lname != i + 1) { + label_map[lname] = i + 1; + } + } + if (jl_is_linenode(stmt) || (expr && expr->head == line_sym)) { + ssize_t lno = -1; if (jl_is_linenode(stmt)) { lno = jl_linenode_line(stmt); } - else if (jl_is_expr(stmt)) { + else { lno = jl_unbox_long(jl_exprarg(stmt,0)); } MDNode *inlinedAt = NULL; - if (DI_loc_stack.size() > 0) { + if (DI_stack.size() > 0) { #ifdef LLVM37 - inlinedAt = DI_loc_stack.back(); + inlinedAt = DI_stack.back().loc; #else - inlinedAt = DI_loc_stack.back().getAsMDNode(jl_LLVMContext); + inlinedAt = DI_stack.back().loc.getAsMDNode(jl_LLVMContext); #endif } if (ctx.debug_enabled) - builder.SetCurrentDebugLocation(DebugLoc::get(lno, 0, SP, inlinedAt)); + cur_prop.loc = DebugLoc::get(lno, 0, SP, inlinedAt); + cur_prop.line = lno; + cur_prop.loc_changed = true; } - else if (ctx.debug_enabled && jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == meta_sym && jl_array_len(((jl_expr_t*)stmt)->args) >= 1) { - jl_expr_t *stmt_e = (jl_expr_t*)stmt; - jl_value_t *meta_arg = jl_exprarg(stmt_e, 0); + else if (expr && expr->head == meta_sym && + jl_array_len(expr->args) >= 1) { + jl_value_t *meta_arg = jl_exprarg(expr, 0); if (meta_arg == (jl_value_t*)jl_symbol("push_loc")) { - std::string new_filename = "<missing>"; - assert(jl_array_len(stmt_e->args) > 1); - jl_sym_t *filesym = (jl_sym_t*)jl_exprarg(stmt_e, 1); + const char *new_filename = "<missing>"; + assert(jl_array_len(expr->args) > 1); + jl_sym_t *filesym = (jl_sym_t*)jl_exprarg(expr, 1); if (filesym != empty_sym) new_filename = jl_symbol_name(filesym); #ifdef LLVM37 - DIFile *new_file = dbuilder.createFile(new_filename, "."); + DIFile *new_file = nullptr; #else - DIFile new_file = dbuilder.createFile(new_filename, "."); + DIFile new_file; #endif - DI_sp_stack.push_back(SP); - DI_loc_stack.push_back(builder.getCurrentDebugLocation()); - std::string inl_name; + if (ctx.debug_enabled) + new_file = dbuilder.createFile(new_filename, "."); + DI_stack.push_back({cur_prop.loc, SP, + cur_prop.file, cur_prop.line}); + const char *inl_name = ""; int inlined_func_lineno = 0; - if (jl_array_len(stmt_e->args) > 2) { - size_t ii; - for(ii=2; ii < jl_array_len(stmt_e->args); ii++) { - jl_value_t *arg = jl_exprarg(stmt_e, ii); + if (jl_array_len(expr->args) > 2) { + for (size_t ii = 2; ii < jl_array_len(expr->args); ii++) { + jl_value_t *arg = jl_exprarg(expr, ii); if (jl_is_symbol(arg)) inl_name = jl_symbol_name((jl_sym_t*)arg); else if (jl_is_int32(arg)) @@ -4647,57 +4588,211 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func else { inl_name = "macro expansion"; } - SP = dbuilder.createFunction(new_file, - inl_name + ";", - inl_name, - new_file, - 0, - jl_di_func_sig, - false, - true, - 0, - 0, - true, - nullptr); - MDNode *inlinedAt = NULL; + if (ctx.debug_enabled) { + SP = dbuilder.createFunction(new_file, + std::string(inl_name) + ";", + inl_name, + new_file, + 0, + jl_di_func_sig, + false, + true, + 0, + 0, + true, + nullptr); + MDNode *inlinedAt = NULL; #ifdef LLVM37 - inlinedAt = builder.getCurrentDebugLocation(); + inlinedAt = cur_prop.loc; #else - inlinedAt = builder.getCurrentDebugLocation().getAsMDNode(jl_LLVMContext); + inlinedAt = cur_proc.loc.getAsMDNode(jl_LLVMContext); #endif - builder.SetCurrentDebugLocation(DebugLoc::get(inlined_func_lineno, 0, SP, inlinedAt)); + cur_prop.loc = DebugLoc::get(inlined_func_lineno, + 0, SP, inlinedAt); + } + cur_prop.file = new_filename; + cur_prop.line = inlined_func_lineno; + cur_prop.loc_changed = true; } else if (meta_arg == (jl_value_t*)jl_symbol("pop_loc")) { - SP = DI_sp_stack.back(); - DI_sp_stack.pop_back(); // because why not make pop a void function - builder.SetCurrentDebugLocation(DI_loc_stack.back()); - DI_loc_stack.pop_back(); + cur_prop.is_poploc = true; + auto &DI = DI_stack.back(); + SP = DI.sp; + cur_prop.loc = DI.loc; + cur_prop.file = DI.file; + cur_prop.line = DI.line; + DI_stack.pop_back(); + cur_prop.loc_changed = true; } } - - if (do_coverage) - coverageVisitLine(filename, lno); - if (jl_is_labelnode(stmt)) { - if (prevlabel) continue; - prevlabel = true; + if (expr) { + jl_value_t **args = (jl_value_t**)jl_array_data(expr->args); + if (expr->head == boundscheck_sym) { + if (jl_array_len(expr->args) > 0) { + jl_value_t *arg = args[0]; + if (arg == jl_true) { + boundsCheck_stack.push_back(true); + } + else if (arg == jl_false) { + boundsCheck_stack.push_back(false); + } + else { + if (!boundsCheck_stack.empty()) { + boundsCheck_stack.pop_back(); + } + } + } + } + else if (cur_prop.enabled && expr->head == inbounds_sym) { + // manipulate inbounds stack + // note that when entering an inbounds context, we must also update + // the boundsCheck context to be false + if (jl_array_len(expr->args) > 0) { + jl_value_t *arg = args[0]; + if (arg == jl_true) { + inbounds_stack.push_back(true); + boundsCheck_stack.push_back(false); + } + else if (arg == jl_false) { + inbounds_stack.push_back(false); + boundsCheck_stack.push_back(false); + } + else { + if (!inbounds_stack.empty()) + inbounds_stack.pop_back(); + if (!boundsCheck_stack.empty()) + boundsCheck_stack.pop_back(); + } + } + } + } + bool is_ib = is_inbounds(); + if (is_ib && is_bounds_check_block() && + jl_options.check_bounds != JL_OPTIONS_CHECK_BOUNDS_ON) { + // elide bounds check blocks in inbounds context + cur_prop.enabled = false; + } + else if (is_bounds_check_block() && + jl_options.check_bounds == JL_OPTIONS_CHECK_BOUNDS_OFF) { + // elide bounds check blocks when turned off by options + cur_prop.enabled = false; } else { - prevlabel = false; - } - if (do_malloc_log) { - // Check memory allocation after finishing a line or hitting the next branch - if (lno != prevlno || - (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == goto_ifnot_sym) || - jl_is_gotonode(stmt)) { - if (prevlno != -1) - mallocVisitLine(filename, prevlno); - prevlno = lno; + cur_prop.enabled = true; + } + cur_prop.is_inbounds = is_ib; + stmtprops[i] = cur_prop; + } + DI_stack.clear(); + boundsCheck_stack.clear(); + inbounds_stack.clear(); + + // step 12. Do codegen in control flow order + std::vector<std::pair<int,BasicBlock*>> workstack; + int cursor = 0; + auto find_next_stmt = [&] (int seq_next) { + // `seq_next` is the next statement we want to emit + // i.e. if it exists, it's the next one following control flow and + // should be emitted into the current insert point. + if (seq_next >= 0 && seq_next < stmtslen) { + cursor = seq_next; + return; + } + if (!builder.GetInsertBlock()->getTerminator()) + builder.CreateUnreachable(); + if (workstack.empty()) { + cursor = -1; + return; + } + auto &item = workstack.back(); + builder.SetInsertPoint(item.second); + cursor = item.first; + workstack.pop_back(); + }; + auto add_to_list = [&] (int pos, BasicBlock *bb) { + if (pos >= stmtslen) + return; + workstack.push_back({pos, bb}); + }; + // returns the corresponding basic block. + // if `unconditional` a unconditional branch is created to the target + // label and the cursor is set to the next statement to process + auto handle_label = [&] (int lname, bool unconditional) { + auto it = label_map.find(lname); + if (it != label_map.end()) + lname = it->second; + auto &bb = labels[lname]; + BasicBlock *cur_bb = builder.GetInsertBlock(); + // Check if we've already visited this label + if (bb) { + // Already in the work list + // branch to it and pop one from the work list + if (unconditional) { + if (!cur_bb->getTerminator()) + builder.CreateBr(bb); + find_next_stmt(-1); } + return bb; } - if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == return_sym) { - jl_expr_t *ex = (jl_expr_t*)stmt; - Value *retval; - bool retboxed; + // If this is a label node in an empty bb + if (lname == cursor + 1 && cur_bb->begin() == cur_bb->end()) { + assert(unconditional); + // Use this bb as the one for the new label. + bb = cur_bb; + } + else { + // Otherwise, create a new BB + // use the label name as the BB name. + bb = BasicBlock::Create(jl_LLVMContext, + "L" + std::to_string(lname), f); + if (unconditional) { + if (!cur_bb->getTerminator()) + builder.CreateBr(bb); + builder.SetInsertPoint(bb); + } + else { + add_to_list(lname, bb); + } + } + if (unconditional) + find_next_stmt(lname); + return bb; + }; + + // If the first expresion changes the line number, we need to visit + // the start of the function. This can happen when the first line is + // a inlined function call. + if (stmtprops[0].loc_changed && do_coverage) { + if (ctx.debug_enabled) + builder.SetCurrentDebugLocation(topdebugloc); + coverageVisitLine(filename, toplineno); + } + stmtprops[0].loc_changed = true; + while (cursor != -1) { + auto &props = stmtprops[cursor]; + if (props.loc_changed) { + if (ctx.debug_enabled) + builder.SetCurrentDebugLocation(props.loc); + // Disable coverage for pop_loc, it doesn't start a new expression + if (do_coverage && props.enabled && !props.is_poploc) { + coverageVisitLine(props.file, props.line); + } + } + ctx.is_inbounds = props.is_inbounds; + jl_value_t *stmt = jl_array_ptr_ref(stmts, cursor); + jl_expr_t *expr = jl_is_expr(stmt) ? (jl_expr_t*)stmt : nullptr; + if (jl_is_labelnode(stmt)) { + // Label node + int lname = jl_labelnode_label(stmt); + handle_label(lname, true); + continue; + } + if (!props.enabled) { + find_next_stmt(cursor + 1); + continue; + } + if (expr && expr->head == return_sym) { + bool retboxed = false; Type *retty; if (specsig) { retty = julia_type_to_llvm(jlrettype, &retboxed); @@ -4706,7 +4801,8 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func retty = T_pjlvalue; retboxed = true; } - jl_cgval_t retvalinfo = emit_expr(jl_exprarg(ex,0), &ctx); + jl_cgval_t retvalinfo = emit_expr(jl_exprarg(expr, 0), &ctx); + Value *retval; if (retboxed) { retval = boxed(retvalinfo, &ctx, false); // skip the gcroot on the return path assert(!ctx.sret); @@ -4715,52 +4811,77 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func retval = emit_unbox(retty, retvalinfo, jlrettype, ctx.sret ? &*ctx.f->arg_begin() : NULL); } - else // undef return type + else { // undef return type retval = NULL; - if (do_malloc_log && lno != -1) - mallocVisitLine(filename, lno); + } + if (do_malloc_log && props.line != -1) + mallocVisitLine(props.file, props.line); if (type_is_ghost(retty) || ctx.sret) builder.CreateRetVoid(); else builder.CreateRet(retval); - if (i != stmtslen-1) { - BasicBlock *bb = - BasicBlock::Create(jl_LLVMContext, "ret", ctx.f); - builder.SetInsertPoint(bb); - } - } - else if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == boundscheck_sym) { - // always emit expressions that update the boundscheck stack - emit_stmtpos(stmt, &ctx); + find_next_stmt(-1); + continue; } - else if (is_inbounds(&ctx) && is_bounds_check_block(&ctx) && - jl_options.check_bounds != JL_OPTIONS_CHECK_BOUNDS_ON) { - // elide bounds check blocks in inbounds context + if (jl_is_gotonode(stmt)) { + int lname = jl_gotonode_label(stmt); + handle_label(lname, true); + continue; } - else if (is_bounds_check_block(&ctx) && - jl_options.check_bounds == JL_OPTIONS_CHECK_BOUNDS_OFF) { - // elide bounds check blocks when turned off by options + if (expr && expr->head == goto_ifnot_sym) { + jl_value_t **args = (jl_value_t**)jl_array_data(expr->args); + jl_value_t *cond = args[0]; + int lname = jl_unbox_long(args[1]); + Value *isfalse = emit_condition(cond, "if", &ctx); + if (do_malloc_log && props.line != -1) + mallocVisitLine(props.file, props.line); + BasicBlock *ifso = BasicBlock::Create(jl_LLVMContext, "if", f); + BasicBlock *ifnot = handle_label(lname, false); + // Any branches treated as constant in type inference should be + // eliminated before running + builder.CreateCondBr(isfalse, ifnot, ifso); + builder.SetInsertPoint(ifso); + } + else if (expr && expr->head == enter_sym) { + jl_value_t **args = (jl_value_t**)jl_array_data(expr->args); + assert(jl_is_long(args[0])); + int lname = jl_unbox_long(args[0]); + CallInst *sj = builder.CreateCall(prepare_call(except_enter_func)); + // We need to mark this on the call site as well. See issue #6757 + sj->setCanReturnTwice(); + Value *isz = builder.CreateICmpEQ(sj, ConstantInt::get(T_int32, 0)); + BasicBlock *tryblk = BasicBlock::Create(jl_LLVMContext, "try", f); + BasicBlock *handlr = handle_label(lname, false); +#ifdef _OS_WINDOWS_ + BasicBlock *cond_resetstkoflw_blk = BasicBlock::Create(jl_LLVMContext, "cond_resetstkoflw", f); + BasicBlock *resetstkoflw_blk = BasicBlock::Create(jl_LLVMContext, "resetstkoflw", f); + builder.CreateCondBr(isz, tryblk, cond_resetstkoflw_blk); + builder.SetInsertPoint(cond_resetstkoflw_blk); + builder.CreateCondBr(builder.CreateICmpEQ( + literal_pointer_val(jl_stackovf_exception), + builder.CreateLoad(emit_exc_in_transit(&ctx), true)), + resetstkoflw_blk, handlr); + builder.SetInsertPoint(resetstkoflw_blk); + builder.CreateCall(prepare_call(resetstkoflw_func) +# ifdef LLVM37 + , {} +# endif + ); + builder.CreateBr(handlr); +#else + builder.CreateCondBr(isz, tryblk, handlr); +#endif + builder.SetInsertPoint(tryblk); } else { emit_stmtpos(stmt, &ctx); + if (do_malloc_log && props.line != -1) { + mallocVisitLine(props.file, props.line); + } } + find_next_stmt(cursor + 1); } - builder.SetCurrentDebugLocation(noDbg); - - // sometimes we have dangling labels after the end - if (builder.GetInsertBlock()->getTerminator() == NULL) { - builder.CreateUnreachable(); - } - - // patch up dangling BasicBlocks from skipped labels - for (std::map<int,BasicBlock*>::iterator it = labels.begin(); it != labels.end(); ++it) { - if (it->second->getTerminator() == NULL) { - builder.SetInsertPoint(it->second); - builder.CreateUnreachable(); - } - } - builder.ClearInsertionPoint(); // step 13, Apply LLVM level inlining @@ -4780,7 +4901,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func // step 14. Perform any delayed instantiations if (ctx.debug_enabled) { - ctx.dbuilder->finalize(); + dbuilder.finalize(); } JL_GC_POP(); diff --git a/test/core.jl b/test/core.jl index d0295b804d87b..c4e7544ff01a7 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4117,6 +4117,18 @@ test_metadata_matches(f4, Tuple{}) end +# SSA value where the assignment is after the user in syntactic order +let f = function(a, b) + @goto a + @label b + return j[1] + j[2] * 2 + @label a + j = (a, b) + @goto b +end + @test f(1, 2) == 5 +end + # issue #8712 type Issue8712; end @test isa(invoke(Issue8712, ()), Issue8712) From b4bf0a4f4dd5e74330225f5a8f90cc0b2a3e0f22 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 22 Aug 2016 17:14:29 -0400 Subject: [PATCH 1031/1117] incremental deserialize: handle LambdaInfo identity uniquing this works to avoid having `Expr(:invoke)` creating unintentional copies of LambdaInfo objects when they show up in the system image fix #18184 --- doc/manual/modules.rst | 8 +++ src/alloc.c | 9 ++- src/codegen.cpp | 2 +- src/dump.c | 148 +++++++++++++++++++++++++++++++++-------- src/gf.c | 35 +++++----- src/julia_internal.h | 2 +- test/compile.jl | 5 ++ 7 files changed, 160 insertions(+), 49 deletions(-) diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index 72241c2ea00a7..4947aba6e6bd4 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -422,6 +422,14 @@ A few other points to be aware of: 4. WeakRef objects and finalizers are not currently handled properly by the serializer (this will be fixed in an upcoming release). +5. It is usually best to avoid capturing references to instances of internal metadata objects such as + Method, LambdaInfo, MethodTable, TypeMapLevel, TypeMapEntry + and fields of those objects, as this can confuse the serializer + and may not lead to the outcome you desire. + It is not necessarily an error to do this, + but you simply need to be prepared that the system will + try to copy some of these and to create a single unique instance of others. + It is sometimes helpful during module development to turn off incremental precompilation. The command line flag ``--compilecache={yes|no}`` enables you to toggle module precompilation on and off. When Julia is started with ``--compilecache=no`` the serialized modules in the compile cache are ignored when loading modules and module dependencies. diff --git a/src/alloc.c b/src/alloc.c index 79da428f63028..6255e056abb29 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -553,7 +553,7 @@ static jl_lambda_info_t *jl_copy_lambda(jl_lambda_info_t *linfo) } // return a new lambda-info that has some extra static parameters merged in -JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp) +JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp, int allow_exec) { jl_lambda_info_t *linfo = m->lambda_template; jl_lambda_info_t *new_linfo; @@ -565,6 +565,13 @@ JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t new_linfo->def = m; new_linfo->sparam_vals = sp; } + else if (!allow_exec) { + new_linfo = jl_copy_lambda(linfo); + new_linfo->specTypes = types; + new_linfo->def = m; + new_linfo->sparam_vals = sp; + jl_set_lambda_code_null(new_linfo); + } else { new_linfo = jl_instantiate_staged(m, types, sp); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 3477bc2fe2ee2..90153be89fe5e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1173,7 +1173,7 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) linfo = jl_get_specialization1(tt); if (linfo == NULL) { linfo = jl_method_lookup_by_type( - ((jl_datatype_t*)jl_tparam0(tt))->name->mt, tt, 0, 0); + ((jl_datatype_t*)jl_tparam0(tt))->name->mt, tt, 0, 0, 1); if (linfo == NULL || jl_has_call_ambiguities(tt, linfo->def)) { JL_GC_POP(); return NULL; diff --git a/src/dump.c b/src/dump.c index 58905f0f3136f..6ee8d8980c83b 100644 --- a/src/dump.c +++ b/src/dump.c @@ -854,6 +854,11 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) writetag(s->s, jl_method_type); jl_method_t *m = (jl_method_t*)v; union jl_typemap_t *tf = &m->specializations; + if (s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK) { + int external = !module_in_worklist(m->module); + if (external) + jl_error("support for serializing a direct reference to an external Method not implemented"); + } if (tf->unknown && tf->unknown != jl_nothing) { // go through the t-func cache, replacing ASTs with just return // types for abstract argument types. these ASTs are generally @@ -879,6 +884,19 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) else if (jl_is_lambda_info(v)) { writetag(s->s, jl_lambda_info_type); jl_lambda_info_t *li = (jl_lambda_info_t*)v; + jl_serialize_value(s, (jl_value_t*)li->specTypes); + write_int8(s->s, li->inferred); + if (s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK) { + int external = li->def && !module_in_worklist(li->def->module); + write_uint8(s->s, external); + if (external) { + // also flag this in the backref table as special + uintptr_t *bp = (uintptr_t*)ptrhash_bp(&backref_table, v); + assert(*bp != (uintptr_t)HT_NOTFOUND); + *bp |= 1; assert(((uintptr_t)HT_NOTFOUND)|1); + return; + } + } if (li->jlcall_api == 2) jl_serialize_value(s, jl_nothing); else @@ -890,8 +908,6 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) jl_serialize_value(s, li->rettype); jl_serialize_value(s, (jl_value_t*)li->sparam_syms); jl_serialize_value(s, (jl_value_t*)li->sparam_vals); - jl_serialize_value(s, (jl_value_t*)li->specTypes); - write_int8(s->s, li->inferred); write_int8(s->s, li->pure); write_int8(s->s, li->inlineable); write_int8(s->s, li->isva); @@ -1389,7 +1405,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta isunboxed = !(elsize>>15); elsize = elsize&0x7fff; } - int pos = backref_list.len; + uintptr_t pos = backref_list.len; if (usetable) arraylist_push(&backref_list, NULL); size_t *dims = (size_t*)alloca(ndims*sizeof(size_t)); @@ -1452,6 +1468,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta jl_method_t *m = (jl_method_t*)jl_gc_alloc(ptls, sizeof(jl_method_t), jl_method_type); + memset(m, 0, sizeof(jl_method_type)); if (usetable) arraylist_push(&backref_list, m); m->specializations.unknown = jl_deserialize_value(s, (jl_value_t**)&m->specializations); @@ -1490,8 +1507,42 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta jl_lambda_info_t *li = (jl_lambda_info_t*)jl_gc_alloc(ptls, sizeof(jl_lambda_info_t), jl_lambda_info_type); + memset(li, 0, sizeof(jl_lambda_info_t)); + uintptr_t pos = backref_list.len; if (usetable) arraylist_push(&backref_list, li); + + li->specTypes = (jl_tupletype_t*)jl_deserialize_value(s, (jl_value_t**)&li->specTypes); + if (li->specTypes) jl_gc_wb(li, li->specTypes); + int inferred = read_int8(s->s); + li->inferred = inferred; + + if (s->mode == MODE_MODULE) { + int external = read_uint8(s->s); + if (external) { + assert(loc != NULL); + arraylist_push(&flagref_list, loc); + arraylist_push(&flagref_list, (void*)pos); + return (jl_value_t*)li; + } + } + if (s->mode == MODE_MODULE_POSTWORK) { + int external = read_uint8(s->s); + if (external) { + jl_datatype_t *ftype = jl_first_argument_datatype((jl_value_t*)li->specTypes); + jl_methtable_t *mt = ftype->name->mt; + li = jl_method_lookup_by_type(mt, li->specTypes, 1, 0, 0); + assert(li); + backref_list.items[pos] = li; + // if it can be inferred but isn't, encourage codegen to infer it + if (inferred && !li->inferred) { + jl_set_lambda_code_null(li); + li->inferred = 1; + } + return (jl_value_t*)li; + } + } + li->code = jl_deserialize_value(s, &li->code); jl_gc_wb(li, li->code); li->slotnames = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&li->slotnames); jl_gc_wb(li, li->slotnames); li->slottypes = jl_deserialize_value(s, &li->slottypes); jl_gc_wb(li, li->slottypes); @@ -1503,10 +1554,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta jl_gc_wb(li, li->sparam_syms); li->sparam_vals = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&li->sparam_vals); jl_gc_wb(li, li->sparam_vals); - li->specTypes = (jl_tupletype_t*)jl_deserialize_value(s, (jl_value_t**)&li->specTypes); - if (li->specTypes) jl_gc_wb(li, li->specTypes); li->unspecialized_ducttape = NULL; - li->inferred = read_int8(s->s); li->pure = read_int8(s->s); li->inlineable = read_int8(s->s); li->isva = read_int8(s->s); @@ -1530,7 +1578,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta return (jl_value_t*)li; } else if (vtag == (jl_value_t*)jl_module_type) { - int pos = backref_list.len; + uintptr_t pos = backref_list.len; if (usetable) arraylist_push(&backref_list, NULL); jl_sym_t *mname = (jl_sym_t*)jl_deserialize_value(s, NULL); @@ -1620,7 +1668,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)SmallDataType_tag) { int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s->s) : read_int32(s->s)); jl_value_t *v = jl_gc_alloc(ptls, sz, NULL); - int pos = backref_list.len; + uintptr_t pos = backref_list.len; if (usetable) arraylist_push(&backref_list, v); jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_astaggedvalue(v)->type); @@ -2327,30 +2375,72 @@ static void jl_recache_types(void) int offs = (int)(intptr_t)flagref_list.items[i++]; jl_value_t *v, *o = loc ? *loc : (jl_value_t*)backref_list.items[offs]; jl_datatype_t *dt, *t; - if (jl_is_datatype(o)) { - dt = (jl_datatype_t*)o; - v = dt->instance; - assert(dt->uid == -1); - t = jl_recache_type(dt, i, NULL); - } - else { - dt = (jl_datatype_t*)jl_typeof(o); + if (jl_is_lambda_info(o)) { + // lookup the real LambdaInfo based on the placeholder specTypes + jl_lambda_info_t *li = (jl_lambda_info_t*)o; + int inferred = li->inferred; + jl_datatype_t *argtypes = jl_recache_type(li->specTypes, i, NULL); + jl_datatype_t *ftype = jl_first_argument_datatype((jl_value_t*)argtypes); + jl_methtable_t *mt = ftype->name->mt; + jl_set_typeof(li, (void*)(intptr_t)0x30); // invalidate the old value to help catch errors + li = jl_method_lookup_by_type(mt, argtypes, 1, 0, 0); + assert(li); + // if it can be inferred but isn't, encourage codegen to infer it + if (inferred && !li->inferred) { + jl_set_lambda_code_null(li); + li->inferred = 1; + } + // update the backref list + if (loc) *loc = (jl_value_t*)li; + if (offs > 0) backref_list.items[offs] = li; v = o; - t = jl_recache_type(dt, i, v); - } - assert(dt); - if (t != dt) { - jl_set_typeof(dt, (void*)(intptr_t)0x10); // invalidate the old value to help catch errors - if ((jl_value_t*)dt == o) { - if (loc) *loc = (jl_value_t*)t; - if (offs > 0) backref_list.items[offs] = t; + size_t j = i; + while (j < flagref_list.len) { + jl_value_t **loc = (jl_value_t**)flagref_list.items[j]; + int offs = (int)(intptr_t)flagref_list.items[j+1]; + jl_value_t *o = loc ? *loc : (jl_value_t*)backref_list.items[offs]; + if ((jl_value_t*)v == o) { // same item, update this entry + if (loc) *loc = (jl_value_t*)li; + if (offs > 0) backref_list.items[offs] = li; + // delete this item from the flagref list, so it won't be re-encountered later + flagref_list.len -= 2; + if (j >= flagref_list.len) + break; + flagref_list.items[j+0] = flagref_list.items[flagref_list.len+0]; + flagref_list.items[j+1] = flagref_list.items[flagref_list.len+1]; + } + else { + j += 2; + } } } - if (t->instance != v) { - jl_set_typeof(v, (void*)(intptr_t)0x20); // invalidate the old value to help catch errors - if (v == o) { - *loc = t->instance; - if (offs > 0) backref_list.items[offs] = t->instance; + else { + if (jl_is_datatype(o)) { + dt = (jl_datatype_t*)o; + v = dt->instance; + assert(dt->uid == -1); + t = jl_recache_type(dt, i, NULL); + } + else { + dt = (jl_datatype_t*)jl_typeof(o); + v = o; + assert(dt->instance); + t = jl_recache_type(dt, i, v); + } + assert(dt); + if (t != dt) { + jl_set_typeof(dt, (void*)(intptr_t)0x10); // invalidate the old value to help catch errors + if ((jl_value_t*)dt == o) { + if (loc) *loc = (jl_value_t*)t; + if (offs > 0) backref_list.items[offs] = t; + } + } + if (t->instance != v) { + jl_set_typeof(v, (void*)(intptr_t)0x20); // invalidate the old value to help catch errors + if (v == o) { + *loc = t->instance; + if (offs > 0) backref_list.items[offs] = t->instance; + } } } } diff --git a/src/gf.c b/src/gf.c index 225ecf7569640..0e69c74090e73 100644 --- a/src/gf.c +++ b/src/gf.c @@ -117,7 +117,7 @@ static int8_t jl_cachearg_offset(jl_methtable_t *mt) /// ----- Insertion logic for special entries ----- /// -JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp); +JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp, int allow_exec); // get or create the LambdaInfo for a specialization JL_DLLEXPORT jl_lambda_info_t *jl_specializations_get_linfo(jl_method_t *m, jl_tupletype_t *type, jl_svec_t *sparams) @@ -128,7 +128,7 @@ JL_DLLEXPORT jl_lambda_info_t *jl_specializations_get_linfo(jl_method_t *m, jl_t JL_UNLOCK(&m->writelock); return (jl_lambda_info_t*)sf->func.value; } - jl_lambda_info_t *li = jl_get_specialized(m, type, sparams); + jl_lambda_info_t *li = jl_get_specialized(m, type, sparams, 1); JL_GC_PUSH1(&li); // TODO: fuse lookup and insert steps jl_typemap_insert(&m->specializations, (jl_value_t*)m, type, jl_emptysvec, NULL, jl_emptysvec, (jl_value_t*)li, 0, &tfunc_cache, NULL); @@ -611,7 +611,8 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_tupletype_t *type, // the specialized type signature for type lambda jl_tupletype_t *tt, // the original tupletype of the signature jl_typemap_entry_t *m, - jl_svec_t *sparams) + jl_svec_t *sparams, + int allow_exec) { // caller must hold the mt->writelock jl_method_t *definition = m->func.method; @@ -797,13 +798,13 @@ static jl_lambda_info_t *cache_method(jl_methtable_t *mt, union jl_typemap_t *ca jl_typemap_insert(cache, parent, origtype, jl_emptysvec, type, guardsigs, (jl_value_t*)newmeth, jl_cachearg_offset(mt), &lambda_cache, NULL); - if (definition->traced && jl_method_tracer) + if (definition->traced && jl_method_tracer && allow_exec) jl_call_tracer(jl_method_tracer, (jl_value_t*)newmeth); JL_GC_POP(); return newmeth; } -static jl_lambda_info_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t *tt, int cache, int inexact) +static jl_lambda_info_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t *tt, int cache, int inexact, int allow_exec) { // caller must hold the mt->writelock jl_typemap_entry_t *entry = NULL; @@ -826,10 +827,10 @@ static jl_lambda_info_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype_t * sig = join_tsig(tt, entry->sig); jl_lambda_info_t *nf; if (!cache) { - nf = jl_get_specialized(m, sig, env); + nf = jl_get_specialized(m, sig, env, allow_exec); } else { - nf = cache_method(mt, &mt->cache, (jl_value_t*)mt, sig, tt, entry, env); + nf = cache_method(mt, &mt->cache, (jl_value_t*)mt, sig, tt, entry, env, allow_exec); } JL_GC_POP(); return nf; @@ -1143,7 +1144,7 @@ jl_tupletype_t *arg_type_tuple(jl_value_t **args, size_t nargs) } jl_lambda_info_t *jl_method_lookup_by_type(jl_methtable_t *mt, jl_tupletype_t *types, - int cache, int inexact) + int cache, int inexact, int allow_exec) { jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(mt->cache, types, NULL, 0, 1, jl_cachearg_offset(mt)); if (entry) @@ -1156,7 +1157,7 @@ jl_lambda_info_t *jl_method_lookup_by_type(jl_methtable_t *mt, jl_tupletype_t *t } if (jl_is_leaf_type((jl_value_t*)types)) cache = 1; - jl_lambda_info_t *sf = jl_mt_assoc_by_type(mt, types, cache, inexact); + jl_lambda_info_t *sf = jl_mt_assoc_by_type(mt, types, cache, inexact, allow_exec); if (cache) { JL_UNLOCK(&mt->writelock); } @@ -1170,7 +1171,7 @@ jl_lambda_info_t *jl_method_lookup_by_type(jl_methtable_t *mt, jl_tupletype_t *t JL_DLLEXPORT int jl_method_exists(jl_methtable_t *mt, jl_tupletype_t *types) { - return jl_method_lookup_by_type(mt, types, 0, 0) != NULL; + return jl_method_lookup_by_type(mt, types, 0, 0, 1) != NULL; } jl_lambda_info_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t nargs, int cache) @@ -1188,7 +1189,7 @@ jl_lambda_info_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t jl_tupletype_t *tt = arg_type_tuple(args, nargs); jl_lambda_info_t *sf = NULL; JL_GC_PUSH2(&tt, &sf); - sf = jl_mt_assoc_by_type(mt, tt, cache, 0); + sf = jl_mt_assoc_by_type(mt, tt, cache, 0, 1); if (cache) { JL_UNLOCK(&mt->writelock); } @@ -1229,7 +1230,7 @@ static jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method) JL_GC_POP(); } if (def->needs_sparam_vals_ducttape) { - method->unspecialized_ducttape = jl_get_specialized(def, method->specTypes, method->sparam_vals); + method->unspecialized_ducttape = jl_get_specialized(def, method->specTypes, method->sparam_vals, 1); jl_gc_wb(method, method->unspecialized_ducttape); method->unspecialized_ducttape->unspecialized_ducttape = method->unspecialized_ducttape; } @@ -1347,7 +1348,7 @@ jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types) // not be the case JL_GC_PUSH1(&sf); JL_TRY { - sf = jl_method_lookup_by_type(mt, types, 1, 1); + sf = jl_method_lookup_by_type(mt, types, 1, 1, 1); } JL_CATCH { sf = NULL; } @@ -1379,8 +1380,8 @@ JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) JL_DLLEXPORT jl_value_t *jl_get_spec_lambda(jl_tupletype_t *types) { - jl_value_t *li = (jl_value_t*)jl_get_specialization1(types); - return li ? li : jl_nothing; + jl_lambda_info_t *li = jl_get_specialization1(types); + return li ? (jl_value_t*)li : jl_nothing; } JL_DLLEXPORT int jl_has_call_ambiguities(jl_tupletype_t *types, jl_method_t *m) @@ -1909,7 +1910,7 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) JL_TIMING(METHOD_LOOKUP_SLOW); jl_tupletype_t *tt = arg_type_tuple(args, nargs); JL_GC_PUSH1(&tt); - mfunc = jl_mt_assoc_by_type(mt, tt, 1, 0); + mfunc = jl_mt_assoc_by_type(mt, tt, 1, 0, 1); JL_GC_POP(); } JL_UNLOCK(&mt->writelock); @@ -2000,7 +2001,7 @@ jl_value_t *jl_gf_invoke(jl_tupletype_t *types0, jl_value_t **args, size_t nargs if (func->invokes.unknown == NULL) func->invokes.unknown = jl_nothing; - mfunc = cache_method(mt, &func->invokes, entry->func.value, sig, tt, entry, tpenv); + mfunc = cache_method(mt, &func->invokes, entry->func.value, sig, tt, entry, tpenv, 1); } JL_UNLOCK(&method->writelock); } diff --git a/src/julia_internal.h b/src/julia_internal.h index d4e0f32e51058..618ac56bb1f86 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -309,7 +309,7 @@ int jl_is_toplevel_only_expr(jl_value_t *e); jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr); jl_lambda_info_t *jl_method_lookup_by_type(jl_methtable_t *mt, jl_tupletype_t *types, - int cache, int inexact); + int cache, int inexact, int allow_exec); jl_lambda_info_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t nargs, int cache); jl_value_t *jl_gf_invoke(jl_tupletype_t *types, jl_value_t **args, size_t nargs); diff --git a/test/compile.jl b/test/compile.jl index 09dc51390edc6..d639ac6cf5e46 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -74,6 +74,9 @@ try (::Type{Vector{NominalValue{T, R}}}){T, R}() = 3 (::Type{Vector{NominalValue{T, T}}}){T}() = 4 (::Type{Vector{NominalValue{Int, Int}}})() = 5 + + #const some_method = @which Base.include("string") // FIXME: support for serializing a direct reference to an external Method not implemented + const some_linfo = @code_typed Base.include("string") end """) @test_throws ErrorException Core.kwfunc(Base.nothing) # make sure `nothing` didn't have a kwfunc (which would invalidate the attempted test) @@ -129,6 +132,8 @@ try Val{3}, Val{nothing}}, 0:25) + + @test Foo.some_linfo === @code_typed Base.include("string") end Baz_file = joinpath(dir, "Baz.jl") From 17c266be2853e304dabeb0aae7006428cf701893 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 15 Aug 2016 18:00:27 -0400 Subject: [PATCH 1032/1117] cleanup stacktraces method signatures ensure that we are using consistent types and introduce ProfileFormat to contain all of the user options --- base/profile.jl | 197 ++++++++++++++++++++++++++------------------ base/stacktraces.jl | 4 +- 2 files changed, 117 insertions(+), 84 deletions(-) diff --git a/base/profile.jl b/base/profile.jl index 861f2c7e2fe93..25965d4c69f42 100644 --- a/base/profile.jl +++ b/base/profile.jl @@ -66,6 +66,23 @@ Clear any existing backtraces from the internal buffer. """ clear() = ccall(:jl_profile_clear_data, Void, ()) +typealias LineInfoDict Dict{UInt64, Vector{StackFrame}} +typealias LineInfoFlatDict Dict{UInt64, StackFrame} + +immutable ProfileFormat + maxdepth::Int + sortedby::Symbol + combine::Bool + C::Bool + function ProfileFormat(; + C = false, + combine = true, + maxdepth::Int = typemax(Int), + sortedby::Symbol = :filefuncline) + return new(maxdepth, sortedby, combine, C) + end +end + """ print([io::IO = STDOUT,] [data::Vector]; format = :tree, C = false, combine = true, maxdepth = typemax(Int), sortedby = :filefuncline) @@ -79,24 +96,32 @@ format, while `sortedby` can be used to control the order in `:flat` format (`:filefuncline` sorts by the source line, whereas `:count` sorts in order of number of collected samples). """ -function print{T<:Unsigned}(io::IO, data::Vector{T} = fetch(), lidict::Dict = getdict(data); +function print{T<:Unsigned}(io::IO, data::Vector{T} = fetch(), lidict::LineInfoDict = getdict(data); format = :tree, C = false, combine = true, maxdepth::Int = typemax(Int), sortedby::Symbol = :filefuncline) - cols = Base.displaysize(io)[2] + print(io, data, lidict, ProfileFormat(C = C, + combine = combine, + maxdepth = maxdepth, + sortedby = sortedby), + format) +end + +function print{T<:Unsigned}(io::IO, data::Vector{T}, lidict::LineInfoDict, fmt::ProfileFormat, format::Symbol) + cols::Int = Base.displaysize(io)[2] if format == :tree - tree(io, data, lidict, C, combine, cols, maxdepth) + tree(io, data, lidict, cols, fmt) elseif format == :flat - flat(io, data, lidict, C, combine, cols, sortedby) + flat(io, data, lidict, cols, fmt) else throw(ArgumentError("output format $(repr(format)) not recognized")) end end """ - print([io::IO = STDOUT,] data::Vector, lidict::Dict; kwargs) + print([io::IO = STDOUT,] data::Vector, lidict::LineInfoDict; kwargs) Prints profiling results to `io`. This variant is used to examine results exported by a previous call to [`retrieve`](:func:`retrieve`). Supply the vector `data` of backtraces and @@ -104,7 +129,7 @@ a dictionary `lidict` of line information. See `Profile.print([io], data)` for an explanation of the valid keyword arguments. """ -print{T<:Unsigned}(data::Vector{T} = fetch(), lidict::Dict = getdict(data); kwargs...) = print(STDOUT, data, lidict; kwargs...) +print{T<:Unsigned}(data::Vector{T} = fetch(), lidict::LineInfoDict = getdict(data); kwargs...) = print(STDOUT, data, lidict; kwargs...) """ retrieve() -> data, lidict @@ -116,16 +141,16 @@ allows you to save profiling results for future analysis. """ function retrieve() data = fetch() - copy(data), getdict(data) + return (copy(data), getdict(data)) end function getdict(data::Vector{UInt}) uip = unique(data) - Dict{UInt, Vector{StackFrame}}(ip=>lookup(ip) for ip in uip) + return LineInfoDict(UInt64(ip)=>lookup(ip) for ip in uip) end """ - flatten(btdata, lidict) -> (newdata, newdict) + flatten(btdata, lidict) -> (newdata::Vector{UInt64}, newdict::LineInfoFlatDict) Produces "flattened" backtrace data. Individual instruction pointers sometimes correspond to a multi-frame backtrace due to inlining; in @@ -133,17 +158,17 @@ such cases, this function inserts fake instruction pointers for the inlined calls, and returns a dictionary that is a 1-to-1 mapping between instruction pointers and a single StackFrame. """ -function flatten(data::Vector{UInt}, lidict::Dict{UInt,Vector{StackFrame}}) +function flatten(data::Vector, lidict::LineInfoDict) # Makes fake instruction pointers, counting down from typemax(UInt) - newip = typemax(UInt) + newip = typemax(UInt64) - 1 taken = Set(keys(lidict)) # make sure we don't pick one that's already used - newdict = Dict{UInt,StackFrame}() - newmap = Dict{UInt,Vector{UInt}}() + newdict = Dict{UInt64,StackFrame}() + newmap = Dict{UInt64,Vector{UInt64}}() for (ip, trace) in lidict if length(trace) == 1 newdict[ip] = trace[1] else - newm = UInt[] + newm = UInt64[] for sf in trace while newip ∈ taken && newip > 0 newip -= 1 @@ -156,15 +181,16 @@ function flatten(data::Vector{UInt}, lidict::Dict{UInt,Vector{StackFrame}}) newmap[ip] = newm end end - newdata = UInt[] + newdata = UInt64[] for ip in data + local ip::UInt64 if haskey(newmap, ip) append!(newdata, newmap[ip]) else push!(newdata, ip) end end - newdata, newdict + return (newdata, newdict) end """ @@ -179,7 +205,7 @@ profile buffer is used. """ function callers end -function callers(funcname::String, bt::Vector{UInt}, lidict; filename = nothing, linerange = nothing) +function callers(funcname::String, bt::Vector, lidict::LineInfoDict; filename = nothing, linerange = nothing) if filename === nothing && linerange === nothing return callersf(li -> li.func == funcname, bt, lidict) end @@ -192,7 +218,7 @@ function callers(funcname::String, bt::Vector{UInt}, lidict; filename = nothing, end callers(funcname::String; kwargs...) = callers(funcname, retrieve()...; kwargs...) -callers(func::Function, bt::Vector{UInt}, lidict; kwargs...) = callers(string(func), bt, lidict; kwargs...) +callers(func::Function, bt::Vector, lidict::LineInfoDict; kwargs...) = callers(string(func), bt, lidict; kwargs...) callers(func::Function; kwargs...) = callers(string(func), retrieve()...; kwargs...) ## @@ -244,9 +270,11 @@ function fetch() len = len_data() maxlen = maxlen_data() if (len == maxlen) - warn("The profile data buffer is full; profiling probably terminated\nbefore your program finished. To profile for longer runs, call Profile.init\nwith a larger buffer and/or larger delay.") + warn("""The profile data buffer is full; profiling probably terminated + before your program finished. To profile for longer runs, call Profile.init + with a larger buffer and/or larger delay.""") end - unsafe_wrap(Array, get_data_pointer(), (len,)) + return unsafe_wrap(Array, get_data_pointer(), (len,)) end @@ -276,10 +304,10 @@ function count_flat{T<:Unsigned}(data::Vector{T}) push!(iplist, k) push!(n, v) end - return iplist, n + return (iplist, n) end -function parse_flat(iplist, n, lidict, C::Bool) +function parse_flat(iplist, n, lidict::LineInfoFlatDict, C::Bool) # Convert instruction pointers to names & line numbers lilist = [lidict[ip] for ip in iplist] # Keep only the interpretable ones @@ -289,11 +317,11 @@ function parse_flat(iplist, n, lidict, C::Bool) keep = !Bool[x == UNKNOWN || x.line == 0 || (x.from_c && !C) for x in lilist] n = n[keep] lilist = lilist[keep] - lilist, n + return (lilist, n) end -function flat{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,StackFrame}, C::Bool, combine::Bool, cols::Integer, sortedby) - if !C +function flat(io::IO, data::Vector, lidict::LineInfoFlatDict, cols::Int, fmt::ProfileFormat) + if !fmt.C data = purgeC(data, lidict) end iplist, n = count_flat(data) @@ -301,20 +329,22 @@ function flat{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,StackFrame}, warning_empty() return end - lilist, n = parse_flat(iplist, n, lidict, C) - print_flat(io, lilist, n, combine, cols, sortedby) + lilist, n = parse_flat(iplist, n, lidict, fmt.C) + print_flat(io, lilist, n, cols, fmt) + nothing end -function flat{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,Vector{StackFrame}}, C::Bool, combine::Bool, cols::Integer, sortedby) +function flat(io::IO, data::Vector, lidict::LineInfoDict, cols::Int, fmt::ProfileFormat) newdata, newdict = flatten(data, lidict) - flat(io, newdata, newdict, C, combine, cols, sortedby) + flat(io, newdata, newdict, cols, fmt) + nothing end -function print_flat(io::IO, lilist::Vector{StackFrame}, n::Vector{Int}, combine::Bool, cols::Integer, sortedby) +function print_flat(io::IO, lilist::Vector{StackFrame}, n::Vector{Int}, cols::Int, fmt::ProfileFormat) p = liperm(lilist) lilist = lilist[p] n = n[p] - if combine + if fmt.combine j = 1 for i = 2:length(lilist) if lilist[i] == lilist[j] @@ -328,7 +358,7 @@ function print_flat(io::IO, lilist::Vector{StackFrame}, n::Vector{Int}, combine: n = n[keep] lilist = lilist[keep] end - if sortedby == :count + if fmt.sortedby == :count p = sortperm(n) n = n[p] lilist = lilist[p] @@ -349,8 +379,8 @@ function print_flat(io::IO, lilist::Vector{StackFrame}, n::Vector{Int}, combine: wfile = maxfile wfunc = maxfunc else - wfile = floor(Integer,2*ntext/5) - wfunc = floor(Integer,3*ntext/5) + wfile = floor(Integer, 2*ntext/5) + wfunc = floor(Integer, 3*ntext/5) end println(io, lpad("Count", wcounts, " "), " ", rpad("File", wfile, " "), " ", lpad("Line", wline, " "), " ", rpad("Function", wfunc, " ")) for i = 1:length(n) @@ -365,44 +395,45 @@ function print_flat(io::IO, lilist::Vector{StackFrame}, n::Vector{Int}, combine: Base.print(io, rpad(ltruncto(fname, wfunc), wfunc, " ")) println(io) end + nothing end ## A tree representation # Identify and counts repetitions of all unique backtraces -function tree_aggregate{T<:Unsigned}(data::Vector{T}) +function tree_aggregate(data::Vector{UInt64}) iz = find(data .== 0) # find the breaks between backtraces - treecount = Dict{Vector{T},Int}() - istart = 1+btskip + treecount = Dict{Vector{UInt64},Int}() + istart = 1 + btskip for iend in iz - tmp = data[iend-1:-1:istart] - treecount[tmp] = get(treecount, tmp, 0)+1 - istart = iend+1+btskip + tmp = data[iend - 1 : -1 : istart] + treecount[tmp] = get(treecount, tmp, 0) + 1 + istart = iend + 1 + btskip end - bt = Array{Vector{T}}(0) + bt = Array{Vector{UInt64}}(0) counts = Array{Int}(0) - for (k,v) in treecount + for (k, v) in treecount if !isempty(k) push!(bt, k) push!(counts, v) end end - bt, counts + return (bt, counts) end -tree_format_linewidth(x::StackFrame) = ndigits(x.line)+6 +tree_format_linewidth(x::StackFrame) = ndigits(x.line) + 6 -function tree_format(lilist::Vector{StackFrame}, counts::Vector{Int}, level::Int, cols::Integer) +function tree_format(lilist::Vector{StackFrame}, counts::Vector{Int}, level::Int, cols::Int) nindent = min(cols>>1, level) ndigcounts = ndigits(maximum(counts)) ndigline = maximum([tree_format_linewidth(x) for x in lilist]) - ntext = cols-nindent-ndigcounts-ndigline-5 - widthfile = floor(Integer,0.4ntext) - widthfunc = floor(Integer,0.6ntext) + ntext = cols - nindent - ndigcounts - ndigline - 5 + widthfile = floor(Integer, 0.4ntext) + widthfunc = floor(Integer, 0.6ntext) strs = Array{String}(length(lilist)) showextra = false if level > nindent - nextra = level-nindent - nindent -= ndigits(nextra)+2 + nextra = level - nindent + nindent -= ndigits(nextra) + 2 showextra = true end for i = 1:length(lilist) @@ -437,26 +468,26 @@ function tree_format(lilist::Vector{StackFrame}, counts::Vector{Int}, level::Int strs[i] = "" end end - strs + return strs end # Print a "branch" starting at a particular level. This gets called recursively. -function tree{T<:Unsigned}(io::IO, bt::Vector{Vector{T}}, counts::Vector{Int}, lidict::Dict, level::Int, combine::Bool, cols::Integer, maxdepth) - if level > maxdepth +function tree(io::IO, bt::Vector{Vector{UInt64}}, counts::Vector{Int}, lidict::LineInfoFlatDict, level::Int, cols::Int, fmt::ProfileFormat) + if level > fmt.maxdepth return end # Organize backtraces into groups that are identical up to this level - if combine + if fmt.combine # Combine based on the line information d = Dict{StackFrame,Vector{Int}}() for i = 1:length(bt) - ip = bt[i][level+1] + ip = bt[i][level + 1] key = lidict[ip] indx = Base.ht_keyindex(d, key) - if indx == -1 - d[key] = [i] + if haskey(d, key) + push!(d[key], i) else - push!(d.vals[indx], i) + d[key] = [i] end end # Generate counts @@ -473,14 +504,13 @@ function tree{T<:Unsigned}(io::IO, bt::Vector{Vector{T}}, counts::Vector{Int}, l end else # Combine based on the instruction pointer - d = Dict{T,Vector{Int}}() + d = Dict{UInt64,Vector{Int}}() for i = 1:length(bt) key = bt[i][level+1] - indx = Base.ht_keyindex(d, key) - if indx == -1 - d[key] = [i] + if haskey(d, key) + push!(d[key], i) else - push!(d.vals[indx], i) + d[key] = [i] end end # Generate counts, and do the code lookup @@ -515,13 +545,14 @@ function tree{T<:Unsigned}(io::IO, bt::Vector{Vector{T}}, counts::Vector{Int}, l keep = len[idx] .> level+1 if any(keep) idx = idx[keep] - tree(io, bt[idx], counts[idx], lidict, level+1, combine, cols, maxdepth) + tree(io, bt[idx], counts[idx], lidict, level + 1, cols, fmt) end end + nothing end -function tree{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,StackFrame}, C::Bool, combine::Bool, cols::Integer, maxdepth) - if !C +function tree(io::IO, data::Vector{UInt64}, lidict::LineInfoFlatDict, cols::Int, fmt::ProfileFormat) + if !fmt.C data = purgeC(data, lidict) end bt, counts = tree_aggregate(data) @@ -532,15 +563,17 @@ function tree{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,StackFrame}, level = 0 len = Int[length(x) for x in bt] keep = len .> 0 - tree(io, bt[keep], counts[keep], lidict, level, combine, cols, maxdepth) + tree(io, bt[keep], counts[keep], lidict, level, cols, fmt) + nothing end -function tree{T<:Unsigned}(io::IO, data::Vector{T}, lidict::Dict{T,Vector{StackFrame}}, C::Bool, combine::Bool, cols::Integer, maxdepth) +function tree(io::IO, data::Vector, lidict::LineInfoDict, cols::Int, fmt::ProfileFormat) newdata, newdict = flatten(data, lidict) - tree(io, newdata, newdict, C, combine, cols, maxdepth) + tree(io, newdata, newdict, cols, fmt) + nothing end -function callersf(matchfunc::Function, bt::Vector{UInt}, lidict) +function callersf(matchfunc::Function, bt::Vector, lidict::LineInfoDict) counts = Dict{StackFrame, Int}() lastmatched = false for id in bt @@ -561,23 +594,23 @@ function callersf(matchfunc::Function, bt::Vector{UInt}, lidict) k = collect(keys(counts)) v = collect(values(counts)) p = sortperm(v, rev=true) - [(v[i], k[i]) for i in p] + return [(v[i], k[i]) for i in p] end # Utilities function rtruncto(str::String, w::Int) - ret = str - if length(str) > w - ret = string("...", str[end-w+4:end]) + if length(str) <= w + return str + else + return string("...", str[end-w+4:end]) end - ret end function ltruncto(str::String, w::Int) - ret = str - if length(str) > w - ret = string(str[1:w-4], "...") + if length(str) <= w + return str + else + return string(str[1:w-4], "...") end - ret end @@ -594,7 +627,7 @@ function liperm(lilist::Vector{StackFrame}) comb[i] = "zzz" end end - sortperm(comb) + return sortperm(comb) end warning_empty() = warn(""" @@ -602,9 +635,9 @@ warning_empty() = warn(""" running it multiple times), or adjust the delay between samples with Profile.init().""") -function purgeC(data, lidict) +function purgeC(data::Vector{UInt64}, lidict::LineInfoFlatDict) keep = Bool[d == 0 || lidict[d].from_c == false for d in data] - data[keep] + return data[keep] end end # module diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 55a7d901225ad..dbc64382e540a 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -109,7 +109,7 @@ function deserialize(s::AbstractSerializer, ::Type{StackFrame}) line = read(s.io, Int) from_c = read(s.io, Bool) inlined = read(s.io, Bool) - pointer = read(s.io, Int64) + pointer = read(s.io, UInt64) return StackFrame(func, file, line, Nullable{LambdaInfo}(), from_c, inlined, pointer) end @@ -131,7 +131,7 @@ function lookup(pointer::Ptr{Void}) li = info[4] === nothing ? Nullable{LambdaInfo}() : Nullable{LambdaInfo}(info[4]) res[i] = StackFrame(info[1], info[2], info[3], li, info[5], info[6], info[7]) end - res + return res end lookup(pointer::UInt) = lookup(convert(Ptr{Void}, pointer)) From 5ee9dbd43954aa0cb019297900975213748d3020 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 15 Aug 2016 18:31:45 -0400 Subject: [PATCH 1033/1117] profile: add mincount argument the min-count argument to Profile.print can be used to filter out lines from the display that are insignificant --- base/profile.jl | 24 ++++++++++++++++-------- doc/stdlib/profile.rst | 6 +++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/base/profile.jl b/base/profile.jl index 25965d4c69f42..6d8d2c82fdae1 100644 --- a/base/profile.jl +++ b/base/profile.jl @@ -71,6 +71,7 @@ typealias LineInfoFlatDict Dict{UInt64, StackFrame} immutable ProfileFormat maxdepth::Int + mincount::Int sortedby::Symbol combine::Bool C::Bool @@ -78,33 +79,38 @@ immutable ProfileFormat C = false, combine = true, maxdepth::Int = typemax(Int), + mincount::Int = 0, sortedby::Symbol = :filefuncline) - return new(maxdepth, sortedby, combine, C) + return new(maxdepth, mincount, sortedby, combine, C) end end """ - print([io::IO = STDOUT,] [data::Vector]; format = :tree, C = false, combine = true, maxdepth = typemax(Int), sortedby = :filefuncline) + print([io::IO = STDOUT,] [data::Vector]; format = :tree, C = false, combine = true, maxdepth = typemax(Int), sortedby = :filefuncline, mincount = 0) Prints profiling results to `io` (by default, `STDOUT`). If you do not supply a `data` vector, the internal buffer of accumulated backtraces -will be used. `format` can be `:tree` or `:flat`. If `C==true`, -backtraces from C and Fortran code are shown. `combine==true` merges -instruction pointers that correspond to the same line of -code. `maxdepth` can be used to limit the depth of printing in `:tree` -format, while `sortedby` can be used to control the order in `:flat` -format (`:filefuncline` sorts by the source line, whereas `:count` +will be used. `format` can be `:tree` or `:flat`. +If `C==true`, backtraces from C and Fortran code are shown. +`combine==true` merges instruction pointers that correspond to the same line of code. +`maxdepth` can be used to limit the depth of printing in `:tree` format, +while `sortedby` can be used to control the order in `:flat` format +(`:filefuncline` sorts by the source line, whereas `:count` sorts in order of number of collected samples). +`mincount` can also be used to limit the printout to only those +lines with at least mincount occurances. """ function print{T<:Unsigned}(io::IO, data::Vector{T} = fetch(), lidict::LineInfoDict = getdict(data); format = :tree, C = false, combine = true, maxdepth::Int = typemax(Int), + mincount::Int = 0, sortedby::Symbol = :filefuncline) print(io, data, lidict, ProfileFormat(C = C, combine = combine, maxdepth = maxdepth, + mincount = mincount, sortedby = sortedby), format) end @@ -384,6 +390,7 @@ function print_flat(io::IO, lilist::Vector{StackFrame}, n::Vector{Int}, cols::In end println(io, lpad("Count", wcounts, " "), " ", rpad("File", wfile, " "), " ", lpad("Line", wline, " "), " ", rpad("Function", wfunc, " ")) for i = 1:length(n) + n[i] < fmt.mincount && continue li = lilist[i] Base.print(io, lpad(string(n[i]), wcounts, " "), " ") Base.print(io, rpad(rtruncto(string(li.file), wfile), wfile, " "), " ") @@ -538,6 +545,7 @@ function tree(io::IO, bt::Vector{Vector{UInt64}}, counts::Vector{Int}, lidict::L # Recurse to the next level len = Int[length(x) for x in bt] for i = 1:length(lilist) + n[i] < fmt.mincount && continue if !isempty(strs[i]) println(io, strs[i]) end diff --git a/doc/stdlib/profile.rst b/doc/stdlib/profile.rst index 1ebe73407e40b..61335ac704ff3 100644 --- a/doc/stdlib/profile.rst +++ b/doc/stdlib/profile.rst @@ -24,13 +24,13 @@ The methods in :mod:`Base.Profile` are not exported and need to be called e.g. a Clear any existing backtraces from the internal buffer. -.. function:: print([io::IO = STDOUT,] [data::Vector]; format = :tree, C = false, combine = true, maxdepth = typemax(Int), sortedby = :filefuncline) +.. function:: print([io::IO = STDOUT,] [data::Vector]; format = :tree, C = false, combine = true, maxdepth = typemax(Int), sortedby = :filefuncline, mincount = 0) .. Docstring generated from Julia source - Prints profiling results to ``io`` (by default, ``STDOUT``\ ). If you do not supply a ``data`` vector, the internal buffer of accumulated backtraces will be used. ``format`` can be ``:tree`` or ``:flat``\ . If ``C==true``\ , backtraces from C and Fortran code are shown. ``combine==true`` merges instruction pointers that correspond to the same line of code. ``maxdepth`` can be used to limit the depth of printing in ``:tree`` format, while ``sortedby`` can be used to control the order in ``:flat`` format (``:filefuncline`` sorts by the source line, whereas ``:count`` sorts in order of number of collected samples). + Prints profiling results to ``io`` (by default, ``STDOUT``\ ). If you do not supply a ``data`` vector, the internal buffer of accumulated backtraces will be used. ``format`` can be ``:tree`` or ``:flat``\ . If ``C==true``\ , backtraces from C and Fortran code are shown. ``combine==true`` merges instruction pointers that correspond to the same line of code. ``maxdepth`` can be used to limit the depth of printing in ``:tree`` format, while ``sortedby`` can be used to control the order in ``:flat`` format (``:filefuncline`` sorts by the source line, whereas ``:count`` sorts in order of number of collected samples). ``mincount`` can also be used to limit the printout to only those lines with at least mincount occurances. -.. function:: print([io::IO = STDOUT,] data::Vector, lidict::Dict; kwargs) +.. function:: print([io::IO = STDOUT,] data::Vector, lidict::LineInfoDict; kwargs) .. Docstring generated from Julia source From f1dd9e15827694f44ca23b4110469634c2d8c6dd Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Tue, 23 Aug 2016 14:11:59 -0400 Subject: [PATCH 1034/1117] profile: add noisefloor argument the noise-floor argument to Profile.print can be used to filter out lines from the display that are insignificant, based on the heuristic `n <= noisefloor * sqrt(N)` --- base/profile.jl | 47 ++++++++++++++++++++++++++++-------------- doc/stdlib/profile.rst | 15 +++++++++++--- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/base/profile.jl b/base/profile.jl index 6d8d2c82fdae1..e3e2154ece3f8 100644 --- a/base/profile.jl +++ b/base/profile.jl @@ -72,6 +72,7 @@ typealias LineInfoFlatDict Dict{UInt64, StackFrame} immutable ProfileFormat maxdepth::Int mincount::Int + noisefloor::Float64 sortedby::Symbol combine::Bool C::Bool @@ -80,25 +81,38 @@ immutable ProfileFormat combine = true, maxdepth::Int = typemax(Int), mincount::Int = 0, + noisefloor = 0, sortedby::Symbol = :filefuncline) - return new(maxdepth, mincount, sortedby, combine, C) + return new(maxdepth, mincount, noisefloor, sortedby, combine, C) end end """ - print([io::IO = STDOUT,] [data::Vector]; format = :tree, C = false, combine = true, maxdepth = typemax(Int), sortedby = :filefuncline, mincount = 0) + print([io::IO = STDOUT,] [data::Vector]; kwargs...) Prints profiling results to `io` (by default, `STDOUT`). If you do not supply a `data` vector, the internal buffer of accumulated backtraces -will be used. `format` can be `:tree` or `:flat`. -If `C==true`, backtraces from C and Fortran code are shown. -`combine==true` merges instruction pointers that correspond to the same line of code. -`maxdepth` can be used to limit the depth of printing in `:tree` format, -while `sortedby` can be used to control the order in `:flat` format -(`:filefuncline` sorts by the source line, whereas `:count` -sorts in order of number of collected samples). -`mincount` can also be used to limit the printout to only those -lines with at least mincount occurances. +will be used. + +The keyword arguments can be any combination of: + + - `format` can be `:tree` (default) or `:flat`. + + - If `C` is `true`, backtraces from C and Fortran code are shown (normally they are excluded). + + - If `combine` is `true` (default), instruction pointers are merged that correspond to the same line of code. + + - `maxdepth` can be used to limit the depth of printing in `:tree` format, + while `sortedby` can be used to control the order in `:flat` format + `:filefuncline` (default) sorts by the source line, whereas `:count` + sorts in order of number of collected samples. + + - `noisefloor` only shows frames that exceed the heuristic noise floor of the sample (only applies to format `:tree`). + A suggested value to try for this is 2.0 (the default is 0). This parameters hides samples for which `n <= noisefloor * √N`, + where `n` is the number of samples on this line, and `N` is the number of samples for the callee. + + - `mincount` can also be used to limit the printout to only those + lines with at least mincount occurrences. """ function print{T<:Unsigned}(io::IO, data::Vector{T} = fetch(), lidict::LineInfoDict = getdict(data); format = :tree, @@ -106,11 +120,13 @@ function print{T<:Unsigned}(io::IO, data::Vector{T} = fetch(), lidict::LineInfoD combine = true, maxdepth::Int = typemax(Int), mincount::Int = 0, + noisefloor = 0, sortedby::Symbol = :filefuncline) print(io, data, lidict, ProfileFormat(C = C, combine = combine, maxdepth = maxdepth, mincount = mincount, + noisefloor = noisefloor, sortedby = sortedby), format) end @@ -127,7 +143,7 @@ function print{T<:Unsigned}(io::IO, data::Vector{T}, lidict::LineInfoDict, fmt:: end """ - print([io::IO = STDOUT,] data::Vector, lidict::LineInfoDict; kwargs) + print([io::IO = STDOUT,] data::Vector, lidict::LineInfoDict; kwargs...) Prints profiling results to `io`. This variant is used to examine results exported by a previous call to [`retrieve`](:func:`retrieve`). Supply the vector `data` of backtraces and @@ -479,7 +495,7 @@ function tree_format(lilist::Vector{StackFrame}, counts::Vector{Int}, level::Int end # Print a "branch" starting at a particular level. This gets called recursively. -function tree(io::IO, bt::Vector{Vector{UInt64}}, counts::Vector{Int}, lidict::LineInfoFlatDict, level::Int, cols::Int, fmt::ProfileFormat) +function tree(io::IO, bt::Vector{Vector{UInt64}}, counts::Vector{Int}, lidict::LineInfoFlatDict, level::Int, cols::Int, fmt::ProfileFormat, noisefloor::Int) if level > fmt.maxdepth return end @@ -546,6 +562,7 @@ function tree(io::IO, bt::Vector{Vector{UInt64}}, counts::Vector{Int}, lidict::L len = Int[length(x) for x in bt] for i = 1:length(lilist) n[i] < fmt.mincount && continue + n[i] < noisefloor && continue if !isempty(strs[i]) println(io, strs[i]) end @@ -553,7 +570,7 @@ function tree(io::IO, bt::Vector{Vector{UInt64}}, counts::Vector{Int}, lidict::L keep = len[idx] .> level+1 if any(keep) idx = idx[keep] - tree(io, bt[idx], counts[idx], lidict, level + 1, cols, fmt) + tree(io, bt[idx], counts[idx], lidict, level + 1, cols, fmt, fmt.noisefloor > 0 ? floor(Int, fmt.noisefloor * sqrt(n[i])) : 0) end end nothing @@ -571,7 +588,7 @@ function tree(io::IO, data::Vector{UInt64}, lidict::LineInfoFlatDict, cols::Int, level = 0 len = Int[length(x) for x in bt] keep = len .> 0 - tree(io, bt[keep], counts[keep], lidict, level, cols, fmt) + tree(io, bt[keep], counts[keep], lidict, level, cols, fmt, 0) nothing end diff --git a/doc/stdlib/profile.rst b/doc/stdlib/profile.rst index 61335ac704ff3..7d39fbccb4693 100644 --- a/doc/stdlib/profile.rst +++ b/doc/stdlib/profile.rst @@ -24,13 +24,22 @@ The methods in :mod:`Base.Profile` are not exported and need to be called e.g. a Clear any existing backtraces from the internal buffer. -.. function:: print([io::IO = STDOUT,] [data::Vector]; format = :tree, C = false, combine = true, maxdepth = typemax(Int), sortedby = :filefuncline, mincount = 0) +.. function:: print([io::IO = STDOUT,] [data::Vector]; kwargs...) .. Docstring generated from Julia source - Prints profiling results to ``io`` (by default, ``STDOUT``\ ). If you do not supply a ``data`` vector, the internal buffer of accumulated backtraces will be used. ``format`` can be ``:tree`` or ``:flat``\ . If ``C==true``\ , backtraces from C and Fortran code are shown. ``combine==true`` merges instruction pointers that correspond to the same line of code. ``maxdepth`` can be used to limit the depth of printing in ``:tree`` format, while ``sortedby`` can be used to control the order in ``:flat`` format (``:filefuncline`` sorts by the source line, whereas ``:count`` sorts in order of number of collected samples). ``mincount`` can also be used to limit the printout to only those lines with at least mincount occurances. + Prints profiling results to ``io`` (by default, ``STDOUT``\ ). If you do not supply a ``data`` vector, the internal buffer of accumulated backtraces will be used. -.. function:: print([io::IO = STDOUT,] data::Vector, lidict::LineInfoDict; kwargs) + The keyword arguments can be any combination of: + + * ``format`` can be ``:tree`` (default) or ``:flat``\ . + * If ``C`` is ``true``\ , backtraces from C and Fortran code are shown (normally they are excluded). + * If ``combine`` is ``true`` (default), instruction pointers are merged that correspond to the same line of code. + * ``maxdepth`` can be used to limit the depth of printing in ``:tree`` format, while ``sortedby`` can be used to control the order in ``:flat`` format ``:filefuncline`` (default) sorts by the source line, whereas ``:count`` sorts in order of number of collected samples. + * ``noisefloor`` only shows frames that exceed the heuristic noise floor of the sample (only applies to format ``:tree``\ ). A suggested value to try for this is 2.0 (the default is 0). This parameters hides samples for which ``n <= noisefloor * √N``\ , where ``n`` is the number of samples on this line, and ``N`` is the number of samples for the callee. + * ``mincount`` can also be used to limit the printout to only those lines with at least mincount occurrences. + +.. function:: print([io::IO = STDOUT,] data::Vector, lidict::LineInfoDict; kwargs...) .. Docstring generated from Julia source From 485f9592fd53f90f2bacccb75f183d01abe46525 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 22 Aug 2016 18:01:57 -0400 Subject: [PATCH 1035/1117] incremental deserialize: optimize many simple common cases of flagref_list many types are easy to compute during deserialization, and don't need the full complexity of the flagref list to handle them some of these are easy to detect ahead-of-time, saving a small amount of deserialization effort --- src/dump.c | 87 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/src/dump.c b/src/dump.c index 6ee8d8980c83b..0ffed80001294 100644 --- a/src/dump.c +++ b/src/dump.c @@ -472,6 +472,45 @@ static int module_in_worklist(jl_module_t *mod) return 0; } +// compute whether a type references something internal to worklist +// and thus could not have existed before deserialize +// and thus does not need delayed unique-ing +static int type_in_worklist(jl_datatype_t *dt) +{ + if (module_in_worklist(dt->name->module)) + return 1; + int i, l = jl_svec_len(dt->parameters); + for (i = 0; i < l; i++) { + jl_value_t *p = jl_tparam(dt, i); + if (type_in_worklist((jl_datatype_t*)(jl_is_datatype(p) ? p : jl_typeof(p)))) + return 1; + } + return 0; +} + +// returns true if all of the parameters are tag 6 or 7 +static int type_recursively_external(jl_datatype_t *dt) +{ + if (dt->uid == 0) + return 0; + if (jl_svec_len(dt->parameters) == 0) + return 1; + + int i, l = jl_svec_len(dt->parameters); + for (i = 0; i < l; i++) { + jl_datatype_t *p = (jl_datatype_t*)jl_tparam(dt, i); + if (!jl_is_datatype(p)) + return 0; + if (module_in_worklist(p->name->module)) + return 0; + if (p->name->primary != (jl_value_t*)p) { + if (!type_recursively_external(p)) + return 0; + } + } + return 1; +} + static int jl_prune_specializations(jl_typemap_entry_t *ml, void *closure) { jl_value_t *ret = ml->func.value; @@ -496,31 +535,31 @@ static void jl_serialize_datatype(jl_serializer_state *s, jl_datatype_t *dt) } else if (s->mode == MODE_MODULE) { int internal = module_in_worklist(dt->name->module); - int i, l = jl_array_len(serializer_worklist); - for (i = 0; i < l; i++) { - jl_module_t *mod = (jl_module_t*)jl_array_ptr_ref(serializer_worklist, i); - if (jl_is_module(mod) && jl_is_submodule(dt->name->module, mod)) { - internal = 1; - break; - } - } if (!internal && dt->name->primary == (jl_value_t*)dt) { tag = 6; // external primary type } else if (dt->uid == 0) { tag = 0; // normal struct } - else if (!internal && jl_svec_len(dt->parameters) == 0) { + else if (internal) { + if (dt->name->primary == (jl_value_t*)dt) // comes up often since functions create types + tag = 5; // internal, and not in the typename cache (just needs uid reassigned) + else + tag = 10; // anything else that's internal (just needs uid reassigned and possibly recaching) + } + else if (type_recursively_external(dt)) { tag = 7; // external type that can be immediately recreated (with apply_type) } + else if (type_in_worklist(dt)) { + tag = 10; // external, but definitely new (still needs uid and caching, but not full unique-ing) + } else { - tag = 5; // anything else (needs uid assigned later) - if (!internal) { - // also flag this in the backref table as special - uintptr_t *bp = (uintptr_t*)ptrhash_bp(&backref_table, dt); - assert(*bp != (uintptr_t)HT_NOTFOUND); - *bp |= 1; assert(((uintptr_t)HT_NOTFOUND)|1); - } + // this'll need a uid and unique-ing later + // flag this in the backref table as special + uintptr_t *bp = (uintptr_t*)ptrhash_bp(&backref_table, dt); + assert(*bp != (uintptr_t)HT_NOTFOUND); + *bp |= 1; + tag = 10; } } else if (dt == jl_int32_type) @@ -534,7 +573,7 @@ static void jl_serialize_datatype(jl_serializer_state *s, jl_datatype_t *dt) if (strncmp(jl_symbol_name(dt->name->name), "#kw#", 4) == 0) { /* XXX: yuck, but the auto-generated kw types from the serializer isn't a real type, so we *must* be very careful */ - assert(tag == 0 || tag == 5 || tag == 6); + assert(tag == 0 || tag == 5 || tag == 6 || tag == 10); if (tag == 6) { jl_methtable_t *mt = dt->name->mt; jl_datatype_t *primarydt = (jl_datatype_t*)jl_get_global(mt->module, mt->name); @@ -576,7 +615,7 @@ static void jl_serialize_datatype(jl_serializer_state *s, jl_datatype_t *dt) } } - if (has_layout) { + if (has_layout) { uint8_t layout = 0; if (dt->layout == jl_array_type->layout) { layout = 1; @@ -893,7 +932,7 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) // also flag this in the backref table as special uintptr_t *bp = (uintptr_t*)ptrhash_bp(&backref_table, v); assert(*bp != (uintptr_t)HT_NOTFOUND); - *bp |= 1; assert(((uintptr_t)HT_NOTFOUND)|1); + *bp |= 1; return; } } @@ -940,11 +979,12 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) } else { if (v == t->instance) { - if (s->mode == MODE_MODULE) { + if (s->mode == MODE_MODULE && !type_in_worklist(t)) { // also flag this in the backref table as special + // if it might not be unique (is external) uintptr_t *bp = (uintptr_t*)ptrhash_bp(&backref_table, v); assert(*bp != (uintptr_t)HT_NOTFOUND); - *bp |= 1; assert(((uintptr_t)HT_NOTFOUND)|1); + *bp |= 1; } writetag(s->s, (jl_value_t*)Singleton_tag); jl_serialize_value(s, t); @@ -1206,7 +1246,7 @@ static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_v dt = jl_int64_type; else if (tag == 8) dt = jl_uint8_type; - else if (tag == 0 || tag == 5) + else if (tag == 0 || tag == 5 || tag == 10) dt = jl_new_uninitialized_datatype(); else assert(0); @@ -1267,6 +1307,9 @@ static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_v } if (tag == 5) { + dt->uid = jl_assign_type_uid(); + } + else if (tag == 10) { assert(pos > 0); assert(s->mode != MODE_MODULE_POSTWORK); arraylist_push(&flagref_list, loc); From 910fd1acbc33f46731fdcaae16dd2073ee6b52d8 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 22 Aug 2016 18:33:47 -0400 Subject: [PATCH 1036/1117] precompile: major LLVM speed improvement fix #15048 --- src/codegen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 90153be89fe5e..6e931439ac3b7 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5951,7 +5951,7 @@ extern "C" void jl_init_codegen(void) #ifdef DISABLE_OPT .setOptLevel(CodeGenOpt::None) #else - .setOptLevel(CodeGenOpt::Aggressive) + .setOptLevel(jl_options.opt_level == 0 ? CodeGenOpt::None : CodeGenOpt::Aggressive) #endif #if defined(USE_MCJIT) && !defined(LLVM36) .setUseMCJIT(true) From f086b7b81f685fa6d316a0135f923daeced0f289 Mon Sep 17 00:00:00 2001 From: Jameson Nash <jameson@juliacomputing.com> Date: Mon, 22 Aug 2016 19:57:02 -0400 Subject: [PATCH 1037/1117] typemap: handle vararg tuple subtyping in jl_typemap_assoc_by_type Fixes a crash while loading ASTInterpreter --- src/typemap.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/typemap.c b/src/typemap.c index 9d308db03d336..b7f959ebf9da4 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -595,6 +595,7 @@ int sigs_eq(jl_value_t *a, jl_value_t *b, int useenv) static jl_typemap_entry_t *jl_typemap_assoc_by_type_(jl_typemap_entry_t *ml, jl_tupletype_t *types, int8_t inexact, jl_svec_t **penv) { size_t n = jl_field_count(types); + int typesisva = n == 0 ? 0 : jl_is_vararg_type(jl_tparam(types, n-1)); while (ml != (void*)jl_nothing) { size_t lensig = jl_field_count(ml->sig); if (lensig == n || (ml->va && lensig <= n+1)) { @@ -611,10 +612,10 @@ static jl_typemap_entry_t *jl_typemap_assoc_by_type_(jl_typemap_entry_t *ml, jl_ if (ismatch == 0) ; // nothing - else if (ml->isleafsig) + else if (ml->isleafsig && !typesisva) ismatch = sig_match_by_type_leaf(jl_svec_data(types->parameters), ml->sig, lensig); - else if (ml->issimplesig) + else if (ml->issimplesig && !typesisva) ismatch = sig_match_by_type_simple(jl_svec_data(types->parameters), n, ml->sig, lensig, ml->va); else if (ml->tvars == jl_emptysvec) From bf79aa44a7ae5a1ab4b1992d4e4cbdcc52d1677d Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Tue, 23 Aug 2016 16:18:15 -0400 Subject: [PATCH 1038/1117] inline native Julia scalars encountered in f.(args...) AST --- src/ast.c | 18 ++++++++++++++++++ src/julia-syntax.scm | 2 +- test/broadcast.jl | 7 +++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/ast.c b/src/ast.c index 4799810c41550..0956723653b1c 100644 --- a/src/ast.c +++ b/src/ast.c @@ -191,11 +191,29 @@ value_t fl_invoke_julia_macro(fl_context_t *fl_ctx, value_t *args, uint32_t narg return scmresult; } +// Check whether v is a scalar for purposes of inlining fused-broadcast +// arguments when lowering; should agree with broadcast.jl on what is a +// scalar. When in doubt, return false, since this is only an optimization. +// (TODO: update after #16966 is resolved.) +int fl_julia_scalar(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) +{ + argcount(fl_ctx, "julia-scalar?", nargs, 1); + if (fl_isnumber(fl_ctx, args[0])) + return fl_ctx->T; + else if (iscvalue(args[0]) && fl_ctx->jl_sym == cv_type((cvalue_t*)ptr(args[0]))) { + jl_value_t *v = *(jl_value_t**)cptr(args[0]); + if (jl_subtype(v,(jl_value_t*)jl_number_type,1)) + return fl_ctx->T; + } + return fl_ctx->F; +} + static const builtinspec_t julia_flisp_ast_ext[] = { { "defined-julia-global", fl_defined_julia_global }, { "invoke-julia-macro", fl_invoke_julia_macro }, { "current-julia-module", fl_current_julia_module }, { "current-julia-module-counter", fl_current_module_counter }, + { "julia-scalar?", fl_julia_scalar }, { NULL, NULL } }; diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 0d14e75fd6327..1ed42e952c75a 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1679,7 +1679,7 @@ new-fargs new-args (cons (cons (cadr farg) (cadr varfarg)) renames) varfarg vararg) (error "multiple splatted args cannot be fused into a single broadcast")))) - ((number? arg) ; inline numeric literals + ((julia-scalar? arg) ; inline numeric literals etc. (cf (cdr old-fargs) (cdr old-args) new-fargs new-args (cons (cons farg arg) renames) diff --git a/test/broadcast.jl b/test/broadcast.jl index dfbbeed1c8950..ec4d4ebf111ef 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -289,6 +289,13 @@ let identity = error, x = [1,2,3] @test x == [1,1,1] end +# make sure scalars are inlined, which causes f.(x,scalar) to lower to a "thunk" +import Base.Meta: isexpr +@test isexpr(expand(:(f.(x,y))), :call) +@test isexpr(expand(:(f.(x,1))), :thunk) +@test isexpr(expand(:(f.(x,1.0))), :thunk) +@test isexpr(expand(:(f.(x,$π))), :thunk) + # PR 16988 @test Base.promote_op(+, Bool) === Int @test isa(broadcast(+, [true]), Array{Int,1}) From d3cddc9e8282548efdaa8c207416a0c2bd468c1d Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Mon, 22 Aug 2016 20:01:33 +0800 Subject: [PATCH 1039/1117] Make sure `:push_loc` meta always has a corresponding `:pop_loc` * Preserve meta node and line number info during dead code elimination * Insert `:push_loc` and `:pop_loc` in pairs during lowering Fix #16578 --- base/inference.jl | 15 ++---- src/ast.scm | 4 ++ src/julia-syntax.scm | 35 ++++++++++---- test/parse.jl | 107 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 18 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 2649514a32bd1..99b9e46ff56c7 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2042,15 +2042,9 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::InferenceState, undefs, pass) end function expr_cannot_delete(ex::Expr) - # This alone should be enough for any sane use of - # `Expr(:inbounds)` and `Expr(:boundscheck)`. However, it is still possible - # to have these embeded in other expressions (e.g. `return @inbounds ...`) - # so we check recursively if there's a matching expression - (ex.head === :inbounds || ex.head === :boundscheck) && return true - for arg in ex.args - isa(arg, Expr) && expr_cannot_delete(arg::Expr) && return true - end - return false + head = ex.head + return (head === :inbounds || head === :boundscheck || head === :meta || + head === :line) end # annotate types of all symbols in AST @@ -2081,7 +2075,8 @@ function type_annotate!(linfo::LambdaInfo, states::Array{Any,1}, sv::ANY, nargs) record_slot_type!(id, widenconst(states[i+1][id].typ), linfo.slottypes) end elseif optimize - if isa(expr, Expr) && expr_cannot_delete(expr::Expr) + if ((isa(expr, Expr) && expr_cannot_delete(expr::Expr)) || + isa(expr, LineNumberNode)) i += 1 continue end diff --git a/src/ast.scm b/src/ast.scm index fc01cb05d35ff..8337b8dec6362 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -211,6 +211,10 @@ (define (make-assignment l r) `(= ,l ,r)) (define (assignment? e) (and (pair? e) (eq? (car e) '=))) (define (return? e) (and (pair? e) (eq? (car e) 'return))) +(define (complex-return? e) (and (return? e) + (let ((x (cadr e))) + (not (or (simple-atom? x) (ssavalue? x) + (equal? x '(null))))))) (define (eq-sym? a b) (or (eq? a b) (and (ssavalue? a) (ssavalue? b) (eqv? (cdr a) (cdr b))))) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 1ed42e952c75a..bf467d1838d65 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3184,20 +3184,39 @@ f(x) = yt(x) (fname (if (and (length> e 1) (pair? fnm) (eq? (car fnm) 'line) (length> fnm 2)) (caddr fnm) - filename))) - (if (not (eq? fname last-fname)) - (begin (set! filename fname) - ;; don't need a filename node for start of function - (if (not (eq? e (lam:body lam))) (emit `(meta push_loc ,fname))))) + filename)) + (file-diff (not (eq? fname last-fname))) + ;; don't need a filename node for start of function + (need-meta (and file-diff + (not (eq? e (lam:body lam)))))) + (if file-diff (set! filename fname)) + (if need-meta (emit `(meta push_loc ,fname))) (begin0 (let loop ((xs (cdr e))) (if (null? (cdr xs)) (compile (car xs) break-labels value tail) (begin (compile (car xs) break-labels #f #f) (loop (cdr xs))))) - (if (not (eq? fname last-fname)) - (begin (if (not tail) (emit `(meta pop_loc))) - (set! filename last-fname)))))) + (if need-meta + (if (or (not tail) + (and (pair? (car code)) + (or (eq? (cdar code) 'meta) + (eq? (cdar code) 'line)))) + (emit '(meta pop_loc)) + ;; If we need to return the last non-meta expression + ;; splice the pop before the result + (let ((retv (car code)) + (body (cdr code))) + (set! code body) + (if (complex-return? retv) + (let ((tmp (make-ssavalue))) + (emit `(= ,tmp ,(cadr retv))) + (emit '(meta pop_loc)) + (emit `(return ,tmp))) + (begin + (emit '(meta pop_loc)) + (emit retv)))))) + (if file-diff (set! filename last-fname))))) ((return) (compile (cadr e) break-labels #t #t) '(null)) diff --git a/test/parse.jl b/test/parse.jl index 50a6b61f437ae..04d953e4ae2db 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -670,3 +670,110 @@ end # issue 15896 and PR 15913 @test_throws ErrorException eval(:(macro test15896(d; y=0) end)) + +# Issue #16578 (Lowering) mismatch between push_loc and pop_loc +module TestMeta_16578 +using Base.Test +function get_expr_list(ex) + if isa(ex, LambdaInfo) + return Base.uncompressed_ast(ex) + elseif ex.head == :thunk + return get_expr_list(ex.args[1]) + else + return ex.args + end +end + +function count_meta_loc(exprs) + push_count = 0 + pop_count = 0 + for expr in exprs + Meta.isexpr(expr, :meta) || continue + expr = expr::Expr + if expr.args[1] === :push_loc + push_count += 1 + elseif expr.args[1] === :pop_loc + pop_count += 1 + end + @test push_count >= pop_count + end + @test push_count == pop_count + return push_count +end + +function is_return_ssavalue(ex::Expr) + ex.head === :return && isa(ex.args[1], SSAValue) +end + +function is_pop_loc(ex::Expr) + ex.head === :meta && ex.args[1] === :pop_loc +end + +# Macros +macro m1() + quote + sin(1) + end +end +macro m2() + quote + 1 + end +end +include_string(""" +macro m3() + quote + @m1 + end +end +macro m4() + quote + @m2 + end +end +""", "another_file.jl") +m1_exprs = get_expr_list(expand(:(@m1))) +m2_exprs = get_expr_list(expand(:(@m2))) +m3_exprs = get_expr_list(expand(:(@m3))) +m4_exprs = get_expr_list(expand(:(@m4))) + +# Check the expanded expresion has expected number of matching push/pop +# and the return is handled correctly +@test count_meta_loc(m1_exprs) == 1 +@test is_return_ssavalue(m1_exprs[end]) +@test is_pop_loc(m1_exprs[end - 1]) + +@test count_meta_loc(m2_exprs) == 1 +@test m2_exprs[end] == :(return 1) +@test is_pop_loc(m2_exprs[end - 1]) + +@test count_meta_loc(m3_exprs) == 2 +@test is_return_ssavalue(m3_exprs[end]) +@test is_pop_loc(m3_exprs[end - 1]) + +@test count_meta_loc(m4_exprs) == 2 +@test m4_exprs[end] == :(return 1) +@test is_pop_loc(m4_exprs[end - 1]) + +function f1(a) + b = a + 100 + b +end + +@generated function f2(a) + quote + b = a + 100 + b + end +end + +f1_exprs = get_expr_list(@code_typed f1(1)) +@test count_meta_loc(f1_exprs) == 0 +@test Meta.isexpr(f1_exprs[end], :return) + +f2_exprs = get_expr_list(@code_typed f2(1)) +@test count_meta_loc(f2_exprs) == 1 +@test is_pop_loc(f2_exprs[end - 1]) +@test Meta.isexpr(f2_exprs[end], :return) + +end From 932f4ff4ef2ef3c3aa4fc76e2b6bab1d8fa1dcd4 Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Wed, 24 Aug 2016 12:07:26 +0200 Subject: [PATCH 1040/1117] Move specialized isequal definitions to avoid hitting #265 Fixes #18213. --- base/deprecated.jl | 5 ++--- base/operators.jl | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 0bac394c53cef..cad9ea951bd59 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -461,12 +461,11 @@ end @deprecate ==(x::Char, y::Integer) UInt32(x) == y @deprecate ==(x::Integer, y::Char) x == UInt32(y) +# Note: when these deprecations are deleted, the specialized definitions isequal(x::Char, y::Integer) +# and isequal(x::Integer, y::Char) in operators.jl can be deleted, too @deprecate isless(x::Char, y::Integer) UInt32(x) < y @deprecate isless(x::Integer, y::Char) x < UInt32(y) -# delete these methods along with deprecations: -isequal(x::Char, y::Integer) = false -isequal(x::Integer, y::Char) = false #6674 and #4233 macro windows(qm,ex) diff --git a/base/operators.jl b/base/operators.jl index be5ccf5c5354f..8127281eed78b 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -15,6 +15,11 @@ supertype(T::DataType) = T.super ==(x, y) = x === y isequal(x, y) = x == y +# TODO: these can be deleted once the deprecations of ==(x::Char, y::Integer) and +# ==(x::Integer, y::Char) are gone and the above returns false anyway +isequal(x::Char, y::Integer) = false +isequal(x::Integer, y::Char) = false + ## minimally-invasive changes to test == causing NotComparableError # export NotComparableError # =={T}(x::T, y::T) = x === y From f7419a7a3dbd2b478180e805f7d4aae22ee3597d Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Fri, 19 Aug 2016 14:16:25 -0400 Subject: [PATCH 1041/1117] fix sort / fallback order for incremental precompile fix #18069 --- base/loading.jl | 97 ++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 6bda601ef549f..317ffd1800af5 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -115,9 +115,9 @@ find_in_path(name::AbstractString, wd = pwd()) = find_in_path(String(name), wd) function find_in_node_path(name::String, srcpath, node::Int=1) if myid() == node - find_in_path(name, srcpath) + return find_in_path(name, srcpath) else - remotecall_fetch(find_in_path, node, name, srcpath) + return remotecall_fetch(find_in_path, node, name, srcpath) end end @@ -126,7 +126,7 @@ function find_source_file(file::String) file2 = find_in_path(file) file2 !== nothing && return file2 file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file) - isfile(file2) ? file2 : nothing + return isfile(file2) ? file2 : nothing end function find_all_in_cache_path(mod::Symbol) @@ -138,21 +138,22 @@ function find_all_in_cache_path(mod::Symbol) push!(paths, path) end end - paths + return paths end function _include_from_serialized(content::Vector{UInt8}) - return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8},Int), content, sizeof(content)) + return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content)) +end + +function _include_from_serialized(path::String) + return ccall(:jl_restore_incremental, Any, (Cstring,), path) end # returns an array of modules loaded, or nothing if failed function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool) - if JLOptions().use_compilecache == 0 - return nothing - end - restored = nothing + local restored = nothing + local content::Vector{UInt8} if toplevel_load && myid() == 1 && nprocs() > 1 - recompile_stale(mod, path_to_try) # broadcast top-level import/using from node 1 (only) if node == myid() content = open(read, path_to_try) @@ -170,13 +171,11 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t end end elseif node == myid() - myid() == 1 && recompile_stale(mod, path_to_try) - restored = ccall(:jl_restore_incremental, Any, (Cstring,), path_to_try) + restored = _include_from_serialized(path_to_try) else content = remotecall_fetch(open, node, read, path_to_try) restored = _include_from_serialized(content) end - # otherwise, continue search if restored !== nothing for M in restored @@ -188,17 +187,20 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t return restored end -function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool) - if JLOptions().use_compilecache == 0 - return nothing - end +# returns `true` if require found a precompile cache for this mod, but couldn't load it +# returns `false` if the module isn't known to be precompilable +# returns the set of modules restored if the cache load succeeded +function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::String, toplevel_load::Bool) if node == myid() paths = find_all_in_cache_path(mod) else paths = @fetchfrom node find_all_in_cache_path(mod) end - sort!(paths, by=mtime, rev=true) # try newest cachefiles first + for path_to_try in paths + if stale_cachefile(sourcepath, path_to_try) + continue + end restored = _require_from_serialized(node, mod, path_to_try, toplevel_load) if restored === nothing warn("deserialization checks failed while attempting to load cache from $path_to_try") @@ -206,7 +208,7 @@ function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool) return restored end end - return nothing + return !isempty(paths) end # to synchronize multiple tasks trying to import/using something @@ -329,22 +331,30 @@ function require(mod::Symbol) last = toplevel_load::Bool try toplevel_load = false - if nothing !== _require_from_serialized(1, mod, last) - return - end - if JLOptions().incremental != 0 - # spawn off a new incremental precompile task from node 1 for recursive `require` calls - cachefile = compilecache(mod) - if nothing === _require_from_serialized(1, mod, cachefile, last) - warn("require failed to create a precompiled cache file") - end - return - end name = string(mod) path = find_in_node_path(name, nothing, 1) if path === nothing - throw(ArgumentError("$name not found in path.\nRun Pkg.add(\"$name\") to install the $name package")) + throw(ArgumentError("module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package.")) end + + doneprecompile = false + if JLOptions().use_compilecache != 0 + doneprecompile = _require_search_from_serialized(1, mod, path, last) + if !isa(doneprecompile, Bool) + return # success + elseif doneprecompile === true || JLOptions().incremental != 0 + # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls + # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) + cachefile = compilecache(mod) + if nothing === _require_from_serialized(1, mod, cachefile, last) + warn("compilecache failed to create a usable precompiled cache file for module $name.") + else + return # success + end + end + # fall-through to attempting to load the source file + end + try if last && myid() == 1 && nprocs() > 1 # include on node 1 first to check for PrecompilableErrors @@ -357,13 +367,12 @@ function require(mod::Symbol) eval(Main, :(Base.include_from_node1($path))) end catch ex - if !precompilableerror(ex, true) + if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true) rethrow() # rethrow non-precompilable=true errors end - isinteractive() && info("Precompiling module $mod...") cachefile = compilecache(mod) if nothing === _require_from_serialized(1, mod, cachefile, last) - error("__precompile__(true) but require failed to create a precompiled cache file") + error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.") end end finally @@ -501,6 +510,13 @@ function compilecache(name::String) mkpath(cachepath) end cachefile = abspath(cachepath, name*".ji") + if isinteractive() + if isfile(cachepath) + info("Recompiling stale cache file $cachefile for module $name.") + else + info("Precompiling module $name.") + end + end if !success(create_expr_cache(path, cachefile)) error("Failed to precompile $name to $cachefile") end @@ -550,7 +566,7 @@ function stale_cachefile(modpath, cachefile) if files[1][1] != modpath return true # cache file was compiled from a different path end - for (f,ftime) in files + for (f, ftime) in files # Issue #13606: compensate for Docker images rounding mtimes if mtime(f) ∉ (ftime, floor(ftime)) return true @@ -570,14 +586,3 @@ function stale_cachefile(modpath, cachefile) close(io) end end - -function recompile_stale(mod, cachefile) - path = find_in_path(string(mod), nothing) - if path === nothing - error("module $mod not found in current path; you should rm(\"$(escape_string(cachefile))\") to remove the orphaned cache file") - end - if stale_cachefile(path, cachefile) - info("Recompiling stale cache file $cachefile for module $mod.") - compilecache(mod) - end -end From 5982c1ac46837549fe2eee0fe851ddbc9dda32b8 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Fri, 19 Aug 2016 15:47:01 -0400 Subject: [PATCH 1042/1117] change incremental serialize to use deferred errors this gives the caller more control over when an how errors get printed --- base/loading.jl | 70 ++++++++++++++++++++++++++++---------------- src/builtins.c | 24 +++++++++++---- src/dump.c | 62 ++++++++++++++++++--------------------- src/julia_internal.h | 1 + 4 files changed, 93 insertions(+), 64 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 317ffd1800af5..bbc2b926fdaa8 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -141,15 +141,18 @@ function find_all_in_cache_path(mod::Symbol) return paths end +# these return either the array of modules loaded from the path / content given +# or an Exception that describes why it couldn't be loaded function _include_from_serialized(content::Vector{UInt8}) return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content)) end - function _include_from_serialized(path::String) return ccall(:jl_restore_incremental, Any, (Cstring,), path) end -# returns an array of modules loaded, or nothing if failed +# returns an array of modules loaded, or an Exception that describes why it failed +# and also attempts to load the same file across all nodes (if toplevel_node and myid() == master) +# and it reconnects the Base.Docs.META function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, toplevel_load::Bool) local restored = nothing local content::Vector{UInt8} @@ -161,13 +164,23 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t content = remotecall_fetch(open, node, read, path_to_try) end restored = _include_from_serialized(content) - if restored !== nothing - others = filter(x -> x != myid(), procs()) - refs = Any[ @spawnat p (nothing !== _include_from_serialized(content)) for p in others] - for (id, ref) in zip(others, refs) - if !fetch(ref) - warn("node state is inconsistent: node $id failed to load cache from $path_to_try") - end + isa(restored, Exception) && return restored + others = filter(x -> x != myid(), procs()) + refs = Any[ + (p, @spawnat(p, + let m = try + _include_from_serialized(content) + catch ex + isa(ex, Exception) ? ex : ErrorException(string(ex)) + end + isa(m, Exception) ? m : nothing + end)) + for p in others ] + for (id, ref) in refs + m = fetch(ref) + if m !== nothing + warn("Node state is inconsistent: node $id failed to load cache from $path_to_try. Got:") + warn(m) end end elseif node == myid() @@ -177,8 +190,8 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t restored = _include_from_serialized(content) end - if restored !== nothing - for M in restored + if !isa(restored, Exception) + for M in restored::Vector{Any} if isdefined(M, Base.Docs.META) push!(Base.Docs.modules, M) end @@ -197,17 +210,29 @@ function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::Str paths = @fetchfrom node find_all_in_cache_path(mod) end + local restored = nothing for path_to_try in paths if stale_cachefile(sourcepath, path_to_try) continue end restored = _require_from_serialized(node, mod, path_to_try, toplevel_load) - if restored === nothing - warn("deserialization checks failed while attempting to load cache from $path_to_try") + if isa(restored, Exception) + if isa(restored, ErrorException) && endswith(restored.msg, " uuid did not match cache file.") + # can't use this cache due to a module uuid mismatch, + # defer reporting error until after trying all of the possible matches + continue + end + warn("Deserialization checks failed while attempting to load cache from $path_to_try.") + error(restored) else return restored end end + if isa(restored, Exception) + warn("""Deserialization checks failed while attempting to load cache from $path_to_try. + This is likely because module %s does not support precompilation but is imported by a module that does.""") + warn(restored) + end return !isempty(paths) end @@ -346,8 +371,10 @@ function require(mod::Symbol) # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) cachefile = compilecache(mod) - if nothing === _require_from_serialized(1, mod, cachefile, last) - warn("compilecache failed to create a usable precompiled cache file for module $name.") + m = _require_from_serialized(1, mod, cachefile, last) + if !isa(m, Exception) + warn("Compilecache failed to create a usable precompiled cache file for module $name. Got:") + warn(m) else return # success end @@ -371,7 +398,9 @@ function require(mod::Symbol) rethrow() # rethrow non-precompilable=true errors end cachefile = compilecache(mod) - if nothing === _require_from_serialized(1, mod, cachefile, last) + m = _require_from_serialized(1, mod, cachefile, last) + if isa(m, Exception) + warn(m) error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.") end end @@ -572,15 +601,6 @@ function stale_cachefile(modpath, cachefile) return true end end - # files are not stale, so module list is valid and needs checking - for (M,uuid) in modules - if !isdefined(Main, M) - require(M) # should recursively recompile module M if stale - end - if module_uuid(getfield(Main, M)) != uuid - return true - end - end return false # fresh cachefile finally close(io) diff --git a/src/builtins.c b/src/builtins.c index 4698cab3afcbb..a89403454fb86 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -44,8 +44,8 @@ JL_DLLEXPORT void JL_NORETURN jl_error(const char *str) extern int vasprintf(char **str, const char *fmt, va_list ap); -static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type, - const char *fmt, va_list args) +static jl_value_t *jl_vexceptionf(jl_datatype_t *exception_type, + const char *fmt, va_list args) { if (exception_type == NULL) { jl_printf(JL_STDERR, "ERROR: "); @@ -64,15 +64,18 @@ static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type, free(str); } JL_GC_PUSH1(&msg); - jl_throw(jl_new_struct(exception_type, msg)); + jl_value_t *e = jl_new_struct(exception_type, msg); + JL_GC_POP(); + return e; } JL_DLLEXPORT void JL_NORETURN jl_errorf(const char *fmt, ...) { va_list args; va_start(args, fmt); - jl_vexceptionf(jl_errorexception_type, fmt, args); + jl_value_t *e = jl_vexceptionf(jl_errorexception_type, fmt, args); va_end(args); + jl_throw(e); } JL_DLLEXPORT void JL_NORETURN jl_exceptionf(jl_datatype_t *exception_type, @@ -80,8 +83,19 @@ JL_DLLEXPORT void JL_NORETURN jl_exceptionf(jl_datatype_t *exception_type, { va_list args; va_start(args, fmt); - jl_vexceptionf(exception_type, fmt, args); + jl_value_t *e = jl_vexceptionf(exception_type, fmt, args); + va_end(args); + jl_throw(e); +} + +jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, + const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + jl_value_t *e = jl_vexceptionf(exception_type, fmt, args); va_end(args); + return e; } JL_DLLEXPORT void JL_NORETURN jl_too_few_args(const char *fname, int min) diff --git a/src/dump.c b/src/dump.c index 0ffed80001294..074428a19af58 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1802,16 +1802,16 @@ static void jl_deserialize_lambdas_from_mod(jl_serializer_state *s) } } -static int read_verify_mod_list(ios_t *s) +static jl_value_t *read_verify_mod_list(ios_t *s) { if (!jl_main_module->uuid) { - jl_printf(JL_STDERR, "ERROR: Main module uuid state is invalid for module deserialization.\n"); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Main module uuid state is invalid for module deserialization."); } while (1) { size_t len = read_int32(s); if (len == 0) - return 1; + return NULL; char *name = (char*)alloca(len+1); ios_read(s, name, len); name[len] = '\0'; @@ -1833,20 +1833,17 @@ static int read_verify_mod_list(ios_t *s) m = (jl_module_t*)jl_get_global(jl_main_module, sym); } if (!m) { - jl_printf(JL_STDERR, "ERROR: requiring \"%s\" did not define a corresponding module\n", name); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Requiring \"%s\" did not define a corresponding module.", name); } if (!jl_is_module(m)) { ios_close(s); - jl_errorf("invalid module path (%s does not name a module)", name); + return jl_get_exceptionf(jl_errorexception_type, + "Invalid module path (%s does not name a module).", name); } if (m->uuid != uuid) { - jl_printf(JL_STDERR, - "WARNING: Module %s uuid did not match cache file\n" - " This is likely because module %s does not support\n" - " precompilation but is imported by a module that does.\n", - name, name); - return 0; + return jl_get_exceptionf(jl_errorexception_type, + "Module %s uuid did not match cache file.", name); } } } @@ -1916,13 +1913,13 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) jl_declare_constant(b); // this can throw if (b->value != NULL) { if (!jl_is_module(b->value)) { - jl_errorf("invalid redefinition of constant %s", + jl_errorf("Invalid redefinition of constant %s.", jl_symbol_name(mod->name)); // this also throws } if (jl_generating_output() && jl_options.incremental) { - jl_errorf("cannot replace module %s during incremental precompile", jl_symbol_name(mod->name)); + jl_errorf("Cannot replace module %s during incremental precompile.", jl_symbol_name(mod->name)); } - jl_printf(JL_STDERR, "WARNING: replacing module %s\n", + jl_printf(JL_STDERR, "WARNING: replacing module %s.\n", jl_symbol_name(mod->name)); } b->value = v; @@ -2189,14 +2186,14 @@ JL_DLLEXPORT void jl_restore_system_image(const char *fname) int err = jl_load_sysimg_so(); if (err != 0) { if (jl_sysimg_handle == 0) - jl_errorf("system image file \"%s\" not found", fname); - jl_errorf("library \"%s\" does not contain a valid system image", fname); + jl_errorf("System image file \"%s\" not found.", fname); + jl_errorf("Library \"%s\" does not contain a valid system image.", fname); } } else { ios_t f; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) - jl_errorf("system image file \"%s\" not found", fname); + jl_errorf("System image file \"%s\" not found.", fname); JL_SIGATOMIC_BEGIN(); jl_restore_system_image_from_stream(&f); ios_close(&f); @@ -2496,16 +2493,17 @@ static int trace_method(jl_typemap_entry_t *entry, void *closure) return 1; } -static jl_array_t *_jl_restore_incremental(ios_t *f) +static jl_value_t *_jl_restore_incremental(ios_t *f) { - if (ios_eof(f)) { + if (ios_eof(f) || !jl_read_verify_header(f)) { ios_close(f); - return NULL; + return jl_get_exceptionf(jl_errorexception_type, + "Precompile file header verification checks failed."); } - if (!jl_read_verify_header(f) || - !read_verify_mod_list(f)) { + jl_value_t *verify_error = read_verify_mod_list(f); + if (verify_error) { ios_close(f); - return NULL; + return verify_error; } size_t deplen = read_uint64(f); ios_skip(f, deplen); // skip past the dependency list @@ -2551,28 +2549,24 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) jl_init_restored_modules(init_order); JL_GC_POP(); - return restored; + return (jl_value_t*)restored; } JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz) { ios_t f; - jl_array_t *modules; ios_static_buffer(&f, (char*)buf, sz); - modules = _jl_restore_incremental(&f); - return modules ? (jl_value_t*) modules : jl_nothing; + return _jl_restore_incremental(&f); } JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname) { ios_t f; - jl_array_t *modules; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { - jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname); - return jl_nothing; + return jl_get_exceptionf(jl_errorexception_type, + "Cache file \"%s\" not found.\n", fname); } - modules = _jl_restore_incremental(&f); - return modules ? (jl_value_t*) modules : jl_nothing; + return _jl_restore_incremental(&f); } // --- init --- diff --git a/src/julia_internal.h b/src/julia_internal.h index 618ac56bb1f86..f9015882d3738 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -240,6 +240,7 @@ void jl_set_gs_ctr(uint32_t ctr); void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args); void JL_NORETURN jl_method_error(jl_function_t *f, jl_value_t **args, size_t na); +jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const char *fmt, ...); JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t); From 0809e417a5c8233e0af2ee8211cdf6a5fbac4957 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Fri, 19 Aug 2016 16:10:09 -0400 Subject: [PATCH 1043/1117] keep track of full dependency state while building incremental precompile caches this ensures that we only recompile a dependency if either: it can be loaded into the current session, or the user explicitly requests it (via reload) thereby reducing the occurrences and improving the accuracy of the "uuid didn't match" error message :) --- base/loading.jl | 166 ++++++++++++++++++++++++++++++----------- doc/manual/modules.rst | 52 ++++++++----- src/dump.c | 37 ++++++++- src/module.c | 4 +- test/compile.jl | 108 ++++++++++++++++++++++----- 5 files changed, 281 insertions(+), 86 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index bbc2b926fdaa8..ddca6283d848f 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -131,7 +131,7 @@ end function find_all_in_cache_path(mod::Symbol) name = string(mod) - paths = AbstractString[] + paths = String[] for prefix in LOAD_CACHE_PATH path = joinpath(prefix, name*".ji") if isfile_casesensitive(path) @@ -180,7 +180,7 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::String, t m = fetch(ref) if m !== nothing warn("Node state is inconsistent: node $id failed to load cache from $path_to_try. Got:") - warn(m) + warn(m, prefix="WARNING: ") end end elseif node == myid() @@ -210,8 +210,8 @@ function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::Str paths = @fetchfrom node find_all_in_cache_path(mod) end - local restored = nothing - for path_to_try in paths + local restored = nothing, failedpath = "" + for path_to_try in paths::Vector{String} if stale_cachefile(sourcepath, path_to_try) continue end @@ -220,18 +220,18 @@ function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::Str if isa(restored, ErrorException) && endswith(restored.msg, " uuid did not match cache file.") # can't use this cache due to a module uuid mismatch, # defer reporting error until after trying all of the possible matches + failedpath = path_to_try continue end warn("Deserialization checks failed while attempting to load cache from $path_to_try.") - error(restored) + throw(restored) else return restored end end if isa(restored, Exception) - warn("""Deserialization checks failed while attempting to load cache from $path_to_try. - This is likely because module %s does not support precompilation but is imported by a module that does.""") - warn(restored) + warn("Deserialization checks failed while attempting to load cache from $failedpath.") + warn(restored, prefix="WARNING: ") end return !isempty(paths) end @@ -240,12 +240,13 @@ end const package_locks = Dict{Symbol,Condition}() # used to optionally track dependencies when requiring a module: -const _require_dependencies = Tuple{String,Float64}[] -const _track_dependencies = [false] +const _concrete_dependencies = Any[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them +const _require_dependencies = Any[] # a list of (path, mtime) tuples that are the file dependencies of the module currently being precompiled +const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies function _include_dependency(_path::AbstractString) prev = source_path(nothing) - path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev),_path) - if myid() == 1 && _track_dependencies[1] + path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev), _path) + if myid() == 1 && _track_dependencies[] apath = abspath(path) push!(_require_dependencies, (apath, mtime(apath))) end @@ -341,8 +342,8 @@ toplevel_load = true function require(mod::Symbol) # dependency-tracking is only used for one top-level include(path), # and is not applied recursively to imported modules: - old_track_dependencies = _track_dependencies[1] - _track_dependencies[1] = false + old_track_dependencies = _track_dependencies[] + _track_dependencies[] = false global toplevel_load loading = get(package_locks, mod, false) @@ -356,32 +357,46 @@ function require(mod::Symbol) last = toplevel_load::Bool try toplevel_load = false + # perform the search operation to select the module file require intends to load name = string(mod) path = find_in_node_path(name, nothing, 1) if path === nothing throw(ArgumentError("module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package.")) end + # attempt to load the module file via the precompile cache locations doneprecompile = false if JLOptions().use_compilecache != 0 doneprecompile = _require_search_from_serialized(1, mod, path, last) if !isa(doneprecompile, Bool) return # success - elseif doneprecompile === true || JLOptions().incremental != 0 - # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls - # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) - cachefile = compilecache(mod) - m = _require_from_serialized(1, mod, cachefile, last) - if !isa(m, Exception) - warn("Compilecache failed to create a usable precompiled cache file for module $name. Got:") - warn(m) - else - return # success - end end - # fall-through to attempting to load the source file end + # if the module being required was supposed to have a particular version + # but it was not handled by the precompile loader, complain + for (concrete_mod, concrete_uuid) in _concrete_dependencies + if mod === concrete_mod + warn("""Module $mod with uuid $concrete_uuid is missing from the cache. + This may mean module $mod does not support precompilation but is imported by a module that does.""") + end + end + + if doneprecompile === true || JLOptions().incremental != 0 + # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls + # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) + cachefile = compilecache(mod) + m = _require_from_serialized(1, mod, cachefile, last) + if isa(m, Exception) + warn("Compilecache failed to create a usable precompiled cache file for module $name. Got:") + warn(m, prefix="WARNING: ") + else + return # success + end + end + + # just load the file normally via include + # for unknown dependencies try if last && myid() == 1 && nprocs() > 1 # include on node 1 first to check for PrecompilableErrors @@ -397,10 +412,11 @@ function require(mod::Symbol) if doneprecompile === true || JLOptions().use_compilecache == 0 || !precompilableerror(ex, true) rethrow() # rethrow non-precompilable=true errors end + # the file requested `__precompile__`, so try to build a cache file and use that cachefile = compilecache(mod) m = _require_from_serialized(1, mod, cachefile, last) if isa(m, Exception) - warn(m) + warn(m, prefix="WARNING: ") error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.") end end @@ -408,7 +424,7 @@ function require(mod::Symbol) toplevel_load = last loading = pop!(package_locks, mod) notify(loading, all=true) - _track_dependencies[1] = old_track_dependencies + _track_dependencies[] = old_track_dependencies end nothing end @@ -452,7 +468,8 @@ task-local include path is set to the directory containing the file. Nested call in parallel, and files will be fetched from node 1. This function is typically used to load source interactively, or to combine files in packages that are broken into multiple source files. """ -function include_from_node1(_path::AbstractString) +include_from_node1(path::AbstractString) = include_from_node1(String(path)) +function include_from_node1(_path::String) path, prev = _include_dependency(_path) tls = task_local_storage() tls[:SOURCE_PATH] = path @@ -486,7 +503,7 @@ function evalfile(path::AbstractString, args::Vector{String}=String[]) end evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...]) -function create_expr_cache(input::AbstractString, output::AbstractString) +function create_expr_cache(input::String, output::String, concrete_deps::Vector{Any}) rm(output, force=true) # Remove file if it exists code_object = """ while !eof(STDIN) @@ -507,6 +524,9 @@ function create_expr_cache(input::AbstractString, output::AbstractString) append!(Base.LOAD_CACHE_PATH, $LOAD_CACHE_PATH) empty!(Base.DL_LOAD_PATH) append!(Base.DL_LOAD_PATH, $DL_LOAD_PATH) + empty!(Base._concrete_dependencies) + append!(Base._concrete_dependencies, $concrete_deps) + Base._track_dependencies[] = true end) source = source_path(nothing) if source !== nothing @@ -514,7 +534,6 @@ function create_expr_cache(input::AbstractString, output::AbstractString) task_local_storage()[:SOURCE_PATH] = $(source) end) end - serialize(io, :(Base._track_dependencies[1] = true)) serialize(io, :(Base.include($(abspath(input))))) if source !== nothing serialize(io, :(delete!(task_local_storage(), :SOURCE_PATH))) @@ -532,21 +551,38 @@ end compilecache(mod::Symbol) = compilecache(string(mod)) function compilecache(name::String) myid() == 1 || error("can only precompile from node 1") + # decide where to get the source file from path = find_in_path(name, nothing) path === nothing && throw(ArgumentError("$name not found in path")) + path = String(path) + # decide where to put the resulting cache file cachepath = LOAD_CACHE_PATH[1] if !isdir(cachepath) mkpath(cachepath) end - cachefile = abspath(cachepath, name*".ji") + cachefile::String = abspath(cachepath, name*".ji") + # build up the list of modules that we want the precompile process to preserve + concrete_deps = copy(_concrete_dependencies) + for existing in names(Main) + if isdefined(Main, existing) + mod = getfield(Main, existing) + if isa(mod, Module) && !(mod === Main || mod === Core || mod === Base) + mod = mod::Module + if module_parent(mod) === Main && module_name(mod) === existing + push!(concrete_deps, (existing, module_uuid(mod))) + end + end + end + end + # run the expression and cache the result if isinteractive() - if isfile(cachepath) + if isfile(cachefile) info("Recompiling stale cache file $cachefile for module $name.") else info("Precompiling module $name.") end end - if !success(create_expr_cache(path, cachefile)) + if !success(create_expr_cache(path, cachefile, concrete_deps)) error("Failed to precompile $name to $cachefile") end return cachefile @@ -556,26 +592,55 @@ module_uuid(m::Module) = ccall(:jl_module_uuid, UInt64, (Any,), m) isvalid_cache_header(f::IOStream) = 0 != ccall(:jl_read_verify_header, Cint, (Ptr{Void},), f.ios) -function cache_dependencies(f::IO) - modules = Tuple{Symbol,UInt64}[] - files = Tuple{String,Float64}[] +function parse_cache_header(f::IO) + modules = Dict{Symbol,UInt64}() while true n = ntoh(read(f, Int32)) n == 0 && break - push!(modules, - (Symbol(read(f, n)), # module symbol - ntoh(read(f, UInt64)))) # module UUID (timestamp) + sym = Symbol(read(f, n)) # module symbol + uuid = ntoh(read(f, UInt64)) # module UUID (mostly just a timestamp) + modules[sym] = uuid end - read(f, Int64) # total bytes for file dependencies + totbytes = ntoh(read(f, Int64)) # total bytes for file dependencies + # read the list of files + files = Tuple{String,Float64}[] while true n = ntoh(read(f, Int32)) n == 0 && break + totbytes -= 4 + n + 8 + if n < 0 # probably means this wasn't a valid file to be read by Base.parse_cache_header + error("EOF while reading cache header") + end push!(files, (String(read(f, n)), ntoh(read(f, Float64)))) end + @assert totbytes == 4 "header of cache file appears to be corrupt" return modules, files end -function cache_dependencies(cachefile::AbstractString) +function parse_cache_header(cachefile::String) + io = open(cachefile, "r") + try + !isvalid_cache_header(io) && throw(ArgumentError("invalid cache file $cachefile")) + return parse_cache_header(io) + finally + close(io) + end +end + +function cache_dependencies(f::IO) + defs, files = parse_cache_header(f) + modules = [] + while true + n = ntoh(read(f, Int32)) + n == 0 && break + sym = Symbol(read(f, n)) # module symbol + uuid = ntoh(read(f, UInt64)) # module UUID (mostly just a timestamp) + push!(modules, (sym, uuid)) + end + return modules, files +end + +function cache_dependencies(cachefile::String) io = open(cachefile, "r") try !isvalid_cache_header(io) && throw(ArgumentError("invalid cache file $cachefile")) @@ -585,13 +650,26 @@ function cache_dependencies(cachefile::AbstractString) end end -function stale_cachefile(modpath, cachefile) +function stale_cachefile(modpath::String, cachefile::String) io = open(cachefile, "r") try if !isvalid_cache_header(io) return true # invalid cache file end - modules, files = cache_dependencies(io) + modules, files = parse_cache_header(io) + + # check if this file is going to provide one of our concrete dependencies + provides_concrete = false + for (mod, uuid) in _concrete_dependencies + if get(modules, mod, UInt64(0)) === uuid + provides_concrete = true + else + return false # cachefile doesn't provide the required version of the dependency + end + end + provides_concrete && return false # this is the file we want + + # now check if this file is fresh relative to its source files if files[1][1] != modpath return true # cache file was compiled from a different path end diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index 4947aba6e6bd4..de0f3f9ea6b81 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -257,24 +257,33 @@ the statements in a module often involves compiling a large amount of code. Julia provides the ability to create precompiled versions of modules to reduce this time. -There are two mechanisms that can achieve this: -incremental compile and custom system image. - -To create a custom system image that can be used when starting Julia with the ``-J`` option, -recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules. - To create an incremental precompiled module file, add -``__precompile__()`` at the top of your module file (before the -``module`` starts). This will cause it to be automatically compiled -the first time it is imported. Alternatively, you can manually call -``Base.compilecache(modulename)``. The resulting cache files will be -stored in ``Base.LOAD_CACHE_PATH[1]``. Subsequently, the module is -automatically recompiled upon ``import`` whenever any of its -dependencies change; dependencies are modules it imports, the Julia -build, files it includes, or explicit dependencies declared by -``include_dependency(path)`` in the module file(s). Precompiling a -module also recursively precompiles any modules that are imported -therein. If you know that it is *not* safe to precompile your module +``__precompile__()`` at the top of your module file +(before the ``module`` starts). +This will cause it to be automatically compiled the first time it is imported. +Alternatively, you can manually call ``Base.compilecache(modulename)``. +The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``. +Subsequently, the module is automatically recompiled upon ``import`` +whenever any of its dependencies change; +dependencies are modules it imports, the Julia build, files it includes, +or explicit dependencies declared by ``include_dependency(path)`` in the module file(s). + +For file dependencies, a change is determined by examining whether the modification time (mtime) +of each file loaded by ``include`` or added explicity by ``include_dependency`` is unchanged, +or equal to the modification time truncated to the nearest second +(to accommodate systems that can't copy mtime with sub-second accuracy). +It also takes into account whether the path to the file chosen by the search logic in ``require`` +matches the path that had created the precompile file. + +It also takes into account the set of dependencies already loaded into the current process +and won't recompile those modules, even if their files change or disappear, +in order to avoid creating incompatibilities between the running system and the precompile cache. +If you want to have changes to the source reflected in the running system, +you should call ``reload("Module")`` on the module you changed, +and any module that depended on it in which you want to see the change reflected. + +Precompiling a module also recursively precompiles any modules that are imported therein. +If you know that it is *not* safe to precompile your module (for the reasons described below), you should put ``__precompile__(false)`` in the module file to cause ``Base.compilecache`` to throw an error (and thereby prevent the module from being imported by @@ -312,14 +321,17 @@ time) because the pointer address will change from run to run. You could accomplish this by defining the following ``__init__`` function in your module:: + const foo_data_ptr = Ref{Ptr{Void}}(0) function __init__() - ccall((:foo_init,:libfoo), Void, ()) - global const foo_data_ptr = ccall((:foo_data,:libfoo), Ptr{Void}, ()) + ccall((:foo_init, :libfoo), Void, ()) + foo_data_ptr[] = ccall((:foo_data, :libfoo), Ptr{Void}, ()) end Notice that it is perfectly possible to define a global inside a function like ``__init__``; this is one of the advantages of using a -dynamic language. +dynamic language. But by making it a constant at global scope, +we can ensure that the type is known to the compiler and allow it to generate +better optimized code. Obviously, any other globals in your module that depends on ``foo_data_ptr`` would also have to be initialized in ``__init__``. diff --git a/src/dump.c b/src/dump.c index 074428a19af58..3f16463d56afe 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1140,7 +1140,7 @@ static void write_mod_list(ios_t *s) } // "magic" string and version header of .ji file -static const int JI_FORMAT_VERSION = 2; +static const int JI_FORMAT_VERSION = 3; static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature static const uint16_t BOM = 0xFEFF; // byte-order marker static void write_header(ios_t *s) @@ -1157,6 +1157,22 @@ static void write_header(ios_t *s) ios_write(s, commit, strlen(commit)+1); } +// serialize information about the result of deserializing this file +static void write_work_list(ios_t *s) +{ + int i, l = jl_array_len(serializer_worklist); + for (i = 0; i < l; i++) { + jl_module_t *workmod = (jl_module_t*)jl_array_ptr_ref(serializer_worklist, i); + if (workmod->parent == jl_main_module) { + size_t l = strlen(jl_symbol_name(workmod->name)); + write_int32(s, l); + ios_write(s, jl_symbol_name(workmod->name), l); + write_uint64(s, workmod->uuid); + } + } + write_int32(s, 0); +} + // serialize the global _require_dependencies array of pathnames that // are include depenencies static void write_dependency_list(ios_t *s) @@ -2288,8 +2304,10 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) } serializer_worklist = worklist; write_header(&f); - write_mod_list(&f); // this can throw, keep it early (before any actual initialization) + write_work_list(&f); write_dependency_list(&f); + write_mod_list(&f); // this can return errors during deserialize, + // best to keep it early (before any actual initialization) arraylist_new(&reinit_list, 0); htable_new(&backref_table, 5000); @@ -2500,13 +2518,24 @@ static jl_value_t *_jl_restore_incremental(ios_t *f) return jl_get_exceptionf(jl_errorexception_type, "Precompile file header verification checks failed."); } + { // skip past the mod list + size_t len; + while ((len = read_int32(f))) + ios_skip(f, len + sizeof(uint64_t)); + } + { // skip past the dependency list + size_t deplen = read_uint64(f); + ios_skip(f, deplen); + } + + // verify that the system state is valid jl_value_t *verify_error = read_verify_mod_list(f); if (verify_error) { ios_close(f); return verify_error; } - size_t deplen = read_uint64(f); - ios_skip(f, deplen); // skip past the dependency list + + // prepare to deserialize arraylist_new(&backref_list, 4000); arraylist_push(&backref_list, jl_main_module); arraylist_new(&flagref_list, 0); diff --git a/src/module.c b/src/module.c index 6fbee0f969345..ac134d6e1ad5b 100644 --- a/src/module.c +++ b/src/module.c @@ -26,7 +26,9 @@ JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name) m->name = name; m->parent = NULL; m->istopmod = 0; - m->uuid = jl_hrtime(); + static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing + m->uuid = jl_hrtime() + (++mcounter); + if (!m->uuid) m->uuid++; // uuid 0 is invalid m->counter = 0; htable_new(&m->bindings, 0); arraylist_new(&m->usings, 0); diff --git a/test/compile.jl b/test/compile.jl index d639ac6cf5e46..f49de8f02d590 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -2,10 +2,18 @@ using Base.Test -function redirected_stderr() +function redirected_stderr(expected) rd, wr = redirect_stderr() - @async readstring(rd) # make sure the kernel isn't being forced to buffer the output - nothing + t = @async begin + read = readstring(rd) # also makes sure the kernel isn't being forced to buffer the output + if !contains(read, expected) + @show expected + @show read + @test false + end + nothing + end + return t end olderr = STDERR @@ -89,12 +97,14 @@ try # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: + t = redirected_stderr("WARNING: replacing module Foo4b3a94a1a081a8cb.\nWARNING: Method definition ") try - redirected_stderr() - @test nothing !== Base._require_from_serialized(myid(), Foo_module, #=broadcast-load=#false) + @test isa(Base._require_from_serialized(myid(), Foo_module, cachefile, #=broadcast-load=#false), Array{Any,1}) finally + close(STDERR) redirect_stderr(olderr) end + wait(t) let Foo = eval(Main, Foo_module) @test Foo.foo(17) == 18 @@ -104,10 +114,14 @@ try @test stringmime("text/plain", Base.Docs.doc(Foo.foo)) == "foo function\n" @test stringmime("text/plain", Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" - deps = Base.cache_dependencies(cachefile) - @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))), - [:Base,:Core,:Main]) - @test map(x -> x[1], sort(deps[2])) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")] + modules, deps = Base.parse_cache_header(cachefile) + @test modules == Dict(Foo_module => Base.module_uuid(Foo)) + @test map(x -> x[1], sort(deps)) == [Foo_file, joinpath(dir, "bar.jl"), joinpath(dir, "foo.jl")] + + modules, deps1 = Base.cache_dependencies(cachefile) + @test sort(modules) == map(s -> (s, Base.module_uuid(eval(s))), + [:Base, :Core, :Main]) + @test deps == deps1 @test current_task()(0x01, 0x4000, 0x30031234) == 2 @test nothing(0x01, 0x4000, 0x30031234) == 52 @@ -144,17 +158,28 @@ try end """) + t = redirected_stderr("ERROR: LoadError: __precompile__(false) is not allowed in files that are being precompiled\n in __precompile__") try - redirected_stderr() Base.compilecache("Baz") # from __precompile__(false) error("__precompile__ disabled test failed") catch exc + close(STDERR) redirect_stderr(olderr) isa(exc, ErrorException) || rethrow(exc) !isempty(search(exc.msg, "__precompile__(false)")) && rethrow(exc) end + wait(t) # Issue #12720 + FooBar1_file = joinpath(dir, "FooBar1.jl") + write(FooBar1_file, + """ + __precompile__(true) + module FooBar1 + using FooBar + end + """) + sleep(2) # give FooBar and FooBar1 different timestamps, in reverse order too FooBar_file = joinpath(dir, "FooBar.jl") write(FooBar_file, """ @@ -164,17 +189,64 @@ try """) Base.compilecache("FooBar") - sleep(2) @test isfile(joinpath(dir, "FooBar.ji")) + @test !Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) + @test !isdefined(Main, :FooBar) + @test !isdefined(Main, :FooBar1) - touch(FooBar_file) + @eval using FooBar + fb_uuid = Base.module_uuid(Main.FooBar) + sleep(2); touch(FooBar_file) insert!(Base.LOAD_CACHE_PATH, 1, dir2) - Base.recompile_stale(:FooBar, joinpath(dir, "FooBar.ji")) - sleep(2) + @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) + @eval using FooBar1 + @test !isfile(joinpath(dir2, "FooBar.ji")) + @test !isfile(joinpath(dir, "FooBar1.ji")) + @test isfile(joinpath(dir2, "FooBar1.ji")) + @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) + @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) + @test fb_uuid == Base.module_uuid(Main.FooBar) + fb_uuid1 = Base.module_uuid(Main.FooBar1) + @test fb_uuid != fb_uuid1 + + t = redirected_stderr("WARNING: replacing module FooBar.") + try + reload("FooBar") + finally + close(STDERR) + redirect_stderr(olderr) + end + wait(t) + @test fb_uuid != Base.module_uuid(Main.FooBar) + @test fb_uuid1 == Base.module_uuid(Main.FooBar1) + fb_uuid = Base.module_uuid(Main.FooBar) + @test isfile(joinpath(dir2, "FooBar.ji")) + @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) + @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) + @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji")) + + t = redirected_stderr(""" + WARNING: Deserialization checks failed while attempting to load cache from $(joinpath(dir2, "FooBar1.ji")). + WARNING: Module FooBar uuid did not match cache file. + WARNING: replacing module FooBar1. + """) + try + reload("FooBar1") + finally + close(STDERR) + redirect_stderr(olderr) + end + wait(t) + @test fb_uuid == Base.module_uuid(Main.FooBar) + @test fb_uuid1 != Base.module_uuid(Main.FooBar1) + @test isfile(joinpath(dir2, "FooBar.ji")) + @test isfile(joinpath(dir2, "FooBar1.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji")) @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji")) + @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) + # test behavior of precompile modules that throw errors write(FooBar_file, """ __precompile__(true) @@ -182,18 +254,20 @@ try error("break me") end """) - + t = redirected_stderr("ERROR: LoadError: break me\n in error") try - redirected_stderr() Base.require(:FooBar) error("\"LoadError: break me\" test failed") catch exc + close(STDERR) redirect_stderr(olderr) isa(exc, ErrorException) || rethrow(exc) !isempty(search(exc.msg, "ERROR: LoadError: break me")) && rethrow(exc) end + wait(t) finally if STDERR != olderr + close(STDERR) redirect_stderr(olderr) end splice!(Base.LOAD_CACHE_PATH, 1:2) @@ -249,7 +323,7 @@ end let module_name = string("a",randstring()) insert!(LOAD_PATH, 1, pwd()) file_name = string(module_name, ".jl") - touch(file_name) + sleep(2); touch(file_name) code = """module $(module_name)\nend\n""" write(file_name, code) reload(module_name) From 631c302d0ccccd6295a8004144c3272994bdf887 Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Mon, 22 Aug 2016 13:37:15 -0400 Subject: [PATCH 1044/1117] document __init__ behavior during precompile fix #18115 --- doc/manual/modules.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index de0f3f9ea6b81..9ccc2276a6695 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -299,12 +299,22 @@ initialization steps that must occur at *runtime* from steps that can occur at *compile time*. For this purpose, Julia allows you to define an ``__init__()`` function in your module that executes any initialization steps that must occur at runtime. +This function will not be called during compilation +(``--output-*`` or ``__precompile__()``). +You may, of course, call it manually if necessary, +but the default is to assume this function deals with computing state for +the local machine, which does not need to be -- or even should not be -- +captured in the compiled image. +It will be called after the module is loaded into a process, +including if it is being loaded into an incremental compile +(``--output-incremental=yes``), but not if it is being loaded +into a full-compilation process. In particular, if you define a ``function __init__()`` in a module, then Julia will call ``__init__()`` immediately *after* the module is loaded (e.g., by ``import``, ``using``, or ``require``) at runtime for the *first* time (i.e., ``__init__`` is only called once, and only -after all statements in the module have been executed). Because it is +after all statements in the module have been executed). Because it is called after the module is fully imported, any submodules or other imported modules have their ``__init__`` functions called *before* the ``__init__`` of the enclosing module. @@ -313,7 +323,7 @@ Two typical uses of ``__init__`` are calling runtime initialization functions of external C libraries and initializing global constants that involve pointers returned by external libraries. For example, suppose that we are calling a C library ``libfoo`` that requires us -to call a ``foo_init()`` initialization function at runtime. Suppose +to call a ``foo_init()`` initialization function at runtime. Suppose that we also want to define a global constant ``foo_data_ptr`` that holds the return value of a ``void *foo_data()`` function defined by ``libfoo`` — this constant must be initialized at runtime (not at compile From 517497945ed7d8c06f0590d0a565d9ecb591d3fe Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Wed, 24 Aug 2016 13:57:08 -0400 Subject: [PATCH 1045/1117] inliner needs to do method match tests early fix #18222 --- base/inference.jl | 51 ++++++++++++++++++++++++----------------------- test/inference.jl | 7 +++++++ 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 99b9e46ff56c7..0906187f8aa58 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2519,26 +2519,10 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference return invoke_NF() end - (linfo, ty, inferred) = typeinf(method, metharg, methsp, false) - if linfo === nothing || !inferred - return invoke_NF() - end - if linfo !== nothing && linfo.jlcall_api == 2 - # in this case function can be inlined to a constant - return inline_as_constant(linfo.constval, argexprs, enclosing) - elseif linfo !== nothing && !linfo.inlineable - return invoke_NF() - elseif linfo === nothing || linfo.code === nothing - (linfo, ty, inferred) = typeinf(method, metharg, methsp, true) - end - if linfo === nothing || !inferred || !linfo.inlineable || (ast = linfo.code) === nothing - return invoke_NF() - end - - na = linfo.nargs + na = method.lambda_template.nargs # check for vararg function isva = false - if na > 0 && linfo.isva + if na > 0 && method.lambda_template.isva @assert length(argexprs) >= na-1 # construct tuple-forming expression for argument tail vararg = mk_tuplecall(argexprs[na:end], sv) @@ -2554,17 +2538,34 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference @assert na == length(argexprs) - spvals = Any[] for i = 1:length(methsp) si = methsp[i] - if isa(si, TypeVar) - return NF - end - push!(spvals, si) + isa(si, TypeVar) && return NF + end + + (linfo, ty, inferred) = typeinf(method, metharg, methsp, false) + if linfo === nothing || !inferred + return invoke_NF() + end + if linfo !== nothing && linfo.jlcall_api == 2 + # in this case function can be inlined to a constant + return inline_as_constant(linfo.constval, argexprs, enclosing) + elseif linfo !== nothing && !linfo.inlineable + return invoke_NF() + elseif linfo === nothing || linfo.code === nothing + (linfo, ty, inferred) = typeinf(method, metharg, methsp, true) + end + if linfo === nothing || !inferred || !linfo.inlineable || (ast = linfo.code) === nothing + return invoke_NF() + end + + spvals = Any[] + for i = 1:length(methsp) + push!(spvals, methsp[i]) end - for i=1:length(spvals) + for i = 1:length(spvals) si = spvals[i] - if isa(si,Symbol) || isa(si,SSAValue) || isa(si,Slot) + if isa(si, Symbol) || isa(si, SSAValue) || isa(si, Slot) spvals[i] = QuoteNode(si) end end diff --git a/test/inference.jl b/test/inference.jl index a324716b2511b..6d61d6d657a4a 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -337,3 +337,10 @@ let tri = Triple18015(1, 2, 3) setabc18015!(tri, b18015(tri), c18015(tri), a18015(tri)) @test tri.a === 2 && tri.b === 3 && tri.c === 1 end + +# issue #18222 +f18222{T<:AbstractFloat}(::Union{T, Int}) = false +f18222(x) = true +g18222(x) = f18222(x) +@test f18222(1) == g18222(1) == true +@test f18222(1.0) == g18222(1.0) == false From 400f21e27c525c0ce16fec2a174f79cff4a5089c Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 24 Aug 2016 14:18:00 -0400 Subject: [PATCH 1046/1117] improve string docs to define String etc (#18216) * improve string docs to define String etc * tweak --- doc/manual/strings.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index e8e3c0813e843..de17e181bdc21 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -34,28 +34,29 @@ handle non-ASCII data is straightforward. There are a few noteworthy high-level features about Julia's strings: -- :obj:`AbstractString` is an abstraction, not a concrete type — many different - representations can implement the :obj:`AbstractString` interface, but they can - easily be used together and interact transparently. Any string type - can be used in any function expecting a :obj:`AbstractString`. +- The built-in concrete type used for strings (and string literals) in Julia is :obj:`String`. + This supports the full range of `Unicode <https://en.wikipedia.org/wiki/Unicode>`_ characters + via the `UTF-8 <https://en.wikipedia.org/wiki/UTF-8>`_ encoding. + (A :func:`transcode` function is provided to convert to/from other Unicode encodings.) +- All string types are subtypes of the abstract type :obj:`AbstractString`, + and external packages define additional :obj:`AbstractString` subtypes + (e.g. for other encodings). If you define a function expecting + a string argument, you should declare the type as :obj:`AbstractString` in + order to accept any string type. - Like C and Java, but unlike most dynamic languages, Julia has a first-class type representing a single character, called :obj:`Char`. This is just a special kind of 32-bit bitstype whose numeric value represents a Unicode code point. -- As in Java, strings are immutable: the value of a :obj:`AbstractString` object +- As in Java, strings are immutable: the value of an :obj:`AbstractString` object cannot be changed. To construct a different string value, you construct a new string from parts of other strings. - Conceptually, a string is a *partial function* from indices to - characters — for some index values, no character value is returned, + characters: for some index values, no character value is returned, and instead an exception is thrown. This allows for efficient indexing into strings by the byte index of an encoded representation rather than by a character index, which cannot be implemented both efficiently and simply for variable-width encodings of Unicode strings. -- Julia supports the full range of - `Unicode <https://en.wikipedia.org/wiki/Unicode>`_ characters: literal - strings are always `UTF-8 <https://en.wikipedia.org/wiki/UTF-8>`_ but - other encodings for strings from external sources can be supported. .. _man-characters: From 83e7a5c225d7b7ffc0676b0f1af94ccc06290108 Mon Sep 17 00:00:00 2001 From: Michael Hatherly <michaelhatherly@gmail.com> Date: Wed, 24 Aug 2016 22:30:39 +0200 Subject: [PATCH 1047/1117] Don't require blank line before markdown lists Fixes #11249 to follow the CommonMark spec for lists which does not require a blank line prior to the start of a list. --- base/markdown/Common/block.jl | 1 + test/markdown.jl | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/base/markdown/Common/block.jl b/base/markdown/Common/block.jl index 598096f1e4181..52a470b2241e9 100644 --- a/base/markdown/Common/block.jl +++ b/base/markdown/Common/block.jl @@ -263,6 +263,7 @@ isordered(list::List) = list.ordered >= 0 const BULLETS = r"^ {0,3}(\*|\+|-)( |$)" const NUM_OR_BULLETS = r"^ {0,3}(\*|\+|-|\d+(\.|\)))( |$)" +@breaking true -> function list(stream::IO, block::MD) withstream(stream) do bullet = startswith(stream, NUM_OR_BULLETS; eat = false) diff --git a/test/markdown.jl b/test/markdown.jl index ba517a8360c55..391e35e8e042d 100644 --- a/test/markdown.jl +++ b/test/markdown.jl @@ -166,6 +166,33 @@ let doc = md""" @test doc.content[2].items[3][1].content[1] == "zombie" end +let doc = Markdown.parse( + """ + A paragraph... + - one + - two + * three + * four + ... another paragraph. + """ + ) + + @test length(doc.content) === 3 + @test isa(doc.content[1], Markdown.Paragraph) + @test isa(doc.content[2], Markdown.List) + @test isa(doc.content[3], Markdown.Paragraph) + + @test length(doc.content[2].items) === 2 + @test doc.content[2].items[1][1].content[1] == "one" + @test length(doc.content[2].items[2]) == 2 + @test doc.content[2].items[2][1].content[1] == "two" + + @test isa(doc.content[2].items[2][2], Markdown.List) + @test length(doc.content[2].items[2][2].items) === 2 + @test doc.content[2].items[2][2].items[1][1].content[1] == "three" + @test doc.content[2].items[2][2].items[2][1].content[1] == "four" +end + @test md"Foo [bar]" == MD(Paragraph("Foo [bar]")) @test md"Foo [bar](baz)" != MD(Paragraph("Foo [bar](baz)")) @test md"Foo \[bar](baz)" == MD(Paragraph("Foo [bar](baz)")) From 9bfea055c58d13164c9e1bc4e5744ed09ced40cd Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Wed, 24 Aug 2016 17:50:49 -0400 Subject: [PATCH 1048/1117] fix typo in ##18205 --- src/ast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast.c b/src/ast.c index 0956723653b1c..fb24abe2cf281 100644 --- a/src/ast.c +++ b/src/ast.c @@ -195,7 +195,7 @@ value_t fl_invoke_julia_macro(fl_context_t *fl_ctx, value_t *args, uint32_t narg // arguments when lowering; should agree with broadcast.jl on what is a // scalar. When in doubt, return false, since this is only an optimization. // (TODO: update after #16966 is resolved.) -int fl_julia_scalar(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) +value_t fl_julia_scalar(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) { argcount(fl_ctx, "julia-scalar?", nargs, 1); if (fl_isnumber(fl_ctx, args[0])) From 57a6055c3f279c2bfedcdc547f4858882cb8da26 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Wed, 24 Aug 2016 21:05:58 -0700 Subject: [PATCH 1049/1117] Fix chol for Symmetric(A, :L). (#18142) --- base/linalg/cholesky.jl | 4 ++-- test/linalg/cholesky.jl | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index 246f1b3c8c140..46e7ac367514b 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -140,7 +140,7 @@ function chol(A::Hermitian) if A.uplo == 'U' copy!(AA, A.data) else - Base.ccopy!(AA, A.data) + Base.ctranspose!(AA, A.data) end chol!(Hermitian(AA, :U)) end @@ -150,7 +150,7 @@ function chol{T<:Real,S<:AbstractMatrix}(A::Symmetric{T,S}) if A.uplo == 'U' copy!(AA, A.data) else - Base.ccopy!(AA, A.data) + Base.ctranspose!(AA, A.data) end chol!(Hermitian(AA, :U)) end diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index 1c61f12060e5f..c747ff06a8f8a 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -25,7 +25,9 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) a = eltya == Int ? rand(1:7, n, n) : convert(Matrix{eltya}, eltya <: Complex ? complex(areal, aimg) : areal) a2 = eltya == Int ? rand(1:7, n, n) : convert(Matrix{eltya}, eltya <: Complex ? complex(a2real, a2img) : a2real) apd = a'*a # symmetric positive-definite - apds = Symmetric(apd) + + apds = Symmetric(apd) + apdsL = Symmetric(apd, :L) ε = εa = eps(abs(float(one(eltya)))) @inferred cholfact(apd) @@ -78,11 +80,15 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test tril(lapd.factors) ≈ capd[:L] if eltya <: Real capds = cholfact(apds) - lapds = cholfact(Symmetric(apds.data, :L)) + lapds = cholfact(apdsL) + cl = chol(apdsL) ls = lapds[:L] @test ls*ls' ≈ apd @test triu(capds.factors) ≈ lapds[:U] @test tril(lapds.factors) ≈ capds[:L] + @test istriu(cl) + @test cl'cl ≈ apds + @test cl'cl ≈ apdsL end #pivoted upper Cholesky From c00113a86242306d7921f94b99d1d0e92ec9bdc1 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Wed, 24 Aug 2016 21:07:25 -0700 Subject: [PATCH 1050/1117] Don't throw in eigs on semidefinite Bs for generalized eigenproblems. (#17873) This makes it possible to solve problems with semidefinite B via explicit shift. --- base/linalg/arnoldi.jl | 9 +++------ test/linalg/arnoldi.jl | 10 +++++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index baf05f4098126..7b1e9bd6036bd 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -160,7 +160,7 @@ function _eigs(A, B; T = eltype(A) iscmplx = T <: Complex isgeneral = B !== I - sym = issymmetric(A) && !iscmplx + sym = issymmetric(A) && issymmetric(B) && !iscmplx nevmax=sym ? n-1 : n-2 if nevmax <= 0 throw(ArgumentError("Input matrix A is too small. Use eigfact instead.")) @@ -178,9 +178,6 @@ function _eigs(A, B; ncv = ncvmin end ncv = BlasInt(min(ncv, n)) - if isgeneral && !isposdef(B) - throw(PosDefException(0)) - end bmat = isgeneral ? "G" : "I" isshift = sigma !== nothing @@ -251,7 +248,7 @@ function _eigs(A, B; solveSI = x->x else # Shift-invert mode mode = 3 - F = factorize(sigma==zero(T) ? A : A - UniformScaling(sigma)) + F = factorize(A - UniformScaling(sigma)) solveSI = x -> F \ x end else # Generalized eigenproblem @@ -262,7 +259,7 @@ function _eigs(A, B; solveSI = x -> F \ x else # Shift-invert mode mode = 3 - F = factorize(sigma==zero(T) ? A : A-sigma*B) + F = factorize(A - sigma*B) solveSI = x -> F \ x end end diff --git a/test/linalg/arnoldi.jl b/test/linalg/arnoldi.jl index 81dc9217972ef..61bdf485ec849 100644 --- a/test/linalg/arnoldi.jl +++ b/test/linalg/arnoldi.jl @@ -72,7 +72,6 @@ let @test_throws ArgumentError eigs(a+a.',which=:LI) @test_throws ArgumentError eigs(a,sigma=rand(Complex64)) end - @test_throws Base.LinAlg.PosDefException eigs(a,b) end end @@ -232,3 +231,12 @@ svds(rand(1:10, 10, 8)) @test_throws MethodError eigs(big(rand(1:10, 10, 10))) @test_throws MethodError eigs(big(rand(1:10, 10, 10)), rand(1:10, 10, 10)) @test_throws MethodError svds(big(rand(1:10, 10, 8))) + +# Symmetric generalized with singular B +let + n = 10 + k = 3 + A = randn(n,n); A = A'A + B = randn(n,k); B = B*B' + @test sort(eigs(A, B, nev = k, sigma = 1.0)[1]) ≈ sort(eigvals(A, B)[1:k]) +end From e664560cfc9223c38c6f7674e84c7008c825c712 Mon Sep 17 00:00:00 2001 From: Keno Fischer <keno@juliacomputing.com> Date: Wed, 17 Aug 2016 18:00:03 +0000 Subject: [PATCH 1051/1117] In cgmemmgr, allow writing permissions on the read page for the debugger The permissions specified in MapViewOfFile specify the maximum permissions the file can ever have, which means that even permissions bypass by the debugger is not allowed to write it. Instead, set the maximum permissions to RWX and VirtualProtect down to the permissions we need. This is the same behavior as we use on Mac/Linux. --- src/cgmemmgr.cpp | 9 +++++++-- test/misc.jl | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/cgmemmgr.cpp b/src/cgmemmgr.cpp index a85dfa57a227b..62d12229a07e9 100644 --- a/src/cgmemmgr.cpp +++ b/src/cgmemmgr.cpp @@ -141,13 +141,18 @@ static intptr_t init_shared_map() static void *alloc_shared_page(size_t size, size_t *id, bool exec) { assert(size % jl_page_size == 0); - auto file_mode = exec ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE; + DWORD file_mode = exec ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE; HANDLE hdl = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, file_mode, 0, size, NULL); *id = (size_t)hdl; - auto map_mode = FILE_MAP_READ | (exec ? FILE_MAP_EXECUTE : 0); + // We set the maximum permissions for this to the maximum for this file, and then + // VirtualProtect, such that the debugger can still access these + // pages and set breakpoints if it wants to. + DWORD map_mode = FILE_MAP_ALL_ACCESS | (exec ? FILE_MAP_EXECUTE : 0); void *addr = MapViewOfFile(hdl, map_mode, 0, 0, size); assert(addr && "Cannot map RO view"); + DWORD protect_mode = exec ? PAGE_EXECUTE_READ : PAGE_READONLY; + VirtualProtect(addr, size, protect_mode, &file_mode); return addr; } #else // _OS_WINDOWS_ diff --git a/test/misc.jl b/test/misc.jl index 114c0f2b9079f..a5f89d8512cf7 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -435,3 +435,18 @@ let creds = Base.LibGit2.CachedCredentials() securezero!(creds) @test LibGit2.get_creds!(creds, "foo", nothing).pass == "\0\0\0" end + +# Test that we can VirtualProtect jitted code to writable +if is_windows() + @noinline function WeVirtualProtectThisToRWX(x, y) + x+y + end + + let addr = cfunction(WeVirtualProtectThisToRWX, UInt64, (UInt64, UInt64)) + addr = addr-(UInt64(addr)%4096) + const PAGE_EXECUTE_READWRITE = 0x40 + oldPerm = Ref{UInt32}() + err = ccall(:VirtualProtect,stdcall,Cint,(Ptr{Void}, Csize_t, UInt32, Ptr{UInt32}), addr, 4096, PAGE_EXECUTE_READWRITE, oldPerm) + err == 0 && error(Libc.GetLastError()) + end +end From a9e3ba69fcaf1ccb2a785be27b7c810184b5a62a Mon Sep 17 00:00:00 2001 From: rened <github@donner.at> Date: Thu, 25 Aug 2016 09:12:52 +0200 Subject: [PATCH 1052/1117] add missing "Threads." in example (#18219) --- doc/manual/parallel-computing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 588a364ff997a..d61d7787f2b20 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -1056,7 +1056,7 @@ Julia supports parallel loops using the :obj:`Threads.@threads` macro. This macr of a ``for`` loop to indicate to Julia that the loop is a multi-threaded region. :: Threads.@threads for i = 1:10 - a[i] = threadid() + a[i] = Threads.threadid() end The iteration space is split amongst the threads, after which each thread writes its thread ID to its assigned locations.:: From 6e3d16b06271cc5e17720d429c766c15a57b92f2 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <Sacha0@users.noreply.github.com> Date: Thu, 25 Aug 2016 09:43:20 -0700 Subject: [PATCH 1053/1117] Deprecate methods that convert from Diagonal and Bidiagonal to <:AbstractTriangular. Remove tests of those convert methods. (#17723) --- base/deprecated.jl | 58 +++++++++++++++++++++++++++++++++++++++++ base/linalg/diagonal.jl | 2 -- base/linalg/special.jl | 20 ++------------ test/linalg/special.jl | 18 +++++-------- 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index cad9ea951bd59..151709861bb00 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -794,4 +794,62 @@ end @deprecate (-)(J::UniformScaling, x::Number) J.λ - x @deprecate (-)(x::Number, J::UniformScaling) x - J.λ +# Deprecate methods that convert Diagonal and Bidiagonal to <:AbstractTriangular. +function convert(::Type{UpperTriangular}, A::Diagonal) + depwarn(string("`convert(::Type{UpperTriangular}, A::Diagonal)` and other methods ", + "that convert `Diagonal`/`Bidiagonal` to `<:AbstractTriangular` are deprecated. ", + "Consider calling the `UpperTriangular` constructor directly ", + "(`UpperTriangular(A)`) instead."), :convert) + UpperTriangular(A) +end +function convert(::Type{LowerTriangular}, A::Diagonal) + depwarn(string("`convert(::Type{LowerTriangular}, A::Diagonal)` and other methods ", + "that convert `Diagonal`/`Bidiagonal` to `<:AbstractTriangular` are deprecated. ", + "Consider calling the `LowerTriangular` constructor directly ", + "(`LowerTriangular(A)`) instead."), :convert) + LowerTriangular(A) +end +function convert(::Type{Base.LinAlg.UnitUpperTriangular}, A::Diagonal) + depwarn(string("`convert(::Type{UnitUpperTriangular}, A::Diagonal)` and other methods ", + "that convert `Diagonal`/`Bidiagonal` to `<:AbstractTriangular` are deprecated. ", + "Consider calling the `UnitUpperTriangular` constructor directly ", + "(`Base.LinAlg.UnitUpperTriangular(A)`) instead."), :convert) + if !all(A.diag .== one(eltype(A))) + throw(ArgumentError("matrix cannot be represented as UnitUpperTriangular")) + end + Base.LinAlg.UnitUpperTriangular(full(A)) +end +function convert(::Type{Base.LinAlg.UnitLowerTriangular}, A::Diagonal) + depwarn(string("`convert(::Type{UnitLowerTriangular}, A::Diagonal)` and other methods ", + "that convert `Diagonal`/`Bidiagonal` to `<:AbstractTriangular` are deprecated. ", + "Consider calling the `UnitLowerTriangular` constructor directly ", + "(`Base.LinAlg.UnitLowerTriangular(A)`) instead."), :convert) + if !all(A.diag .== one(eltype(A))) + throw(ArgumentError("matrix cannot be represented as UnitLowerTriangular")) + end + Base.LinAlg.UnitLowerTriangular(full(A)) +end +function convert(::Type{LowerTriangular}, A::Bidiagonal) + depwarn(string("`convert(::Type{LowerTriangular}, A::Bidiagonal)` and other methods ", + "that convert `Diagonal`/`Bidiagonal` to `<:AbstractTriangular` are deprecated. ", + "Consider calling the `LowerTriangular` constructor directly (`LowerTriangular(A)`) ", + "instead."), :convert) + if !A.isupper + LowerTriangular(full(A)) + else + throw(ArgumentError("Bidiagonal matrix must have lower off diagonal to be converted to LowerTriangular")) + end +end +function convert(::Type{UpperTriangular}, A::Bidiagonal) + depwarn(string("`convert(::Type{UpperTriangular}, A::Bidiagonal)` and other methods ", + "that convert `Diagoinal`/`Bidiagonal` to `<:AbstractTriangular` are deprecated. ", + "Consider calling the `UpperTriangular` constructor directly (`UpperTriangular(A)`) ", + "instead."), :convert) + if A.isupper + UpperTriangular(full(A)) + else + throw(ArgumentError("Bidiagonal matrix must have upper off diagonal to be converted to UpperTriangular")) + end +end + # End deprecations scheduled for 0.6 diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 0d377c4b9308f..6dd6a9b735c22 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -21,8 +21,6 @@ Diagonal(V::AbstractVector) = Diagonal(collect(V)) convert{T}(::Type{Diagonal{T}}, D::Diagonal{T}) = D convert{T}(::Type{Diagonal{T}}, D::Diagonal) = Diagonal{T}(convert(Vector{T}, D.diag)) convert{T}(::Type{AbstractMatrix{T}}, D::Diagonal) = convert(Diagonal{T}, D) -convert{T}(::Type{UpperTriangular}, A::Diagonal{T}) = UpperTriangular(A) -convert{T}(::Type{LowerTriangular}, A::Diagonal{T}) = LowerTriangular(A) convert(::Type{Matrix}, D::Diagonal) = diagm(D.diag) convert(::Type{Array}, D::Diagonal) = convert(Matrix, D) full(D::Diagonal) = convert(Array, D) diff --git a/base/linalg/special.jl b/base/linalg/special.jl index f70d67130807b..69b13517045f5 100644 --- a/base/linalg/special.jl +++ b/base/linalg/special.jl @@ -6,22 +6,6 @@ convert{T}(::Type{Bidiagonal}, A::Diagonal{T})=Bidiagonal(A.diag, zeros(T, size(A.diag,1)-1), true) convert{T}(::Type{SymTridiagonal}, A::Diagonal{T})=SymTridiagonal(A.diag, zeros(T, size(A.diag,1)-1)) convert{T}(::Type{Tridiagonal}, A::Diagonal{T})=Tridiagonal(zeros(T, size(A.diag,1)-1), A.diag, zeros(T, size(A.diag,1)-1)) -convert(::Type{LowerTriangular}, A::Bidiagonal) = !A.isupper ? LowerTriangular(full(A)) : throw(ArgumentError("Bidiagonal matrix must have lower off diagonal to be converted to LowerTriangular")) -convert(::Type{UpperTriangular}, A::Bidiagonal) = A.isupper ? UpperTriangular(full(A)) : throw(ArgumentError("Bidiagonal matrix must have upper off diagonal to be converted to UpperTriangular")) - -function convert(::Type{UnitUpperTriangular}, A::Diagonal) - if !all(A.diag .== one(eltype(A))) - throw(ArgumentError("matrix cannot be represented as UnitUpperTriangular")) - end - UnitUpperTriangular(full(A)) -end - -function convert(::Type{UnitLowerTriangular}, A::Diagonal) - if !all(A.diag .== one(eltype(A))) - throw(ArgumentError("matrix cannot be represented as UnitLowerTriangular")) - end - UnitLowerTriangular(full(A)) -end function convert(::Type{Diagonal}, A::Union{Bidiagonal, SymTridiagonal}) if !all(A.ev .== 0) @@ -147,8 +131,8 @@ for op in (:+, :-) (:LowerTriangular,:LowerTriangular), (:UnitLowerTriangular,:LowerTriangular)) @eval begin - ($op)(A::($matrixtype1), B::($matrixtype2)) = ($op)(convert(($matrixtype3), A), B) - ($op)(A::($matrixtype2), B::($matrixtype1)) = ($op)(A, convert(($matrixtype3), B)) + ($op)(A::($matrixtype1), B::($matrixtype2)) = ($op)(($matrixtype3)(A), B) + ($op)(A::($matrixtype2), B::($matrixtype1)) = ($op)(A, ($matrixtype3)(B)) end end end diff --git a/test/linalg/special.jl b/test/linalg/special.jl index 05105acd149a8..6194dcdbbafbb 100644 --- a/test/linalg/special.jl +++ b/test/linalg/special.jl @@ -9,30 +9,30 @@ srand(1) debug && println("Test interconversion between special matrix types") let a=[1.0:n;] A=Diagonal(a) - for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Tridiagonal, LowerTriangular, UpperTriangular, Matrix] + for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Tridiagonal, Matrix] debug && println("newtype is $(newtype)") @test full(convert(newtype, A)) == full(A) end - for newtype in [Base.LinAlg.UnitUpperTriangular, Base.LinAlg.UnitLowerTriangular] - @test_throws ArgumentError convert(newtype, A) - @test full(convert(newtype, Diagonal(ones(n)))) == eye(n) - end for isupper in (true, false) debug && println("isupper is $(isupper)") A=Bidiagonal(a, [1.0:n-1;], isupper) - for newtype in [Bidiagonal, Tridiagonal, isupper ? UpperTriangular : LowerTriangular, Matrix] + for newtype in [Bidiagonal, Tridiagonal, Matrix] debug && println("newtype is $(newtype)") @test full(convert(newtype, A)) == full(A) @test full(newtype(A)) == full(A) end @test_throws ArgumentError convert(SymTridiagonal, A) + tritype = isupper ? UpperTriangular : LowerTriangular + @test full(tritype(A)) == full(A) + A=Bidiagonal(a, zeros(n-1), isupper) #morally Diagonal - for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Tridiagonal, isupper ? UpperTriangular : LowerTriangular, Matrix] + for newtype in [Diagonal, Bidiagonal, SymTridiagonal, Tridiagonal, Matrix] debug && println("newtype is $(newtype)") @test full(convert(newtype, A)) == full(A) @test full(newtype(A)) == full(A) end + @test full(tritype(A)) == full(A) end A = SymTridiagonal(a, [1.0:n-1;]) @@ -78,10 +78,6 @@ let a=[1.0:n;] for newtype in [Diagonal, Bidiagonal, Tridiagonal, SymTridiagonal] @test_throws ArgumentError convert(newtype,A) end - A = Diagonal(a) - for newtype in [UpperTriangular, LowerTriangular] - @test full(convert(newtype,A)) == full(A) - end end # Binary ops among special types From da52f96c7d9c5298f324b7fece4e7a0e1b43f984 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Thu, 25 Aug 2016 13:11:52 -0700 Subject: [PATCH 1054/1117] hopefully unbreak osx travis llvm linking (#18241) ref https://github.com/staticfloat/homebrew-julia/commit/f57dcc20464c9ce4758a6fca088164a591daf1cd#diff-633fd9f8448015391714f35e04d39b5bL154 [av skip] --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1f7ff08e6711f..56ef0a5351d90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,7 +80,7 @@ before_install: brew rm --force $(brew deps --HEAD julia); brew install -v staticfloat/juliadeps/libgfortran; brew install -v --only-dependencies --HEAD julia; - BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm37-julia)/bin/llvm-config-3.7.1 LLVM_SIZE=$(brew --prefix llvm37-julia)/bin/llvm-size-3.7.1"; + BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm37-julia)/bin/llvm-config-3.7 LLVM_SIZE=$(brew --prefix llvm37-julia)/bin/llvm-size-3.7"; BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1"; BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas"; for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND; do From d4def98503869d90c96d88c728fd958485a2c76a Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Sat, 2 Jul 2016 09:27:14 -0700 Subject: [PATCH 1055/1117] Rewrite vectorized unary functions over SparseMatrixCSCs, leveraging higher order functions and multiple dispatch to displace eval. Fixes some apparent type instabilities. --- base/sparse/sparsematrix.jl | 218 +++++++++++++++++------------------- 1 file changed, 101 insertions(+), 117 deletions(-) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index d8d3fa47d0f42..61a28a0207c0d 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1305,134 +1305,118 @@ end ## Unary arithmetic and boolean operators -macro _unary_op_nz2z_z2z(op,A,Tv,Ti) - esc(quote - nfilledA = nnz($A) - colptrB = Array{$Ti}($A.n+1) - rowvalB = Array{$Ti}(nfilledA) - nzvalB = Array{$Tv}(nfilledA) - - nzvalA = $A.nzval - colptrA = $A.colptr - rowvalA = $A.rowval - - k = 0 # number of additional zeros introduced by op(A) - @inbounds for i = 1 : $A.n - colptrB[i] = colptrA[i] - k - for j = colptrA[i] : colptrA[i+1]-1 - opAj = $(op)(nzvalA[j]) - if opAj == 0 - k += 1 - else - rowvalB[j - k] = rowvalA[j] - nzvalB[j - k] = opAj - end - end - end - colptrB[end] = $A.colptr[end] - k - deleteat!(rowvalB, colptrB[end]:nfilledA) - deleteat!(nzvalB, colptrB[end]:nfilledA) - return SparseMatrixCSC($A.m, $A.n, colptrB, rowvalB, nzvalB) - end) # quote -end - -# Operations that may map nonzeros to zero, and zero to zero -# Result is sparse -for op in (:ceil, :floor, :trunc, :round, - :sin, :tan, :asin, :atan, - :sinh, :tanh, :asinh, :atanh, - :sinpi, :cosc, - :sind, :tand, :asind, :atand) - @eval begin - $(op){Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = @_unary_op_nz2z_z2z($op,A,Tv,Ti) - end # quote -end # macro - -for op in (:real, :imag) - @eval begin - ($op){Tv<:Complex,Ti}(A::SparseMatrixCSC{Tv,Ti}) = @_unary_op_nz2z_z2z($op,A,Tv.parameters[1],Ti) - end # quote -end # macro -real{Tv<:Number,Ti}(A::SparseMatrixCSC{Tv,Ti}) = copy(A) -imag{Tv<:Number,Ti}(A::SparseMatrixCSC{Tv,Ti}) = spzeros(Tv, Ti, A.m, A.n) - -for op in (:ceil, :floor, :trunc, :round) - @eval begin - ($op){T,Tv,Ti}(::Type{T},A::SparseMatrixCSC{Tv,Ti}) = @_unary_op_nz2z_z2z($op,A,T,Ti) - end # quote -end # macro - - -# Operations that map nonzeros to nonzeros, and zeros to zeros -# Result is sparse -for op in (:-, :log1p, :expm1) - @eval begin - - function ($op)(A::SparseMatrixCSC) - B = similar(A) - nzvalB = B.nzval - nzvalA = A.nzval - @simd for i=1:length(nzvalB) - @inbounds nzvalB[i] = ($op)(nzvalA[i]) - end - return B - end - - end -end - -function abs{Tv<:Complex,Ti}(A::SparseMatrixCSC{Tv,Ti}) - T = Tv.parameters[1] - (T <: Integer) && (T = (T <: BigInt) ? BigFloat : Float64) - @_unary_op_nz2z_z2z(abs,A,T,Ti) -end -abs2{Tv<:Complex,Ti}(A::SparseMatrixCSC{Tv,Ti}) = @_unary_op_nz2z_z2z(abs2,A,Tv.parameters[1],Ti) -for op in (:abs, :abs2) - @eval begin - function ($op){Tv<:Number,Ti}(A::SparseMatrixCSC{Tv,Ti}) - B = similar(A) - nzvalB = B.nzval - nzvalA = A.nzval - @simd for i=1:length(nzvalB) - @inbounds nzvalB[i] = ($op)(nzvalA[i]) +""" +Helper macro for the unary broadcast definitions below. Takes parent method `fp` and a set +of desired child methods `fcs`, and builds an expression defining each of the child methods +such that `fc(A::SparseMatrixCSC) = fp(fc, A)`. +""" +macro _enumerate_childmethods(fp, fcs...) + fcexps = Expr(:block) + for fc in fcs + push!(fcexps.args, :( $(esc(fc))(A::SparseMatrixCSC) = $(esc(fp))($(esc(fc)), A) ) ) + end + return fcexps +end + +# Operations that map zeros to zeros and may map nonzeros to zeros, yielding a sparse matrix +""" +Takes unary function `f` that maps zeros to zeros and may map nonzeros to zeros, and returns +a new `SparseMatrixCSC{TiA,TvB}` `B` generated by applying `f` to each nonzero entry in +`A` and retaining only the resulting nonzeros. +""" +function _broadcast_unary_nz2z_z2z_T{TvA,TiA,TvB}(f::Function, A::SparseMatrixCSC{TvA,TiA}, ::Type{TvB}) + Bcolptr = Array{TiA}(A.n + 1) + Browval = Array{TiA}(nnz(A)) + Bnzval = Array{TvB}(nnz(A)) + Bk = 1 + @inbounds for j in 1:A.n + Bcolptr[j] = Bk + for Ak in nzrange(A, j) + x = f(A.nzval[Ak]) + if x != 0 + Browval[Bk] = A.rowval[Ak] + Bnzval[Bk] = x + Bk += 1 end - return B end end -end - + Bcolptr[A.n + 1] = Bk + resize!(Browval, Bk - 1) + resize!(Bnzval, Bk - 1) + return SparseMatrixCSC(A.m, A.n, Bcolptr, Browval, Bnzval) +end +function _broadcast_unary_nz2z_z2z{Tv}(f::Function, A::SparseMatrixCSC{Tv}) + _broadcast_unary_nz2z_z2z_T(f, A, Tv) +end +@_enumerate_childmethods(_broadcast_unary_nz2z_z2z, + sin, sinh, sind, asin, asinh, asind, + tan, tanh, tand, atan, atanh, atand, + sinpi, cosc, ceil, floor, trunc, round) +real(A::SparseMatrixCSC) = copy(A) +imag{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = spzeros(Tv, Ti, A.m, A.n) +real{TTv}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2z_z2z_T(real, A, TTv) +imag{TTv}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2z_z2z_T(imag, A, TTv) +ceil{To}(::Type{To}, A::SparseMatrixCSC) = _broadcast_unary_nz2z_z2z_T(ceil, A, To) +floor{To}(::Type{To}, A::SparseMatrixCSC) = _broadcast_unary_nz2z_z2z_T(floor, A, To) +trunc{To}(::Type{To}, A::SparseMatrixCSC) = _broadcast_unary_nz2z_z2z_T(trunc, A, To) +round{To}(::Type{To}, A::SparseMatrixCSC) = _broadcast_unary_nz2z_z2z_T(round, A, To) + +# Operations that map zeros to zeros and map nonzeros to nonzeros, yielding a sparse matrix +""" +Takes unary function `f` that maps zeros to zeros and nonzeros to nonzeros, and returns a +new `SparseMatrixCSC{TiA,TvB}` `B` generated by applying `f` to each nonzero entry in `A`. +""" +function _broadcast_unary_nz2nz_z2z_T{TvA,TiA,TvB}(f::Function, A::SparseMatrixCSC{TvA,TiA}, ::Type{TvB}) + Bcolptr = Vector{TiA}(A.n + 1) + Browval = Vector{TiA}(nnz(A)) + Bnzval = Vector{TvB}(nnz(A)) + copy!(Bcolptr, 1, A.colptr, 1, A.n + 1) + copy!(Browval, 1, A.rowval, 1, nnz(A)) + @inbounds @simd for k in 1:nnz(A) + Bnzval[k] = f(A.nzval[k]) + end + return SparseMatrixCSC(A.m, A.n, Bcolptr, Browval, Bnzval) +end +function _broadcast_unary_nz2nz_z2z{Tv}(f::Function, A::SparseMatrixCSC{Tv}) + _broadcast_unary_nz2nz_z2z_T(f, A, Tv) +end +@_enumerate_childmethods(_broadcast_unary_nz2nz_z2z, + log1p, expm1, abs, abs2, conj) +abs2{TTv}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs2, A, TTv) +abs{TTv}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, TTv) +abs{TTv<:Integer}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, Float64) +abs{TTv<:BigInt}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, BigFloat) function conj!(A::SparseMatrixCSC) - nzvalA = A.nzval - @simd for i=1:length(nzvalA) - @inbounds nzvalA[i] = conj(nzvalA[i]) + @inbounds @simd for k in 1:nnz(A) + A.nzval[k] = conj(A.nzval[k]) end return A end -conj(A::SparseMatrixCSC) = conj!(copy(A)) - -# Operations that map nonzeros to nonzeros, and zeros to nonzeros -# Result is dense -for op in (:cos, :cosh, :acos, :sec, :csc, :cot, :acot, :sech, - :csch, :coth, :asech, :acsch, :cospi, :sinc, :cosd, - :cotd, :cscd, :secd, :acosd, :acotd, :log, :log2, :log10, - :exp, :exp2, :exp10) - @eval begin - - function ($op){Tv}(A::SparseMatrixCSC{Tv}) - B = fill($(op)(zero(Tv)), size(A)) - @inbounds for col = 1 : A.n - for j = A.colptr[col] : A.colptr[col+1]-1 - row = A.rowval[j] - nz = A.nzval[j] - B[row,col] = $(op)(nz) - end - end - return B +# Operations that map both zeros and nonzeros to zeros, yielding a dense matrix +""" +Takes unary function `f` that maps both zeros and nonzeros to nonzeros, constructs a new +`Matrix{TvB}` `B`, populates all entries of `B` with the result of a single `f(one(zero(Tv)))` +call, and then, for each stored entry in `A`, calls `f` on the entry's value and stores the +result in the corresponding location in `B`. +""" +function _broadcast_unary_nz2nz_z2nz{Tv}(f::Function, A::SparseMatrixCSC{Tv}) + B = fill(f(zero(Tv)), size(A)) + @inbounds for j in 1:A.n + for k in nzrange(A, j) + i = A.rowval[k] + x = A.nzval[k] + B[i,j] = f(x) end - end + return B end +@_enumerate_childmethods(_broadcast_unary_nz2nz_z2nz, + log, log2, log10, exp, exp2, exp10, sinc, cospi, + cos, cosh, cosd, acos, acosd, + cot, coth, cotd, acot, acotd, + sec, sech, secd, asech, + csc, csch, cscd, acsch) ## Broadcasting kernels specialized for returning a SparseMatrixCSC From 69dfbd3fbb184e1b6451f288e6e915650c046bdd Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Sun, 3 Jul 2016 16:38:41 -0700 Subject: [PATCH 1056/1117] Transition vectorized unary functions over `SparseMatrixCSC`s to compact broadcast syntax, accordingly revise and expand the associated tests, and add deprecations for the vectorized syntax. --- base/deprecated.jl | 13 +++++++ base/sparse/sparsematrix.jl | 22 ++++++------ test/sparsedir/sparse.jl | 67 ++++++++++++++++++++++--------------- 3 files changed, 64 insertions(+), 38 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 151709861bb00..667bd10c5c4e7 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -852,4 +852,17 @@ function convert(::Type{UpperTriangular}, A::Bidiagonal) end end +# Deprecate vectorized unary functions over sparse matrices in favor of compact broadcast syntax (#17265). +for f in (:sin, :sinh, :sind, :asin, :asinh, :asind, + :tan, :tanh, :tand, :atan, :atanh, :atand, + :sinpi, :cosc, :ceil, :floor, :trunc, :round, :real, :imag, + :log1p, :expm1, :abs, :abs2, :conj, + :log, :log2, :log10, :exp, :exp2, :exp10, :sinc, :cospi, + :cos, :cosh, :cosd, :acos, :acosd, + :cot, :coth, :cotd, :acot, :acotd, + :sec, :sech, :secd, :asech, + :csc, :csch, :cscd, :acsch) + @eval @deprecate $f(A::SparseMatrixCSC) $f.(A) +end + # End deprecations scheduled for 0.6 diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 61a28a0207c0d..87cb03485d304 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1308,12 +1308,12 @@ end """ Helper macro for the unary broadcast definitions below. Takes parent method `fp` and a set of desired child methods `fcs`, and builds an expression defining each of the child methods -such that `fc(A::SparseMatrixCSC) = fp(fc, A)`. +such that `broadcast(::typeof(fc), A::SparseMatrixCSC) = fp(fc, A)`. """ macro _enumerate_childmethods(fp, fcs...) fcexps = Expr(:block) for fc in fcs - push!(fcexps.args, :( $(esc(fc))(A::SparseMatrixCSC) = $(esc(fp))($(esc(fc)), A) ) ) + push!(fcexps.args, :( broadcast(::typeof($(esc(fc))), A::SparseMatrixCSC) = $(esc(fp))($(esc(fc)), A) ) ) end return fcexps end @@ -1352,10 +1352,10 @@ end sin, sinh, sind, asin, asinh, asind, tan, tanh, tand, atan, atanh, atand, sinpi, cosc, ceil, floor, trunc, round) -real(A::SparseMatrixCSC) = copy(A) -imag{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}) = spzeros(Tv, Ti, A.m, A.n) -real{TTv}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2z_z2z_T(real, A, TTv) -imag{TTv}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2z_z2z_T(imag, A, TTv) +broadcast(::typeof(real), A::SparseMatrixCSC) = copy(A) +broadcast{Tv,Ti}(::typeof(imag), A::SparseMatrixCSC{Tv,Ti}) = spzeros(Tv, Ti, A.m, A.n) +broadcast{TTv}(::typeof(real), A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2z_z2z_T(real, A, TTv) +broadcast{TTv}(::typeof(imag), A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2z_z2z_T(imag, A, TTv) ceil{To}(::Type{To}, A::SparseMatrixCSC) = _broadcast_unary_nz2z_z2z_T(ceil, A, To) floor{To}(::Type{To}, A::SparseMatrixCSC) = _broadcast_unary_nz2z_z2z_T(floor, A, To) trunc{To}(::Type{To}, A::SparseMatrixCSC) = _broadcast_unary_nz2z_z2z_T(trunc, A, To) @@ -1382,10 +1382,10 @@ function _broadcast_unary_nz2nz_z2z{Tv}(f::Function, A::SparseMatrixCSC{Tv}) end @_enumerate_childmethods(_broadcast_unary_nz2nz_z2z, log1p, expm1, abs, abs2, conj) -abs2{TTv}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs2, A, TTv) -abs{TTv}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, TTv) -abs{TTv<:Integer}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, Float64) -abs{TTv<:BigInt}(A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, BigFloat) +broadcast{TTv}(::typeof(abs2), A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs2, A, TTv) +broadcast{TTv}(::typeof(abs), A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, TTv) +broadcast{TTv<:Integer}(::typeof(abs), A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, Float64) +broadcast{TTv<:BigInt}(::typeof(abs), A::SparseMatrixCSC{Complex{TTv}}) = _broadcast_unary_nz2nz_z2z_T(abs, A, BigFloat) function conj!(A::SparseMatrixCSC) @inbounds @simd for k in 1:nnz(A) A.nzval[k] = conj(A.nzval[k]) @@ -1718,7 +1718,7 @@ end # macro (.^)(A::SparseMatrixCSC, B::Number) = B==0 ? sparse(ones(typeof(one(eltype(A)).^B), A.m, A.n)) : SparseMatrixCSC(A.m, A.n, copy(A.colptr), copy(A.rowval), A.nzval .^ B) -(.^)(::Irrational{:e}, B::SparseMatrixCSC) = exp(B) +(.^)(::Irrational{:e}, B::SparseMatrixCSC) = exp.(B) (.^)(A::Number, B::SparseMatrixCSC) = (.^)(A, full(B)) (.^)(A::SparseMatrixCSC, B::Array) = (.^)(full(A), B) (.^)(A::Array, B::SparseMatrixCSC) = (.^)(A, full(B)) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 4cfc2b248097d..98c10af32dd02 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -286,7 +286,7 @@ end # conj cA = sprandn(5,5,0.2) + im*sprandn(5,5,0.2) -@test full(conj(cA)) == conj(full(cA)) +@test full(conj.(cA)) == conj(full(cA)) # Test SparseMatrixCSC [c]transpose[!] and permute[!] methods let smalldim = 5, largedim = 10, nzprob = 0.4 @@ -461,22 +461,47 @@ end @test maximum(sparse(-ones(3,3))) == -1 @test minimum(sparse(ones(3,3))) == 1 -# Unary functions -a = sprand(5,15, 0.5) -afull = full(a) -for op in (:sin, :cos, :tan, :ceil, :floor, :abs, :abs2) - @eval begin - @test ($op)(afull) == full($(op)(a)) - end -end - -for op in (:ceil, :floor) - @eval begin - @test ($op)(Int,afull) == full($(op)(Int,a)) +# Test unary functions with specialized broadcast over SparseMatrixCSCs +let + A = sprand(5, 15, 0.5) + C = A + im*A + Afull = full(A) + Cfull = full(C) + # Test representatives of [unary functions that map zeros to zeros and may map nonzeros to zeros] + @test sin.(Afull) == full(sin.(A)) + @test tan.(Afull) == full(tan.(A)) # should be redundant with sin test + @test ceil.(Afull) == full(ceil.(A)) + @test floor.(Afull) == full(floor.(A)) # should be redundant with ceil test + @test real.(Afull) == full(real.(A)) + @test imag.(Afull) == full(imag.(A)) + @test real.(Cfull) == full(real.(C)) + @test imag.(Cfull) == full(imag.(C)) + # Test representatives of [unary functions that map zeros to zeros and nonzeros to nonzeros] + @test expm1.(Afull) == full(expm1.(A)) + @test abs.(Afull) == full(abs.(A)) + @test abs2.(Afull) == full(abs2.(A)) + @test abs.(Cfull) == full(abs.(C)) + @test abs2.(Cfull) == full(abs2.(C)) + # Test representatives of [unary functions that map both zeros and nonzeros to nonzeros] + @test cos.(Afull) == full(cos.(A)) + # Test representatives of remaining vectorized-nonbroadcast unary functions + @test ceil(Int, Afull) == full(ceil(Int, A)) + @test floor(Int, Afull) == full(floor(Int, A)) + # Tests of real, imag, abs, and abs2 for SparseMatrixCSC{Int,X}s previously elsewhere + for T in (Int, Float16, Float32, Float64, BigInt, BigFloat) + R = rand(T[1:100;], 2, 2) + I = rand(T[1:100;], 2, 2) + D = R + I*im + S = sparse(D) + @test R == real.(S) + @test I == imag.(S) + @test real.(sparse(R)) == R + @test nnz(imag.(sparse(R))) == 0 + @test abs.(S) == abs(D) + @test abs2.(S) == abs2(D) end end - # getindex tests ni = 23 nj = 32 @@ -872,7 +897,7 @@ end @test_throws ArgumentError sparsevec(Dict(-1=>1,1=>2)) # issue #8976 -@test conj(sparse([1im])) == sparse(conj([1im])) +@test conj.(sparse([1im])) == sparse(conj([1im])) @test conj!(sparse([1im])) == sparse(conj!([1im])) # issue #9525 @@ -1038,18 +1063,6 @@ end x = speye(100) @test_throws BoundsError x[-10:10] -for T in (Int, Float16, Float32, Float64, BigInt, BigFloat) - let R=rand(T[1:100;],2,2), I=rand(T[1:100;],2,2) - D = R + I*im - S = sparse(D) - @test R == real(S) - @test I == imag(S) - @test real(sparse(R)) == R - @test nnz(imag(sparse(R))) == 0 - @test abs(S) == abs(D) - @test abs2(S) == abs2(D) - end -end # issue #10407 @test maximum(spzeros(5, 5)) == 0.0 From d52b18e880a2a1926a62c8d63f1ad1d927b61150 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Thu, 25 Aug 2016 19:06:03 +0800 Subject: [PATCH 1057/1117] Handle `VecElement` in `julia_const_to_llvm` Also clean up `julia_const_to_llvm` to avoid boxing due to `getfield`. Fix #18236 --- src/intrinsics.cpp | 125 +++++++++++++++++++++------------------------ test/core.jl | 22 ++++++++ 2 files changed, 80 insertions(+), 67 deletions(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 1d9844e20e944..47452dc53ead3 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -155,55 +155,52 @@ static Value *uint_cnvt(Type *to, Value *x) } #define LLVM_FP(a,b) APFloat(a,b) -static Constant *julia_const_to_llvm(jl_value_t *e, bool nested=false) +static Constant *julia_const_to_llvm(void *ptr, jl_value_t *bt) { - jl_value_t *jt = jl_typeof(e); - jl_datatype_t *bt = (jl_datatype_t*)jt; + // assume `jl_isbits(bt)`. + // `ptr` can point to a inline field, do not read the tag from it. + if (bt == (jl_value_t*)jl_bool_type) + return ConstantInt::get(T_int8, (*(uint8_t*)ptr) ? 1 : 0); - if (!jl_is_datatype(bt) || bt == jl_ssavalue_type) + if (bt == (jl_value_t*)jl_ssavalue_type) return NULL; - if (e == jl_true) - return ConstantInt::get(T_int8, 1); - if (e == jl_false) - return ConstantInt::get(T_int8, 0); + if (jl_is_vecelement_type(bt)) + bt = jl_tparam0(bt); - if (jl_is_cpointer_type(jt)) - return ConstantExpr::getIntToPtr(ConstantInt::get(T_size, jl_unbox_long(e)), julia_type_to_llvm((jl_value_t*)bt)); - if (jl_is_bitstype(jt)) { + if (jl_is_cpointer_type(bt)) + return ConstantExpr::getIntToPtr(ConstantInt::get(T_size, *(uintptr_t*)ptr), julia_type_to_llvm(bt)); + if (jl_is_bitstype(bt)) { int nb = jl_datatype_size(bt); - //TODO: non-power-of-2 size datatypes may not be interpreted correctly on big-endian systems + // TODO: non-power-of-2 size datatypes may not be interpreted correctly on big-endian systems switch (nb) { case 1: { - uint8_t data8 = *(uint8_t*)jl_data_ptr(e); + uint8_t data8 = *(uint8_t*)ptr; return ConstantInt::get(T_int8, data8); } case 2: { - uint16_t data16 = *(uint16_t*)jl_data_ptr(e); + uint16_t data16 = *(uint16_t*)ptr; #ifndef DISABLE_FLOAT16 - if (jl_is_float(e)) { - return ConstantFP::get(jl_LLVMContext,LLVM_FP(APFloat::IEEEhalf,APInt(16,data16))); - } + if (jl_is_floattype(bt)) + return ConstantFP::get(jl_LLVMContext, LLVM_FP(APFloat::IEEEhalf,APInt(16,data16))); #endif return ConstantInt::get(T_int16, data16); } case 4: { - uint32_t data32 = *(uint32_t*)jl_data_ptr(e); - if (jl_is_float(e)) { - return ConstantFP::get(jl_LLVMContext,LLVM_FP(APFloat::IEEEsingle,APInt(32,data32))); - } + uint32_t data32 = *(uint32_t*)ptr; + if (jl_is_floattype(bt)) + return ConstantFP::get(jl_LLVMContext, LLVM_FP(APFloat::IEEEsingle,APInt(32,data32))); return ConstantInt::get(T_int32, data32); } case 8: { - uint64_t data64 = *(uint64_t*)jl_data_ptr(e); - if (jl_is_float(e)) { - return ConstantFP::get(jl_LLVMContext,LLVM_FP(APFloat::IEEEdouble,APInt(64,data64))); - } + uint64_t data64 = *(uint64_t*)ptr; + if (jl_is_floattype(bt)) + return ConstantFP::get(jl_LLVMContext, LLVM_FP(APFloat::IEEEdouble,APInt(64,data64))); return ConstantInt::get(T_int64, data64); } default: size_t nw = (nb+sizeof(uint64_t)-1)/sizeof(uint64_t); - uint64_t *data = (uint64_t*)jl_data_ptr(e); + uint64_t *data = (uint64_t*)ptr; APInt val; #if !defined(_P64) // malloc may not be 16-byte aligned on P32, @@ -218,54 +215,48 @@ static Constant *julia_const_to_llvm(jl_value_t *e, bool nested=false) else #endif val = APInt(8*nb, ArrayRef<uint64_t>(data, nw)); - if (nb == 16 && jl_is_float(e)) { + if (nb == 16 && jl_is_floattype(bt)) { return ConstantFP::get(jl_LLVMContext,LLVM_FP(APFloat::IEEEquad,val)); // If we have a floating point type that's not hardware supported, just treat it like an integer for LLVM purposes } return ConstantInt::get(IntegerType::get(jl_LLVMContext,8*nb),val); } } - if (jl_isbits(jt)) { - size_t nf = jl_datatype_nfields(bt), i; - size_t llvm_nf = 0; - Constant **fields = (Constant**)alloca(nf * sizeof(Constant*)); - jl_value_t *f=NULL; - JL_GC_PUSH1(&f); - for(i=0; i < nf; i++) { - f = jl_get_nth_field(e, i); - Constant *val; - if (f == jl_true) - val = ConstantInt::get(T_int8,1); - else if (f == jl_false) - val = ConstantInt::get(T_int8,0); - else - val = julia_const_to_llvm(f, true); - if (val == NULL) { - JL_GC_POP(); - return NULL; - } - fields[llvm_nf++] = val; - } - JL_GC_POP(); - - Type *t = julia_struct_to_llvm(jt, NULL); - if (type_is_ghost(t)) - return UndefValue::get(NoopType); - if (t->isVectorTy()) - return ConstantVector::get(ArrayRef<Constant*>(fields,llvm_nf)); - if (t->isStructTy()) { - StructType *st = dyn_cast<StructType>(t); - assert(st); - return ConstantStruct::get(st, ArrayRef<Constant*>(fields,llvm_nf)); - } - else { - assert(t->isArrayTy()); - ArrayType *at = dyn_cast<ArrayType>(t); - assert(at); - return ConstantArray::get(at, ArrayRef<Constant*>(fields,llvm_nf)); - } + size_t nf = jl_datatype_nfields(bt); + Constant **fields = (Constant**)alloca(nf * sizeof(Constant*)); + for (size_t i = 0; i < nf; i++) { + size_t offs = jl_field_offset((jl_datatype_t*)bt, i); + jl_value_t *ft = jl_field_type(bt, i); + Constant *val = julia_const_to_llvm((char*)ptr + offs, ft); + if (val == NULL) + return NULL; + fields[i] = val; } - return NULL; + + Type *t = julia_struct_to_llvm(bt, NULL); + if (type_is_ghost(t)) + return UndefValue::get(NoopType); + if (t->isVectorTy()) + return ConstantVector::get(ArrayRef<Constant*>(fields, nf)); + if (StructType *st = dyn_cast<StructType>(t)) { + return ConstantStruct::get(st, ArrayRef<Constant*>(fields, nf)); + } + else { + ArrayType *at = cast<ArrayType>(t); + return ConstantArray::get(at, ArrayRef<Constant*>(fields, nf)); + } +} + +static Constant *julia_const_to_llvm(jl_value_t *e) +{ + if (e == jl_true) + return ConstantInt::get(T_int8, 1); + if (e == jl_false) + return ConstantInt::get(T_int8, 0); + jl_value_t *bt = jl_typeof(e); + if (!jl_isbits(bt)) + return NULL; + return julia_const_to_llvm(e, bt); } static jl_cgval_t ghostValue(jl_value_t *ty); diff --git a/test/core.jl b/test/core.jl index c4e7544ff01a7..db74b3bd47bba 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4497,6 +4497,28 @@ for (f,g) in ((:asin,:sin), (:acos,:cos)) end @test f18085(Val{:asin},3) === (0.0,) +# issue #18236 constant VecElement in ast triggers codegen assertion/undef +# VecElement of scalar +v18236 = VecElement(1.0) +ptr18236 = cfunction(identity, VecElement{Float64}, Tuple{VecElement{Float64}}) +@eval @noinline f18236(ptr) = ccall(ptr, VecElement{Float64}, + (VecElement{Float64},), $v18236) +@test f18236(ptr18236) === v18236 +@test !contains(sprint(code_llvm, f18236, Tuple{Ptr{Void}}), "double undef") +# VecElement of struct type, not necessarily useful but does have special +# ABI so should be handled correctly +# This struct should be small enough to be passed by value in C ABI +# in order to trigger the problematic code path. +# We should be at least testing this on some platforms. +# Not sure if there's a better way to trigger unboxing in codegen. +v18236_2 = VecElement((Int8(1), Int8(2))) +ptr18236_2 = cfunction(identity, VecElement{NTuple{2,Int8}}, + Tuple{VecElement{NTuple{2,Int8}}}) +@eval @noinline f18236_2(ptr) = ccall(ptr, VecElement{NTuple{2,Int8}}, + (VecElement{NTuple{2,Int8}},), + $v18236_2) +@test f18236_2(ptr18236_2) === v18236_2 + # issue #18173 function f18173() identity(()->successflag) From e5eb0b675ecad2b701c7d598121827950ed5022c Mon Sep 17 00:00:00 2001 From: Erik Schnetter <schnetter@gmail.com> Date: Fri, 26 Aug 2016 14:40:03 -0400 Subject: [PATCH 1058/1117] Clarifiy performance results (#18242) Closes #18231. --- doc/manual/performance-tips.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index 70d3bb9820b6b..d8be03c636f7e 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -1068,20 +1068,21 @@ Here is an example with both :obj:`@inbounds` and :obj:`@simd` markup:: time = @elapsed for j in 1:reps s+=inner(x,y) end - println("GFlop = ",2.0*n*reps/time*1E-9) + println("GFlop/sec = ",2.0*n*reps/time*1E-9) time = @elapsed for j in 1:reps s+=innersimd(x,y) end - println("GFlop (SIMD) = ",2.0*n*reps/time*1E-9) + println("GFlop/sec (SIMD) = ",2.0*n*reps/time*1E-9) end timeit(1000,1000) On a computer with a 2.4GHz Intel Core i5 processor, this produces:: - GFlop = 1.9467069505224963 - GFlop (SIMD) = 17.578554163920018 + GFlop/sec = 1.9467069505224963 + GFlop/sec (SIMD) = 17.578554163920018 +(``GFlop/sec`` measures the performance, and larger numbers are better.) The range for a ``@simd for`` loop should be a one-dimensional range. A variable used for accumulating, such as ``s`` in the example, is called a *reduction variable*. By using :obj:`@simd`, you are asserting several @@ -1167,13 +1168,11 @@ evaluates the L2-norm of the result:: On a computer with a 2.7 GHz Intel Core i7 processor, this produces:: - $ julia wave.jl + $ julia wave.jl; elapsed time: 1.207814709 seconds (0 bytes allocated) - 4.443986180758243 - $ julia --math-mode=ieee wave.jl + $ julia --math-mode=ieee wave.jl; elapsed time: 4.487083643 seconds (0 bytes allocated) - 4.443986180758243 Here, the option ``--math-mode=ieee`` disables the :obj:`@fastmath` macro, so that we can compare results. From e46401e81c8ccc03a43d4623a308e7b63287c6df Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 28 Aug 2016 08:26:19 -0500 Subject: [PATCH 1059/1117] Speed up indices for SubArray --- base/subarray.jl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/base/subarray.jl b/base/subarray.jl index 1f1be5b260a72..a675fd75ebd37 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -273,11 +273,20 @@ end # they are taken from the range/vector # Since bounds-checking is performance-critical and uses # indices, it's worth optimizing these implementations thoroughly -indices(S::SubArray) = (@_inline_pure_meta; _indices_sub(S, indices(S.parent), S.indexes...)) +indices(S::SubArray) = (@_inline_meta; _indices_sub(S, indices(S.parent), S.indexes...)) _indices_sub(S::SubArray, pinds) = () -_indices_sub(S::SubArray, pinds, ::Real, I...) = _indices_sub(S, tail(pinds), I...) -_indices_sub(S::SubArray, pinds, ::Colon, I...) = (pinds[1], _indices_sub(S, tail(pinds), I...)...) -_indices_sub(S::SubArray, pinds, i1::AbstractArray, I...) = (unsafe_indices(i1)..., _indices_sub(S, tail(pinds), I...)...) +function _indices_sub(S::SubArray, pinds, ::Real, I...) + @_inline_meta + _indices_sub(S, tail(pinds), I...) +end +function _indices_sub(S::SubArray, pinds, ::Colon, I...) + @_inline_meta + (pinds[1], _indices_sub(S, tail(pinds), I...)...) +end +function _indices_sub(S::SubArray, pinds, i1::AbstractArray, I...) + @_inline_meta + (unsafe_indices(i1)..., _indices_sub(S, tail(pinds), I...)...) +end ## Compatability # deprecate? From 54e6cfb8addf41a5e95514a525a93007dfac9e4f Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Sun, 28 Aug 2016 08:31:52 -0500 Subject: [PATCH 1060/1117] Use inbounds in reductions with generic iterators --- base/reduce.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reduce.jl b/base/reduce.jl index 6ba2d8e457544..c0ba39c7310fd 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -45,7 +45,7 @@ function mapfoldl_impl(f, op, v0, itr, i) (x, i) = next(itr, i) v = op(r_promote(op, v0), f(x)) while !done(itr, i) - (x, i) = next(itr, i) + @inbounds (x, i) = next(itr, i) v = op(v, f(x)) end return v From 4710fedca80602eaf8d66b3a70051b499c22fac8 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Mon, 29 Aug 2016 08:20:03 +0200 Subject: [PATCH 1061/1117] add faster done for strings (#18259) --- base/strings/string.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/strings/string.jl b/base/strings/string.jl index 9ff67987338e7..c7b9dd9237bf1 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -80,6 +80,8 @@ end return Char(c), i end +done(s::String, state) = state > endof(s.data) + @inline function next(s::String, i::Int) # function is split into this critical fast-path # for pure ascii data, such as parsing numbers, From 9b4f47e83165a6d770e3316effff183bc8fa4f20 Mon Sep 17 00:00:00 2001 From: Martin Holters <martin.holters@hsu-hh.de> Date: Mon, 29 Aug 2016 12:14:47 +0200 Subject: [PATCH 1062/1117] Test for Dict not triggering a deprecation due to ::Integer==::Char comparison (#18234) See #18213. --- test/dict.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/dict.jl b/test/dict.jl index 0af6b4d3807d7..f2ee31b8cfe7c 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -555,3 +555,6 @@ let badKeys = UInt16[0xb800,0xa501,0xcdff,0x6303,0xe40a,0xcf0e,0xf3df,0xae99,0x9 end end end + +# #18213 +Dict(1 => rand(2,3), 'c' => "asdf") # just make sure this does not trigger a deprecation From eb687ffb780c3b7caccbfb460310dcbaf41e02d7 Mon Sep 17 00:00:00 2001 From: garrettthomaskth <garrettt@kth.se> Date: Mon, 29 Aug 2016 14:23:47 +0200 Subject: [PATCH 1063/1117] lufact for sparse matrix pivot option error (#18246) * lufact for sparse matrix pivot option error base/sparse/umfpack.jl includes the following method definition for lufact `lufact(A::SparseMatrixCSC, pivot::Type{Val{false}}) = lufact(A)` This should likely be `lufact(A::SparseMatrixCSC, pivot::Type{Val{true}}) = lufact(A)` because in lufact pivoting is on by default. The error is shown in the following example ``` A = speye(4) A[1:2,1:2] = [-.01 -200; 200 .001] F = lufact(A,Val{false}) F[:p] ``` which returns ``` julia> F[:q] 4-element Array{Int64,1}: 3 4 1 2 ``` However it should return ``` julia> F[:q] 4-element Array{Int64,1}: 1 2 3 4 ``` because pivoting was turned off. * Added test for #18246 and #18244 * 4 space indent oops my bad * remove unnecessary lufact method * update test for removed lufact method definition --- base/sparse/umfpack.jl | 1 - test/sparsedir/umfpack.jl | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/base/sparse/umfpack.jl b/base/sparse/umfpack.jl index 3b426bb9f234d..25f4b9935a7d1 100644 --- a/base/sparse/umfpack.jl +++ b/base/sparse/umfpack.jl @@ -157,7 +157,6 @@ lufact{T<:AbstractFloat}(A::Union{SparseMatrixCSC{T},SparseMatrixCSC{Complex{T}} "sparse floating point LU using UMFPACK or lufact(full(A)) for generic ", "dense LU."))) lufact(A::SparseMatrixCSC) = lufact(float(A)) -lufact(A::SparseMatrixCSC, pivot::Type{Val{false}}) = lufact(A) size(F::UmfpackLU) = (F.m, F.n) diff --git a/test/sparsedir/umfpack.jl b/test/sparsedir/umfpack.jl index 6db636a20353e..6ce7877e58011 100644 --- a/test/sparsedir/umfpack.jl +++ b/test/sparsedir/umfpack.jl @@ -136,4 +136,12 @@ let @test_throws ArgumentError Base.SparseArrays.UMFPACK.solve!(a, lufact(speye(5,5)), a, Base.SparseArrays.UMFPACK.UMFPACK_A) aa = complex(a) @test_throws ArgumentError Base.SparseArrays.UMFPACK.solve!(aa, lufact(complex(speye(5,5))), aa, Base.SparseArrays.UMFPACK.UMFPACK_A) -end \ No newline at end of file +end + +#18246,18244-lufact sparse pivot +let + A = speye(4) + A[1:2,1:2] = [-.01 -200; 200 .001] + F = lufact(A) + @test F[:p] == [3 ; 4 ; 2 ; 1] +end From 382de0b6acce3310499e4de67e6ea58f33c4d1fb Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson <kristoffer.carlsson@chalmers.se> Date: Mon, 29 Aug 2016 14:57:54 +0200 Subject: [PATCH 1064/1117] Revert #182599 "add faster done for strings" (#18275) * Revert "add faster done for strings (#18259)" This reverts commit 30ee10b0fff2cf7c0f14ba5531b8f980a7f5c975. * add test that failed with #18259 --- base/strings/string.jl | 2 -- test/strings/basic.jl | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/strings/string.jl b/base/strings/string.jl index c7b9dd9237bf1..9ff67987338e7 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -80,8 +80,6 @@ end return Char(c), i end -done(s::String, state) = state > endof(s.data) - @inline function next(s::String, i::Int) # function is split into this critical fast-path # for pure ascii data, such as parsing numbers, diff --git a/test/strings/basic.jl b/test/strings/basic.jl index e736da9e69890..55398736d9976 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -427,3 +427,7 @@ foobaz(ch) = reinterpret(Char, typemax(UInt32)) # issue #17624, missing getindex method for String @test "abc"[:] == "abc" + +# PR #18259 reverted because failing test below +a = [x for x in String([0xcf, 0x83, 0x83, 0x83, 0x83])] +@test a[1] == 'σ' From a142b996b39cba01ef33610145885c5a40619211 Mon Sep 17 00:00:00 2001 From: Christof Stocker <stocker.christof@gmail.com> Date: Mon, 29 Aug 2016 17:19:07 +0200 Subject: [PATCH 1065/1117] Add missing convert(Vector, ...) and convert(Matrix, ...) methods (#17848) This commit will allow a user to call: convert(Vector, my_abstractvector_of_eltype_Foo) convert(Matrix, my_abstractmatrix_of_eltype_Foo) which is useful because - for the case that the variable is a subarray of the same dimensionality - for consistency with PR #17066 --- base/array.jl | 3 +++ test/abstractarray.jl | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/base/array.jl b/base/array.jl index 2e0f5b4feb98b..49b9758424f59 100644 --- a/base/array.jl +++ b/base/array.jl @@ -226,6 +226,9 @@ end ## Conversions ## +convert{T}(::Type{Vector}, x::AbstractVector{T}) = convert(Vector{T}, x) +convert{T}(::Type{Matrix}, x::AbstractMatrix{T}) = convert(Matrix{T}, x) + convert{T,n}(::Type{Array{T}}, x::Array{T,n}) = x convert{T,n}(::Type{Array{T,n}}, x::Array{T,n}) = x diff --git a/test/abstractarray.jl b/test/abstractarray.jl index db26e0099f896..807e5da5a7398 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -405,7 +405,19 @@ function test_primitives{T}(::Type{T}, shape, ::Type{TestAbstractArray}) # convert{T, N}(::Type{Array}, A::AbstractArray{T, N}) X = [1:10...] + Y = [1 2; 3 4] @test convert(Array, X) == X + @test convert(Array, Y) == Y + + # convert{T}(::Type{Vector}, A::AbstractVector{T}) + @test convert(Vector, X) == X + @test convert(Vector, view(X, 2:4)) == [2,3,4] + @test_throws MethodError convert(Vector, Y) + + # convert{T}(::Type{Matrix}, A::AbstractMatrix{T}) + @test convert(Matrix, Y) == Y + @test convert(Matrix, view(Y, 1:2, 1:2)) == Y + @test_throws MethodError convert(Matrix, X) end let From 1d792f7d6dbda84bd616151fd25f71ea9a29dc7d Mon Sep 17 00:00:00 2001 From: Pablo Zubieta <pabloferz@yahoo.com.mx> Date: Mon, 29 Aug 2016 21:10:22 +0200 Subject: [PATCH 1066/1117] Some broadcast fixes (#18200) * Fix #18176 (broadcast over mixtures of arrays and numeric scalars) * Fix #17984 (broadcast behavior over empty arrays) --- base/abstractarray.jl | 3 ++- base/broadcast.jl | 2 +- test/abstractarray.jl | 9 +++++++++ test/broadcast.jl | 12 +++++++++++- test/inference.jl | 8 -------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index fbecc43efa4f8..408f05f4645ef 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1656,12 +1656,13 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector) end # These are needed because map(eltype, As) is not inferrable -promote_eltype_op(::Any) = (@_pure_meta; Bottom) +promote_eltype_op(::Any) = (@_pure_meta; Any) promote_eltype_op(op, A) = (@_pure_meta; promote_op(op, eltype(A))) promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T)) promote_eltype_op{T}(op, ::AbstractArray{T}, A) = (@_pure_meta; promote_op(op, T, eltype(A))) promote_eltype_op{T}(op, A, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, eltype(A), T)) promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S)) +promote_eltype_op(op, A, B) = (@_pure_meta; promote_op(op, eltype(A), eltype(B))) promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_eltype_op(op, promote_eltype_op(op, A, B), C, D...)) ## 1 argument diff --git a/base/broadcast.jl b/base/broadcast.jl index e42f2a67edfbc..aee5920bb9386 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -213,7 +213,7 @@ function broadcast_t(f, ::Type{Any}, As...) shape = broadcast_shape(As...) iter = CartesianRange(shape) if isempty(iter) - return similar(Array{Union{}}, shape) + return similar(Array{Any}, shape) end nargs = length(As) keeps, Idefaults = map_newindexer(shape, As) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 807e5da5a7398..f8f3e158b8cf8 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -746,3 +746,12 @@ end @test ndims(Diagonal(rand(1:5,5))) == 2 @test ndims(Diagonal{Float64}) == 2 @test Base.elsize(Diagonal(rand(1:5,5))) == sizeof(Int) + +# Issue #17811 +let A17811 = Integer[] + I = [abs(x) for x in A17811] + @test isa(I, Array{Any,1}) + push!(I, 1) + @test I == Any[1] + @test isa(map(abs, A17811), Array{Any,1}) +end diff --git a/test/broadcast.jl b/test/broadcast.jl index ec4d4ebf111ef..bbc2afb6d560c 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -311,9 +311,19 @@ end let f17314 = x -> x < 0 ? false : x @test eltype(broadcast(f17314, 1:3)) === Int @test eltype(broadcast(f17314, -1:1)) === Integer - @test eltype(broadcast(f17314, Int[])) === Union{} + @test eltype(broadcast(f17314, Int[])) === Any end let io = IOBuffer() broadcast(x->print(io,x), 1:5) # broadcast with side effects @test takebuf_array(io) == [0x31,0x32,0x33,0x34,0x35] end + +# Issue 18176 +let f18176(a, b, c) = a + b + c + @test f18176.(1.0:2, 3, 4) == f18176.(3.0, 1.0:2, 4.0) == broadcast(f18176, 3, 4, 1.0:2) +end + +# Issue #17984 +let A17984 = [] + @test isa(abs.(A17984), Array{Any,1}) +end diff --git a/test/inference.jl b/test/inference.jl index 6d61d6d657a4a..8a2e883a77d66 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -275,14 +275,6 @@ let f(x) = (x===nothing) ? 1 : 1.0 @test Base.return_types(f, (Void,)) == Any[Int] end -# Issue #17811 -let I = Integer[] - I = abs(I) - @test typeof(I) == Array{Any,1} - push!(I, 1) - @test I == Any[1] -end - # issue #16530 type Foo16530a{dim} c::Vector{NTuple{dim, Float64}} From c45b35ee6299b3d95a17e03ca648deab913f4563 Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Mon, 29 Aug 2016 12:13:14 -0700 Subject: [PATCH 1067/1117] Rename the FooBar type in test/core.jl (#18279) so `JULIA_CPU_CORES=1 ./julia test/runtests.jl core compile` passes after #18150 --- test/core.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/core.jl b/test/core.jl index db74b3bd47bba..3790769b4d1dd 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1019,20 +1019,20 @@ let @test_throws InexactError unsafe_wrap(Array, pointer(a), -3) end -immutable FooBar +immutable FooBar2515 foo::Int bar::Int end let local X, p - X = FooBar[ FooBar(3,1), FooBar(4,4) ] + X = FooBar2515[ FooBar2515(3,1), FooBar2515(4,4) ] p = pointer(X) - @test unsafe_load(p) == FooBar(3,1) - @test unsafe_load(p, 2) == FooBar(4,4) - unsafe_store!(p, FooBar(8,4)) - @test X[1] == FooBar(8,4) - unsafe_store!(p, FooBar(7,3), 1) - @test X[1] == FooBar(7,3) + @test unsafe_load(p) == FooBar2515(3,1) + @test unsafe_load(p, 2) == FooBar2515(4,4) + unsafe_store!(p, FooBar2515(8,4)) + @test X[1] == FooBar2515(8,4) + unsafe_store!(p, FooBar2515(7,3), 1) + @test X[1] == FooBar2515(7,3) end # issue #1287, combinations of try, catch, return From 2a26531f0688cb8a5995542ac6bdb63353d2adec Mon Sep 17 00:00:00 2001 From: Tim Holy <tim.holy@gmail.com> Date: Mon, 29 Aug 2016 14:15:20 -0500 Subject: [PATCH 1068/1117] Add a more efficient implementation of in(::CartesianIndex, ::CartesianRange) (#18277) --- base/multidimensional.jl | 8 +++++++- test/arrayops.jl | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 0a2b9489cec51..10a495213f639 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -3,7 +3,7 @@ ### Multidimensional iterators module IteratorsMD -import Base: eltype, length, size, start, done, next, last, getindex, setindex!, linearindexing, min, max, zero, one, isless, eachindex, ndims, iteratorsize +import Base: eltype, length, size, start, done, next, last, in, getindex, setindex!, linearindexing, min, max, zero, one, isless, eachindex, ndims, iteratorsize importall ..Base.Operators import Base: simd_outer_range, simd_inner_length, simd_index using Base: LinearFast, LinearSlow, AbstractCartesianIndex, fill_to_length, tail @@ -130,6 +130,12 @@ length(iter::CartesianRange) = prod(size(iter)) last(iter::CartesianRange) = iter.stop +@inline function in{I<:CartesianIndex}(i::I, r::CartesianRange{I}) + _in(true, i.I, r.start.I, r.stop.I) +end +_in(b, ::Tuple{}, ::Tuple{}, ::Tuple{}) = b +@inline _in(b, i, start, stop) = _in(b & (start[1] <= i[1] <= stop[1]), tail(i), tail(start), tail(stop)) + simd_outer_range(iter::CartesianRange{CartesianIndex{0}}) = iter function simd_outer_range{I}(iter::CartesianRange{I}) start = CartesianIndex(tail(iter.start.I)) diff --git a/test/arrayops.jl b/test/arrayops.jl index 7b5e3f628213c..cb0a30d788a8f 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1327,6 +1327,14 @@ indexes = collect(R) @test length(indexes) == 12 @test length(R) == 12 @test ndims(R) == 2 +@test in(CartesianIndex((2,3)), R) +@test in(CartesianIndex((3,3)), R) +@test in(CartesianIndex((3,5)), R) +@test in(CartesianIndex((5,5)), R) +@test !in(CartesianIndex((1,3)), R) +@test !in(CartesianIndex((3,2)), R) +@test !in(CartesianIndex((3,6)), R) +@test !in(CartesianIndex((6,5)), R) @test CartesianRange((3:5,-7:7)) == CartesianRange(CartesianIndex{2}(3,-7),CartesianIndex{2}(5,7)) @test CartesianRange((3,-7:7)) == CartesianRange(CartesianIndex{2}(3,-7),CartesianIndex{2}(3,7)) From da75407cc00f27830997f86c3adc608edf8dc132 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat <nalimilan@club.fr> Date: Mon, 29 Aug 2016 21:16:16 +0200 Subject: [PATCH 1069/1117] Add tests for conversions from ReshapedArray to Array/Vector/Matrix (#18262) These were not tested systematically, and are more generally useful to test conversions from AbstractArray. --- test/arrayops.jl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/arrayops.jl b/test/arrayops.jl index cb0a30d788a8f..24a1e1b730d7f 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -146,6 +146,22 @@ for idx in ((3,), (2,2), (Base.ReshapedIndex(1),)) end end +# conversion from ReshapedArray to Array (#18262) +a = Base.ReshapedArray(1:3, (3, 1), ()) +@test convert(Array, a) == a +@test convert(Array{Int}, a) == a +@test convert(Array{Float64}, a) == a +@test convert(Matrix, a) == a +@test convert(Matrix{Int}, a) == a +@test convert(Matrix{Float64}, a) == a +b = Base.ReshapedArray(1:3, (3,), ()) +@test convert(Array, b) == b +@test convert(Array{Int}, b) == b +@test convert(Array{Float64}, b) == b +@test convert(Vector, b) == b +@test convert(Vector{Int}, b) == b +@test convert(Vector{Float64}, b) == b + # operations with LinearFast ReshapedArray b = collect(1:12) a = Base.ReshapedArray(b, (4,3), ()) From 3a3fd89f3903346eb3d06c81e9c9c6c11e279be6 Mon Sep 17 00:00:00 2001 From: Tracy Wadleigh <tracy.wadleigh@gmail.com> Date: Mon, 29 Aug 2016 12:22:27 -0700 Subject: [PATCH 1070/1117] Install libopenlibm.dll.a in windows. Bump openlibm to v0.5.4. (#18265) --- Makefile | 1 + contrib/julia-config.jl | 2 +- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/openlibm.version | 4 ++-- 7 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 deps/checksums/openlibm-1581174c85f7b645b15ba1ac1c3a98fb601f0fe7.tar.gz/md5 create mode 100644 deps/checksums/openlibm-1581174c85f7b645b15ba1ac1c3a98fb601f0fe7.tar.gz/sha512 delete mode 100644 deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 delete mode 100644 deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 diff --git a/Makefile b/Makefile index c10090e4b73c2..812fc5d35a1d2 100644 --- a/Makefile +++ b/Makefile @@ -338,6 +338,7 @@ ifeq ($(OS),WINNT) -$(INSTALL_M) $(build_bindir)/*.dll $(DESTDIR)$(bindir)/ -$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ else -cp -a $(build_libexecdir) $(DESTDIR)$(prefix) diff --git a/contrib/julia-config.jl b/contrib/julia-config.jl index 6e98c5c0938fe..e954b854d2242 100755 --- a/contrib/julia-config.jl +++ b/contrib/julia-config.jl @@ -52,7 +52,7 @@ function ldlibs() if is_unix() return replace("""-Wl,-rpath,$(libDir()) -ljulia""","\\","\\\\") else - return replace("""-ljulia""","\\","\\\\") + return "-ljulia -lopenlibm" end end diff --git a/deps/checksums/openlibm-1581174c85f7b645b15ba1ac1c3a98fb601f0fe7.tar.gz/md5 b/deps/checksums/openlibm-1581174c85f7b645b15ba1ac1c3a98fb601f0fe7.tar.gz/md5 new file mode 100644 index 0000000000000..e52954a4a931a --- /dev/null +++ b/deps/checksums/openlibm-1581174c85f7b645b15ba1ac1c3a98fb601f0fe7.tar.gz/md5 @@ -0,0 +1 @@ +46654ecc67adba1d86426b1d9dfa3bc9 diff --git a/deps/checksums/openlibm-1581174c85f7b645b15ba1ac1c3a98fb601f0fe7.tar.gz/sha512 b/deps/checksums/openlibm-1581174c85f7b645b15ba1ac1c3a98fb601f0fe7.tar.gz/sha512 new file mode 100644 index 0000000000000..ff14e9f45d268 --- /dev/null +++ b/deps/checksums/openlibm-1581174c85f7b645b15ba1ac1c3a98fb601f0fe7.tar.gz/sha512 @@ -0,0 +1 @@ +64bb73800f3ff9219d7094b9f5f224468ee4cb34136bef77e1ae8b849e18d5f5fc1001f2e8dd92bcf0018f78152c3b52b9284e431bbafcea21d4f594c05e19ad diff --git a/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 deleted file mode 100644 index aa9ed22e8bb6d..0000000000000 --- a/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -1bde34205734b9b30bd6a92768bd916b diff --git a/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 deleted file mode 100644 index 5af90ecd2b687..0000000000000 --- a/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -9a324bf989fa513b6d513794114d08a24e77a4ff15da577c950c178322e34121f9eba6bf01e1520cd12a926723cda4687bb8efd6aa9de987cd9b3a1e6230d2ce diff --git a/deps/openlibm.version b/deps/openlibm.version index af5b13cbea355..9ba6e633d0d04 100644 --- a/deps/openlibm.version +++ b/deps/openlibm.version @@ -1,2 +1,2 @@ -OPENLIBM_BRANCH=v0.5.3 -OPENLIBM_SHA1=71e79eb6f74d3f04ce724195b8ef5846a70d281e +OPENLIBM_BRANCH=v0.5.4 +OPENLIBM_SHA1=1581174c85f7b645b15ba1ac1c3a98fb601f0fe7 From 476da8bdafc00b7823e3f2e1176cd3b7dbe9d4eb Mon Sep 17 00:00:00 2001 From: Elliot Saba <staticfloat@gmail.com> Date: Mon, 29 Aug 2016 22:05:29 -0700 Subject: [PATCH 1071/1117] Small build system modifications to make ppc64le compileable (#18258) * Small build system modifications to make ppc64le compileable * Make things more explicit for big-endian PPC64 --- Make.inc | 52 +++++++++++++++++++++++++++---------------------- src/threading.c | 5 ++--- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/Make.inc b/Make.inc index 5f93cfe3a9a29..9520d3267b13e 100644 --- a/Make.inc +++ b/Make.inc @@ -600,27 +600,6 @@ $(error Pre-SSE2 CPU targets not supported. To create a generic 32-bit x86 binar pass 'MARCH=pentium4'.) endif -ifneq ($(MARCH),) -CC += -march=$(MARCH) -CXX += -march=$(MARCH) -FC += -march=$(MARCH) -JULIA_CPU_TARGET ?= $(MARCH) -ifeq ($(OS),Darwin) -# on Darwin, the standalone `as` program doesn't know -# how to handle AVX instructions, but it does know how -# to dispatch to the clang assembler (if we ask it to) -ifeq ($(USECLANG),1) -CC += -integrated-as -CXX += -integrated-as -else -CC += -Wa,-q -CXX += -Wa,-q -endif -FC += -Wa,-q -AS += -q -endif -endif - JULIA_CPU_TARGET ?= native # We map amd64 to x86_64 for compatibility with systems that identify 64-bit systems as such @@ -651,13 +630,19 @@ else ISX86:=0 endif -# If we are running on powerpc64 or ppc64, set certain options automatically -ifneq (,$(filter $(ARCH), powerpc64 ppc64le)) +# If we are running on powerpc64le or ppc64le, set certain options automatically +ifneq (,$(filter $(ARCH), powerpc64le ppc64le)) JCFLAGS += -fsigned-char override LLVM_VER:=3.8.1 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=POWER8 +# GCC doesn't do -march= on ppc64le +override MARCH= +endif +# If we are running on powerpc64 or ppc64, fail out dramatically +ifneq (,$(filter $(ARCH), powerpc64 ppc64)) +$(error Big-endian PPC64 is not supported, to ignore this error, set ARCH=ppc64le) endif # If we are running on ARM, set certain options automatically @@ -669,7 +654,28 @@ override USE_BLAS64:=0 override OPENBLAS_DYNAMIC_ARCH:=0 override OPENBLAS_TARGET_ARCH:=ARMV7 override USE_SYSTEM_LIBM:=1 +endif +# Set MARCH-specific flags +ifneq ($(MARCH),) +CC += -march=$(MARCH) +CXX += -march=$(MARCH) +FC += -march=$(MARCH) +JULIA_CPU_TARGET ?= $(MARCH) +ifeq ($(OS),Darwin) +# on Darwin, the standalone `as` program doesn't know +# how to handle AVX instructions, but it does know how +# to dispatch to the clang assembler (if we ask it to) +ifeq ($(USECLANG),1) +CC += -integrated-as +CXX += -integrated-as +else +CC += -Wa,-q +CXX += -Wa,-q +endif +FC += -Wa,-q +AS += -q +endif endif # Set some ARCH-specific flags diff --git a/src/threading.c b/src/threading.c index 664a28df8880a..b3638be9b74f0 100644 --- a/src/threading.c +++ b/src/threading.c @@ -137,7 +137,7 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) // The general solution is to add one more indirection in the C entry point // (see `jl_get_ptls_states_wrapper`). // -// When `ifunc` is availabe, we can use it to trick the linker to use the +// When `ifunc` is available, we can use it to trick the linker to use the // real address (`jl_get_ptls_states_static`) directly as the symbol address. // (see `jl_get_ptls_states_resolve`). // @@ -145,8 +145,7 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) // is not guaranteed to be reliable, we still need to fallback to the wrapper // version as the symbol address if we didn't find the static version in `ifunc`. #if defined(__GLIBC__) && (defined(_CPU_X86_64_) || defined(_CPU_X86_) || \ - ((defined(_CPU_AARCH64_) || defined(_CPU_ARM_) || \ - defined(_CPU_PPC64_) || defined(_CPU_PPC_)) && \ + ((defined(_CPU_AARCH64_) || defined(_CPU_ARM_)) && \ __GNUC__ >= 6)) // Only enable this on architectures that are tested. // For example, GCC doesn't seem to support the `ifunc` attribute on power yet. From f2c6690b9f6ee8326971e011d0aec059f057dbd9 Mon Sep 17 00:00:00 2001 From: Helge Eichhorn <git@helgeeichhorn.de> Date: Tue, 30 Aug 2016 07:05:58 +0200 Subject: [PATCH 1072/1117] Fix deprecated flag in Pkg tests. (#18283) --- test/pkg.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 7fb6848d270eb..2399078032dbe 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -439,13 +439,13 @@ temp_pkg_dir() do touch(depsbuild) # Pkg.build works without the src directory now # but it's probably fine to require it. - msg = readstring(`$(Base.julia_cmd()) -f -e 'redirect_stderr(STDOUT); Pkg.build("BuildFail")'`) + msg = readstring(`$(Base.julia_cmd()) --startup-file=no -e 'redirect_stderr(STDOUT); Pkg.build("BuildFail")'`) @test contains(msg, "Building BuildFail") @test !contains(msg, "ERROR") open(depsbuild, "w") do fd println(fd, "error(\"Throw build error\")") end - msg = readstring(`$(Base.julia_cmd()) -f -e 'redirect_stderr(STDOUT); Pkg.build("BuildFail")'`) + msg = readstring(`$(Base.julia_cmd()) --startup-file=no -e 'redirect_stderr(STDOUT); Pkg.build("BuildFail")'`) @test contains(msg, "Building BuildFail") @test contains(msg, "ERROR") @test contains(msg, "Pkg.build(\"BuildFail\")") @@ -456,7 +456,7 @@ temp_pkg_dir() do let package = "Example" Pkg.rm(package) # Remove package if installed @test Pkg.installed(package) === nothing # Registered with METADATA but not installed - msg = readstring(ignorestatus(`$(Base.julia_cmd()) -f -e "redirect_stderr(STDOUT); Pkg.build(\"$package\")"`)) + msg = readstring(ignorestatus(`$(Base.julia_cmd()) --startup-file=no -e "redirect_stderr(STDOUT); Pkg.build(\"$package\")"`)) @test contains(msg, "$package is not an installed package") @test !contains(msg, "signal (15)") end From 6fce364b1ffdc1f56a82150c5b407f6662f297e0 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt <kshyatt@users.noreply.github.com> Date: Tue, 30 Aug 2016 06:50:34 -0700 Subject: [PATCH 1073/1117] Add tests for Hermitian cholesky methods (#18284) --- test/linalg/cholesky.jl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index c747ff06a8f8a..bd54fd05101de 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -28,6 +28,8 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) apds = Symmetric(apd) apdsL = Symmetric(apd, :L) + apdh = Hermitian(apd) + apdhL = Hermitian(apd, :L) ε = εa = eps(abs(float(one(eltya)))) @inferred cholfact(apd) @@ -64,6 +66,10 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) capds = cholfact(apds) @test inv(capds)*apds ≈ eye(n) @test abs((det(capds) - det(apd))/det(capds)) <= ε*κ*n + else + capdh = cholfact(apdh) + @test inv(capdh)*apdh ≈ eye(n) + @test abs((det(capdh) - det(apd))/det(capdh)) <= ε*κ*n end # test chol of 2x2 Strang matrix @@ -89,6 +95,17 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test istriu(cl) @test cl'cl ≈ apds @test cl'cl ≈ apdsL + else + capdh = cholfact(apdh) + lapdh = cholfact(apdhL) + cl = chol(apdhL) + ls = lapdh[:L] + @test ls*ls' ≈ apd + @test triu(capdh.factors) ≈ lapdh[:U] + @test tril(lapdh.factors) ≈ capdh[:L] + @test istriu(cl) + @test cl'cl ≈ apdh + @test cl'cl ≈ apdhL end #pivoted upper Cholesky @@ -226,4 +243,5 @@ end @test_throws ArgumentError cholfact(complex(randn(5,5), randn(5,5))) @test_throws ArgumentError Base.LinAlg.chol!(randn(5,5)) @test_throws ArgumentError Base.LinAlg.cholfact!(randn(5,5),:U,Val{false}) +@test_throws ArgumentError Base.LinAlg.cholfact!(randn(5,5),:U,Val{true}) @test_throws ArgumentError cholfact(randn(5,5),:U,Val{false}) From 092cfec24a5feb6d6f40050d32d4344953957e99 Mon Sep 17 00:00:00 2001 From: Josh Langsfeld <jdlangs@gmail.com> Date: Tue, 30 Aug 2016 11:53:21 -0400 Subject: [PATCH 1074/1117] Remove comments from same line of variable declarations in Make.inc --- Make.inc | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Make.inc b/Make.inc index 9520d3267b13e..e7869501702d1 100644 --- a/Make.inc +++ b/Make.inc @@ -4,6 +4,10 @@ ## It is generally preferable to change these options, for ## your local machine, in a file named `Make.user` in the toplevel ## and build directories +## +## For developers, take care to not insert comments on the same line as +## variable declarations. The spaces between the variable value and the +## comment will be included in the value. # OPENBLAS build options OPENBLAS_TARGET_ARCH:= @@ -58,7 +62,8 @@ USE_INTEL_JITEVENTS ?= 0 USEICC ?= 0 USEIFC ?= 0 -JULIA_THREADS := 1 # Enable threading with one thread +# Enable threading with one thread +JULIA_THREADS := 1 ifeq ($(USE_MKL), 1) $(warning "The julia make variable USE_MKL has been renamed to USE_INTEL_MKL") @@ -427,12 +432,16 @@ endif ifeq ($(USECCACHE), 1) # expand CC and CXX at declaration time because we will redefine them -CC_ARG := $(CC) # Expand CC and CXX here already because we want -CXX_ARG := $(CXX) # the original definition and not the ccache version. -CC_FULL := ccache $(CC) # Expand CC and CXX here already to avoid recursive -CXX_FULL := ccache $(CXX) # referencing. -CC := $(CC_FULL) # Add an extra indirection to make CC/CXX non-simple -CXX := $(CXX_FULL) # vars (because of how -m$(BINARY) is added later on). +# Expand CC and CXX here already because we want the original definition and not the ccache version. +CC_ARG := $(CC) +CXX_ARG := $(CXX) +# Expand CC and CXX here already to avoid recursive referencing. +CC_FULL := ccache $(CC) +CXX_FULL := ccache $(CXX) +# Add an extra indirection to make CC/CXX non-simple vars +# (because of how -m$(BINARY) is added later on). +CC := $(CC_FULL) +CXX := $(CXX_FULL) CC_BASE := ccache CXX_BASE := ccache ifeq ($(USECLANG),1) From b784bc8f728bea27d0c63e9cdf4f9b896bf23eeb Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Wed, 31 Aug 2016 02:13:36 -0700 Subject: [PATCH 1075/1117] ppc64 build cleanups --- deps/llvm.mk | 4 ++-- deps/openblas.version | 5 +++++ deps/unwind.mk | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 7a6e99bf2287c..e7813c9d94728 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -148,9 +148,9 @@ LLVM_CMAKE += -DLLDB_DISABLE_PYTHON=ON endif # LLDB_DISABLE_PYTHON endif # BUILD_LLDB -ifeq ($(ARCH), ppc64) +ifneq (,$(filter $(ARCH), powerpc64le ppc64le)) LLVM_CXXFLAGS += -mminimal-toc -endif # ARCH == ppc64 +endif # LLVM bug #24157 ifeq ($(USE_LLVM_SHLIB),1) diff --git a/deps/openblas.version b/deps/openblas.version index 35276b8ba7994..fd955f169428f 100644 --- a/deps/openblas.version +++ b/deps/openblas.version @@ -1,2 +1,7 @@ +ifneq (,$(filter $(ARCH), powerpc64le ppc64le)) +OPENBLAS_BRANCH=develop +OPENBLAS_SHA1=515bc56ea92cf7afa6c8316eb0f760439944300b +else OPENBLAS_BRANCH=v0.2.18 OPENBLAS_SHA1=12ab1804b6ebcd38b26960d65d254314d8bc33d6 +endif \ No newline at end of file diff --git a/deps/unwind.mk b/deps/unwind.mk index 1fadb5c581c37..59cd6482080d1 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -27,7 +27,7 @@ endif #todo: libunwind tests are known to fail, so they aren't run $(LIBUNWIND_TARGET_OBJ): $(LIBUNWIND_TARGET_SOURCE) $(call make-install,libunwind-$(UNWIND_VER),) -ifeq ($(ARCH), ppc64) +ifneq (,$(filter $(ARCH), powerpc64le ppc64le)) @# workaround for configure script bug mv $(build_prefix)/lib64/libunwind*.a $(build_libdir) endif From 29160e359a599a1acb9c8e70944bbe093bbfee10 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 31 Aug 2016 08:41:13 -0400 Subject: [PATCH 1076/1117] tiny cleanup (#18233) just a random thing I noticed --- base/cartesian.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/cartesian.jl b/base/cartesian.jl index 2d1065b955953..b8b96b51c0d77 100644 --- a/base/cartesian.jl +++ b/base/cartesian.jl @@ -258,7 +258,7 @@ function inlineanonymous(ex::Expr, val) end # Given :i and 3, this generates :i_3 -inlineanonymous(base::Symbol, ext) = Symbol(base,"_",string(ext)) +inlineanonymous(base::Symbol, ext) = Symbol(base,'_',ext) # Replace a symbol by a value or a "coded" symbol # E.g., for d = 3, From b03168782d8c3fe6031ac8bee5882b8aeda62213 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Wed, 31 Aug 2016 11:39:16 -0400 Subject: [PATCH 1077/1117] display something useful for text/plain output of invalid String (#18296) --- base/replutil.jl | 10 ++++++++++ test/show.jl | 3 +++ 2 files changed, 13 insertions(+) diff --git a/base/replutil.jl b/base/replutil.jl index fbd14f79b4608..6a5002a4efaba 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -153,6 +153,16 @@ end show(io::IO, ::MIME"text/plain", X::AbstractArray) = showarray(io, X, false) show(io::IO, ::MIME"text/plain", r::Range) = show(io, r) # always use the compact form for printing ranges +# display something useful even for strings containing arbitrary +# (non-UTF8) binary data: +function show(io::IO, ::MIME"text/plain", s::String) + if isvalid(s) + show(io, s) + else + println(io, sizeof(s), "-byte String of invalid UTF-8 data:") + showarray(io, s.data, false; header=false) + end +end # showing exception objects as descriptive error messages diff --git a/test/show.jl b/test/show.jl index d839f7ce8933c..f74a9ad13ac67 100644 --- a/test/show.jl +++ b/test/show.jl @@ -603,3 +603,6 @@ end @test repr(NTuple{7,Int64}) == "NTuple{7,Int64}" @test repr(Tuple{Float64, Float64, Float64, Float64}) == "NTuple{4,Float64}" @test repr(Tuple{Float32, Float32, Float32}) == "Tuple{Float32,Float32,Float32}" + +# Test that REPL/mime display of invalid UTF-8 data doesn't throw an exception: +@test isa(stringmime("text/plain", String(UInt8[0x00:0xff;])), String) From b76a0f18b92d76f0d498e428dc8fd6b86f3a5dcd Mon Sep 17 00:00:00 2001 From: Tony Kelman <tony@kelman.net> Date: Wed, 31 Aug 2016 13:38:09 -0700 Subject: [PATCH 1078/1117] set JULIA_CPU_TARGET default after MARCH-dependent block otherwise #18258 causes "Target architecture mismatch" errors in generic linux binaries --- Make.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Make.inc b/Make.inc index e7869501702d1..b2b171972abf8 100644 --- a/Make.inc +++ b/Make.inc @@ -609,8 +609,6 @@ $(error Pre-SSE2 CPU targets not supported. To create a generic 32-bit x86 binar pass 'MARCH=pentium4'.) endif -JULIA_CPU_TARGET ?= native - # We map amd64 to x86_64 for compatibility with systems that identify 64-bit systems as such ifeq ($(ARCH),amd64) override ARCH := x86_64 @@ -687,6 +685,8 @@ AS += -q endif endif +JULIA_CPU_TARGET ?= native + # Set some ARCH-specific flags ifneq ($(USEICC),1) ifeq ($(ISX86),1) From ee6c1367cd128843db63ae02e972453af8307176 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" <viral@mayin.org> Date: Wed, 31 Aug 2016 22:40:36 -0700 Subject: [PATCH 1079/1117] Update OpenBLAS to v0.2.19 for Power8. --- .../md5 | 1 + .../sha512 | 1 + deps/openblas.version | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 deps/checksums/openblas-85636ff1a015d04d3a8f960bc644b85ee5157135.tar.gz/md5 create mode 100644 deps/checksums/openblas-85636ff1a015d04d3a8f960bc644b85ee5157135.tar.gz/sha512 diff --git a/deps/checksums/openblas-85636ff1a015d04d3a8f960bc644b85ee5157135.tar.gz/md5 b/deps/checksums/openblas-85636ff1a015d04d3a8f960bc644b85ee5157135.tar.gz/md5 new file mode 100644 index 0000000000000..0be161ae7995b --- /dev/null +++ b/deps/checksums/openblas-85636ff1a015d04d3a8f960bc644b85ee5157135.tar.gz/md5 @@ -0,0 +1 @@ +4839fd98bce2900aa780cf1b1d387478 diff --git a/deps/checksums/openblas-85636ff1a015d04d3a8f960bc644b85ee5157135.tar.gz/sha512 b/deps/checksums/openblas-85636ff1a015d04d3a8f960bc644b85ee5157135.tar.gz/sha512 new file mode 100644 index 0000000000000..564e69b1983c6 --- /dev/null +++ b/deps/checksums/openblas-85636ff1a015d04d3a8f960bc644b85ee5157135.tar.gz/sha512 @@ -0,0 +1 @@ +2be2a191817959841bcfb07972601f74f3112744fe8ac8c6f280525fd282ecbe4060513f72f5bf5386296be43489405567b04b418617152e6be54ac42e34dd62 diff --git a/deps/openblas.version b/deps/openblas.version index fd955f169428f..d0847fe4c9ba0 100644 --- a/deps/openblas.version +++ b/deps/openblas.version @@ -1,6 +1,6 @@ ifneq (,$(filter $(ARCH), powerpc64le ppc64le)) -OPENBLAS_BRANCH=develop -OPENBLAS_SHA1=515bc56ea92cf7afa6c8316eb0f760439944300b +OPENBLAS_BRANCH=v0.2.19 +OPENBLAS_SHA1=85636ff1a015d04d3a8f960bc644b85ee5157135 else OPENBLAS_BRANCH=v0.2.18 OPENBLAS_SHA1=12ab1804b6ebcd38b26960d65d254314d8bc33d6 From ed6a96d1c7b1533d427e812ea8a586ef23a54106 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@alum.mit.edu> Date: Thu, 1 Sep 2016 07:59:08 -0400 Subject: [PATCH 1080/1117] rm stray : in manual --- doc/manual/arrays.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 5644e2d047bf6..ef400cd751bc2 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -566,7 +566,7 @@ function elementwise: 1.71056 0.847604 1.73659 0.873631 -Elementwise operators such as ``.+`` and ``.*`` perform broadcasting if necessary. There is also a :func:`broadcast!` function to specify an explicit destination, and :func:`broadcast_getindex` and :func:`broadcast_setindex!` that broadcast the indices before indexing. Moreover, ``f.(args...)`` is equivalent to ``broadcast(f, args...)``, providing a convenient syntax to broadcast any function (:ref:`man-dot-vectorizing`:). +Elementwise operators such as ``.+`` and ``.*`` perform broadcasting if necessary. There is also a :func:`broadcast!` function to specify an explicit destination, and :func:`broadcast_getindex` and :func:`broadcast_setindex!` that broadcast the indices before indexing. Moreover, ``f.(args...)`` is equivalent to ``broadcast(f, args...)``, providing a convenient syntax to broadcast any function (:ref:`man-dot-vectorizing`). Implementation -------------- From 95aeb9180aeb18e881d6b15ce6f1ecd8a91b6b80 Mon Sep 17 00:00:00 2001 From: Andreas Noack <andreasnoackjensen@gmail.com> Date: Thu, 1 Sep 2016 09:15:24 -0400 Subject: [PATCH 1081/1117] Avoid aliasing in in UniformScaling*AbstractMatrix (#18286) * Avoid aliasing in in UniformScaling*AbstractMatrix ...and remove unnecessary UniformScaling*SparseMatrixCSC methods * Broaden the tests for non-commutative multiplication * Add Quaternion test case for q*[q] and clean up the Quaternion test type --- base/linalg/uniformscaling.jl | 5 +- base/sparse/linalg.jl | 5 -- test/linalg/generic.jl | 25 ++++++---- test/linalg/uniformscaling.jl | 94 ++++++++++++++++++----------------- 4 files changed, 66 insertions(+), 63 deletions(-) diff --git a/base/linalg/uniformscaling.jl b/base/linalg/uniformscaling.jl index f2fce351438a8..1d02b756099f4 100644 --- a/base/linalg/uniformscaling.jl +++ b/base/linalg/uniformscaling.jl @@ -127,9 +127,8 @@ inv(J::UniformScaling) = UniformScaling(inv(J.λ)) *(J1::UniformScaling, J2::UniformScaling) = UniformScaling(J1.λ*J2.λ) *(B::BitArray{2}, J::UniformScaling) = *(Array(B), J::UniformScaling) *(J::UniformScaling, B::BitArray{2}) = *(J::UniformScaling, Array(B)) -*(A::AbstractMatrix, J::UniformScaling) = J.λ == 1 ? A : J.λ*A -*(J::UniformScaling, A::AbstractVecOrMat) = J.λ == 1 ? A : J.λ*A - +*(A::AbstractMatrix, J::UniformScaling) = A*J.λ +*(J::UniformScaling, A::AbstractVecOrMat) = J.λ*A *(x::Number, J::UniformScaling) = UniformScaling(x*J.λ) *(J::UniformScaling, x::Number) = UniformScaling(J.λ*x) diff --git a/base/sparse/linalg.jl b/base/sparse/linalg.jl index a80e5f4a6cb51..d50e0f94f369c 100644 --- a/base/sparse/linalg.jl +++ b/base/sparse/linalg.jl @@ -18,11 +18,6 @@ function increment!{T<:Integer}(A::AbstractArray{T}) end increment{T<:Integer}(A::AbstractArray{T}) = increment!(copy(A)) -## Multiplication with UniformScaling (scaled identity matrices) - -(*)(S::SparseMatrixCSC, J::UniformScaling) = J.λ == 1 ? S : J.λ*S -(*){Tv,Ti}(J::UniformScaling, S::SparseMatrixCSC{Tv,Ti}) = J.λ == 1 ? S : S*J.λ - ## sparse matrix multiplication function (*){TvA,TiA,TvB,TiB}(A::SparseMatrixCSC{TvA,TiA}, B::SparseMatrixCSC{TvB,TiB}) diff --git a/test/linalg/generic.jl b/test/linalg/generic.jl index 5e8f3b515a6ff..e16df15b36e98 100644 --- a/test/linalg/generic.jl +++ b/test/linalg/generic.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -import Base: * +import Base: -, * using Base.Test # A custom Quaternion type with minimal defined interface and methods. @@ -10,16 +10,19 @@ immutable Quaternion{T<:Real} <: Number v1::T v2::T v3::T - norm::Bool end -Quaternion(s::Real, v1::Real, v2::Real, v3::Real, n::Bool = false) = - Quaternion( promote(s, v1, v2, v3)..., n) -Quaternion(a::Vector) = Quaternion(0, a[1], a[2], a[3]) +Quaternion(s::Real, v1::Real, v2::Real, v3::Real) = Quaternion(promote(s, v1, v2, v3)...) +Base.abs2(q::Quaternion) = q.s*q.s + q.v1*q.v1 + q.v2*q.v2 + q.v3*q.v3 +Base.abs(q::Quaternion) = sqrt(abs2(q)) +Base.real{T}(::Type{Quaternion{T}}) = T +Base.conj(q::Quaternion) = Quaternion(q.s, -q.v1, -q.v2, -q.v3) + +(-)(ql::Quaternion, qr::Quaternion) = + Quaternion(ql.s - qr.s, ql.v1 - qr.v1, ql.v2 - qr.v2, ql.v3 - qr.v3) (*)(q::Quaternion, w::Quaternion) = Quaternion(q.s*w.s - q.v1*w.v1 - q.v2*w.v2 - q.v3*w.v3, q.s*w.v1 + q.v1*w.s + q.v2*w.v3 - q.v3*w.v2, q.s*w.v2 - q.v1*w.v3 + q.v2*w.s + q.v3*w.v1, - q.s*w.v3 + q.v1*w.v2 - q.v2*w.v1 + q.v3*w.s, - q.norm && w.norm) + q.s*w.v3 + q.v1*w.v2 - q.v2*w.v1 + q.v3*w.s) debug = false @@ -199,10 +202,12 @@ end @test isequal(BigFloat[1.0] * 2.0f0im, Complex{BigFloat}[2.0im]) # test scale and scale! for non-commutative multiplication -q = Quaternion([0.44567, 0.755871, 0.882548, 0.423612]) -qmat = [] -push!(qmat, Quaternion([0.015007, 0.355067, 0.418645, 0.318373])) +q = Quaternion(0.44567, 0.755871, 0.882548, 0.423612) +qmat = [Quaternion(0.015007, 0.355067, 0.418645, 0.318373)] @test scale!(q, copy(qmat)) != scale!(copy(qmat), q) +## Test * because it doesn't dispatch to scale! +@test q*qmat != qmat*q +@test conj(q*qmat) ≈ conj(qmat)*conj(q) # test ops on Numbers for elty in [Float32,Float64,Complex64,Complex128] diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index e82245c941b2b..f81de94697689 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -53,77 +53,81 @@ let AA = randn(2, 2) S = view(SS, 1:3, 1:3) end - @test A + I == A + eye(A) - @test I + A == A + eye(A) - @test I - I === UniformScaling(0) - @test B - I == B - eye(B) - @test I - B == eye(B) - B - @test A - I == A - eye(A) - @test I - A == eye(A) - A - @test I*J === UniformScaling(λ) - @test B*J == B*λ - @test J*B == B*λ + @test @inferred(A + I) == A + eye(A) + @test @inferred(I + A) == A + eye(A) + @test @inferred(I - I) === UniformScaling(0) + @test @inferred(B - I) == B - eye(B) + @test @inferred(I - B) == eye(B) - B + @test @inferred(A - I) == A - eye(A) + @test @inferred(I - A) == eye(A) - A + @test @inferred(I*J) === UniformScaling(λ) + @test @inferred(B*J) == B*λ + @test @inferred(J*B) == B*λ + @test @inferred(I*A) !== A # Don't alias + @test @inferred(I*S) !== S # Don't alias + @test @inferred(A*I) !== A # Don't alias + @test @inferred(S*I) !== S # Don't alias - @test S*J == S*λ - @test J*S == S*λ - @test A*J == A*λ - @test J*A == A*λ - @test J*ones(3) == ones(3)*λ - @test λ*J === UniformScaling(λ*J.λ) - @test J*λ === UniformScaling(λ*J.λ) - @test J/I === J - @test I/A == inv(A) - @test A/I == A - @test I/λ === UniformScaling(1/λ) - @test I\J === J + @test @inferred(S*J) == S*λ + @test @inferred(J*S) == S*λ + @test @inferred(A*J) == A*λ + @test @inferred(J*A) == A*λ + @test @inferred(J*ones(3)) == ones(3)*λ + @test @inferred(λ*J) === UniformScaling(λ*J.λ) + @test @inferred(J*λ) === UniformScaling(λ*J.λ) + @test @inferred(J/I) === J + @test @inferred(I/A) == inv(A) + @test @inferred(A/I) == A + @test @inferred(I/λ) === UniformScaling(1/λ) + @test @inferred(I\J) === J if atype == "Array" T = LowerTriangular(randn(3,3)) else T = LowerTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == full(T) + J - @test J + T == J + full(T) - @test T - J == full(T) - J - @test J - T == J - full(T) - @test T\I == inv(T) + @test @inferred(T + J) == full(T) + J + @test @inferred(J + T) == J + full(T) + @test @inferred(T - J) == full(T) - J + @test @inferred(J - T) == J - full(T) + @test @inferred(T\I) == inv(T) if atype == "Array" T = LinAlg.UnitLowerTriangular(randn(3,3)) else T = LinAlg.UnitLowerTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == full(T) + J - @test J + T == J + full(T) - @test T - J == full(T) - J - @test J - T == J - full(T) - @test T\I == inv(T) + @test @inferred(T + J) == full(T) + J + @test @inferred(J + T) == J + full(T) + @test @inferred(T - J) == full(T) - J + @test @inferred(J - T) == J - full(T) + @test @inferred(T\I) == inv(T) if atype == "Array" T = UpperTriangular(randn(3,3)) else T = UpperTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == full(T) + J - @test J + T == J + full(T) - @test T - J == full(T) - J - @test J - T == J - full(T) - @test T\I == inv(T) + @test @inferred(T + J) == full(T) + J + @test @inferred(J + T) == J + full(T) + @test @inferred(T - J) == full(T) - J + @test @inferred(J - T) == J - full(T) + @test @inferred(T\I) == inv(T) if atype == "Array" T = LinAlg.UnitUpperTriangular(randn(3,3)) else T = LinAlg.UnitUpperTriangular(view(randn(3,3), 1:3, 1:3)) end - @test T + J == full(T) + J - @test J + T == J + full(T) - @test T - J == full(T) - J - @test J - T == J - full(T) - @test T\I == inv(T) + @test @inferred(T + J) == full(T) + J + @test @inferred(J + T) == J + full(T) + @test @inferred(T - J) == full(T) - J + @test @inferred(J - T) == J - full(T) + @test @inferred(T\I) == inv(T) - @test I\A == A - @test A\I == inv(A) - @test λ\I === UniformScaling(1/λ) + @test @inferred(I\A) == A + @test @inferred(A\I) == inv(A) + @test @inferred(λ\I) === UniformScaling(1/λ) end end end From ff318df1882591baeeec5d10dd01c9270732573f Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 27 Aug 2016 11:08:10 +0800 Subject: [PATCH 1082/1117] Improve inlining cost model Consistently ignore line numbers and metadata nodes in all cases. --- base/inference.jl | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 0906187f8aa58..35458d03a655d 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2162,7 +2162,9 @@ end function occurs_more(e::ANY, pred, n) if isa(e,Expr) e = e::Expr - e.head === :line && return 0 + head = e.head + (head === :line || head === :meta || head === :inbounds || + head === :boundscheck) && return 0 c = 0 for a = e.args c += occurs_more(a, pred, n) @@ -2766,10 +2768,16 @@ inline_worthy(body::ANY, cost::Integer) = true # should the expression be part of the inline cost model function inline_ignore(ex::ANY) - isa(ex, LineNumberNode) || - ex === nothing || - isa(ex, Expr) && ((ex::Expr).head === :line || - (ex::Expr).head === :meta) + if isa(ex, LineNumberNode) || ex === nothing + return true + end + if isa(ex, Expr) + ex = ex::Expr + head = ex.head + return (head === :line || head === :meta || head === :inbounds || + head === :boundscheck) + end + return false end function inline_worthy(body::Expr, cost::Integer=1000) # precondition: 0 < cost; nominal cost = 1000 @@ -2779,7 +2787,7 @@ function inline_worthy(body::Expr, cost::Integer=1000) # precondition: 0 < cost; symlim = 1000 + 5_000_000 ÷ cost nstmt = 0 for stmt in body.args - if !inline_ignore(stmt) + if !(isa(stmt, SSAValue) || inline_ignore(stmt)) nstmt += 1 end end From 54ad8ed52afb9d9b39401824c3bb7a033be57ce1 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Wed, 24 Aug 2016 16:19:42 +0800 Subject: [PATCH 1083/1117] Pop noinline meta before label reindexing --- base/inference.jl | 5 ++++- src/codegen.cpp | 12 +++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 35458d03a655d..251137c34bd6b 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1902,6 +1902,7 @@ function finish(me::InferenceState) type_annotate!(me.linfo, me.stmt_types, me, me.nargs) # run optimization passes on fulltree + force_noinline = false if me.optimize # This pass is required for the AST to be valid in codegen # if any `SSAValue` is created by type inference. Ref issue #6068 @@ -1915,6 +1916,8 @@ function finish(me::InferenceState) getfield_elim_pass!(me.linfo, me) # remove placeholders filter!(x->x!==nothing, me.linfo.code) + # Pop metadata before label reindexing + force_noinline = popmeta!(me.linfo.code::Array{Any,1}, :noinline)[1] reindex_labels!(me.linfo, me) end widen_all_consts!(me.linfo) @@ -1949,7 +1952,7 @@ function finish(me::InferenceState) end # determine and cache inlineability - if !me.linfo.inlineable + if !me.linfo.inlineable && !force_noinline me.linfo.inlineable = me.linfo.jlcall_api==2 || isinlineable(me.linfo) end diff --git a/src/codegen.cpp b/src/codegen.cpp index 6e931439ac3b7..c58a7a84206c5 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4519,20 +4519,21 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func return inbounds; }; StmtProp cur_prop{topdebugloc, filename, toplineno, true, false, true, false}; - // this should not be necessary but it seems that meta node can cause - // an offset between the label number and the statement number - std::map<int, int> label_map; for (i = 0; i < stmtslen; i++) { cur_prop.loc_changed = false; cur_prop.is_poploc = false; jl_value_t *stmt = jl_array_ptr_ref(stmts, i); jl_expr_t *expr = jl_is_expr(stmt) ? (jl_expr_t*)stmt : nullptr; +#ifndef NDEBUG if (jl_is_labelnode(stmt)) { int lname = jl_labelnode_label(stmt); if (lname != i + 1) { - label_map[lname] = i + 1; + jl_safe_printf("Label number mismatch.\n"); + jl_(stmts); + abort(); } } +#endif if (jl_is_linenode(stmt) || (expr && expr->head == line_sym)) { ssize_t lno = -1; if (jl_is_linenode(stmt)) { @@ -4718,9 +4719,6 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func // if `unconditional` a unconditional branch is created to the target // label and the cursor is set to the next statement to process auto handle_label = [&] (int lname, bool unconditional) { - auto it = label_map.find(lname); - if (it != label_map.end()) - lname = it->second; auto &bb = labels[lname]; BasicBlock *cur_bb = builder.GetInsertBlock(); // Check if we've already visited this label From a08395187da5b08d0226a606c73d9640e657eaa1 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Wed, 24 Aug 2016 21:12:14 +0800 Subject: [PATCH 1084/1117] Store `propragate_inbounds` in lambda info Move unexported `jl_sym_t*` GVs from `julia.h` to `julia_internal.h` --- base/inference.jl | 3 ++- src/alloc.c | 5 +++++ src/dump.c | 2 ++ src/jltypes.c | 9 ++++++--- src/julia.h | 30 +----------------------------- src/julia_internal.h | 31 +++++++++++++++++++++++++++++++ 6 files changed, 47 insertions(+), 33 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 251137c34bd6b..70ae2c3035aa0 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1604,6 +1604,7 @@ function typeinf_ext(linfo::LambdaInfo) linfo.ssavaluetypes = code.ssavaluetypes linfo.pure = code.pure linfo.inlineable = code.inlineable + linfo.propagate_inbounds = code.propagate_inbounds ccall(:jl_set_lambda_rettype, Void, (Any, Any), linfo, code.rettype) if code.jlcall_api == 2 linfo.constval = code.constval @@ -2587,7 +2588,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference body = Expr(:block) body.args = ast - propagate_inbounds, _ = popmeta!(body, :propagate_inbounds) + propagate_inbounds = linfo.propagate_inbounds # see if each argument occurs only once in the body expression stmts = Any[] diff --git a/src/alloc.c b/src/alloc.c index 6255e056abb29..289bc8a12e4ce 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -101,6 +101,7 @@ jl_sym_t *meta_sym; jl_sym_t *compiler_temp_sym; jl_sym_t *inert_sym; jl_sym_t *vararg_sym; jl_sym_t *unused_sym; jl_sym_t *static_parameter_sym; jl_sym_t *polly_sym; jl_sym_t *inline_sym; +jl_sym_t *propagate_inbounds_sym; typedef struct { int64_t a; @@ -349,6 +350,8 @@ static void jl_lambda_info_set_ast(jl_lambda_info_t *li, jl_expr_t *ast) li->pure = 1; else if (ma == (jl_value_t*)inline_sym) li->inlineable = 1; + else if (ma == (jl_value_t*)propagate_inbounds_sym) + li->propagate_inbounds = 1; else jl_array_ptr_set(meta, ins++, ma); } @@ -431,6 +434,7 @@ JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info_uninit(void) li->constval = NULL; li->pure = 0; li->inlineable = 0; + li->propagate_inbounds = 0; return li; } @@ -544,6 +548,7 @@ static jl_lambda_info_t *jl_copy_lambda(jl_lambda_info_t *linfo) new_linfo->sparam_vals = linfo->sparam_vals; new_linfo->pure = linfo->pure; new_linfo->inlineable = linfo->inlineable; + new_linfo->propagate_inbounds = linfo->propagate_inbounds; new_linfo->nargs = linfo->nargs; new_linfo->isva = linfo->isva; new_linfo->rettype = linfo->rettype; diff --git a/src/dump.c b/src/dump.c index 3f16463d56afe..c2c387e7af753 100644 --- a/src/dump.c +++ b/src/dump.c @@ -949,6 +949,7 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) jl_serialize_value(s, (jl_value_t*)li->sparam_vals); write_int8(s->s, li->pure); write_int8(s->s, li->inlineable); + write_int8(s->s, li->propagate_inbounds); write_int8(s->s, li->isva); write_int32(s->s, li->nargs); jl_serialize_value(s, (jl_value_t*)li->def); @@ -1616,6 +1617,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta li->unspecialized_ducttape = NULL; li->pure = read_int8(s->s); li->inlineable = read_int8(s->s); + li->propagate_inbounds = read_int8(s->s); li->isva = read_int8(s->s); li->nargs = read_int32(s->s); li->def = (jl_method_t*)jl_deserialize_value(s, (jl_value_t**)&li->def); diff --git a/src/jltypes.c b/src/jltypes.c index 9caf8b720f0a8..a0a224e737b5a 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3880,7 +3880,7 @@ void jl_init_types(void) jl_lambda_info_type = jl_new_datatype(jl_symbol("LambdaInfo"), jl_any_type, jl_emptysvec, - jl_svec(24, + jl_svec(25, jl_symbol("rettype"), jl_symbol("sparam_syms"), jl_symbol("sparam_vals"), @@ -3898,13 +3898,14 @@ void jl_init_types(void) jl_symbol("inferred"), jl_symbol("pure"), jl_symbol("inlineable"), + jl_symbol("propagate_inbounds"), jl_symbol("inInference"), jl_symbol("inCompile"), jl_symbol("jlcall_api"), jl_symbol(""), jl_symbol("fptr"), jl_symbol(""), jl_symbol("")), - jl_svec(24, + jl_svec(25, jl_any_type, jl_simplevector_type, jl_simplevector_type, @@ -3924,6 +3925,7 @@ void jl_init_types(void) jl_bool_type, jl_bool_type, jl_bool_type, + jl_bool_type, jl_uint8_type, jl_bool_type, jl_any_type, @@ -3991,9 +3993,9 @@ void jl_init_types(void) jl_svecset(jl_methtable_type->types, 6, jl_int32_type); // DWORD #endif jl_svecset(jl_methtable_type->types, 7, jl_int32_type); // uint32_t - jl_svecset(jl_lambda_info_type->types, 21, jl_voidpointer_type); jl_svecset(jl_lambda_info_type->types, 22, jl_voidpointer_type); jl_svecset(jl_lambda_info_type->types, 23, jl_voidpointer_type); + jl_svecset(jl_lambda_info_type->types, 24, jl_voidpointer_type); jl_compute_field_offsets(jl_datatype_type); jl_compute_field_offsets(jl_typename_type); @@ -4071,6 +4073,7 @@ void jl_init_types(void) compiler_temp_sym = jl_symbol("#temp#"); polly_sym = jl_symbol("polly"); inline_sym = jl_symbol("inline"); + propagate_inbounds_sym = jl_symbol("propagate_inbounds"); tttvar = jl_new_typevar(jl_symbol("T"), (jl_value_t*)jl_bottom_type, diff --git a/src/julia.h b/src/julia.h index 6b5c790603c83..61af9fc6930d1 100644 --- a/src/julia.h +++ b/src/julia.h @@ -258,6 +258,7 @@ typedef struct _jl_lambda_info_t { int8_t inferred; int8_t pure; int8_t inlineable; + int8_t propagate_inbounds; int8_t inInference; // flags to tell if inference is running on this function int8_t inCompile; // flag to tell if codegen is running on this function int8_t jlcall_api; // the c-abi for fptr; 0 = jl_fptr_t, 1 = jl_fptr_sparam_t, 2 = constval @@ -555,36 +556,7 @@ extern JL_DLLEXPORT jl_value_t *jl_false; extern JL_DLLEXPORT jl_value_t *jl_nothing; // some important symbols -extern jl_sym_t *call_sym; extern jl_sym_t *invoke_sym; -extern jl_sym_t *empty_sym; extern jl_sym_t *body_sym; -extern jl_sym_t *dots_sym; extern jl_sym_t *vararg_sym; -extern jl_sym_t *quote_sym; extern jl_sym_t *newvar_sym; -extern jl_sym_t *top_sym; extern jl_sym_t *dot_sym; -extern jl_sym_t *line_sym; extern jl_sym_t *toplevel_sym; -extern jl_sym_t *core_sym; extern jl_sym_t *globalref_sym; extern JL_DLLEXPORT jl_sym_t *jl_incomplete_sym; -extern jl_sym_t *error_sym; extern jl_sym_t *amp_sym; -extern jl_sym_t *module_sym; extern jl_sym_t *colons_sym; -extern jl_sym_t *export_sym; extern jl_sym_t *import_sym; -extern jl_sym_t *importall_sym; extern jl_sym_t *using_sym; -extern jl_sym_t *goto_sym; extern jl_sym_t *goto_ifnot_sym; -extern jl_sym_t *label_sym; extern jl_sym_t *return_sym; -extern jl_sym_t *lambda_sym; extern jl_sym_t *assign_sym; -extern jl_sym_t *method_sym; extern jl_sym_t *slot_sym; -extern jl_sym_t *enter_sym; extern jl_sym_t *leave_sym; -extern jl_sym_t *exc_sym; extern jl_sym_t *new_sym; -extern jl_sym_t *compiler_temp_sym; -extern jl_sym_t *const_sym; extern jl_sym_t *thunk_sym; -extern jl_sym_t *anonymous_sym; extern jl_sym_t *underscore_sym; -extern jl_sym_t *abstracttype_sym; extern jl_sym_t *bitstype_sym; -extern jl_sym_t *compositetype_sym; -extern jl_sym_t *global_sym; extern jl_sym_t *unused_sym; -extern jl_sym_t *boundscheck_sym; extern jl_sym_t *inbounds_sym; -extern jl_sym_t *copyast_sym; extern jl_sym_t *fastmath_sym; -extern jl_sym_t *pure_sym; extern jl_sym_t *simdloop_sym; -extern jl_sym_t *meta_sym; extern jl_sym_t *list_sym; -extern jl_sym_t *inert_sym; extern jl_sym_t *static_parameter_sym; -extern jl_sym_t *polly_sym; extern jl_sym_t *inline_sym; // gc ------------------------------------------------------------------------- diff --git a/src/julia_internal.h b/src/julia_internal.h index f9015882d3738..abc4d7323e163 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -766,6 +766,37 @@ JL_DLLEXPORT jl_array_t *jl_array_cconvert_cstring(jl_array_t *a); int isabspath(const char *in); +extern jl_sym_t *call_sym; extern jl_sym_t *invoke_sym; +extern jl_sym_t *empty_sym; extern jl_sym_t *body_sym; +extern jl_sym_t *dots_sym; extern jl_sym_t *vararg_sym; +extern jl_sym_t *quote_sym; extern jl_sym_t *newvar_sym; +extern jl_sym_t *top_sym; extern jl_sym_t *dot_sym; +extern jl_sym_t *line_sym; extern jl_sym_t *toplevel_sym; +extern jl_sym_t *core_sym; extern jl_sym_t *globalref_sym; +extern jl_sym_t *error_sym; extern jl_sym_t *amp_sym; +extern jl_sym_t *module_sym; extern jl_sym_t *colons_sym; +extern jl_sym_t *export_sym; extern jl_sym_t *import_sym; +extern jl_sym_t *importall_sym; extern jl_sym_t *using_sym; +extern jl_sym_t *goto_sym; extern jl_sym_t *goto_ifnot_sym; +extern jl_sym_t *label_sym; extern jl_sym_t *return_sym; +extern jl_sym_t *lambda_sym; extern jl_sym_t *assign_sym; +extern jl_sym_t *method_sym; extern jl_sym_t *slot_sym; +extern jl_sym_t *enter_sym; extern jl_sym_t *leave_sym; +extern jl_sym_t *exc_sym; extern jl_sym_t *new_sym; +extern jl_sym_t *compiler_temp_sym; +extern jl_sym_t *const_sym; extern jl_sym_t *thunk_sym; +extern jl_sym_t *anonymous_sym; extern jl_sym_t *underscore_sym; +extern jl_sym_t *abstracttype_sym; extern jl_sym_t *bitstype_sym; +extern jl_sym_t *compositetype_sym; +extern jl_sym_t *global_sym; extern jl_sym_t *unused_sym; +extern jl_sym_t *boundscheck_sym; extern jl_sym_t *inbounds_sym; +extern jl_sym_t *copyast_sym; extern jl_sym_t *fastmath_sym; +extern jl_sym_t *pure_sym; extern jl_sym_t *simdloop_sym; +extern jl_sym_t *meta_sym; extern jl_sym_t *list_sym; +extern jl_sym_t *inert_sym; extern jl_sym_t *static_parameter_sym; +extern jl_sym_t *polly_sym; extern jl_sym_t *inline_sym; +extern jl_sym_t *propagate_inbounds_sym; + #ifdef __cplusplus } #endif From 2cece0982622b3a33090b88e6d0eb5ffa36cd8c8 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Wed, 24 Aug 2016 21:27:05 +0800 Subject: [PATCH 1085/1117] Add void_use_elim_pass! --- base/inference.jl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/base/inference.jl b/base/inference.jl index 70ae2c3035aa0..e81a0719304a8 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1913,8 +1913,11 @@ function finish(me::InferenceState) gotoifnot_elim_pass!(me.linfo, me) inlining_pass!(me.linfo, me) inbounds_meta_elim_pass!(me.linfo.code) + void_use_elim_pass!(me.linfo, me) alloc_elim_pass!(me.linfo, me) getfield_elim_pass!(me.linfo, me) + # Clean up for `alloc_elim_pass!` and `getfield_elim_pass!` + void_use_elim_pass!(me.linfo, me) # remove placeholders filter!(x->x!==nothing, me.linfo.code) # Pop metadata before label reindexing @@ -3209,6 +3212,29 @@ function occurs_outside_getfield(linfo::LambdaInfo, e::ANY, sym::ANY, return false end +function void_use_elim_pass!(linfo::LambdaInfo, sv) + # Remove top level SSAValue and slots that is `!usedUndef`. + # Also remove some `nothing` while we are at it.... + not_void_use = function (ex::ANY) + if isa(ex, SSAValue) + # Explicitly listed here for clarity + return false + elseif isa(ex, Slot) + return linfo.slotflags[(ex::Slot).id] & Slot_UsedUndef != 0 + elseif isa(ex, GlobalRef) + ex = ex::GlobalRef + return !isdefined(ex.mod, ex.name) + elseif (isa(ex, Expr) || isa(ex, GotoNode) || isa(ex, LineNumberNode) || + isa(ex, NewvarNode) || isa(ex, Symbol) || isa(ex, LabelNode)) + # This is a list of special type handled by the compiler + return true + end + return false + end + filter!(not_void_use, linfo.code::Array{Any,1}) + return +end + # removes inbounds metadata if we never encounter an inbounds=true or # boundscheck context in the method body function inbounds_meta_elim_pass!(code::Array{Any,1}) From d2e732db7ff809943ad6f5be413f5439916ca78a Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 27 Aug 2016 11:32:04 +0800 Subject: [PATCH 1086/1117] Clean up meta expression detection/handling --- base/inference.jl | 50 +++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index e81a0719304a8..a9b6627f5f27b 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1280,6 +1280,13 @@ widenconst(t::ANY) = t issubstate(a::VarState, b::VarState) = (a.typ ⊑ b.typ && a.undef <= b.undef) +# Meta expression head, these generally can't be deleted even when they are +# in a dead branch but can be ignored when analyzing uses/liveness. +is_meta_expr_head(head::Symbol) = + (head === :inbounds || head === :boundscheck || head === :meta || + head === :line) +is_meta_expr(ex::Expr) = is_meta_expr_head(ex.head) + function tmerge(typea::ANY, typeb::ANY) typea ⊑ typeb && return typeb typeb ⊑ typea && return typea @@ -1389,9 +1396,7 @@ function find_ssavalue_uses(e::ANY, uses, line) elseif isa(e,Expr) b = e::Expr head = b.head - if head === :line - return - end + is_meta_expr_head(head) && return if head === :(=) if isa(b.args[1],SSAValue) id = (b.args[1]::SSAValue).id+1 @@ -2032,7 +2037,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::InferenceState, undefs, pass) e = e::Expr head = e.head - if is(head,:line) || is(head,:const) + if is_meta_expr_head(head) || is(head,:const) return e elseif is(head,:(=)) e.args[2] = eval_annotate(e.args[2], vtypes, sv, undefs, pass) @@ -2048,12 +2053,6 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::InferenceState, undefs, pass) return e end -function expr_cannot_delete(ex::Expr) - head = ex.head - return (head === :inbounds || head === :boundscheck || head === :meta || - head === :line) -end - # annotate types of all symbols in AST function type_annotate!(linfo::LambdaInfo, states::Array{Any,1}, sv::ANY, nargs) nslots = length(states[1]) @@ -2082,7 +2081,7 @@ function type_annotate!(linfo::LambdaInfo, states::Array{Any,1}, sv::ANY, nargs) record_slot_type!(id, widenconst(states[i+1][id].typ), linfo.slottypes) end elseif optimize - if ((isa(expr, Expr) && expr_cannot_delete(expr::Expr)) || + if ((isa(expr, Expr) && is_meta_expr(expr::Expr)) || isa(expr, LineNumberNode)) i += 1 continue @@ -2154,9 +2153,10 @@ function substitute!(e::ANY, na, argexprs, spvals, offset) end if isa(e,Expr) e = e::Expr - if e.head === :static_parameter + head = e.head + if head === :static_parameter return spvals[e.args[1]] - elseif e.head !== :line + elseif !is_meta_expr_head(head) for i=1:length(e.args) e.args[i] = substitute!(e.args[i], na, argexprs, spvals, offset) end @@ -2170,8 +2170,7 @@ function occurs_more(e::ANY, pred, n) if isa(e,Expr) e = e::Expr head = e.head - (head === :line || head === :meta || head === :inbounds || - head === :boundscheck) && return 0 + is_meta_expr_head(head) && return 0 c = 0 for a = e.args c += occurs_more(a, pred, n) @@ -2256,8 +2255,7 @@ function effect_free(e::ANY, linfo::LambdaInfo, allow_volatile::Bool) elseif isa(e, Expr) e = e::Expr head = e.head - if head === :static_parameter || head === :meta || head === :line || - head === :inbounds || head === :boundscheck + if head === :static_parameter || is_meta_expr_head(head) return true end ea = e.args @@ -2729,7 +2727,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end if !isempty(stmts) - if all(stmt -> (isa(stmt,Expr) && stmt.head === :line) || isa(stmt, LineNumberNode) || stmt === nothing, + if all(stmt -> (isa(stmt, Expr) && is_meta_expr(stmt::Expr)) || isa(stmt, LineNumberNode) || stmt === nothing, stmts) empty!(stmts) else @@ -2778,13 +2776,7 @@ function inline_ignore(ex::ANY) if isa(ex, LineNumberNode) || ex === nothing return true end - if isa(ex, Expr) - ex = ex::Expr - head = ex.head - return (head === :line || head === :meta || head === :inbounds || - head === :boundscheck) - end - return false + return isa(ex, Expr) && is_meta_expr(ex::Expr) end function inline_worthy(body::Expr, cost::Integer=1000) # precondition: 0 < cost; nominal cost = 1000 @@ -2811,7 +2803,7 @@ end ssavalue_increment(body::ANY, incr) = body ssavalue_increment(body::SSAValue, incr) = SSAValue(body.id + incr) function ssavalue_increment(body::Expr, incr) - if body.head === :line + if is_meta_expr(body) return body end for i in 1:length(body.args) @@ -3179,6 +3171,8 @@ function occurs_outside_getfield(linfo::LambdaInfo, e::ANY, sym::ANY, end if isa(e,Expr) e = e::Expr + head = e.head + is_meta_expr_head(head) && return false if is_known_call(e, getfield, linfo) && symequal(e.args[2],sym) idx = e.args[3] if isa(idx,QuoteNode) && (idx.value in field_names) @@ -3189,11 +3183,11 @@ function occurs_outside_getfield(linfo::LambdaInfo, e::ANY, sym::ANY, end return true end - if is(e.head,:(=)) + if head === :(=) return occurs_outside_getfield(linfo, e.args[2], sym, sv, field_count, field_names) else - if (e.head === :block && isa(sym, Slot) && + if (head === :block && isa(sym, Slot) && linfo.slotflags[(sym::Slot).id] & Slot_UsedUndef == 0) ignore_void = true else From 886a3ab0d113d07b0272b24b7c92aa3f533a1dc0 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 27 Aug 2016 22:19:17 +0800 Subject: [PATCH 1087/1117] Handle boundscheck elimination in type inference Also collapse redundant debug info when coverage is off. Fix incorrect `Expr(:inbounds, false)` elimination. --- base/inference.jl | 262 +++++++++++++++++++++++++++++++++++++-- test/boundscheck_exec.jl | 23 ++++ 2 files changed, 274 insertions(+), 11 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index a9b6627f5f27b..6857ab70dab6e 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1444,6 +1444,7 @@ function unshare_linfo!(li::LambdaInfo) end inlining_enabled() = (JLOptions().can_inline == 1) +coverage_enabled() = (JLOptions().code_coverage != 0) #### entry points for inferring a LambdaInfo given a type signature #### function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtree::Bool, optimize::Bool, cached::Bool, caller) @@ -1917,14 +1918,12 @@ function finish(me::InferenceState) # optimizing and use unoptimized IR in codegen. gotoifnot_elim_pass!(me.linfo, me) inlining_pass!(me.linfo, me) - inbounds_meta_elim_pass!(me.linfo.code) void_use_elim_pass!(me.linfo, me) alloc_elim_pass!(me.linfo, me) getfield_elim_pass!(me.linfo, me) # Clean up for `alloc_elim_pass!` and `getfield_elim_pass!` void_use_elim_pass!(me.linfo, me) - # remove placeholders - filter!(x->x!==nothing, me.linfo.code) + meta_elim_pass!(me.linfo, me.linfo.code::Array{Any,1}) # Pop metadata before label reindexing force_noinline = popmeta!(me.linfo.code::Array{Any,1}, :noinline)[1] reindex_labels!(me.linfo, me) @@ -2479,7 +2478,6 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference if spec_miss !== nothing push!(stmts, merge) end - #println(stmts) return (ret_var, stmts) end else @@ -3229,14 +3227,256 @@ function void_use_elim_pass!(linfo::LambdaInfo, sv) return end -# removes inbounds metadata if we never encounter an inbounds=true or -# boundscheck context in the method body -function inbounds_meta_elim_pass!(code::Array{Any,1}) - if findfirst(x -> isa(x, Expr) && - ((x.head === :inbounds && x.args[1] === true) || x.head === :boundscheck), - code) == 0 - filter!(x -> !(isa(x, Expr) && x.head === :inbounds), code) +function meta_elim_pass!(linfo::LambdaInfo, code::Array{Any,1}) + # 1. Remove place holders + # + # 2. If coverage is off, remove line number nodes that don't mark any + # real expressions. + # + # 3. Remove top level SSAValue + # + # 4. Handle bounds check elision + # + # 4.1. If check_bounds is always on, delete all `Expr(:boundscheck)` + # 4.2. If check_bounds is always off, delete all boundscheck blocks. + # 4.3. If check_bounds is default, figure out whether each checkbounds + # blocks needs to be eliminated or could be eliminated when inlined + # into another function. Delete the blocks that should be eliminated + # and delete the `Expr(:boundscheck)` for blocks that will never be + # deleted. (i.e. the ones that are not eliminated with + # `length(inbounds_stack) >= 2`) + # + # When deleting IR with boundscheck, keep the label node in order to not + # confuse later passes or codegen. (we could also track if any SSAValue + # is deleted while still having uses that are not but that's a little + # expensive). + # + # 5. Clean up `Expr(:inbounds)` + # + # Delete all `Expr(:inbounds)` that is unnecessary, which is all of them + # for non-default check_bounds. For default check_bounds this includes + # + # * `Expr(:inbounds, true)` in `Expr(:inbounds, true)` + # * `Expr(:inbounds, false)` when + # `!is_inbounds && length(inbounds_stack) >= 2` + # + # Functions without `propagate_inbounds` have an implicit `false` on the + # `inbounds_stack` + # + # There are other cases in which we can eliminate `Expr(:inbounds)` or + # `Expr(:boundscheck)` (e.g. when they don't enclose any non-meta + # expressions). Those are a little harder to detect and are hopefully + # not too common. + do_coverage = coverage_enabled() + check_bounds = JLOptions().check_bounds + + inbounds_stack = linfo.propagate_inbounds ? Bool[] : [false] + # Whether the push is deleted (therefore if the pop has to be too) + # Shared for `Expr(:boundscheck)` and `Expr(:inbounds)` + bounds_elim_stack = Bool[] + # The expression index of the push, set to `0` when encountering a + # non-meta expression that might be affect by the push. + # The clearing needs to be propagated up during pop + # This is not pushed to if the push is already eliminated + # Also shared for `Expr(:boundscheck)` and `Expr(:inbounds)` + bounds_push_pos_stack = [0] + # Number of boundscheck pushes in a eliminated boundscheck block + void_boundscheck_depth = 0 + is_inbounds = check_bounds == 2 + enabled = true + + # Position of the last line number node without any non-meta expressions + # in between. + prev_dbg_stack = [0] + # Whether there's any non-meta exprs after the enclosing `push_loc` + push_loc_pos_stack = [0] + + for i in 1:length(code) + ex = code[i] + if ex === nothing + continue + elseif isa(ex, SSAValue) + code[i] = nothing + continue + elseif isa(ex, LabelNode) + prev_dbg_stack[end] = 0 + push_loc_pos_stack[end] = 0 + continue + elseif !do_coverage && (isa(ex, LineNumberNode) || + (isa(ex, Expr) && (ex::Expr).head === :line)) + prev_label = prev_dbg_stack[end] + if prev_label != 0 + code[prev_label] = nothing + end + prev_dbg_stack[end] = i + continue + elseif !isa(ex, Expr) + if enabled + prev_dbg_stack[end] = 0 + push_loc_pos_stack[end] = 0 + bounds_push_pos_stack[end] = 0 + else + code[i] = nothing + end + continue + end + ex = ex::Expr + args = ex.args + head = ex.head + if head === :boundscheck + if !enabled + # we are in an eliminated boundscheck, simply record the number + # of push/pop + if !(args[1] === :pop) + void_boundscheck_depth += 1 + elseif void_boundscheck_depth == 0 + pop!(bounds_elim_stack) + enabled = true + else + void_boundscheck_depth -= 1 + end + code[i] = nothing + elseif args[1] === :pop + # This will also delete pops that don't match + if (isempty(bounds_elim_stack) ? true : + pop!(bounds_elim_stack)) + code[i] = nothing + continue + end + push_idx = bounds_push_pos_stack[end] + if length(bounds_push_pos_stack) > 1 + pop!(bounds_push_pos_stack) + end + if push_idx > 0 + code[push_idx] = nothing + code[i] = nothing + else + bounds_push_pos_stack[end] = 0 + end + elseif is_inbounds + code[i] = nothing + push!(bounds_elim_stack, true) + enabled = false + elseif check_bounds == 1 || length(inbounds_stack) >= 2 + # Not inbounds and at least two levels deep, this will never + # be eliminated when inlined to another function. + code[i] = nothing + push!(bounds_elim_stack, true) + else + push!(bounds_elim_stack, false) + push!(bounds_push_pos_stack, i) + end + continue + end + if !enabled && !(do_coverage && head === :meta) + code[i] = nothing + continue + end + if head === :inbounds + if check_bounds != 0 + code[i] = nothing + continue + end + arg1 = args[1] + if arg1 === true + if inbounds_stack[end] + code[i] = nothing + push!(bounds_elim_stack, true) + else + is_inbounds = true + push!(bounds_elim_stack, false) + push!(bounds_push_pos_stack, i) + end + push!(inbounds_stack, true) + elseif arg1 === false + if is_inbounds + if !inbounds_stack[end] + is_inbounds = false + end + push!(bounds_elim_stack, false) + push!(bounds_push_pos_stack, i) + elseif length(inbounds_stack) >= 2 + code[i] = nothing + push!(bounds_elim_stack, true) + else + push!(bounds_elim_stack, false) + push!(bounds_push_pos_stack, i) + end + push!(inbounds_stack, false) + else + # pop + inbounds_len = length(inbounds_stack) + if inbounds_len != 0 + pop!(inbounds_stack) + inbounds_len -= 1 + end + # This will also delete pops that don't match + if (isempty(bounds_elim_stack) ? true : + pop!(bounds_elim_stack)) + # No need to update `is_inbounds` since the push was a no-op + code[i] = nothing + continue + end + if inbounds_len >= 2 + is_inbounds = (inbounds_stack[inbounds_len] || + inbounds_stack[inbounds_len - 1]) + elseif inbounds_len == 1 + is_inbounds = inbounds_stack[inbounds_len] + else + is_inbounds = false + end + push_idx = bounds_push_pos_stack[end] + if length(bounds_push_pos_stack) > 1 + pop!(bounds_push_pos_stack) + end + if push_idx > 0 + code[push_idx] = nothing + code[i] = nothing + else + bounds_push_pos_stack[end] = 0 + end + end + continue + end + if head !== :meta + prev_dbg_stack[end] = 0 + push_loc_pos_stack[end] = 0 + bounds_push_pos_stack[end] = 0 + continue + end + nargs = length(args) + if do_coverage || nargs == 0 + continue + end + arg1 = args[1] + if arg1 === :push_loc + push!(prev_dbg_stack, 0) + push!(push_loc_pos_stack, i) + elseif arg1 === :pop_loc + prev_dbg = if length(prev_dbg_stack) > 1 + pop!(prev_dbg_stack) + else + prev_dbg_stack[end] + end + if prev_dbg > 0 + code[prev_dbg] = nothing + end + push_loc = if length(push_loc_pos_stack) > 1 + pop!(push_loc_pos_stack) + else + push_loc_pos_stack[end] + end + if push_loc > 0 + code[push_loc] = nothing + code[i] = nothing + else + push_loc_pos_stack[end] = 0 + end + else + continue + end end + filter!(x->x!==nothing, code) end # does the same job as alloc_elim_pass for allocations inline in getfields diff --git a/test/boundscheck_exec.jl b/test/boundscheck_exec.jl index e711fcf172da8..daf602c91c06e 100644 --- a/test/boundscheck_exec.jl +++ b/test/boundscheck_exec.jl @@ -151,4 +151,27 @@ end @test B2() == 0 +# Make sure type inference doesn't incorrectly optimize out +# `Expr(:inbounds, false)` +# Simply `return a[1]` doesn't work due to inlining bug +@inline function f1(a) + # This has to be an arrayget / arrayset since these currently have a + # implicit `Expr(:boundscheck)` that's not visible to type inference + x = a[1] + return x +end +# second level +@inline function g1(a) + x = f1(a) + return x +end +function k1(a) + # This `Expr(:inbounds, true)` shouldn't affect `f1` + @inbounds x = g1(a) + return x +end +if bc_opt != bc_off + @test_throws BoundsError k1(Int[]) +end + end From 6e067341183e2f4d6f3057f610a66e76c73b55a5 Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Sat, 27 Aug 2016 22:46:45 +0800 Subject: [PATCH 1088/1117] Delete boundscheck handling from codegen --- src/codegen.cpp | 59 +++++-------------------------------------------- 1 file changed, 6 insertions(+), 53 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index c58a7a84206c5..8d9f29d663600 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4498,18 +4498,13 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func DebugLoc loc; StringRef file; ssize_t line; - bool enabled; bool is_inbounds; bool loc_changed; bool is_poploc; }; std::vector<StmtProp> stmtprops(stmtslen); std::vector<DbgState> DI_stack; - std::vector<bool> boundsCheck_stack{false}; std::vector<bool> inbounds_stack{false}; - auto is_bounds_check_block = [&] () { - return !boundsCheck_stack.empty() && boundsCheck_stack.back(); - }; auto is_inbounds = [&] () { // inbounds rule is either of top two values on inbounds stack are true size_t sz = inbounds_stack.size(); @@ -4518,7 +4513,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func inbounds |= inbounds_stack[sz - 2]; return inbounds; }; - StmtProp cur_prop{topdebugloc, filename, toplineno, true, false, true, false}; + StmtProp cur_prop{topdebugloc, filename, toplineno, false, true, false}; for (i = 0; i < stmtslen; i++) { cur_prop.loc_changed = false; cur_prop.is_poploc = false; @@ -4628,64 +4623,26 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func } if (expr) { jl_value_t **args = (jl_value_t**)jl_array_data(expr->args); - if (expr->head == boundscheck_sym) { - if (jl_array_len(expr->args) > 0) { - jl_value_t *arg = args[0]; - if (arg == jl_true) { - boundsCheck_stack.push_back(true); - } - else if (arg == jl_false) { - boundsCheck_stack.push_back(false); - } - else { - if (!boundsCheck_stack.empty()) { - boundsCheck_stack.pop_back(); - } - } - } - } - else if (cur_prop.enabled && expr->head == inbounds_sym) { + if (expr->head == inbounds_sym) { // manipulate inbounds stack - // note that when entering an inbounds context, we must also update - // the boundsCheck context to be false if (jl_array_len(expr->args) > 0) { jl_value_t *arg = args[0]; if (arg == jl_true) { inbounds_stack.push_back(true); - boundsCheck_stack.push_back(false); } else if (arg == jl_false) { inbounds_stack.push_back(false); - boundsCheck_stack.push_back(false); } - else { - if (!inbounds_stack.empty()) - inbounds_stack.pop_back(); - if (!boundsCheck_stack.empty()) - boundsCheck_stack.pop_back(); + else if (!inbounds_stack.empty()) { + inbounds_stack.pop_back(); } } } } - bool is_ib = is_inbounds(); - if (is_ib && is_bounds_check_block() && - jl_options.check_bounds != JL_OPTIONS_CHECK_BOUNDS_ON) { - // elide bounds check blocks in inbounds context - cur_prop.enabled = false; - } - else if (is_bounds_check_block() && - jl_options.check_bounds == JL_OPTIONS_CHECK_BOUNDS_OFF) { - // elide bounds check blocks when turned off by options - cur_prop.enabled = false; - } - else { - cur_prop.enabled = true; - } - cur_prop.is_inbounds = is_ib; + cur_prop.is_inbounds = is_inbounds(); stmtprops[i] = cur_prop; } DI_stack.clear(); - boundsCheck_stack.clear(); inbounds_stack.clear(); // step 12. Do codegen in control flow order @@ -4772,7 +4729,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func if (ctx.debug_enabled) builder.SetCurrentDebugLocation(props.loc); // Disable coverage for pop_loc, it doesn't start a new expression - if (do_coverage && props.enabled && !props.is_poploc) { + if (do_coverage && !props.is_poploc) { coverageVisitLine(props.file, props.line); } } @@ -4785,10 +4742,6 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func handle_label(lname, true); continue; } - if (!props.enabled) { - find_next_stmt(cursor + 1); - continue; - } if (expr && expr->head == return_sym) { bool retboxed = false; Type *retty; From b9007345d7342cc09ad1b6ff188b3954d1821262 Mon Sep 17 00:00:00 2001 From: Brandon Edwards <brandon.edwards144@gmail.com> Date: Fri, 2 Sep 2016 09:31:27 -0400 Subject: [PATCH 1089/1117] Fix #18189 (Add deprecation of @unix_only, etc. to NEWS) (#18254) * Fix #18189 (Add deprecation of etc. to NEWS) * Remove mentioning of @static, add as a note * Added reference to Handling Operating System Variation section in manual * Changed reference numbers * Change PR number --- NEWS.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/NEWS.md b/NEWS.md index 530777425f5b8..0cd0ee6f1f31e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -478,6 +478,17 @@ Deprecated or removed * The no-op `transpose` fallback has been deprecated. Consider introducing suitable `transpose` methods or calling `permutedims(x, [2,1])` ([#13171], [#17075], [#17374]). + * The following macros have been deprecated ([#16219]): + * `@windows` is deprecated in favor of `is_windows()` + * `@unix` is deprecated in favor of `is_unix()` + * `@osx` is deprecated in favor of `is_apple()` + * `@linux` is deprecated in favor of `is_linux()` + * `@windows_only` is deprecated in favor of `if is_windows()` + * `@unix_only` is deprecated in favor of `if is_unix()` + * `@osx_only` is deprecated in favor of `if is_apple()` + * `@linux_only` is deprecated in favor of `if is_linux()` + * NOTE: Using `@static` could be useful/necessary when used in a function's local scope. See details at the section entitled [Handling Operating System Variation](http://docs.julialang.org/en/latest/manual/handling-operating-system-variation/#man-handling-operating-system-variation) in the manual. + Command-line option changes --------------------------- From 733c07455b7bf4b0af648d0a35e4514867b8833f Mon Sep 17 00:00:00 2001 From: Yichao Yu <yyc1992@gmail.com> Date: Fri, 2 Sep 2016 11:47:26 -0400 Subject: [PATCH 1090/1117] Fix `jl_get_specialization` signature mismatch Also fix a few other compiler warnings. --- src/codegen.cpp | 12 +++++------- src/gf.c | 2 -- src/julia_internal.h | 1 + 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 8d9f29d663600..671dbd0a17346 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1036,8 +1036,6 @@ void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) // for use in reflection from Julia. // this is paired with jl_dump_function_ir and jl_dump_function_asm in particular ways: // misuse will leak memory or cause read-after-free -extern "C" JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp); - extern "C" JL_DLLEXPORT void *jl_get_llvmf_defn(jl_lambda_info_t *linfo, bool getwrapper) { @@ -1053,7 +1051,7 @@ void *jl_get_llvmf_defn(jl_lambda_info_t *linfo, bool getwrapper) // first copy the linfo to avoid corrupting it and // confusing the compiler about the // validity of the code it already generated - temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals); + temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals, 1); jl_type_infer(temp, 0); if (temp->code == jl_nothing || temp->inInference) { // something went wrong: abort! @@ -1142,7 +1140,7 @@ void *jl_get_llvmf_decl(jl_lambda_info_t *linfo, bool getwrapper) if (linfo->functionObjectsDecls.functionObject == NULL) { jl_lambda_info_t *temp = NULL; JL_GC_PUSH1(&temp); - temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals); + temp = jl_get_specialized(linfo->def, linfo->specTypes, linfo->sparam_vals, 1); jl_type_infer(temp, 0); temp->jlcall_api = 0; temp->constval = jl_nothing; @@ -4521,7 +4519,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func jl_expr_t *expr = jl_is_expr(stmt) ? (jl_expr_t*)stmt : nullptr; #ifndef NDEBUG if (jl_is_labelnode(stmt)) { - int lname = jl_labelnode_label(stmt); + size_t lname = jl_labelnode_label(stmt); if (lname != i + 1) { jl_safe_printf("Label number mismatch.\n"); jl_(stmts); @@ -4652,7 +4650,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func // `seq_next` is the next statement we want to emit // i.e. if it exists, it's the next one following control flow and // should be emitted into the current insert point. - if (seq_next >= 0 && seq_next < stmtslen) { + if (seq_next >= 0 && (unsigned)seq_next < stmtslen) { cursor = seq_next; return; } @@ -4667,7 +4665,7 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func cursor = item.first; workstack.pop_back(); }; - auto add_to_list = [&] (int pos, BasicBlock *bb) { + auto add_to_list = [&] (unsigned pos, BasicBlock *bb) { if (pos >= stmtslen) return; workstack.push_back({pos, bb}); diff --git a/src/gf.c b/src/gf.c index 0e69c74090e73..569a6193cb696 100644 --- a/src/gf.c +++ b/src/gf.c @@ -117,8 +117,6 @@ static int8_t jl_cachearg_offset(jl_methtable_t *mt) /// ----- Insertion logic for special entries ----- /// -JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp, int allow_exec); - // get or create the LambdaInfo for a specialization JL_DLLEXPORT jl_lambda_info_t *jl_specializations_get_linfo(jl_method_t *m, jl_tupletype_t *type, jl_svec_t *sparams) { diff --git a/src/julia_internal.h b/src/julia_internal.h index abc4d7323e163..87d323fa363f3 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -426,6 +426,7 @@ JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); JL_DLLEXPORT int jl_has_call_ambiguities(jl_tupletype_t *types, jl_method_t *m); +JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp, int allow_exec); uint32_t jl_module_next_counter(jl_module_t *m); void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig); From 247225d730ef0a623871911fb260199f03f68b9d Mon Sep 17 00:00:00 2001 From: Tracy Wadleigh <tracy.wadleigh@gmail.com> Date: Thu, 1 Sep 2016 21:10:10 -0700 Subject: [PATCH 1091/1117] Repeatedly divide read buffer size by 8 until success. Fixes #11481. Dividing by 8 leads to success on the second try in the case of the failure of --lisp on Windows 7. --- src/support/ios.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/support/ios.c b/src/support/ios.c index f05c6128d2f6a..385a24258bfeb 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -99,14 +99,20 @@ static int _os_read(long fd, void *buf, size_t n, size_t *nread) { ssize_t r; + n = LIMIT_IO_SIZE(n); while (1) { set_io_wait_begin(1); - r = read((int)fd, buf, LIMIT_IO_SIZE(n)); + r = read((int)fd, buf, n); set_io_wait_begin(0); if (r > -1) { *nread = (size_t)r; return 0; } + // This test is a hack to fix #11481 for Windows 7. Unnecessary for Windows 10. + if (errno == ENOMEM && n > 80) { + n >>= 3; + continue; + } if (!_enonfatal(errno)) { *nread = 0; return errno; From 55a840956488d6c2d034262a86f874ebda01cdbf Mon Sep 17 00:00:00 2001 From: Jameson Nash <vtjnash@gmail.com> Date: Sat, 3 Sep 2016 00:34:25 -0400 Subject: [PATCH 1092/1117] make compile fast we really badly need a heuristic for detecting simple methods like this. --- base/essentials.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/essentials.jl b/base/essentials.jl index 30b8d83148f46..38880994435cf 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -23,6 +23,7 @@ macro _propagate_inbounds_meta() Expr(:meta, :inline, :propagate_inbounds) end +convert(::Type{Any}, x::ANY) = x convert{T}(::Type{T}, x::T) = x convert(::Type{Tuple{}}, ::Tuple{}) = () From 538760882f33985ab7af7e866207ecc78e1c7c55 Mon Sep 17 00:00:00 2001 From: Matthieu Gomez <gomez.matthieu@gmail.com> Date: Sat, 3 Sep 2016 09:57:03 -0400 Subject: [PATCH 1093/1117] Update utf8proc.jl (#18327) add ! suffix to isgraphemebreak! to clarify that the state is changed --- base/strings/utf8proc.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/strings/utf8proc.jl b/base/strings/utf8proc.jl index 6bc7d94329079..aadcf97ba83da 100644 --- a/base/strings/utf8proc.jl +++ b/base/strings/utf8proc.jl @@ -187,7 +187,7 @@ isgraphemebreak(c1::Char, c2::Char) = # Stateful grapheme break required by Unicode-9 rules: the string # must be processed in sequence, with state initialized to Ref{Int32}(0). # Requires utf8proc v2.0 or later. -isgraphemebreak(c1::Char, c2::Char, state::Ref{Int32}) = +isgraphemebreak!(state::Ref{Int32}, c1::Char, c2::Char) = ccall(:utf8proc_grapheme_break_stateful, Bool, (UInt32, UInt32, Ref{Int32}), c1, c2, state) immutable GraphemeIterator{S<:AbstractString} @@ -202,7 +202,7 @@ function length(g::GraphemeIterator) n = 0 state = Ref{Int32}(0) for c in g.s - n += isgraphemebreak(c0, c, state) + n += isgraphemebreak!(state, c0, c) c0 = c end return n @@ -218,7 +218,7 @@ function next(g::GraphemeIterator, i_) c0, k = next(s, i) while !done(s, k) # loop until next grapheme is s[i:j] c, ℓ = next(s, k) - isgraphemebreak(c0, c, state) && break + isgraphemebreak!(state, c0, c) && break j = k k = ℓ c0 = c From b60c97568b19d3bd370b4af9ee3daed08a6f4c43 Mon Sep 17 00:00:00 2001 From: Katie Hyatt <kshyatt@physics.ucsb.edu> Date: Mon, 15 Aug 2016 15:13:12 -0700 Subject: [PATCH 1094/1117] Moved so many docs out of HelpDB. More examples. --- base/Enums.jl | 17 + base/abstractarray.jl | 181 +- base/abstractarraymath.jl | 61 +- base/array.jl | 191 +- base/arraymath.jl | 30 + base/base64.jl | 28 + base/broadcast.jl | 102 + base/client.jl | 8 + base/complex.jl | 33 + base/dict.jl | 150 + base/docs/helpdb/Base.jl | 4863 +------------------ base/dsp.jl | 41 + base/env.jl | 5 + base/essentials.jl | 7 + base/expr.jl | 40 +- base/file.jl | 137 +- base/float.jl | 5 + base/int.jl | 91 + base/interactiveutil.jl | 81 +- base/intfuncs.jl | 175 + base/intset.jl | 6 + base/io.jl | 171 +- base/iobuffer.jl | 38 + base/iostream.jl | 48 + base/iterator.jl | 86 +- base/libc.jl | 10 + base/linalg/dense.jl | 143 +- base/linalg/generic.jl | 489 +- base/linalg/matmul.jl | 24 + base/loading.jl | 53 + base/math.jl | 135 + base/mpfr.jl | 24 + base/number.jl | 44 + base/operators.jl | 364 ++ base/path.jl | 142 + base/permuteddimsarray.jl | 16 + base/poll.jl | 37 + base/process.jl | 98 + base/profile.jl | 6 + base/random.jl | 112 +- base/range.jl | 45 + base/reduce.jl | 195 +- base/reflection.jl | 174 +- base/rounding.jl | 18 + base/sharedarray.jl | 17 + base/socket.jl | 80 + base/sort.jl | 64 + base/stat.jl | 149 + base/statistics.jl | 16 + base/stream.jl | 44 +- base/strings/basic.jl | 129 + base/strings/io.jl | 74 +- base/strings/search.jl | 58 +- base/strings/types.jl | 20 + base/strings/utf8proc.jl | 133 + base/strings/util.jl | 196 + base/sysinfo.jl | 10 + base/tuple.jl | 9 +- base/util.jl | 17 + doc/manual/arrays.rst | 2 +- doc/manual/complex-and-rational-numbers.rst | 3 +- doc/manual/control-flow.rst | 2 +- doc/manual/mathematical-operations.rst | 2 +- doc/manual/stacktraces.rst | 2 +- doc/manual/strings.rst | 4 +- doc/stdlib/arrays.rst | 253 +- doc/stdlib/base.rst | 100 +- doc/stdlib/c.rst | 6 +- doc/stdlib/collections.rst | 160 +- doc/stdlib/file.rst | 99 +- doc/stdlib/io-network.rst | 155 +- doc/stdlib/linalg.rst | 353 +- doc/stdlib/math.rst | 150 +- doc/stdlib/numbers.rst | 100 +- doc/stdlib/parallel.rst | 18 +- doc/stdlib/sort.rst | 30 +- doc/stdlib/strings.rst | 248 +- doc/stdlib/test.rst | 2 +- 78 files changed, 6358 insertions(+), 5071 deletions(-) diff --git a/base/Enums.jl b/base/Enums.jl index ba1073cd36f72..44d77e76771bb 100644 --- a/base/Enums.jl +++ b/base/Enums.jl @@ -26,6 +26,23 @@ end @noinline enum_argument_error(typename, x) = throw(ArgumentError(string("invalid value for Enum $(typename): $x"))) +""" + @enum EnumName EnumValue1[=x] EnumValue2[=y] + +Create an [`Enum`](:obj:`Enum`) type with name `EnumName` and enum member values of +`EnumValue1` and `EnumValue2` with optional assigned values of `x` and `y`, respectively. +`EnumName` can be used just like other types and enum member values as regular values, such as + +```jldoctest +julia> @enum FRUIT apple=1 orange=2 kiwi=3 + +julia> f(x::FRUIT) = "I'm a FRUIT with value: \$(Int(x))" +f (generic function with 1 method) + +julia> f(apple) +"I'm a FRUIT with value: 1" +``` +""" macro enum(T,syms...) if isempty(syms) throw(ArgumentError("no arguments given for Enum $T")) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 408f05f4645ef..cfce65719ae94 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -51,6 +51,13 @@ size{N}(x, d1::Integer, d2::Integer, dx::Vararg{Integer, N}) = (size(x, d1), siz indices(A, d) Returns the valid range of indices for array `A` along dimension `d`. + +```jldoctest +julia> A = ones(5,6,7); + +julia> indices(A,2) +Base.OneTo(6) +``` """ function indices{T,N}(A::AbstractArray{T,N}, d) @_inline_meta @@ -61,6 +68,13 @@ end indices(A) Returns the tuple of valid indices for array `A`. + +```jldoctest +julia> A = ones(5,6,7); + +julia> indices(A) +(Base.OneTo(5),Base.OneTo(6),Base.OneTo(7)) +``` """ function indices(A) @_inline_meta @@ -88,6 +102,15 @@ is `indices(A, 1)`. Calling this function is the "safe" way to write algorithms that exploit linear indexing. + +```jldoctest +julia> A = ones(5,6,7); + +julia> b = linearindices(A); + +julia> extrema(b) +(1,210) +``` """ linearindices(A) = (@_inline_meta; OneTo(_length(A))) linearindices(A::AbstractVector) = (@_inline_meta; indices1(A)) @@ -261,9 +284,9 @@ linearindexing(::LinearIndexing, ::LinearIndexing) = LinearSlow() Return `true` if the specified indices `I` are in bounds for the given array `A`. Subtypes of `AbstractArray` should specialize this method if they need to provide custom bounds checking behaviors; however, in -many cases one can rely on `A`'s indices and `checkindex`. +many cases one can rely on `A`'s indices and [`checkindex`](:func:`checkindex`). -See also `checkindex`. +See also [`checkindex`](:func:`checkindex`). """ function checkbounds(::Type{Bool}, A::AbstractArray, I...) @_inline_meta @@ -297,7 +320,7 @@ usually in a 1-for-1 fashion, checkbounds_indices(Bool, (IA1, IA...), (I1, I...)) = checkindex(Bool, IA1, I1) & checkbounds_indices(Bool, IA, I) -Note that `checkindex` is being used to perform the actual +Note that [`checkindex`](:func:`checkindex`) is being used to perform the actual bounds-check for a single dimension of the array. There are two important exceptions to the 1-1 rule: linear indexing and @@ -362,6 +385,14 @@ Return `true` if the given `index` is within the bounds of `inds`. Custom types that would like to behave as indices for all arrays can extend this method in order to provide a specialized bounds checking implementation. + +```jldoctest +julia> checkindex(Bool,1:20,8) +true + +julia> checkindex(Bool,1:20,21) +false +``` """ checkindex(::Type{Bool}, inds::AbstractUnitRange, i) = throw(ArgumentError("unable to check bounds for indices of type $(typeof(i))")) checkindex(::Type{Bool}, inds::AbstractUnitRange, i::Real) = (first(inds) <= i) & (i <= last(inds)) @@ -648,11 +679,6 @@ function copy_transpose!{R,S}(B::AbstractVecOrMat{R}, ir_dest::Range{Int}, jr_de return B end -function copymutable(a::AbstractArray) - @_propagate_inbounds_meta - copy!(similar(a), a) -end -copymutable(itr) = collect(itr) """ copymutable(a) @@ -661,7 +687,11 @@ this is equivalent to `copy(a)`, but for other array types it may differ depending on the type of `similar(a)`. For generic iterables this is equivalent to `collect(a)`. """ -copymutable +function copymutable(a::AbstractArray) + @_propagate_inbounds_meta + copy!(similar(a), a) +end +copymutable(itr) = collect(itr) zero{T}(x::AbstractArray{T}) = fill!(similar(x), zero(T)) @@ -677,6 +707,51 @@ done(A::AbstractArray,i) = (@_propagate_inbounds_meta; done(i[1], i[2])) # eachindex iterates over all indices. LinearSlow definitions are later. eachindex(A::AbstractVector) = (@_inline_meta(); indices1(A)) + +""" + eachindex(A...) + +Creates an iterable object for visiting each index of an AbstractArray `A` in an efficient +manner. For array types that have opted into fast linear indexing (like `Array`), this is +simply the range `1:length(A)`. For other array types, this returns a specialized Cartesian +range to efficiently index into the array with indices specified for every dimension. For +other iterables, including strings and dictionaries, this returns an iterator object +supporting arbitrary index types (e.g. unevenly spaced or non-integer indices). + +Example for a sparse 2-d array: + +```jldoctest +julia> A = sparse([1, 1, 2], [1, 3, 1], [1, 2, -5]) +2×3 sparse matrix with 3 Int64 nonzero entries: + [1, 1] = 1 + [2, 1] = -5 + [1, 3] = 2 + +julia> for iter in eachindex(A) + @show iter.I[1], iter.I[2] + @show A[iter] + end +(iter.I[1],iter.I[2]) = (1,1) +A[iter] = 1 +(iter.I[1],iter.I[2]) = (2,1) +A[iter] = -5 +(iter.I[1],iter.I[2]) = (1,2) +A[iter] = 0 +(iter.I[1],iter.I[2]) = (2,2) +A[iter] = 0 +(iter.I[1],iter.I[2]) = (1,3) +A[iter] = 2 +(iter.I[1],iter.I[2]) = (2,3) +A[iter] = 0 +``` + +If you supply more than one `AbstractArray` argument, `eachindex` will create an +iterable object that is fast for all arguments (a `UnitRange` if all inputs have fast +linear indexing, a [`CartesianRange`](:obj`CartesianRange`) otherwise). +If the arrays have different sizes and/or +dimensionalities, `eachindex` returns an iterable that spans the largest range along each +dimension. +""" eachindex(A::AbstractArray) = (@_inline_meta(); eachindex(linearindexing(A), A)) function eachindex(A::AbstractArray, B::AbstractArray) @@ -1419,6 +1494,22 @@ function sub2ind(A::AbstractArray, I...) @_inline_meta sub2ind(indices(A), I...) end + +""" + ind2sub(a, index) -> subscripts + +Returns a tuple of subscripts into array `a` corresponding to the linear index `index`. + +```jldoctest +julia> A = ones(5,6,7); + +julia> ind2sub(A,35) +(5,1,2) + +julia> ind2sub(A,70) +(5,2,3) +``` +""" function ind2sub(A::AbstractArray, ind) @_inline_meta ind2sub(indices(A), ind) @@ -1430,6 +1521,20 @@ sub2ind(::DimsInteger) = 1 sub2ind(::Indices) = 1 sub2ind(::Tuple{}, I::Integer...) = (@_inline_meta; _sub2ind((), 1, 1, I...)) # Generic cases + +""" + sub2ind(dims, i, j, k...) -> index + +The inverse of [`ind2sub`](:func:`ind2sub`), returns the linear index corresponding to the provided subscripts. + +```jldoctest +julia> sub2ind((5,6,7),1,2,3) +66 + +julia> sub2ind((5,6,7),1,6,3) +86 +``` +""" sub2ind(dims::DimsInteger, I::Integer...) = (@_inline_meta; _sub2ind(dims, 1, 1, I...)) sub2ind(inds::Indices, I::Integer...) = (@_inline_meta; _sub2ind(inds, 1, 1, I...)) # In 1d, there's a question of whether we're doing cartesian indexing @@ -1455,6 +1560,32 @@ offsetin(i, l::Integer) = i-1 offsetin(i, r::AbstractUnitRange) = i-first(r) ind2sub(::Tuple{}, ind::Integer) = (@_inline_meta; ind == 1 ? () : throw(BoundsError())) + +""" + ind2sub(dims, index) -> subscripts + +Returns a tuple of subscripts into an array with dimensions `dims`, +corresponding to the linear index `index`. + +**Example**: + +``` +i, j, ... = ind2sub(size(A), indmax(A)) +``` + +provides the indices of the maximum element. + +```jldoctest +julia> ind2sub((3,4),2) +(2,1) + +julia> ind2sub((3,4),3) +(3,1) + +julia> ind2sub((3,4),4) +(1,2) +``` +""" ind2sub(dims::DimsInteger, ind::Integer) = (@_inline_meta; _ind2sub(dims, ind-1)) ind2sub(inds::Indices, ind::Integer) = (@_inline_meta; _ind2sub(inds, ind-1)) ind2sub(inds::Indices{1}, ind::Integer) = throw(ArgumentError("Linear indexing is not defined for one-dimensional arrays")) @@ -1666,6 +1797,12 @@ promote_eltype_op(op, A, B) = (@_pure_meta; promote_op(op, eltype(A), eltype(B)) promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_eltype_op(op, promote_eltype_op(op, A, B), C, D...)) ## 1 argument + +""" + map!(function, collection) + +In-place version of [`map`](:func:`map`). +""" map!{F}(f::F, A::AbstractArray) = map!(f, A, A) function map!{F}(f::F, dest::AbstractArray, A::AbstractArray) for (i,j) in zip(eachindex(dest),eachindex(A)) @@ -1678,6 +1815,26 @@ end map(f, A::Union{AbstractArray,AbstractSet,Associative}) = collect_similar(A, Generator(f,A)) # default to returning an Array for `map` on general iterators +""" + map(f, c...) -> collection + +Transform collection `c` by applying `f` to each element. For multiple collection arguments, +apply `f` elementwise. + +```jldoctest +julia> map((x) -> x * 2, [1, 2, 3]) +3-element Array{Int64,1}: + 2 + 4 + 6 + +julia> map(+, [1, 2, 3], [10, 20, 30]) +3-element Array{Int64,1}: + 11 + 22 + 33 +``` +""" map(f, A) = collect(Generator(f,A)) ## 2 argument @@ -1700,6 +1857,12 @@ function map_n!{F}(f::F, dest::AbstractArray, As) return dest end +""" + map!(function, destination, collection...) + +Like [`map`](:func:`map`), but stores the result in `destination` rather than a new +collection. `destination` must be at least as large as the first collection. +""" map!{F}(f::F, dest::AbstractArray, As::AbstractArray...) = map_n!(f, dest, As) map(f) = f() diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 8abe1b19d86f9..1cc950b0063e7 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -101,6 +101,25 @@ imag{T<:Real}(x::AbstractArray{T}) = zero(x) \(A::Number, B::AbstractArray) = B ./ A # index A[:,:,...,i,:,:,...] where "i" is in dimension "d" + +""" + slicedim(A, d::Integer, i) + +Return all the data of `A` where the index for dimension `d` equals `i`. Equivalent to +`A[:,:,...,i,:,:,...]` where `i` is in position `d`. + +```jldoctest +julia> A = [1 2 3 4; 5 6 7 8] +2×4 Array{Int64,2}: + 1 2 3 4 + 5 6 7 8 + +julia> slicedim(A,2,3) +2-element Array{Int64,1}: + 3 + 7 +``` +""" function slicedim(A::AbstractArray, d::Integer, i) d >= 1 || throw(ArgumentError("dimension must be ≥ 1")) nd = ndims(A) @@ -114,7 +133,7 @@ function flipdim(A::AbstractVector, d::Integer) end """ - flipdim(A, d) + flipdim(A, d::Integer) Reverse `A` in dimension `d`. @@ -191,7 +210,7 @@ julia> circshift(b, (-1,0)) 1 5 9 13 ``` -See also `circshift!`. +See also [`circshift!`](:func:`circshift!`). """ function circshift(a::AbstractArray, shiftamt) circshift!(similar(a), a, map(Integer, (shiftamt...,))) @@ -222,6 +241,13 @@ end # Uses K-B-N summation # TODO: Needs a separate LinearSlow method, this is only fast for LinearIndexing + +""" + cumsum_kbn(A, [dim::Integer=1]) + +Cumulative sum along a dimension, using the Kahan-Babuska-Neumaier compensated summation +algorithm for additional accuracy. The dimension defaults to 1. +""" function cumsum_kbn{T<:AbstractFloat}(A::AbstractArray{T}, axis::Integer=1) dimsA = size(A) ndimsA = ndims(A) @@ -259,6 +285,11 @@ end ## ipermutedims in terms of permutedims ## +""" + ipermutedims(A, perm) + +Like [`permutedims`](:func:`permutedims`), except the inverse of the given permutation is applied. +""" function ipermutedims(A::AbstractArray,perm) iperm = Array{Int}(length(perm)) for (i,p) = enumerate(perm) @@ -269,6 +300,32 @@ end ## Other array functions ## +""" + repmat(A, m::Int, n::Int=1) + +Construct a matrix by repeating the given matrix `m` times in dimension 1 and `n` times in +dimension 2. + +```jldoctest +julia> repmat([1, 2, 3], 2) +6-element Array{Int64,1}: + 1 + 2 + 3 + 1 + 2 + 3 + +julia> repmat([1, 2, 3], 2, 3) +6×3 Array{Int64,2}: + 1 1 1 + 2 2 2 + 3 3 3 + 1 1 1 + 2 2 2 + 3 3 3 +``` +""" function repmat(a::AbstractVecOrMat, m::Int, n::Int=1) o, p = size(a,1), size(a,2) b = similar(a, o*m, p*n) diff --git a/base/array.jl b/base/array.jl index 49b9758424f59..8dcd68294b1da 100644 --- a/base/array.jl +++ b/base/array.jl @@ -161,6 +161,26 @@ function fill!{T<:Union{Integer,AbstractFloat}}(a::Array{T}, x) return a end + +""" + fill(x, dims) + +Create an array filled with the value `x`. For example, `fill(1.0, (5,5))` returns a 10×10 +array of floats, with each element initialized to `1.0`. + +```jldoctest +julia> fill(1.0, (5,5)) +5×5 Array{Float64,2}: + 1.0 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 1.0 +``` + +If `x` is an object reference, all elements will refer to the same object. `fill(Foo(), +dims)` will return an array filled with the result of evaluating `Foo()` once. +""" fill(v, dims::Dims) = fill!(Array{typeof(v)}(dims), v) fill(v, dims::Integer...) = fill!(Array{typeof(v)}(dims...), v) @@ -185,6 +205,12 @@ function eye(T::Type, m::Integer, n::Integer) end return a end + +""" + eye(m, n) + +`m`-by-`n` identity matrix. +""" eye(m::Integer, n::Integer) = eye(Float64, m, n) eye(T::Type, n::Integer) = eye(T, n, n) """ @@ -269,6 +295,18 @@ _similar_for(c, T, itr, isz) = similar(c, T) Return an `Array` of all items in a collection or iterator. For associative collections, returns `Pair{KeyType, ValType}`. If the argument is array-like or is an iterator with the `HasShape()` trait, the result will have the same shape and number of dimensions as the argument. + +```jldoctest +julia> collect(1:2:13) +7-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + 13 +``` """ collect(itr) = _collect(1:1 #= Array =#, itr, iteratoreltype(itr), iteratorsize(itr)) @@ -495,6 +533,19 @@ function append!{T}(a::Array{T,1}, items::AbstractVector) return a end +""" + prepend!(a::Vector, items) -> collection + +Insert the elements of `items` to the beginning of `a`. + +```jldoctest +julia> prepend!([3],[1,2]) +3-element Array{Int64,1}: + 1 + 2 + 3 +``` +""" function prepend!{T}(a::Array{T,1}, items::AbstractVector) n = length(items) ccall(:jl_array_grow_beg, Void, (Any, UInt), a, n) @@ -506,6 +557,35 @@ function prepend!{T}(a::Array{T,1}, items::AbstractVector) return a end + +""" + resize!(a::Vector, n::Integer) -> Vector + +Resize `a` to contain `n` elements. If `n` is smaller than the current collection +length, the first `n` elements will be retained. If `n` is larger, the new elements are not +guaranteed to be initialized. + +```jldoctest +julia> resize!([6, 5, 4, 3, 2, 1], 3) +3-element Array{Int64,1}: + 6 + 5 + 4 +``` + +```julia +julia> resize!([6, 5, 4, 3, 2, 1], 8) +8-element Array{Int64,1}: + 6 + 5 + 4 + 3 + 2 + 1 + 0 + 0 +``` +""" function resize!(a::Vector, nl::Integer) l = length(a) if nl > l @@ -533,6 +613,22 @@ function pop!(a::Vector) return item end +""" + unshift!(collection, items...) -> collection + +Insert one or more `items` at the beginning of `collection`. + +```jldoctest + julia> unshift!([1, 2, 3, 4], 5, 6) + 6-element Array{Int64,1}: + 5 + 6 + 1 + 2 + 3 + 4 +``` +""" function unshift!{T}(a::Array{T,1}, item) item = convert(T, item) ccall(:jl_array_grow_beg, Void, (Any, UInt), a, 1) @@ -549,6 +645,23 @@ function shift!(a::Vector) return item end +""" + insert!(a::Vector, index::Integer, item) + +Insert an `item` into `a` at the given `index`. `index` is the index of `item` in +the resulting `a`. + +```jldoctest +julia> insert!([6, 5, 4, 2, 1], 4, 3) +6-element Array{Int64,1}: + 6 + 5 + 4 + 3 + 2 + 1 +``` +""" function insert!{T}(a::Array{T,1}, i::Integer, item) # Throw convert error before changing the shape of the array _item = convert(T, item) @@ -597,7 +710,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:611 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:724 ... ``` """ @@ -632,6 +745,54 @@ end const _default_splice = [] +""" + splice!(a::Vector, index::Integer, [replacement]) -> item + +Remove the item at the given index, and return the removed item. +Subsequent items are shifted left to fill the resulting gap. +If specified, replacement values from an ordered +collection will be spliced in place of the removed item. + +```jldoctest +julia> A = [6, 5, 4, 3, 2, 1]; splice!(A, 5) +2 + +julia> A +5-element Array{Int64,1}: + 6 + 5 + 4 + 3 + 1 + +julia> splice!(A, 5, -1) +1 + +julia> A +5-element Array{Int64,1}: + 6 + 5 + 4 + 3 + -1 + +julia> splice!(A, 1, [-1, -2, -3]) +6 + +julia> A +7-element Array{Int64,1}: + -1 + -2 + -3 + 5 + 4 + 3 + -1 +``` + +To insert `replacement` before an index `n` without removing any items, use +`splice!(collection, n:n-1, replacement)`. +""" function splice!(a::Vector, i::Integer, ins=_default_splice) v = a[i] m = length(ins) @@ -650,6 +811,34 @@ function splice!(a::Vector, i::Integer, ins=_default_splice) return v end +""" + splice!(a::Vector, range, [replacement]) -> items + +Remove items in the specified index range, and return a collection containing +the removed items. +Subsequent items are shifted left to fill the resulting gap. +If specified, replacement values from an ordered collection will be spliced in +place of the removed items. + +To insert `replacement` before an index `n` without removing any items, use +`splice!(collection, n:n-1, replacement)`. + +```jldoctest +julia> splice!(A, 4:3, 2) +0-element Array{Int64,1} + +julia> A +8-element Array{Int64,1}: + -1 + -2 + -3 + 2 + 5 + 4 + 3 + -1 +``` +""" function splice!{T<:Integer}(a::Vector, r::UnitRange{T}, ins=_default_splice) v = a[r] m = length(ins) diff --git a/base/arraymath.jl b/base/arraymath.jl index 171d301fb48e9..4d9b9a8f09ed3 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -2,6 +2,13 @@ ## Unary operators ## +""" + conj!(A) + +Transform an array to its complex conjugate in-place. + +See also [`conj`](:func:`conj`). +""" function conj!{T<:Number}(A::AbstractArray{T}) for i in eachindex(A) A[i] = conj(A[i]) @@ -337,7 +344,25 @@ julia> rot180(a,2) rot180(A::AbstractMatrix, k::Integer) = mod(k, 2) == 1 ? rot180(A) : copy(A) ## Transpose ## + +""" + transpose!(dest,src) + +Transpose array `src` and store the result in the preallocated array `dest`, which should +have a size corresponding to `(size(src,2),size(src,1))`. No in-place transposition is +supported and unexpected results will happen if `src` and `dest` have overlapping memory +regions. +""" transpose!(B::AbstractMatrix, A::AbstractMatrix) = transpose_f!(transpose, B, A) + +""" + ctranspose!(dest,src) + +Conjugate transpose array `src` and store the result in the preallocated array `dest`, which +should have a size corresponding to `(size(src,2),size(src,1))`. No in-place transposition +is supported and unexpected results will happen if `src` and `dest` have overlapping memory +regions. +""" ctranspose!(B::AbstractMatrix, A::AbstractMatrix) = transpose_f!(ctranspose, B, A) function transpose!(B::AbstractVector, A::AbstractMatrix) indices(B,1) == indices(A,2) && indices(A,1) == 1:1 || throw(DimensionMismatch("transpose")) @@ -401,6 +426,11 @@ function ccopy!(B, A) end end +""" + transpose(A) + +The transposition operator (`.'`). +""" function transpose(A::AbstractMatrix) ind1, ind2 = indices(A) B = similar(A, (ind2, ind1)) diff --git a/base/base64.jl b/base/base64.jl index 89bd7a525b0e6..f36f78cc492f3 100644 --- a/base/base64.jl +++ b/base/base64.jl @@ -14,6 +14,13 @@ export Base64EncodePipe, Base64DecodePipe, base64encode, base64decode # , while function base64decode is useful for decoding strings ############################################################################# +""" + Base64EncodePipe(ostream) + +Returns a new write-only I/O stream, which converts any bytes written to it into +base64-encoded ASCII bytes written to `ostream`. Calling `close` on the `Base64EncodePipe` stream +is necessary to complete the encoding (but does not close `ostream`). +""" type Base64EncodePipe <: IO io::IO # writing works in groups of 3, so we need to cache last two bytes written @@ -154,6 +161,16 @@ function close(b::Base64EncodePipe) end # like sprint, but returns base64 string +""" + base64encode(writefunc, args...) + base64encode(args...) + +Given a `write`-like function `writefunc`, which takes an I/O stream as its first argument, +`base64encode(writefunc, args...)` calls `writefunc` to write `args...` to a base64-encoded +string, and returns the string. `base64encode(args...)` is equivalent to `base64encode(write, args...)`: +it converts its arguments into bytes using the standard `write` functions and returns the +base64-encoded string. +""" function base64encode(f::Function, args...) s = IOBuffer() b = Base64EncodePipe(s) @@ -165,6 +182,11 @@ base64encode(x...) = base64encode(write, x...) ############################################################################# +""" + Base64DecodePipe(istream) + +Returns a new read-only I/O stream, which decodes base64-encoded data read from `istream`. +""" type Base64DecodePipe <: IO io::IO # reading works in blocks of 4 characters that are decoded into 3 bytes and 2 of them cached @@ -197,6 +219,12 @@ eof(b::Base64DecodePipe) = isempty(b.cache) && eof(b.io) close(b::Base64DecodePipe) = nothing # Decodes a base64-encoded string + +""" + base64decode(string) + +Decodes the base64-encoded `string` and returns a `Vector{UInt8}` of the decoded bytes. +""" function base64decode(s) b = IOBuffer(s) try diff --git a/base/broadcast.jl b/base/broadcast.jl index aee5920bb9386..5914bc21de4d8 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -165,6 +165,15 @@ end end end +""" + broadcast!(f, dest, As...) + +Like [`broadcast`](:func:`broadcast`), but store the result of +`broadcast(f, As...)` in the `dest` array. +Note that `dest` is only used to store the result, and does not supply +arguments to `f` unless it is also listed in the `As`, +as in `broadcast!(f, A, A, B)` to perform `A[:] = broadcast(f, A, B)`. +""" @inline function broadcast!{nargs}(f, B::AbstractArray, As::Vararg{Any,nargs}) shape = indices(B) check_broadcast_shape(shape, As...) @@ -227,6 +236,38 @@ end @inline broadcast_t(f, T, As...) = broadcast!(f, similar(Array{T}, broadcast_shape(As...)), As...) +""" + broadcast(f, As...) + +Broadcasts the arrays `As` to a common size by expanding singleton dimensions, and returns +an array of the results `f(as...)` for each position. + +```jldoctest +julia> A = [1, 2, 3, 4, 5] +5-element Array{Int64,1}: + 1 + 2 + 3 + 4 + 5 + +julia> B = [1 2; 3 4; 5 6; 7 8; 9 10] +5×2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 + 7 8 + 9 10 + +julia> broadcast(+, A, B) +5×2 Array{Int64,2}: + 2 3 + 5 6 + 8 9 + 11 12 + 14 15 +``` +""" @inline broadcast(f, As...) = broadcast_t(f, promote_eltype_op(f, As...), As...) # alternate, more compact implementation; unfortunately slower. @@ -247,8 +288,63 @@ function broadcast(f, As...) end =# +""" + bitbroadcast(f, As...) + +Like [`broadcast`](:func:`broadcast`), but allocates a `BitArray` to store the +result, rather then an `Array`. + +```jldoctest +julia> bitbroadcast(isodd,[1,2,3,4,5]) +5-element BitArray{1}: + true + false + true + false + true +``` +""" @inline bitbroadcast(f, As...) = broadcast!(f, similar(BitArray, broadcast_shape(As...)), As...) +""" + broadcast_getindex(A, inds...) + +Broadcasts the `inds` arrays to a common size like [`broadcast`](:func:`broadcast`) +and returns an array of the results `A[ks...]`, +where `ks` goes over the positions in the broadcast result `A`. + +```jldoctest +julia> A = [1, 2, 3, 4, 5] +5-element Array{Int64,1}: + 1 + 2 + 3 + 4 + 5 + +julia> B = [1 2; 3 4; 5 6; 7 8; 9 10] +5×2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 + 7 8 + 9 10 + +julia> C = broadcast(+,A,B) +5×2 Array{Int64,2}: + 2 3 + 5 6 + 8 9 + 11 12 + 14 15 + +julia> broadcast_getindex(C,[1,2,10]) +3-element Array{Int64,1}: + 2 + 5 + 15 +``` +""" broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex!(similar(Array{eltype(src)}, broadcast_shape(I...)), src, I...) @generated function broadcast_getindex!(dest::AbstractArray, src::AbstractArray, I::AbstractArray...) N = length(I) @@ -266,6 +362,12 @@ broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex end end +""" + broadcast_setindex!(A, X, inds...) + +Broadcasts the `X` and `inds` arrays to a common size and stores the value from each +position in `X` at the indices in `A` given by the same positions in `inds`. +""" @generated function broadcast_setindex!(A::AbstractArray, x, I::AbstractArray...) N = length(I) Isplat = Expr[:(I[$d]) for d = 1:N] diff --git a/base/client.jl b/base/client.jl index 19cc480e4d624..4885bf5112469 100644 --- a/base/client.jl +++ b/base/client.jl @@ -297,6 +297,14 @@ import .REPL const repl_hooks = [] +""" + atreplinit(f) + +Register a one-argument function to be called before the REPL interface is initialized in +interactive sessions; this is useful to customize the interface. The argument of `f` is the +REPL object. This function should be called from within the `.juliarc.jl` initialization +file. +""" atreplinit(f::Function) = (unshift!(repl_hooks, f); nothing) function _atreplinit(repl) diff --git a/base/complex.jl b/base/complex.jl index 7be18a036f951..dbfa77c91fa66 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -33,10 +33,27 @@ promote_rule{T<:Real,S<:Real}(::Type{Complex{T}}, ::Type{Complex{S}}) = widen{T}(::Type{Complex{T}}) = Complex{widen(T)} +""" + real(z) + +Return the real part of the complex number `z`. +""" real(z::Complex) = z.re + +""" + imag(z) + +Return the imaginary part of the complex number `z`. +""" imag(z::Complex) = z.im real(x::Real) = x imag(x::Real) = zero(x) + +""" + reim(z) + +Return both the real and imaginary parts of the complex number `z`. +""" reim(z) = (real(z), imag(z)) real{T<:Real}(::Type{T}) = T @@ -115,6 +132,11 @@ end ## generic functions of complex numbers ## +""" + conj(z) + +Compute the complex conjugate of a complex number `z`. +""" conj(z::Complex) = Complex(real(z),-imag(z)) abs(z::Complex) = hypot(real(z), imag(z)) abs2(z::Complex) = real(z)*real(z) + imag(z)*imag(z) @@ -334,12 +356,23 @@ sqrt(z::Complex) = sqrt(float(z)) # compute exp(im*theta) cis(theta::Real) = Complex(cos(theta),sin(theta)) + +""" + cis(z) + +Return ``\\exp(iz)``. +""" function cis(z::Complex) v = exp(-imag(z)) Complex(v*cos(real(z)), v*sin(real(z))) end @vectorize_1arg Number cis +""" + angle(z) + +Compute the phase angle in radians of a complex number `z`. +""" angle(z::Complex) = atan2(imag(z), real(z)) function log{T<:AbstractFloat}(z::Complex{T}) diff --git a/base/dict.jl b/base/dict.jl index 69c41ece8d8cc..5974cfef126e6 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -115,8 +115,48 @@ end in(k, v::KeyIterator) = !is(get(v.dict, k, secret_table_token), secret_table_token) + +""" + keys(a::Associative) + +Return an iterator over all keys in a collection. +`collect(keys(d))` returns an array of keys. +Since the keys are stored internally in a hash table, +the order in which they are returned may vary. + +```jldoctest +julia> a = Dict('a'=>2, 'b'=>3) +Dict{Char,Int64} with 2 entries: + 'b' => 3 + 'a' => 2 + +julia> collect(keys(a)) +2-element Array{Char,1}: + 'b' + 'a' +``` +""" keys(a::Associative) = KeyIterator(a) eachindex(a::Associative) = KeyIterator(a) + +""" + values(a::Associative) + +Return an iterator over all values in a collection. +`collect(values(d))` returns an array of values. + +```jldoctest +julia> a = Dict('a'=>2, 'b'=>3) +Dict{Char,Int64} with 2 entries: + 'b' => 3 + 'a' => 2 + +julia> collect(values(a)) +2-element Array{Int64,1}: + 3 + 2 +``` +""" values(a::Associative) = ValueIterator(a) function copy(a::Associative) @@ -127,6 +167,12 @@ function copy(a::Associative) return b end +""" + merge!(d::Associative, others::Associative...) + +Update collection with pairs from the other collections. +See also [`merge`](:func:`merge`). +""" function merge!(d::Associative, others::Associative...) for other in others for (k,v) in other @@ -145,12 +191,56 @@ function copy!(dest::Union{Associative,AbstractSet}, src) return dest end +""" + keytype(type) + +Get the key type of an associative collection type. Behaves similarly to [`eltype`](:func:`eltype`). +""" keytype{K,V}(::Type{Associative{K,V}}) = K keytype(a::Associative) = keytype(typeof(a)) keytype{A<:Associative}(::Type{A}) = keytype(supertype(A)) + +""" + valtype(type) + +Get the value type of an associative collection type. Behaves similarly to [`eltype`](:func:`eltype`). +""" valtype{K,V}(::Type{Associative{K,V}}) = V valtype{A<:Associative}(::Type{A}) = valtype(supertype(A)) valtype(a::Associative) = valtype(typeof(a)) + +""" + merge(d::Associative, others::Associative...) + +Construct a merged collection from the given collections. If necessary, the +types of the resulting collection will be promoted to accommodate the types of +the merged collections. If the same key is present in another collection, the +value for that key will be the value it has in the last collection listed. + +```jldoctest +julia> a = Dict("foo" => 0.0, "bar" => 42.0) +Dict{String,Float64} with 2 entries: + "bar" => 42.0 + "foo" => 0.0 + +julia> b = Dict("baz" => 17, "bar" => 4711) +Dict{String,Int64} with 2 entries: + "bar" => 4711 + "baz" => 17 + +julia> merge(a, b) +Dict{String,Float64} with 3 entries: + "bar" => 4711.0 + "baz" => 17.0 + "foo" => 0.0 + +julia> merge(b, a) +Dict{String,Float64} with 3 entries: + "bar" => 42.0 + "baz" => 17.0 + "foo" => 0.0 +``` +""" function merge(d::Associative, others::Associative...) K, V = keytype(d), valtype(d) for other in others @@ -329,6 +419,30 @@ abstract AbstractSerializer const global maxallowedprobe = 16 const global maxprobeshift = 6 +""" + Dict([itr]) + +`Dict{K,V}()` constructs a hash table with keys of type `K` and values of type `V`. + +Given a single iterable argument, constructs a [`Dict`](:obj:`Dict`) whose key-value pairs +are taken from 2-tuples `(key,value)` generated by the argument. + +```jldoctest +julia> Dict([("A", 1), ("B", 2)]) +Dict{String,Int64} with 2 entries: + "B" => 2 + "A" => 1 +``` + +Alternatively, a sequence of pair arguments may be passed. + +```jldoctest +julia> Dict("A"=>1, "B"=>2) +Dict{String,Int64} with 2 entries: + "B" => 2 + "A" => 1 +``` +""" type Dict{K,V} <: Associative{K,V} slots::Array{UInt8,1} keys::Array{K,1} @@ -698,9 +812,45 @@ function get{K,V}(default::Callable, h::Dict{K,V}, key) return (index < 0) ? default() : h.vals[index]::V end +""" + haskey(collection, key) -> Bool + +Determine whether a collection has a mapping for a given key. + +```jldoctest +julia> a = Dict('a'=>2, 'b'=>3) +Dict{Char,Int64} with 2 entries: + 'b' => 3 + 'a' => 2 + +julia> haskey(a,'a') +true + +julia> haskey(a,'c') +false +``` +""" haskey(h::Dict, key) = (ht_keyindex(h, key) >= 0) in{T<:Dict}(key, v::KeyIterator{T}) = (ht_keyindex(v.dict, key) >= 0) +""" + getkey(collection, key, default) + +Return the key matching argument `key` if one exists in `collection`, otherwise return `default`. + +```jldoctest +julia> a = Dict('a'=>2, 'b'=>3) +Dict{Char,Int64} with 2 entries: + 'b' => 3 + 'a' => 2 + +julia> getkey(a,'a',1) +'a' + +julia> getkey(a,'d','a') +'a' +``` +""" function getkey{K,V}(h::Dict{K,V}, key, default) index = ht_keyindex(h, key) return (index<0) ? default : h.keys[index]::K diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index b1e51b084e6d2..43c995fac199f 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -40,13 +40,6 @@ Compute the inverse sine of `x`, where the output is in radians. """ asin -""" - <:(T1, T2) - -Subtype operator, equivalent to `issubtype(T1,T2)`. -""" -Base.:(<:) - """ takebuf_array(b::IOBuffer) @@ -55,40 +48,6 @@ Obtain the contents of an `IOBuffer` as an array, without copying. Afterwards, t """ takebuf_array -""" - download(url,[localfile]) - -Download a file from the given url, optionally renaming it to the given local file name. -Note that this function relies on the availability of external tools such as `curl`, `wget` -or `fetch` to download the file and is provided for convenience. For production use or -situations in which more options are needed, please use a package that provides the desired -functionality instead. -""" -download - -""" - lstrip(string, [chars]) - -Return `string` with any leading whitespace removed. If `chars` (a character, or vector or -set of characters) is provided, instead remove characters contained in it. -""" -lstrip - -""" - powermod(x, p, m) - -Compute ``x^p \\pmod m``. -""" -powermod - -""" - typeintersect(T, S) - -Compute a type that contains the intersection of `T` and `S`. Usually this will be the -smallest such type or one close to it. -""" -typeintersect - """ pointer(array [, index]) @@ -100,20 +59,6 @@ Calling `Ref(array[, index])` is generally preferable to this function. """ pointer -""" - isnan(f) -> Bool - -Test whether a floating point number is not a number (NaN). -""" -isnan - -""" - println(x) - -Print (using [`print`](:func:`print`)) `x` followed by a newline. -""" -println - """ //(num, den) @@ -121,22 +66,6 @@ Divide two integers or rational numbers, giving a `Rational` result. """ Base.:(//) -""" - At_mul_B(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ⋅B``. -""" -At_mul_B - -""" - methods(f, [types]) - -Returns the method table for `f`. - -If `types` is specified, returns an array of methods whose types match. -""" -methods - """ isinteger(x) -> Bool @@ -144,13 +73,6 @@ Test whether `x` or all its elements are numerically equal to some integer """ isinteger -""" - sortrows(A, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) - -Sort the rows of matrix `A` lexicographically. -""" -sortrows - """ ./(x, y) @@ -158,13 +80,6 @@ Element-wise right division operator. """ Base.:(./) -""" - IPv6(host::Integer) -> IPv6 - -Returns IPv6 object from ip address formatted as Integer -""" -IPv6 - """ prod!(r, A) @@ -172,13 +87,6 @@ Multiply elements of `A` over the singleton dimensions of `r`, and write results """ prod! -""" - gensym([tag]) - -Generates a symbol which will not conflict with other variable names. -""" -gensym - """ cummin(A, [dim]) @@ -193,16 +101,6 @@ Compute the minimum absolute values over the singleton dimensions of `r`, and wr """ minabs! -""" - @evalpoly(z, c...) - -Evaluate the polynomial ``\\sum_k c[k] z^{k-1}`` for the coefficients `c[1]`, `c[2]`, ...; -that is, the coefficients are given in ascending order by power of `z`. This macro expands -to efficient inline code that uses either Horner's method or, for complex `z`, a more -efficient Goertzel-like algorithm. -""" -:@evalpoly - """ eigfact!(A, [B]) @@ -218,34 +116,6 @@ Compute hyperbolic cosine of `x`. """ cosh -""" - ipermutedims(A, perm) - -Like [`permutedims`](:func:`permutedims`), except the inverse of the given permutation is applied. -""" -ipermutedims - -""" - dirname(path::AbstractString) -> AbstractString - -Get the directory part of a path. -""" -dirname - -""" - isfile(path) -> Bool - -Returns `true` if `path` is a regular file, `false` otherwise. -""" -isfile - -""" - diff(A, [dim]) - -Finite difference operator of matrix or vector. -""" -diff - """ precision(num::AbstractFloat) @@ -254,46 +124,6 @@ the mantissa. """ precision -""" - readlines(stream::IO) - readlines(filename::AbstractString) - -Read all lines of an I/O stream or a file as a vector of strings. -The text is assumed to be encoded in UTF-8. -""" -readlines - -""" - foldl(op, v0, itr) - -Like [`reduce`](:func:`reduce`), but with guaranteed left associativity. `v0` will be used -exactly once. -""" -foldl(op, v0, itr) - -""" - foldl(op, itr) - -Like `foldl(op, v0, itr)`, but using the first element of `itr` as `v0`. In general, this -cannot be used with empty collections (see `reduce(op, itr)`). -""" -foldl(op, itr) - -""" - Ac_rdiv_B(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ / B``. -""" -Ac_rdiv_B - - -""" - linspace(start, stop, n=50) - -Construct a range of `n` linearly spaced elements from `start` to `stop`. -""" -linspace - """ promote_type(type1, type2) @@ -305,29 +135,6 @@ losslessly, some loss is tolerated; for example, `promote_type(Int64,Float64)` r """ promote_type -""" - ind2sub(dims, index) -> subscripts - -Returns a tuple of subscripts into an array with dimensions `dims`, -corresponding to the linear index `index`. - -**Example**: - -``` -i, j, ... = ind2sub(size(A), indmax(A)) -``` - -provides the indices of the maximum element. -""" -ind2sub(dims::Tuple, index::Int) - -""" - ind2sub(a, index) -> subscripts - -Returns a tuple of subscripts into array `a` corresponding to the linear index `index`. -""" -ind2sub(a, index) - """ ``` .*(x, y) @@ -337,54 +144,6 @@ Element-wise multiplication operator. """ Base.:(.*) -""" - range(start, [step], length) - -Construct a range by length, given a starting value and optional step (defaults to 1). -""" -range - -""" - eltype(type) - -Determine the type of the elements generated by iterating a collection of the given `type`. -For associative collection types, this will be a `Pair{KeyType,ValType}`. The definition -`eltype(x) = eltype(typeof(x))` is provided for convenience so that instances can be passed -instead of types. However the form that accepts a type argument should be defined for new -types. -""" -eltype - -""" - keytype(type) - -Get the key type of an associative collection type. Behaves similarly to `eltype`. -""" -keytype - -""" - valtype(type) - -Get the value type of an associative collection type. Behaves similarly to `eltype`. -""" -valtype - -""" - edit(path::AbstractString, [line]) - -Edit a file or directory optionally providing a line number to edit the file at. Returns to -the `julia` prompt when you quit the editor. -""" -edit(path::AbstractString, line=?) - -""" - edit(function, [types]) - -Edit the definition of a function, optionally specifying a tuple of types to indicate which -method to edit. -""" -edit(::Function, types=?) - """ backtrace() @@ -406,30 +165,6 @@ Subtraction operator. """ -(x, y) -""" - mapfoldr(f, op, v0, itr) - -Like [`mapreduce`](:func:`mapreduce`), but with guaranteed right associativity. `v0` will be -used exactly once. -""" -mapfoldr(f, op, v0, itr) - -""" - mapfoldr(f, op, itr) - -Like `mapfoldr(f, op, v0, itr)`, but using the first element of `itr` as `v0`. In general, -this cannot be used with empty collections (see `reduce(op, itr)`). -""" -mapfoldr(f, op, itr) - -""" - broadcast_setindex!(A, X, inds...) - -Broadcasts the `X` and `inds` arrays to a common size and stores the value from each -position in `X` at the indices given by the same positions in `inds`. -""" -broadcast_setindex! - """ Nullable(x) @@ -476,30 +211,19 @@ getindex(collection, key...) Convert `x` to a value of type `T`, typically by calling `convert(T,x)` -In cases where `x` cannot be safely converted to `T`, unlike `convert`, `cconvert` may +In cases where `x` cannot be safely converted to `T`, unlike [`convert`](:func:`convert`), `cconvert` may return an object of a type different from `T`, which however is suitable for -`unsafe_convert` to handle. +[`unsafe_convert`](:func:`unsafe_convert`) to handle. Neither `convert` nor `cconvert` should take a Julia object and turn it into a `Ptr`. """ cconvert -""" - |>(x, f) - -Applies a function to the preceding argument. This allows for easy function chaining. - -```jldoctest -julia> [1:5;] |> x->x.^2 |> sum |> inv -0.01818181818181818 -``` -""" -Base.:(|>) - """ assert(cond) -Throw an `AssertionError` if `cond` is `false`. Also available as the macro `@assert expr`. +Throw an [`AssertionError`](:obj:`AssertionError`) if `cond` is `false`. +Also available as the macro `@assert expr`. """ assert @@ -510,55 +234,6 @@ Compute the hyperbolic secant of `x` """ sech -""" - filemode(file) - -Equivalent to `stat(file).mode` -""" -filemode - -""" - join(io, items, delim, [last]) - -Print elements of `items` to `io` with `delim` between them. If `last` is specified, it is -used as the final delimiter instead of `delim`. -""" -join(io, items, delim, last) - -""" - deconv(b,a) - -Construct vector `c` such that `b = conv(a,c) + r`. Equivalent to polynomial division. -""" -deconv - -""" - insert!(collection, index, item) - -Insert an `item` into `collection` at the given `index`. `index` is the index of `item` in -the resulting `collection`. - -```jldoctest -julia> insert!([6, 5, 4, 2, 1], 4, 3) -6-element Array{Int64,1}: - 6 - 5 - 4 - 3 - 2 - 1 -``` -""" -insert! - -""" - repmat(A, n, m) - -Construct a matrix by repeating the given matrix `n` times in dimension 1 and `m` times in -dimension 2. -""" -repmat - """ acos(x) @@ -566,22 +241,6 @@ Compute the inverse cosine of `x`, where the output is in radians """ acos -""" - ispath(path) -> Bool - -Returns `true` if `path` is a valid filesystem path, `false` otherwise. -""" -ispath - -""" - fdio([name::AbstractString, ]fd::Integer[, own::Bool]) -> IOStream - -Create an `IOStream` object from an integer file descriptor. If `own` is `true`, closing -this object will close the underlying descriptor. By default, an `IOStream` is closed when -it is garbage collected. `name` allows you to associate the descriptor with a named file. -""" -fdio - """ unsafe_copy!(dest::Ptr{T}, src::Ptr{T}, N) @@ -606,13 +265,6 @@ the same manner as C. """ unsafe_copy!(dest::Array, d, src::Array, so, N) -""" - diag(M[, k]) - -The `k`th diagonal of a matrix, as a vector. Use `diagm` to construct a diagonal matrix. -""" -diag - """ .^(x, y) @@ -620,50 +272,6 @@ Element-wise exponentiation operator. """ Base.:(.^) -""" - isspace(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is any whitespace character. Includes ASCII characters '\\t', -'\\n', '\\v', '\\f', '\\r', and ' ', Latin-1 character U+0085, and characters in Unicode -category Zs. For strings, tests whether this is true for all elements of the string. -""" -isspace - -""" - splitext(path::AbstractString) -> (AbstractString,AbstractString) - -If the last component of a path contains a dot, split the path into everything before the -dot and everything including and after the dot. Otherwise, return a tuple of the argument -unmodified and the empty string. -""" -splitext - -""" - gethostname() -> AbstractString - -Get the local machine's host name. -""" -gethostname - -""" - replace(string, pat, r[, n]) - -Search for the given pattern `pat`, and replace each occurrence with `r`. If `n` is -provided, replace at most `n` occurrences. As with search, the second argument may be a -single character, a vector or a set of characters, a string, or a regular expression. If `r` -is a function, each occurrence is replaced with `r(s)` where `s` is the matched substring. -If `pat` is a regular expression and `r` is a `SubstitutionString`, then capture group -references in `r` are replaced with the corresponding matched text. -""" -replace - -""" - chop(string) - -Remove the last character from a string. -""" -chop - """ Float32(x [, mode::RoundingMode]) @@ -682,38 +290,6 @@ See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. """ Float32 -""" - readuntil(stream::IO, delim) - readuntil(filename::AbstractString, delim) - -Read a string from an I/O stream or a file, up to and including the given delimiter byte. -The text is assumed to be encoded in UTF-8. -""" -readuntil - -""" - isimmutable(v) - -Return `true` iff value `v` is immutable. See [manual](:ref:`man-immutable-composite-types`) -for a discussion of immutability. Note that this function works on values, so if you give it -a type, it will tell you that a value of `DataType` is mutable. -""" -isimmutable - -""" - macroexpand(x) - -Takes the expression `x` and returns an equivalent expression with all macros removed (expanded). -""" -macroexpand - -""" - issticky(path) -> Bool - -Returns `true` if `path` has the sticky bit set, `false` otherwise. -""" -issticky - """ Mmap.mmap(io::Union{IOStream,AbstractString,Mmap.AnonymousMmap}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) Mmap.mmap(type::Type{Array{T,N}}, dims) @@ -805,24 +381,20 @@ any! """ filter!(function, collection) -Update `collection`, removing elements for which `function` is `false`. For associative -collections, the function is passed two arguments (key and value). -""" -filter! - -""" - base64decode(string) - -Decodes the base64-encoded `string` and returns a `Vector{UInt8}` of the decoded bytes. -""" -base64decode - -""" - oct(n, [pad]) +Update `collection`, removing elements for which `function` is `false`. +For associative collections, the function is passed two arguments (key and value). -Convert an integer to an octal string, optionally specifying a number of digits to pad to. +```jldoctest +julia> filter!(isodd, collect(1:10)) +5-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 +``` """ -oct +filter! """ sizeof(T) @@ -831,13 +403,6 @@ Size, in bytes, of the canonical binary representation of the given DataType `T` """ sizeof(::Type) -""" - sizeof(s::AbstractString) - -The number of bytes in string `s`. -""" -sizeof(::AbstractString) - """ ===(x, y) ≡(x,y) @@ -854,84 +419,23 @@ An operation tried to write to memory that is read-only. ReadOnlyMemoryError """ - startswith(string, prefix) - -Returns `true` if `string` starts with `prefix`. If `prefix` is a vector or set -of characters, tests whether the first character of `string` belongs to that set. -""" -startswith + last(coll) +Get the last element of an ordered collection, if it can be computed in O(1) time. This is +accomplished by calling [`endof`](:func:`endof`) to get the last index. Returns the end +point of a [`Range`](:obj:`Range`) even if it is empty. """ - permutedims!(dest, src, perm) +last -Permute the dimensions of array `src` and store the result in the array `dest`. `perm` is a -vector specifying a permutation of length `ndims(src)`. The preallocated array `dest` should -have `size(dest) == size(src)[perm]` and is completely overwritten. No in-place permutation -is supported and unexpected results will happen if `src` and `dest` have overlapping memory -regions. """ -permutedims! + sinh(x) +Compute hyperbolic sine of `x`. """ - functionloc(f::Function, types) +sinh -Returns a tuple `(filename,line)` giving the location of a generic `Function` definition. """ -functionloc(f, types) - -""" - functionloc(m::Method) - -Returns a tuple `(filename,line)` giving the location of a `Method` definition. -""" -functionloc(m) - -""" - last(coll) - -Get the last element of an ordered collection, if it can be computed in O(1) time. This is -accomplished by calling [`endof`](:func:`endof`) to get the last index. Returns the end -point of a [`Range`](:obj:`Range`) even if it is empty. -""" -last - -""" - islink(path) -> Bool - -Returns `true` if `path` is a symbolic link, `false` otherwise. -""" -islink - -""" - istril(A) -> Bool - -Test whether a matrix is lower triangular. -""" -istril - -""" - bin(n, [pad]) - -Convert an integer to a binary string, optionally specifying a number of digits to pad to. -""" -bin - -""" - cis(z) - -Return ``\\exp(iz)``. -""" -cis - -""" - sinh(x) - -Compute hyperbolic sine of `x`. -""" -sinh - -""" - ceil([T,] x, [digits, [base]]) + ceil([T,] x, [digits, [base]]) `ceil(x)` returns the nearest integral value of the same type as `x` that is greater than or equal to `x`. @@ -943,24 +447,6 @@ representable. """ ceil -""" - issocket(path) -> Bool - -Returns `true` if `path` is a socket, `false` otherwise. -""" -issocket - -""" - srand([rng], [seed]) - -Reseed the random number generator. If a `seed` is provided, the RNG will give a -reproducible sequence of numbers, otherwise Julia will get entropy from the system. For -`MersenneTwister`, the `seed` may be a non-negative integer, a vector of `UInt32` integers -or a filename, in which case the seed is read from a file. `RandomDevice` does not support -seeding. -""" -srand - """ oftype(x, y) @@ -976,16 +462,17 @@ Compute the maximum absolute values over the singleton dimensions of `r`, and wr maxabs! """ - nullspace(M) + isfinite(f) -> Bool -Basis for nullspace of `M`. -""" -nullspace +Test whether a number is finite. -""" - isfinite(f) -> Bool +```jldoctest +julia> isfinite(5) +true -Test whether a number is finite +julia> isfinite(NaN32) +false +``` """ isfinite @@ -1014,35 +501,11 @@ push! """ prevpow(a, x) -The largest `a^n` not greater than `x`, where `n` is a non-negative integer. `a` must be -greater than 1, and `x` must not be less than 1. +The largest `a^n` not greater than `x`, where `n` is a non-negative integer. +`a` must be greater than 1, and `x` must not be less than 1. """ prevpow -""" - permutedims(A, perm) - -Permute the dimensions of array `A`. `perm` is a vector specifying a permutation of length -`ndims(A)`. This is a generalization of transpose for multi-dimensional arrays. Transpose is -equivalent to `permutedims(A, [2,1])`. -""" -permutedims - -""" - shuffle!([rng,] v) - -In-place version of [`shuffle`](:func:`shuffle`): randomly permute the array `v` in-place, -optionally supplying the random-number generator `rng`. -""" -shuffle! - -""" - fldmod(x, y) - -The floored quotient and modulus after division. Equivalent to `(fld(x,y), mod(x,y))`. -""" -fldmod - """ promote(xs...) @@ -1050,14 +513,6 @@ Convert all arguments to their common promotion type (if any), and return them a """ promote -""" - gradient(F, [h]) - -Compute differences along vector `F`, using `h` as the spacing between points. The default -spacing is one. -""" -gradient - """ tan(x) @@ -1065,14 +520,6 @@ Compute tangent of `x`, where `x` is in radians. """ tan -""" - sprint(f::Function, args...) - -Call the given function with an I/O stream and the supplied extra arguments. Everything -written to this I/O stream is returned as a string. -""" -sprint - """ fd(stream) @@ -1081,50 +528,6 @@ to synchronous `File`'s and `IOStream`'s not to any of the asynchronous streams. """ fd -""" - require(module::Symbol) - -This function is part of the implementation of `using` / `import`, if a module is not -already defined in `Main`. It can also be called directly to force reloading a module, -regardless of whether it has been loaded before (for example, when interactively developing -libraries). - -Loads a source files, in the context of the `Main` module, on every active node, searching -standard locations for files. `require` is considered a top-level operation, so it sets the -current `include` path but does not use it to search for files (see help for `include`). -This function is typically used to load library code, and is implicitly called by `using` to -load packages. - -When searching for files, `require` first looks for package code under `Pkg.dir()`, -then tries paths in the global array `LOAD_PATH`. `require` is case-sensitive on -all platforms, including those with case-insensitive filesystems like macOS and -Windows. -""" -require - -""" - expand(x) - -Takes the expression `x` and returns an equivalent expression in lowered form. -""" -expand - -""" - peakflops(n; parallel=false) - -`peakflops` computes the peak flop rate of the computer by using double precision -[`gemm!`](:func:`Base.LinAlg.BLAS.gemm!`). By default, if no arguments are specified, it -multiplies a matrix of size `n x n`, where `n = 2000`. If the underlying BLAS is using -multiple threads, higher flop rates are realized. The number of BLAS threads can be set with -`BLAS.set_num_threads(n)`. - -If the keyword argument `parallel` is set to `true`, `peakflops` is run in parallel on all -the worker processors. The flop rate of the entire parallel computer is returned. When -running in parallel, only 1 BLAS thread is used. The argument `n` still refers to the size -of the problem that is solved on each processor. -""" -peakflops - """ ones(type, dims) @@ -1139,13 +542,6 @@ Create an array of all ones with the same element type and shape as `A`. """ ones(A) -""" - ind2chr(string, i) - -Convert a byte index to a character index. -""" -ind2chr - """ reshape(A, dims) @@ -1156,7 +552,8 @@ reshape """ randsubseq!(S, A, p) -Like `randsubseq`, but the results are stored in `S` (which is resized as needed). +Like [`randsubseq`](:func:`randsubseq`), but the results are stored in `S` +(which is resized as needed). """ randsubseq! @@ -1173,20 +570,15 @@ maximum(A,dims) redisplay(mime, x) redisplay(d::Display, mime, x) -By default, the `redisplay` functions simply call `display`. However, some display backends -may override `redisplay` to modify an existing display of `x` (if any). Using `redisplay` is -also a hint to the backend that `x` may be redisplayed several times, and the backend may -choose to defer the display until (for example) the next interactive prompt. +By default, the `redisplay` functions simply call [`display`](:func:`display`). +However, some display backends may override `redisplay` to modify an existing +display of `x` (if any). +Using `redisplay` is also a hint to the backend that `x` may be redisplayed +several times, and the backend may choose to defer the display until +(for example) the next interactive prompt. """ redisplay -""" - A_mul_Bc(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``A⋅Bᴴ``. -""" -A_mul_Bc - """ searchsorted(a, x, [by=<transform>,] [lt=<comparison>,] [rev=false]) @@ -1205,33 +597,6 @@ floating-point results for integer arguments. """ Base.:(/) -""" - connect([host],port) -> TCPSocket - -Connect to the host `host` on port `port`. -""" -connect(host=?, port) - -""" - connect(path) -> PipeEndpoint - -Connect to the named pipe / UNIX domain socket at `path`. -""" -connect(path) - -""" - split(string, [chars]; limit=0, keep=true) - -Return an array of substrings by splitting the given string on occurrences of the given -character delimiters, which may be specified in any of the formats allowed by `search`'s -second argument (i.e. a single character, collection of characters, string, or regular -expression). If `chars` is omitted, it defaults to the set of all space characters, and -`keep` is taken to be `false`. The two keyword arguments are optional: they are a -maximum size for the result and a flag determining whether empty fields should be kept in -the result. -""" -split - """ dump(x) @@ -1239,13 +604,6 @@ Show every part of the representation of a value. """ dump -""" - sumabs(itr) - -Sum absolute values of all elements in a collection. This is equivalent to `sum(abs(itr))` but faster. -""" -sumabs(itr) - """ sumabs(A, dims) @@ -1261,13 +619,6 @@ be passed, to be returned from the last `produce` call in the producer. """ consume -""" - ndigits(n, b = 10) - -Compute the number of digits in number `n` written in base `b`. -""" -ndigits - """ cummax(A, [dim]) @@ -1275,27 +626,6 @@ Cumulative maximum along a dimension. The dimension defaults to 1. """ cummax -""" - watch_file(path, timeout_s::Real) - -Watch file or directory `path` for changes until a change occurs or `timeout_s` seconds have -elapsed. - -The returned value is an object with boolean fields `changed`, `renamed`, and `timedout`, -giving the result of watching the file. - -This behavior of this function varies slightly across platforms. See -<https://nodejs.org/api/fs.html#fs_caveats> for more detailed information. -""" -watch_file - -""" - At_rdiv_Bt(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ / Bᵀ``. -""" -At_rdiv_Bt - """ isinteractive() -> Bool @@ -1303,13 +633,6 @@ Determine whether Julia is running an interactive session. """ isinteractive -""" - At_mul_Bt(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ⋅Bᵀ``. -""" -At_mul_Bt - """ sum!(r, A) @@ -1317,13 +640,6 @@ Sum elements of `A` over the singleton dimensions of `r`, and write results to ` """ sum! -""" - close(stream) - -Close an I/O stream. Performs a `flush` first. -""" -close(stream::IO) - """ parentindexes(A) @@ -1338,7 +654,7 @@ parentindexes display(d::Display, mime, x) Display `x` using the topmost applicable display in the display stack, typically using the -richest supported multimedia output for `x`, with plain-text `STDOUT` output as a fallback. +richest supported multimedia output for `x`, with plain-text [`STDOUT`](:obj:`STDOUT`) output as a fallback. The `display(d, x)` variant attempts to display `x` on the given display `d` only, throwing a `MethodError` if `d` cannot display objects of this type. @@ -1355,7 +671,7 @@ display @spawnat Accepts two arguments, `p` and an expression. A closure is created around the expression and -run asynchronously on process `p`. Returns a `Future` to the result. +run asynchronously on process `p`. Returns a [`Future`](:obj:`Future`) to the result. """ :@spawnat @@ -1367,142 +683,6 @@ digits, of number `x`, ensuring that it would parse to the exact same number. """ print_shortest -""" - merge(collection, others...) - -Construct a merged collection from the given collections. If necessary, the -types of the resulting collection will be promoted to accommodate the types of -the merged collections. If the same key is present in another collection, the -value for that key will be the value it has in the last collection listed. - -```jldoctest -julia> a = Dict("foo" => 0.0, "bar" => 42.0) -Dict{String,Float64} with 2 entries: - "bar" => 42.0 - "foo" => 0.0 - -julia> b = Dict("baz" => 17, "bar" => 4711) -Dict{String,Int64} with 2 entries: - "bar" => 4711 - "baz" => 17 - -julia> merge(a, b) -Dict{String,Float64} with 3 entries: - "bar" => 4711.0 - "baz" => 17.0 - "foo" => 0.0 - -julia> merge(b, a) -Dict{String,Float64} with 3 entries: - "bar" => 42.0 - "baz" => 17.0 - "foo" => 0.0 -``` -""" -merge - -""" - transpose!(dest,src) - -Transpose array `src` and store the result in the preallocated array `dest`, which should -have a size corresponding to `(size(src,2),size(src,1))`. No in-place transposition is -supported and unexpected results will happen if `src` and `dest` have overlapping memory -regions. -""" -transpose! - -""" - isconst([m::Module], s::Symbol) -> Bool - -Determine whether a global is declared `const` in a given `Module`. The default `Module` -argument is `current_module()`. -""" -isconst - -""" - open(command, mode::AbstractString="r", stdio=DevNull) - -Start running `command` asynchronously, and return a tuple `(stream,process)`. If `mode` is -`"r"`, then `stream` reads from the process's standard output and `stdio` optionally -specifies the process's standard input stream. If `mode` is `"w"`, then `stream` writes to -the process's standard input and `stdio` optionally specifies the process's standard output -stream. -""" -open(command::Cmd, mod::AbstractString="r", stdio=DevNull) - -""" - open(f::Function, command, mode::AbstractString="r", stdio=DevNull) - -Similar to `open(command, mode, stdio)`, but calls `f(stream)` on the resulting read or -write stream, then closes the stream and waits for the process to complete. Returns the -value returned by `f`. -""" -open(f::Function, command::Cmd, mod::AbstractString="r", stdio=DevNull) - -""" - open(filename, [read, write, create, truncate, append]) -> IOStream - -Open a file in a mode specified by five boolean arguments. The default is to open files for -reading only. Returns a stream for accessing the file. -""" -open(filename, ::Bool, ::Bool, ::Bool, ::Bool, ::Bool) - -""" - open(filename, [mode]) -> IOStream - -Alternate syntax for open, where a string-based mode specifier is used instead of the five -booleans. The values of `mode` correspond to those from `fopen(3)` or Perl `open`, and are -equivalent to setting the following boolean groups: - -| Mode | Description | -|:-----|:------------------------------| -| r | read | -| r+ | read, write | -| w | write, create, truncate | -| w+ | read, write, create, truncate | -| a | write, create, append | -| a+ | read, write, create, append | -""" -open(filename, mode="r") - -""" - open(f::Function, args...) - -Apply the function `f` to the result of `open(args...)` and close the resulting file -descriptor upon completion. - -**Example**: `open(readstring, "file.txt")` -""" -open(f::Function, args...) - -""" - sort(v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) - -Variant of `sort!` that returns a sorted copy of `v` leaving `v` itself unmodified. -""" -sort(v,?,?,?,?) - -""" - sort(A, dim, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) - -Sort a multidimensional array `A` along the given dimension. -""" -sort(A,dim,?,?,?,?) - -""" - kron(A, B) - -Kronecker tensor product of two vectors or two matrices. -""" -kron - -""" - process_exited(p::Process) - -Determine whether a process has exited. -""" -process_exited - """ tuple(xs...) @@ -1522,51 +702,22 @@ eachmatch """ log10(x) -Compute the logarithm of `x` to base 10. Throws `DomainError` for negative `Real` arguments. +Compute the logarithm of `x` to base 10. +Throws [`DomainError`](:obj:`DomainError`) for negative `Real` arguments. """ log10 -""" - @profile - -`@profile <expression>` runs your expression while taking periodic backtraces. These are -appended to an internal buffer of backtraces. -""" -:@profile - -""" - isdigit(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is a numeric digit (0-9), or whether this is true for all elements -of a string. -""" -isdigit - """ num2hex(f) Get a hexadecimal string of the binary representation of a floating point number. -""" -num2hex - -""" - count_ones(x::Integer) -> Integer - -Number of ones in the binary representation of `x`. ```jldoctest -julia> count_ones(7) -3 +julia> num2hex(2.2) +"400199999999999a" ``` """ -count_ones - -""" - reim(z) - -Return both the real and imaginary parts of the complex number `z`. -""" -reim +num2hex """ displayable(mime) -> Bool @@ -1578,13 +729,6 @@ second variant. """ displayable -""" - sdata(S::SharedArray) - -Returns the actual `Array` object backing `S`. -""" -sdata - """ truncate(file,n) @@ -1593,30 +737,6 @@ previously unallocated space with '\\0' if the file or buffer is grown. """ truncate -""" - stat(file) - -Returns a structure whose fields contain information about the file. The fields of the -structure are: - -| Name | Description | -|:--------|:-------------------------------------------------------------------| -| size | The size (in bytes) of the file | -| device | ID of the device that contains the file | -| inode | The inode number of the file | -| mode | The protection mode of the file | -| nlink | The number of hard links to the file | -| uid | The user id of the owner of the file | -| gid | The group id of the file owner | -| rdev | If this file refers to a device, the ID of the device it refers to | -| blksize | The file-system preferred block size for the file | -| blocks | The number of such blocks allocated | -| mtime | Unix timestamp of when the file was last modified | -| ctime | Unix timestamp of when the file was created | - -""" -stat - """ exp10(x) @@ -1631,37 +751,6 @@ Bitwise and. """ & -""" - PipeBuffer() - -An IOBuffer that allows reading and performs writes by appending. Seeking and truncating are -not supported. See IOBuffer for the available constructors. -""" -PipeBuffer() - -""" - PipeBuffer(data::Vector{UInt8},[maxsize]) - -Create a PipeBuffer to operate on a data vector, optionally specifying a size beyond which -the underlying Array may not be grown. -""" -PipeBuffer(data) - -""" - sortperm(v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) - -Return a permutation vector of indices of `v` that puts it in sorted order. Specify `alg` to -choose a particular sorting algorithm (see Sorting Algorithms). `MergeSort` is used by -default, and since it is stable, the resulting permutation will be the lexicographically -first one that puts the input array into sorted order – i.e. indices of equal elements -appear in ascending order. If you choose a non-stable sorting algorithm such as `QuickSort`, -a different permutation that puts the array into order may be returned. The order is -specified using the same keywords as `sort!`. - -See also [`sortperm!`](:func:`sortperm!`). -""" -sortperm - """ cumsum!(B, A, [dim]) @@ -1670,14 +759,6 @@ to 1. """ cumsum! -""" - logdet(M) - -Log of matrix determinant. Equivalent to `log(det(M))`, but may provide increased accuracy -and/or speed. -""" -logdet - """ select(v, k, [by=<transform>,] [lt=<comparison>,] [rev=false]) @@ -1687,106 +768,31 @@ same thing as `select!` but leaving `v` unmodified. select """ - lpad(string, n, p) + accept(server[,client]) -Make a string at least `n` columns wide when printed, by padding on the left with copies of `p`. +Accepts a connection on the given server and returns a connection to the client. An +uninitialized client stream may be provided, in which case it will be used instead of +creating a new stream. """ -lpad +accept """ - mapreduce(f, op, v0, itr) - -Apply function `f` to each element in `itr`, and then reduce the result using the binary -function `op`. `v0` must be a neutral element for `op` that will be returned for empty -collections. It is unspecified whether `v0` is used for non-empty collections. - -[`mapreduce`](:func:`mapreduce`) is functionally equivalent to calling `reduce(op, v0, -map(f, itr))`, but will in general execute faster since no intermediate collection needs to -be created. See documentation for [`reduce`](:func:`reduce`) and [`map`](:func:`map`). - -```jldoctest -julia> mapreduce(x->x^2, +, [1:3;]) # == 1 + 4 + 9 -14 -``` + readstring(stream::IO) + readstring(filename::AbstractString) -The associativity of the reduction is implementation-dependent. Additionally, some -implementations may reuse the return value of `f` for elements that appear multiple times in -`itr`. Use [`mapfoldl`](:func:`mapfoldl`) or [`mapfoldr`](:func:`mapfoldr`) instead for -guaranteed left or right associativity and invocation of `f` for every value. +Read the entire contents of an I/O stream or a file as a string. +The text is assumed to be encoded in UTF-8. """ -mapreduce(f, op, v0, itr) +readstring """ - mapreduce(f, op, itr) + eachline(stream::IO) + eachline(filename::AbstractString) -Like `mapreduce(f, op, v0, itr)`. In general, this cannot be used with empty collections -(see `reduce(op, itr)`). +Create an iterable object that will yield each line from an I/O stream or a file. +The text is assumed to be encoded in UTF-8. """ -mapreduce(f, op, itr) - -""" - accept(server[,client]) - -Accepts a connection on the given server and returns a connection to the client. An -uninitialized client stream may be provided, in which case it will be used instead of -creating a new stream. -""" -accept - -""" - triu!(M) - -Upper triangle of a matrix, overwriting `M` in the process. -""" -triu!(M) - -""" - triu!(M, k) - -Returns the upper triangle of `M` starting from the `k`th superdiagonal, overwriting `M` in -the process. -""" -triu!(M, k) - -""" - readstring(stream::IO) - readstring(filename::AbstractString) - -Read the entire contents of an I/O stream or a file as a string. -The text is assumed to be encoded in UTF-8. -""" -readstring - -""" - poll_file(path, interval_s::Real, timeout_s::Real) -> (previous::StatStruct, current::StatStruct) - -Monitor a file for changes by polling every `interval_s` seconds until a change occurs or -`timeout_s` seconds have elapsed. The `interval_s` should be a long period; the default is -5.007 seconds. - -Returns a pair of `StatStruct` objects `(previous, current)` when a change is detected. - -To determine when a file was modified, compare `mtime(prev) != mtime(current)` to detect -notification of changes. However, using `watch_file` for this operation is preferred, since -it is more reliable and efficient, although in some situations it may not be available. -""" -poll_file - -""" - eachline(stream::IO) - eachline(filename::AbstractString) - -Create an iterable object that will yield each line from an I/O stream or a file. -The text is assumed to be encoded in UTF-8. -""" -eachline - -""" - isposdef!(A) -> Bool - -Test whether a matrix is positive definite, overwriting `A` in the processes. -""" -isposdef! +eachline """ complex(r, [i]) @@ -1795,16 +801,6 @@ Convert real numbers or arrays to complex. `i` defaults to zero. """ complex -""" - setopt(sock::UDPSocket; multicast_loop = nothing, multicast_ttl=nothing, enable_broadcast=nothing, ttl=nothing) - -Set UDP socket options. `multicast_loop`: loopback for multicast packets (default: `true`). -`multicast_ttl`: TTL for multicast packets. `enable_broadcast`: flag must be set to `true` -if socket will be used for broadcast messages, or else the UDP system will return an access -error (default: `false`). `ttl`: Time-to-live of packets sent on the socket. -""" -setopt - """ Mmap.Anonymous(name, readonly, create) @@ -1813,92 +809,6 @@ for use in `Mmap.mmap`. Used by `SharedArray` for creating shared memory arrays. """ Mmap.Anonymous -""" - A_rdiv_Bc(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``A / Bᴴ``. -""" -A_rdiv_Bc - -""" - strwidth(s) - -Gives the number of columns needed to print a string. -""" -strwidth - -""" - hex(n, [pad]) - -Convert an integer to a hexadecimal string, optionally specifying a number of digits to pad to. -""" -hex - -""" - workspace() - -Replace the top-level module (`Main`) with a new one, providing a clean workspace. The -previous `Main` module is made available as `LastMain`. A previously-loaded package can be -accessed using a statement such as `using LastMain.Package`. - -This function should only be used interactively. -""" -workspace - -""" - tempdir() - -Obtain the path of a temporary directory (possibly shared with other processes). -""" -tempdir - -""" - reduce(op, v0, itr) - -Reduce the given collection `ìtr` with the given binary operator `op`. `v0` must be a -neutral element for `op` that will be returned for empty collections. It is unspecified -whether `v0` is used for non-empty collections. - -Reductions for certain commonly-used operators have special implementations which should be -used instead: `maximum(itr)`, `minimum(itr)`, `sum(itr)`, `prod(itr)`, `any(itr)`, -`all(itr)`. - -The associativity of the reduction is implementation dependent. This means that you can't -use non-associative operations like `-` because it is undefined whether `reduce(-,[1,2,3])` -should be evaluated as `(1-2)-3` or `1-(2-3)`. Use `foldl` or `foldr` instead for guaranteed -left or right associativity. - -Some operations accumulate error, and parallelism will also be easier if the reduction can -be executed in groups. Future versions of Julia might change the algorithm. Note that the -elements are not reordered if you use an ordered collection. -""" -reduce(op, v0, itr) - -""" - reduce(op, itr) - -Like `reduce(op, v0, itr)`. This cannot be used with empty collections, except for some -special cases (e.g. when `op` is one of `+`, `*`, `max`, `min`, `&`, `|`) when Julia can -determine the neutral element of `op`. -""" -reduce(op, itr) - -""" - .>=(x, y) - .≥(x,y) - -Element-wise greater-than-or-equals comparison operator. -""" -Base.:(.>=) - -""" - mv(src::AbstractString,dst::AbstractString; remove_destination::Bool=false) - -Move the file, link, or directory from `src` to `dst`. `remove_destination=true` will first -remove an existing `dst`. -""" -mv - """ erfi(x) @@ -1919,29 +829,6 @@ not representable. """ floor -""" - tril!(M) - -Lower triangle of a matrix, overwriting `M` in the process. -""" -tril!(M) - -""" - tril!(M, k) - -Returns the lower triangle of `M` starting from the `k`th superdiagonal, overwriting `M` in -the process. -""" -tril!(M, k) - -""" - divrem(x, y) - -The quotient and remainder from Euclidean division. Equivalent to `(div(x,y), rem(x,y))` or -`(x÷y, x%y)`. -""" -divrem - """ ErrorException(msg) @@ -1956,13 +843,6 @@ Return a copy of `v` reversed from start to stop. """ reverse -""" - reverse(s::AbstractString) -> AbstractString - -Reverses a string. -""" -reverse(s::AbstractString) - """ reverse!(v [, start=1 [, stop=length(v) ]]) -> v @@ -1977,51 +857,6 @@ Numerator of the rational representation of `x`. """ num -""" - eachindex(A...) - -Creates an iterable object for visiting each index of an AbstractArray `A` in an efficient -manner. For array types that have opted into fast linear indexing (like `Array`), this is -simply the range `1:length(A)`. For other array types, this returns a specialized Cartesian -range to efficiently index into the array with indices specified for every dimension. For -other iterables, including strings and dictionaries, this returns an iterator object -supporting arbitrary index types (e.g. unevenly spaced or non-integer indices). - -Example for a sparse 2-d array: - -```jldoctest -julia> A = sparse([1, 1, 2], [1, 3, 1], [1, 2, -5]) -2×3 sparse matrix with 3 Int64 nonzero entries: - [1, 1] = 1 - [2, 1] = -5 - [1, 3] = 2 - -julia> for iter in eachindex(A) - @show iter.I[1], iter.I[2] - @show A[iter] - end -(iter.I[1],iter.I[2]) = (1,1) -A[iter] = 1 -(iter.I[1],iter.I[2]) = (2,1) -A[iter] = -5 -(iter.I[1],iter.I[2]) = (1,2) -A[iter] = 0 -(iter.I[1],iter.I[2]) = (2,2) -A[iter] = 0 -(iter.I[1],iter.I[2]) = (1,3) -A[iter] = 2 -(iter.I[1],iter.I[2]) = (2,3) -A[iter] = 0 -``` - -If you supply more than one `AbstractArray` argument, `eachindex` will create an -iterable object that is fast for all arguments (a `UnitRange` if all inputs have fast -linear indexing, a CartesianRange otherwise). If the arrays have different sizes and/or -dimensionalities, `eachindex` returns an iterable that spans the largest range along each -dimension. -""" -eachindex - """ .<(x, y) @@ -2043,31 +878,6 @@ Bessel function of the second kind of order 1, ``Y_1(x)``. """ bessely1 -""" - print(x) - -Write (to the default output stream) a canonical (un-decorated) text representation of a -value if there is one, otherwise call `show`. The representation used by `print` includes -minimal formatting and tries to avoid Julia-specific details. -""" -print - -""" - filt(b, a, x, [si]) - -Apply filter described by vectors `a` and `b` to vector `x`, with an optional initial filter -state vector `si` (defaults to zeros). -""" -filt - -""" - indexpids(S::SharedArray) - -Returns the index of the current worker into the `pids` vector, i.e., the list of workers -mapping the SharedArray -""" -indexpids - """ append!(collection, collection2) -> collection. @@ -2098,14 +908,6 @@ themselves in another collection. The result is of the preceding example is equi """ append! - -""" - ctranspose(A) - -The conjugate transposition operator (`'`). -""" -ctranspose - """ skip(s, offset) @@ -2120,13 +922,6 @@ Compute the LU factorization of `A`, such that `A[p,:] = L*U`. """ lu -""" - fld(x, y) - -Largest integer less than or equal to `x/y`. -""" -fld - """ setdiff!(s, iterable) @@ -2134,21 +929,6 @@ Remove each element of `iterable` from set `s` in-place. """ setdiff! -""" - isascii(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character belongs to the ASCII character set, or whether this is true for -all elements of a string. -""" -isascii - -""" - ucfirst(string) - -Returns `string` with the first character converted to uppercase. -""" -ucfirst - """ copysign(x, y) @@ -2156,13 +936,6 @@ Return `x` such that it has the same sign as `y` """ copysign -""" - getaddrinfo(host) - -Gets the IP address of the `host` (may have to do a DNS lookup) -""" -getaddrinfo - """ @show @@ -2184,21 +957,6 @@ test `get(io, :compact, false)` in its normal `show` method. """ showcompact -""" - isleaftype(T) - -Determine whether `T` is a concrete type that can have instances, meaning its only subtypes -are itself and `Union{}` (but `T` itself is not `Union{}`). -""" -isleaftype - -""" - string(xs...) - -Create a string from any values using the `print` function. -""" -string - """ erfc(x) @@ -2206,13 +964,6 @@ Compute the complementary error function of `x`, defined by ``1 - \\operatorname """ erfc -""" - rest(iter, state) - -An iterator that yields the same elements as `iter`, but starting at the given `state`. -""" -rest - """ getfield(value, name::Symbol) @@ -2247,13 +998,6 @@ Compute the maximum value of `A` over the singleton dimensions of `r`, and write """ maximum! -""" - prod(itr) - -Returns the product of all elements of a collection. -""" -prod(itr) - """ prod(A, dims) @@ -2261,13 +1005,6 @@ Multiply elements of an array over the given dimensions. """ prod(A, dims) -""" - isqrt(n) - -Integer square root: the largest integer `m` such that `m*m <= n`. -""" -isqrt - """ log1p(x) @@ -2319,14 +1056,6 @@ See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. """ Float64 -""" - mkpath(path, [mode]) - -Create all directories in the given `path`, with permissions `mode`. `mode` defaults to -`0o777`, modified by the current file creation mask. -""" -mkpath - """ union(s1,s2...) ∪(s1,s2...) @@ -2335,31 +1064,6 @@ Construct the union of two or more sets. Maintains order with arrays. """ union -""" - lstat(file) - -Like stat, but for symbolic links gets the info for the link itself rather than the file it -refers to. This function must be called on a file path rather than a file object or a file -descriptor. -""" -lstat - -""" - mapfoldl(f, op, v0, itr) - -Like [`mapreduce`](:func:`mapreduce`), but with guaranteed left associativity. `v0` will be -used exactly once. -""" -mapfoldl(f, op, v0, itr) - -""" - mapfoldl(f, op, itr) - -Like `mapfoldl(f, op, v0, itr)`, but using the first element of `itr` as `v0`. In general, -this cannot be used with empty collections (see `reduce(op, itr)`). -""" -mapfoldl(f, op, itr) - """ realmax(T) @@ -2367,70 +1071,17 @@ The highest finite value representable by the given floating-point DataType `T`. """ realmax -""" - takebuf_string(b::IOBuffer) - -Obtain the contents of an `IOBuffer` as a string, without copying. Afterwards, the IOBuffer -is reset to its initial state. -""" -takebuf_string - -""" - pipeline(from, to, ...) - -Create a pipeline from a data source to a destination. The source and destination can be -commands, I/O streams, strings, or results of other `pipeline` calls. At least one argument -must be a command. Strings refer to filenames. When called with more than two arguments, -they are chained together from left to right. For example `pipeline(a,b,c)` is equivalent to -`pipeline(pipeline(a,b),c)`. This provides a more concise way to specify multi-stage -pipelines. - -**Examples**: - -```julia -run(pipeline(`ls`, `grep xyz`)) -run(pipeline(`ls`, "out.txt")) -run(pipeline("out.txt", `grep xyz`)) -``` -""" -pipeline(from, to, rest...) - -""" - pipeline(command; stdin, stdout, stderr, append=false) - -Redirect I/O to or from the given `command`. Keyword arguments specify which of the -command's streams should be redirected. `append` controls whether file output appends to the -file. This is a more general version of the 2-argument `pipeline` function. -`pipeline(from, to)` is equivalent to `pipeline(from, stdout=to)` when `from` is a command, -and to `pipeline(to, stdin=from)` when `from` is another kind of data source. - -**Examples**: - -```julia -run(pipeline(`dothings`, stdout="out.txt", stderr="errs.txt")) -run(pipeline(`update`, stdout="log.txt", append=true)) -``` -""" -pipeline(command) - """ serialize(stream, value) Write an arbitrary value to a stream in an opaque format, such that it can be read back by -`deserialize`. The read-back value will be as identical as possible to the original. In +[`deserialize`](:func:`deserialize`). The read-back value will be as identical as possible to the original. In general, this process will not work if the reading and writing are done by different versions of Julia, or an instance of Julia with a different system image. `Ptr` values are serialized as all-zero bit patterns (`NULL`). """ serialize -""" - sum(itr) - -Returns the sum of all elements in a collection. -""" -sum(itr) - """ sum(A, dims) @@ -2438,13 +1089,6 @@ Sum elements of an array over the given dimensions. """ sum(A, dims) -""" - sum(f, itr) - -Sum the results of calling function `f` on each element of `itr`. -""" -sum(f::Function, itr) - """ typemin(T) @@ -2453,222 +1097,79 @@ The lowest value representable by the given (real) numeric DataType `T`. typemin """ - countfrom(start=1, step=1) + typeof(x) -An iterator that counts forever, starting at `start` and incrementing by `step`. +Get the concrete type of `x`. """ -countfrom +typeof """ - eof(stream) -> Bool + log(x) -Tests whether an I/O stream is at end-of-file. If the stream is not yet exhausted, this -function will block to wait for more data if necessary, and then return `false`. Therefore -it is always safe to read one byte after seeing `eof` return `false`. `eof` will return -`false` as long as buffered data is still available, even if the remote end of a connection -is closed. -""" -eof +Compute the natural logarithm of `x`. Throws `DomainError` for negative `Real` arguments. +Use complex negative arguments to obtain complex results. +There is an experimental variant in the `Base.Math.JuliaLibm` module, which is typically +faster and more accurate. """ - mktempdir([parent=tempdir()]) +log(x) -Create a temporary directory in the `parent` directory and return its path. """ -mktempdir() + trunc([T,] x, [digits, [base]]) -""" - mktempdir(f::Function, [parent=tempdir()]) +`trunc(x)` returns the nearest integral value of the same type as `x` whose absolute value +is less than or equal to `x`. -Apply the function `f` to the result of `mktempdir(parent)` and remove the temporary -directory upon completion. -""" -mktempdir(f::Function) +`trunc(T, x)` converts the result to type `T`, throwing an `InexactError` if the value is +not representable. +`digits` and `base` work as for [`round`](:func:`round`). """ - tril(M) +trunc -Lower triangle of a matrix. """ -tril(M) + unsafe_convert(T,x) -""" - tril(M, k) +Convert `x` to a value of type `T` -Returns the lower triangle of `M` starting from the `k`th superdiagonal. -""" -tril(M,k) +In cases where [`convert`](:func:`convert`) would need to take a Julia object +and turn it into a `Ptr`, this function should be used to define and perform +that conversion. +Be careful to ensure that a Julia reference to `x` exists as long as the result of this +function will be used. Accordingly, the argument `x` to this function should never be an +expression, only a variable name or field reference. For example, `x=a.b.c` is acceptable, +but `x=[a,b,c]` is not. +The `unsafe` prefix on this function indicates that using the result of this function after +the `x` argument to this function is no longer accessible to the program may cause undefined +behavior, including program corruption or segfaults, at any later time. """ - subtypes(T::DataType) +unsafe_convert -Return a list of immediate subtypes of DataType `T`. Note that all currently loaded subtypes -are included, including those not visible in the current module. """ -subtypes + erfinv(x) +Compute the inverse error function of a real `x`, defined by ``\\operatorname{erf}(\\operatorname{erfinv}(x)) = x``. """ - digits([T], n, [base], [pad]) +erfinv -Returns an array with element type `T` (default `Int`) of the digits of `n` in the given -base, optionally padded with zeros to a specified size. More significant digits are at -higher indexes, such that `n == sum([digits[k]*base^(k-1) for k=1:length(digits)])`. """ -digits + seek(s, pos) +Seek a stream to the given position. """ - bytes2hex(bin_arr::Array{UInt8, 1}) +seek -Convert an array of bytes to its hexadecimal representation. All characters are in -lower-case. Returns a `String`. """ -bytes2hex + besselj0(x) +Bessel function of the first kind of order 0, ``J_0(x)``. """ - BigFloat(x) - -Create an arbitrary precision floating point number. `x` may be an `Integer`, a `Float64` or -a `BigInt`. The usual mathematical operators are defined for this type, and results are -promoted to a `BigFloat`. - -Note that because decimal literals are converted to floating point numbers when parsed, -`BigFloat(2.1)` may not yield what you expect. You may instead prefer to initialize -constants from strings via [`parse`](:func:`parse`), or using the `big` string literal. - -```jldoctest -julia> BigFloat(2.1) -2.100000000000000088817841970012523233890533447265625000000000000000000000000000 +besselj0 -julia> big"2.1" -2.099999999999999999999999999999999999999999999999999999999999999999999999999986 -``` """ -BigFloat - -""" - xcorr(u,v) - -Compute the cross-correlation of two vectors. -""" -xcorr - -""" - typeof(x) - -Get the concrete type of `x`. -""" -typeof - -""" - log(x) - -Compute the natural logarithm of `x`. Throws `DomainError` for negative `Real` arguments. -Use complex negative arguments to obtain complex results. - -There is an experimental variant in the `Base.Math.JuliaLibm` module, which is typically -faster and more accurate. -""" -log(x) - -""" - log(b,x) - -Compute the base `b` logarithm of `x`. Throws `DomainError` for negative `Real` arguments. -""" -log(b, x) - -""" - trunc([T,] x, [digits, [base]]) - -`trunc(x)` returns the nearest integral value of the same type as `x` whose absolute value -is less than or equal to `x`. - -`trunc(T, x)` converts the result to type `T`, throwing an `InexactError` if the value is -not representable. - -`digits` and `base` work as for [`round`](:func:`round`). -""" -trunc - -""" - unsafe_convert(T,x) - -Convert `x` to a value of type `T` - -In cases where `convert` would need to take a Julia object and turn it into a `Ptr`, this -function should be used to define and perform that conversion. - -Be careful to ensure that a Julia reference to `x` exists as long as the result of this -function will be used. Accordingly, the argument `x` to this function should never be an -expression, only a variable name or field reference. For example, `x=a.b.c` is acceptable, -but `x=[a,b,c]` is not. - -The `unsafe` prefix on this function indicates that using the result of this function after -the `x` argument to this function is no longer accessible to the program may cause undefined -behavior, including program corruption or segfaults, at any later time. -""" -unsafe_convert - -""" - warn(msg) - -Display a warning. Argument `msg` is a string describing the warning to be displayed. -""" -warn - -""" - erfinv(x) - -Compute the inverse error function of a real `x`, defined by ``\\operatorname{erf}(\\operatorname{erfinv}(x)) = x``. -""" -erfinv - -""" - readdir([dir]) -> Vector{String} - -Returns the files and directories in the directory `dir` (or the current working directory if not given). -""" -readdir - -""" - seek(s, pos) - -Seek a stream to the given position. -""" -seek - -""" - triu(M) - -Upper triangle of a matrix. -""" -triu(M) - -""" - triu(M, k) - -Returns the upper triangle of `M` starting from the `k`th superdiagonal. -""" -triu(M, k) - -""" - instances(T::Type) - -Return a collection of all instances of the given type, if applicable. Mostly used for -enumerated types (see `@enum`). -""" -instances - -""" - besselj0(x) - -Bessel function of the first kind of order 0, ``J_0(x)``. -""" -besselj0 - -""" - erfcinv(x) + erfcinv(x) Compute the inverse error complementary function of a real `x`, defined by ``\\operatorname{erfc}(\\operatorname{erfcinv}(x)) = x``. @@ -2691,19 +1192,14 @@ second variant. """ popdisplay -""" - filesize(path...) - -Equivalent to `stat(file).size`. -""" -filesize - """ cglobal((symbol, library) [, type=Void]) Obtain a pointer to a global variable in a C-exported shared library, specified exactly as -in `ccall`. Returns a `Ptr{Type}`, defaulting to `Ptr{Void}` if no Type argument is -supplied. The values can be read or written by `unsafe_load` or `unsafe_store!`, +in [`ccall`](:func:`ccall`). +Returns a `Ptr{Type}`, defaulting to `Ptr{Void}` if no `Type` argument is +supplied. +The values can be read or written by [`unsafe_load`](:func:`unsafe_load`) or [`unsafe_store!`](:func:`unsafe_store!`), respectively. """ cglobal @@ -2716,83 +1212,6 @@ itself). For matrices, returns an identity matrix of the appropriate size and ty """ one -""" - splice!(collection, index, [replacement]) -> item - -Remove the item at the given index, and return the removed item. Subsequent items are -shifted down to fill the resulting gap. If specified, replacement values from an ordered -collection will be spliced in place of the removed item. - -```jldoctest -julia> A = [6, 5, 4, 3, 2, 1]; splice!(A, 5) -2 - -julia> A -5-element Array{Int64,1}: - 6 - 5 - 4 - 3 - 1 - -julia> splice!(A, 5, -1) -1 - -julia> A -5-element Array{Int64,1}: - 6 - 5 - 4 - 3 - -1 - -julia> splice!(A, 1, [-1, -2, -3]) -6 - -julia> A -7-element Array{Int64,1}: - -1 - -2 - -3 - 5 - 4 - 3 - -1 -``` - -To insert `replacement` before an index `n` without removing any items, use -`splice!(collection, n:n-1, replacement)`. -""" -splice!(collection, index, replacement = ?) - -""" - splice!(collection, range, [replacement]) -> items - -Remove items in the specified index range, and return a collection containing the removed -items. Subsequent items are shifted down to fill the resulting gap. If specified, -replacement values from an ordered collection will be spliced in place of the removed items. - -To insert `replacement` before an index `n` without removing any items, use -`splice!(collection, n:n-1, replacement)`. - -```jldoctest -julia> splice!(A, 4:3, 2) -0-element Array{Int64,1} - -julia> A -8-element Array{Int64,1}: - -1 - -2 - -3 - 2 - 5 - 4 - 3 - -1 -``` -""" -splice!(collection, range::Range, replacement) - """ endof(collection) -> Integer @@ -2805,13 +1224,6 @@ julia> endof([1,2,4]) """ endof -""" - isfifo(path) -> Bool - -Returns `true` if `path` is a FIFO, `false` otherwise. -""" -isfifo - """ Channel{T}(sz::Int) @@ -2832,24 +1244,6 @@ For a given iterable object and iteration state, return the current item and the """ next -""" - unshift!(collection, items...) -> collection - -Insert one or more `items` at the beginning of `collection`. - -```jldoctest - julia> unshift!([1, 2, 3, 4], 5, 6) - 6-element Array{Int64,1}: - 5 - 6 - 1 - 2 - 3 - 4 -``` -""" -unshift! - """ log2(x) @@ -2857,29 +1251,6 @@ Compute the logarithm of `x` to base 2. Throws `DomainError` for negative `Real` """ log2 -""" - colon(start, [step], stop) - -Called by `:` syntax for constructing ranges. -""" -colon - -""" - Base64EncodePipe(ostream) - -Returns a new write-only I/O stream, which converts any bytes written to it into -base64-encoded ASCII bytes written to `ostream`. Calling `close` on the `Base64EncodePipe` stream -is necessary to complete the encoding (but does not close `ostream`). -""" -Base64EncodePipe - -""" - issetgid(path) -> Bool - -Returns `true` if `path` has the setgid flag set, `false` otherwise. -""" -issetgid - """ isnull(x) @@ -2902,646 +1273,163 @@ Suggest that collection `s` reserve capacity for at least `n` elements. This can sizehint! """ - ifelse(condition::Bool, x, y) + OutOfMemoryError() -Return `x` if `condition` is `true`, otherwise return `y`. This differs from `?` or `if` in -that it is an ordinary function, so all the arguments are evaluated first. In some cases, -using `ifelse` instead of an `if` statement can eliminate the branch in generated code and -provide higher performance in tight loops. +An operation allocated too much memory for either the system or the garbage collector to +handle properly. """ -ifelse +OutOfMemoryError """ - ispow2(n) -> Bool + finalize(x) -Test whether `n` is a power of two. +Immediately run finalizers registered for object `x`. """ -ispow2 +finalize """ - isgraph(c::Union{Char,AbstractString}) -> Bool + BoundsError([a],[i]) -Tests whether a character is printable, and not a space, or whether this is true for all -elements of a string. Any character that would cause a printer to use ink should be -classified with `isgraph(c)==true`. +An indexing operation into an array, `a`, tried to access an out-of-bounds element, `i`. """ -isgraph +BoundsError """ - OutOfMemoryError() + invoke(f, (types...), args...) -An operation allocated too much memory for either the system or the garbage collector to -handle properly. +Invoke a method for the given generic function matching the specified types (as a tuple), on +the specified arguments. The arguments must be compatible with the specified types. This +allows invoking a method other than the most specific matching method, which is useful when +the behavior of a more general definition is explicitly needed (often as part of the +implementation of a more specific method of the same function). """ -OutOfMemoryError - +invoke """ - binomial(n,k) + parse(str, start; greedy=true, raise=true) -Number of ways to choose `k` out of `n` items. +Parse the expression string and return an expression (which could later be passed to eval +for execution). `start` is the index of the first character to start parsing. If `greedy` is +`true` (default), `parse` will try to consume as much input as it can; otherwise, it will +stop as soon as it has parsed a valid expression. Incomplete but otherwise syntactically +valid expressions will return `Expr(:incomplete, "(error message)")`. If `raise` is `true` +(default), syntax errors other than incomplete expressions will raise an error. If `raise` +is `false`, `parse` will return an expression that will raise an error upon evaluation. """ -binomial +parse(str, start) """ - .<=(x, y) - .≤(x,y) + parse(str; raise=true) -Element-wise less-than-or-equals comparison operator. +Parse the expression string greedily, returning a single expression. An error is thrown if +there are additional characters after the first expression. If `raise` is `true` (default), +syntax errors will raise an error; otherwise, `parse` will return an expression that will +raise an error upon evaluation. """ -Base.:(.<=) +parse(str) """ - rank(M) + parse(type, str, [base]) -Compute the rank of a matrix. +Parse a string as a number. If the type is an integer type, then a base can be specified +(the default is 10). If the type is a floating point type, the string is parsed as a decimal +floating point number. If the string does not contain a valid number, an error is raised. """ -rank +parse(T::Type, str, base=Int) """ - max(x, y, ...) + bkfact!(A) -> BunchKaufman -Return the maximum of the arguments. Operates elementwise over arrays. +`bkfact!` is the same as [`bkfact`](:func:`bkfact`), but saves space by overwriting the +input `A`, instead of creating a copy. """ -max +bkfact! """ - versioninfo([verbose::Bool]) + ^(x, y) -Print information about the version of Julia in use. If the `verbose` argument is `true`, -detailed system information is shown as well. +Exponentiation operator. """ -versioninfo +Base.:(^)(x, y) """ - sort!(v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) + position(s) -Sort the vector `v` in place. `QuickSort` is used by default for numeric arrays while -`MergeSort` is used for other arrays. You can specify an algorithm to use via the `alg` -keyword (see Sorting Algorithms for available algorithms). The `by` keyword lets you provide -a function that will be applied to each element before comparison; the `lt` keyword allows -providing a custom "less than" function; use `rev=true` to reverse the sorting order. These -options are independent and can be used together in all possible combinations: if both `by` -and `lt` are specified, the `lt` function is applied to the result of the `by` function; -`rev=true` reverses whatever ordering specified via the `by` and `lt` keywords. +Get the current position of a stream. """ -sort! +position """ - kill(p::Process, signum=SIGTERM) + selectperm(v, k, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) -Send a signal to a process. The default is to terminate the process. +Return a partial permutation of the vector `v`, according to the order specified by +`by`, `lt` and `rev`, so that `v[output]` returns the first `k` (or range of adjacent values +if `k` is a range) values of a fully sorted version of `v`. If `k` is a single index +(Integer), an array of the first `k` indices is returned; if `k` is a range, an array of +those indices is returned. Note that the handling of integer values for `k` is different +from `select` in that it returns a vector of `k` elements instead of just the `k` th +element. Also note that this is equivalent to, but more efficient than, calling +`sortperm(...)[k]` """ -kill(p::Process, signum=SIGTERM) +selectperm """ - sylvester(A, B, C) + reinterpret(type, A) -Computes the solution `X` to the Sylvester equation `AX + XB + C = 0`, where `A`, `B` and -`C` have compatible dimensions and `A` and `-B` have no eigenvalues with equal real part. +Change the type-interpretation of a block of memory. For example, +`reinterpret(Float32, UInt32(7))` interprets the 4 bytes corresponding to `UInt32(7)` as a +`Float32`. For arrays, this constructs an array with the same binary data as the given +array, but with the specified element type. """ -sylvester +reinterpret """ - broadcast!(f, dest, As...) + ~(x) -Like `broadcast`, but store the result of `broadcast(f, As...)` in the `dest` array. Note -that `dest` is only used to store the result, and does not supply arguments to `f` unless it -is also listed in the `As`, as in `broadcast!(f, A, A, B)` to perform `A[:] = broadcast(f, A, B)`. +Bitwise not. """ -broadcast! +~ """ - cross(x, y) - ×(x,y) + bswap(n) -Compute the cross product of two 3-vectors. +Byte-swap an integer. """ -cross +bswap """ - keys(collection) + sumabs2!(r, A) -Return an iterator over all keys in a collection. `collect(keys(d))` returns an array of keys. +Sum squared absolute values of elements of `A` over the singleton dimensions of `r`, and +write results to `r`. """ -keys +sumabs2! """ - real(z) + @sprintf("%Fmt", args...) -Return the real part of the complex number `z`. -""" -real +Return `@printf` formatted output as string. + + julia> s = @sprintf "this is a %s %15.1f" "test" 34.567; + julia> println(s) + this is a test 34.6 """ - gperm(file) +:@sprintf -Like uperm but gets the permissions of the group owning the file. """ -gperm + tanh(x) +Compute hyperbolic tangent of `x`. """ - nb_available(stream) +tanh -Returns the number of bytes available for reading before a read from this stream or buffer will block. """ -nb_available + maxintfloat(T) +The largest integer losslessly representable by the given floating-point DataType `T`. """ - finalize(x) - -Immediately run finalizers registered for object `x`. -""" -finalize - -""" - rand([rng], [S], [dims...]) - -Pick a random element or array of random elements from the set of values specified by `S`; `S` can be - -* an indexable collection (for example `1:n` or `['x','y','z']`), or -* a type: the set of values to pick from is then equivalent to `typemin(S):typemax(S)` for - integers (this is not applicable to `BigInt`), and to ``[0, 1)`` for floating point numbers; - -`S` defaults to `Float64`. -""" -rand - -""" - base(base, n, [pad]) - -Convert an integer to a string in the given base, optionally specifying a number of digits to pad to. -""" -base - -""" - BoundsError([a],[i]) - -An indexing operation into an array, `a`, tried to access an out-of-bounds element, `i`. -""" -BoundsError - -""" - which(f, types) - -Returns the method of `f` (a `Method` object) that would be called for arguments of the given `types`. - -If `types` is an abstract type, then the method that would be called by `invoke` is returned. -""" -which(f, types) - -""" - which(symbol) - -Return the module in which the binding for the variable referenced by `symbol` was created. -""" -which(symbol) - -""" - conv2(u,v,A) - -2-D convolution of the matrix `A` with the 2-D separable kernel generated by the vectors `u` -and `v`. Uses 2-D FFT algorithm. -""" -conv2(u, v, A) - -""" - conv2(B,A) - -2-D convolution of the matrix `B` with the matrix `A`. Uses 2-D FFT algorithm. -""" -conv2(B, A) - -""" - broadcast_getindex(A, inds...) - -Broadcasts the `inds` arrays to a common size like `broadcast`, and returns an array of the -results `A[ks...]`, where `ks` goes over the positions in the broadcast. -""" -broadcast_getindex - -""" - invoke(f, (types...), args...) - -Invoke a method for the given generic function matching the specified types (as a tuple), on -the specified arguments. The arguments must be compatible with the specified types. This -allows invoking a method other than the most specific matching method, which is useful when -the behavior of a more general definition is explicitly needed (often as part of the -implementation of a more specific method of the same function). -""" -invoke - -""" - parse(str, start; greedy=true, raise=true) - -Parse the expression string and return an expression (which could later be passed to eval -for execution). `start` is the index of the first character to start parsing. If `greedy` is -`true` (default), `parse` will try to consume as much input as it can; otherwise, it will -stop as soon as it has parsed a valid expression. Incomplete but otherwise syntactically -valid expressions will return `Expr(:incomplete, "(error message)")`. If `raise` is `true` -(default), syntax errors other than incomplete expressions will raise an error. If `raise` -is `false`, `parse` will return an expression that will raise an error upon evaluation. -""" -parse(str, start) - -""" - parse(str; raise=true) - -Parse the expression string greedily, returning a single expression. An error is thrown if -there are additional characters after the first expression. If `raise` is `true` (default), -syntax errors will raise an error; otherwise, `parse` will return an expression that will -raise an error upon evaluation. -""" -parse(str) - -""" - parse(type, str, [base]) - -Parse a string as a number. If the type is an integer type, then a base can be specified -(the default is 10). If the type is a floating point type, the string is parsed as a decimal -floating point number. If the string does not contain a valid number, an error is raised. -""" -parse(T::Type, str, base=Int) - -""" - touch(path::AbstractString) - -Update the last-modified timestamp on a file to the current time. -""" -touch - -""" - bkfact!(A) -> BunchKaufman - -`bkfact!` is the same as [`bkfact`](:func:`bkfact`), but saves space by overwriting the -input `A`, instead of creating a copy. -""" -bkfact! - -""" - ^(x, y) - -Exponentiation operator. -""" -Base.:(^)(x, y) - -""" - ^(s, n) - -Repeat `n` times the string `s`. The `repeat` function is an alias to this operator. - -```jldoctest -julia> "Test "^3 -"Test Test Test " -``` -""" -Base.:(^)(s::AbstractString, n::Int) - -""" - position(s) - -Get the current position of a stream. -""" -position - -""" - selectperm(v, k, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) - -Return a partial permutation of the vector `v`, according to the order specified by -`by`, `lt` and `rev`, so that `v[output]` returns the first `k` (or range of adjacent values -if `k` is a range) values of a fully sorted version of `v`. If `k` is a single index -(Integer), an array of the first `k` indices is returned; if `k` is a range, an array of -those indices is returned. Note that the handling of integer values for `k` is different -from `select` in that it returns a vector of `k` elements instead of just the `k` th -element. Also note that this is equivalent to, but more efficient than, calling -`sortperm(...)[k]` -""" -selectperm - -""" - isabspath(path::AbstractString) -> Bool - -Determines whether a path is absolute (begins at the root directory). -""" -isabspath - -""" - hex2bytes(s::AbstractString) - -Convert an arbitrarily long hexadecimal string to its binary representation. Returns an -`Array{UInt8,1}`, i.e. an array of bytes. -""" -hex2bytes - -""" - isdir(path) -> Bool - -Returns `true` if `path` is a directory, `false` otherwise. -""" -isdir - -""" - reinterpret(type, A) - -Change the type-interpretation of a block of memory. For example, -`reinterpret(Float32, UInt32(7))` interprets the 4 bytes corresponding to `UInt32(7)` as a -`Float32`. For arrays, this constructs an array with the same binary data as the given -array, but with the specified element type. -""" -reinterpret - -""" - ~(x) - -Bitwise not. -""" -~ - -""" - rem(x, y) - %(x, y) - -Remainder from Euclidean division, returning a value of the same sign as `x`, and smaller in -magnitude than `y`. This value is always exact. - -```julia -x == div(x,y)*y + rem(x,y) -``` -""" -rem - -""" - info(msg) - -Display an informational message. Argument `msg` is a string describing the information to be displayed. -""" -info - -""" - ltoh(x) - -Converts the endianness of a value from Little-endian to that used by the Host. -""" -ltoh - -""" - evalfile(path::AbstractString) - -Load the file using `include`, evaluate all expressions, and return the value of the last one. -""" -evalfile - -""" - success(command) - -Run a command object, constructed with backticks, and tell whether it was successful (exited -with a code of 0). An exception is raised if the process cannot be started. -""" -success - -""" - sortperm!(ix, v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false,] [initialized=false]) - -Like `sortperm`, but accepts a preallocated index vector `ix`. If `initialized` is `false` -(the default), ix is initialized to contain the values `1:length(v)`. - -See also [`sortperm`](:func:`sortperm`). -""" -sortperm! - -""" - isodd(x::Integer) -> Bool - -Returns `true` if `x` is odd (that is, not divisible by 2), and `false` otherwise. - -```jldoctest -julia> isodd(9) -true - -julia> isodd(10) -false -``` -""" -isodd - -""" - normalize_string(s, normalform::Symbol) - -Normalize the string `s` according to one of the four "normal forms" of the Unicode -standard: `normalform` can be `:NFC`, `:NFD`, `:NFKC`, or `:NFKD`. Normal forms C -(canonical composition) and D (canonical decomposition) convert different visually identical -representations of the same abstract string into a single canonical form, with form C being -more compact. Normal forms KC and KD additionally canonicalize "compatibility equivalents": -they convert characters that are abstractly similar but visually distinct into a single -canonical choice (e.g. they expand ligatures into the individual characters), with form KC -being more compact. - -Alternatively, finer control and additional transformations may be be obtained by calling -`normalize_string(s; keywords...)`, where any number of the following boolean keywords -options (which all default to `false` except for `compose`) are specified: - -* `compose=false`: do not perform canonical composition -* `decompose=true`: do canonical decomposition instead of canonical composition (`compose=true` - is ignored if present) -* `compat=true`: compatibility equivalents are canonicalized -* `casefold=true`: perform Unicode case folding, e.g. for case-insensitive string comparison -* `newline2lf=true`, `newline2ls=true`, or `newline2ps=true`: convert various newline sequences - (LF, CRLF, CR, NEL) into a linefeed (LF), line-separation (LS), or paragraph-separation (PS) - character, respectively -* `stripmark=true`: strip diacritical marks (e.g. accents) -* `stripignore=true`: strip Unicode's "default ignorable" characters (e.g. the soft hyphen - or the left-to-right marker) -* `stripcc=true`: strip control characters; horizontal tabs and form feeds are converted to - spaces; newlines are also converted to spaces unless a newline-conversion flag was specified -* `rejectna=true`: throw an error if unassigned code points are found -* `stable=true`: enforce Unicode Versioning Stability - -For example, NFKC corresponds to the options `compose=true, compat=true, stable=true`. -""" -normalize_string - -""" - cd([dir::AbstractString=homedir()]) - -Set the current working directory. -""" -cd(dir::AbstractString) - -""" - cd(f, [dir=homedir()]) - -Temporarily changes the current working directory and applies function `f` before returning. -""" -cd(f, dir=?) - -""" - hton(x) - -Converts the endianness of a value from that used by the Host to Network byte order (big-endian). -""" -hton - -""" - is(x, y) -> Bool - ===(x,y) -> Bool - ≡(x,y) -> Bool - -Determine whether `x` and `y` are identical, in the sense that no program could distinguish -them. Compares mutable objects by address in memory, and compares immutable objects (such as -numbers) by contents at the bit level. This function is sometimes called `egal`. -""" -is(x,y) - -""" - mark(s) - -Add a mark at the current position of stream `s`. Returns the marked position. - -See also [`unmark`](:func:`unmark`), [`reset`](:func:`reset`), [`ismarked`](:func:`ismarked`). -""" -mark - -""" - cp(src::AbstractString, dst::AbstractString; remove_destination::Bool=false, follow_symlinks::Bool=false) - -Copy the file, link, or directory from *src* to *dest*. `remove_destination=true` will first -remove an existing `dst`. - -If `follow_symlinks=false`, and `src` is a symbolic link, `dst` will be created as a -symbolic link. If `follow_symlinks=true` and `src` is a symbolic link, `dst` will be a copy -of the file or directory `src` refers to. -""" -cp - -""" - bswap(n) - -Byte-swap an integer. -""" -bswap - -""" - resize!(collection, n) -> collection - -Resize `collection` to contain `n` elements. If `n` is smaller than the current collection -length, the first `n` elements will be retained. If `n` is larger, the new elements are not -guaranteed to be initialized. - -```jldoctest -julia> resize!([6, 5, 4, 3, 2, 1], 3) -3-element Array{Int64,1}: - 6 - 5 - 4 -``` - -```julia -julia> resize!([6, 5, 4, 3, 2, 1], 8) -8-element Array{Int64,1}: - 6 - 5 - 4 - 3 - 2 - 1 - 0 - 0 -``` -""" -resize! - -""" - sumabs2!(r, A) - -Sum squared absolute values of elements of `A` over the singleton dimensions of `r`, and -write results to `r`. -""" -sumabs2! - -""" - IPv4(host::Integer) -> IPv4 - -Returns IPv4 object from ip address formatted as Integer. -""" -IPv4 - -""" - trailing_zeros(x::Integer) -> Integer - -Number of zeros trailing the binary representation of `x`. - -```jldoctest -julia> trailing_zeros(2) -1 -``` -""" -trailing_zeros - -""" - isalnum(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is alphanumeric, or whether this is true for all elements of a -string. A character is classified as alphabetic if it belongs to the Unicode general -category Letter or Number, i.e. a character whose category code begins with 'L' or 'N'. -""" -isalnum - -""" - @sprintf("%Fmt", args...) - -Return `@printf` formatted output as string. - - julia> s = @sprintf "this is a %s %15.1f" "test" 34.567; - - julia> println(s) - this is a test 34.6 -""" -:@sprintf - -""" - tanh(x) - -Compute hyperbolic tangent of `x`. -""" -tanh - -""" - repr(x) - -Create a string from any value using the `showall` function. -""" -repr - -""" - maxintfloat(T) - -The largest integer losslessly representable by the given floating-point DataType `T`. -""" -maxintfloat - -""" - promote_shape(s1, s2) - -Check two array shapes for compatibility, allowing trailing singleton dimensions, and return -whichever shape has more dimensions. -""" -promote_shape - -""" - foldr(op, v0, itr) - -Like [`reduce`](:func:`reduce`), but with guaranteed right associativity. `v0` will be used -exactly once. -""" -foldr(op, v0, itr) - -""" - foldr(op, itr) - -Like `foldr(op, v0, itr)`, but using the last element of `itr` as `v0`. In general, this -cannot be used with empty collections (see `reduce(op, itr)`). -""" -foldr(op, itr) +maxintfloat """ delete!(collection, key) @@ -3550,28 +1438,6 @@ Delete the mapping for the given key in a collection, and return the collection. """ delete! -""" - chr2ind(string, i) - -Convert a character index to a byte index. -""" -chr2ind - -""" - fullname(m::Module) - -Get the fully-qualified name of a module as a tuple of symbols. For example, -`fullname(Base.Pkg)` gives `(:Base,:Pkg)`, and `fullname(Main)` gives `()`. -""" -fullname - -""" - isreadable(io) -> Bool - -Returns `true` if the specified IO object is readable (if that can be determined). -""" -isreadable - """ eps(T) @@ -3591,372 +1457,83 @@ eps() eps(x) The distance between `x` and the next larger representable floating-point value of the same -`DataType` as `x`. -""" -eps(::AbstractFloat) - -""" - isalpha(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is alphabetic, or whether this is true for all elements of a -string. A character is classified as alphabetic if it belongs to the Unicode general -category Letter, i.e. a character whose category code begins with 'L'. -""" -isalpha - -""" - transpose(A) - -The transposition operator (`.'`). -""" -transpose - -""" - searchsortedfirst(a, x, [by=<transform>,] [lt=<comparison>,] [rev=false]) - -Returns the index of the first value in `a` greater than or equal to `x`, according to the -specified order. Returns `length(a)+1` if `x` is greater than all values in `a`. -""" -searchsortedfirst - -""" - big(x) - -Convert a number to a maximum precision representation (typically `BigInt` or `BigFloat`). -See `BigFloat` for information about some pitfalls with floating-point numbers. -""" -big - -""" - names(x::Module[, all=false[, imported=false]]) - -Get an array of the names exported by a `Module`, with optionally more `Module` globals -according to the additional parameters. -""" -names - -""" - quit() - -Quit the program indicating that the processes completed successfully. This function calls -`exit(0)` (see [`exit`](:func:`exit`)). -""" -quit - -""" - escape_string(io, str::AbstractString, esc::AbstractString) - -General escaping of traditional C and Unicode escape sequences, plus any characters in esc -are also escaped (with a backslash). -""" -escape_string(io, str, esc) - -""" - typejoin(T, S) - -Compute a type that contains both `T` and `S`. -""" -typejoin - -""" - Base64DecodePipe(istream) - -Returns a new read-only I/O stream, which decodes base64-encoded data read from `istream`. -""" -Base64DecodePipe - -""" - module_parent(m::Module) -> Module - -Get a module's enclosing `Module`. `Main` is its own parent, as is `LastMain` after `workspace()`. -""" -module_parent - -""" - prepend!(collection, items) -> collection - -Insert the elements of `items` to the beginning of `collection`. - -```jldoctest -julia> prepend!([3],[1,2]) -3-element Array{Int64,1}: - 1 - 2 - 3 -``` -""" -prepend! - -""" - sum_kbn(A) - -Returns the sum of all array elements, using the Kahan-Babuska-Neumaier compensated -summation algorithm for additional accuracy. -""" -sum_kbn - -""" - beta(x, y) - -Euler integral of the first kind ``\\operatorname{B}(x,y) = \\Gamma(x)\\Gamma(y)/\\Gamma(x+y)``. -""" -beta - -""" - eye(n) - -`n`-by-`n` identity matrix. -""" -eye(n::Int) - -""" - eye(m, n) - -`m`-by-`n` identity matrix. -""" -eye(m, n) - -""" - diagind(M[, k]) - -A `Range` giving the indices of the `k`th diagonal of the matrix `M`. -""" -diagind - -""" - include_string(code::AbstractString, [filename]) - -Like `include`, except reads code from the given string rather than from a file. Since there -is no file path involved, no path processing or fetching from node 1 is done. -""" -include_string - -""" - chmod(path, mode; recursive=false) - -Change the permissions mode of `path` to `mode`. Only integer `mode`s (e.g. `0o777`) are -currently supported. If `recursive=true` and the path is a directory all permissions in -that directory will be recursively changed. -""" -chmod - -""" - chown(path, owner, group=-1) - -Change the owner and/or group of `path` to `owner` and/or `group`. If the value entered for `owner` or `group` -is `-1` the corresponding ID will not change. Only integer `owner`s and `group`s are currently supported. -""" -chown - -""" - sin(x) - -Compute sine of `x`, where `x` is in radians. -""" -sin - -""" - Base.compilecache(module::String) - -Creates a precompiled cache file for module (see help for `require`) and all of its -dependencies. This can be used to reduce package load times. Cache files are stored in -`LOAD_CACHE_PATH[1]`, which defaults to `~/.julia/lib/VERSION`. See -[Module initialization and precompilation](:ref:`Module initialization and precompilation <man-modules-initialization-precompilation>`) -for important notes. -""" -compilecache - -""" - clipboard() -> AbstractString - -Return a string with the contents of the operating system clipboard ("paste"). -""" -clipboard - -""" - clipboard(x) - -Send a printed form of `x` to the operating system clipboard ("copy"). -""" -clipboard(x) - -""" - values(collection) - -Return an iterator over all values in a collection. `collect(values(d))` returns an array of values. -""" -values - -""" - A_mul_B!(Y, A, B) -> Y - -Calculates the matrix-matrix or matrix-vector product ``A⋅B`` and stores the result in `Y`, -overwriting the existing value of `Y`. Note that `Y` must not be aliased with either `A` or -`B`. - -```jldoctest -julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; Y = similar(B); A_mul_B!(Y, A, B); - -julia> Y -2×2 Array{Float64,2}: - 3.0 3.0 - 7.0 7.0 -``` -""" -A_mul_B! - -""" - ntuple(f::Function, n) - -Create a tuple of length `n`, computing each element as `f(i)`, where `i` is the index of the element. -""" -ntuple - -""" - Ac_rdiv_Bc(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ / Bᴴ``. -""" -Ac_rdiv_Bc - -""" - selectperm!(ix, v, k, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false,] [initialized=false]) - -Like `selectperm`, but accepts a preallocated index vector `ix`. If `initialized` is `false` -(the default), ix is initialized to contain the values `1:length(ix)`. -""" -selectperm! - -""" - .>(x, y) - -Element-wise greater-than comparison operator. -""" -Base.:(.>) - -""" - search(string, chars, [start]) - -Search for the first occurrence of the given characters within the given string. The second -argument may be a single character, a vector or a set of characters, a string, or a regular -expression (though regular expressions are only allowed on contiguous strings, such as ASCII -or UTF-8 strings). The third argument optionally specifies a starting index. The return -value is a range of indexes where the matching sequence is found, such that `s[search(s,x)] == x`: - -`search(string, "substring")` = `start:end` such that `string[start:end] == "substring"`, or -`0:-1` if unmatched. - -`search(string, 'c')` = `index` such that `string[index] == 'c'`, or `0` if unmatched. -""" -search - -""" - contains(haystack, needle) - -Determine whether the second argument is a substring of the first. -""" -contains - -""" - flush(stream) - -Commit all currently buffered writes to the given stream. -""" -flush - -""" - precompile(f,args::Tuple{Vararg{Any}}) - -Compile the given function `f` for the argument tuple (of types) `args`, but do not execute it. -""" -precompile - -""" - asinh(x) - -Compute the inverse hyperbolic sine of `x`. +`DataType` as `x`. """ -asinh +eps(::AbstractFloat) """ - atreplinit(f) + searchsortedfirst(a, x, [by=<transform>,] [lt=<comparison>,] [rev=false]) -Register a one-argument function to be called before the REPL interface is initialized in -interactive sessions; this is useful to customize the interface. The argument of `f` is the -REPL object. This function should be called from within the `.juliarc.jl` initialization -file. +Returns the index of the first value in `a` greater than or equal to `x`, according to the +specified order. Returns `length(a)+1` if `x` is greater than all values in `a`. """ -atreplinit +searchsortedfirst """ - strip(string, [chars]) + big(x) -Return `string` with any leading and trailing whitespace removed. If `chars` (a character, -or vector or set of characters) is provided, instead remove characters contained in it. +Convert a number to a maximum precision representation (typically `BigInt` or `BigFloat`). +See [`BigFloat`](:obj:`BigFloat`) for information about some pitfalls with floating-point numbers. """ -strip +big """ - minimum(A, dims) + quit() -Compute the minimum value of an array over the given dimensions. +Quit the program indicating that the processes completed successfully. This function calls +`exit(0)` (see [`exit`](:func:`exit`)). """ -minimum(A,dims) +quit """ - var(v[, region]) + typejoin(T, S) -Compute the sample variance of a vector or array `v`, optionally along dimensions in -`region`. The algorithm will return an estimator of the generative distribution's variance -under the assumption that each entry of `v` is an IID drawn from that generative -distribution. This computation is equivalent to calculating `sumabs2(v - mean(v)) / -(length(v) - 1)`. Note: Julia does not ignore `NaN` values in the computation. For -applications requiring the handling of missing data, the `DataArray` package is recommended. +Compute a type that contains both `T` and `S`. """ -var +typejoin """ - lcfirst(string) + beta(x, y) -Returns `string` with the first character converted to lowercase. +Euler integral of the first kind ``\\operatorname{B}(x,y) = \\Gamma(x)\\Gamma(y)/\\Gamma(x+y)``. """ -lcfirst +beta """ - readlink(path) -> AbstractString + sin(x) -Returns the value of a symbolic link `path`. +Compute sine of `x`, where `x` is in radians. """ -readlink +sin """ - deg2rad(x) + selectperm!(ix, v, k, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false,] [initialized=false]) -Convert `x` from degrees to radians. +Like `selectperm`, but accepts a preallocated index vector `ix`. If `initialized` is `false` +(the default), ix is initialized to contain the values `1:length(ix)`. """ -deg2rad +selectperm! """ - mktemp([parent=tempdir()]) + precompile(f,args::Tuple{Vararg{Any}}) -Returns `(path, io)`, where `path` is the path of a new temporary file in `parent` and `io` -is an open file object for this path. +Compile the given function `f` for the argument tuple (of types) `args`, but do not execute it. """ -mktemp(?) +precompile """ - mktemp(f::Function, [parent=tempdir()]) + asinh(x) -Apply the function `f` to the result of `mktemp(parent)` and remove the temporary file upon completion. +Compute the inverse hyperbolic sine of `x`. """ -mktemp(::Function, ?) +asinh """ - isreadonly(stream) -> Bool + minimum(A, dims) -Determine whether a stream is read-only. +Compute the minimum value of an array over the given dimensions. """ -isreadonly +minimum(A,dims) """ view(A, inds...) @@ -3968,20 +1545,6 @@ indices to the parent array on the fly without checking bounds. """ view -""" - expanduser(path::AbstractString) -> AbstractString - -On Unix systems, replace a tilde character at the start of a path with the current user's home directory. -""" -expanduser - -""" - haskey(collection, key) -> Bool - -Determine whether a collection has a mapping for a given key. -""" -haskey - """ cot(x) @@ -4014,14 +1577,6 @@ end """ get -""" - .!=(x, y) - .≠(x,y) - -Element-wise not-equals comparison operator. -""" -Base.:(.!=) - """ lufact!(A) -> LU @@ -4032,66 +1587,6 @@ integer types. """ lufact! -""" - IOBuffer() -> IOBuffer - -Create an in-memory I/O stream. -""" -IOBuffer() - -""" - IOBuffer(size::Int) - -Create a fixed size IOBuffer. The buffer will not grow dynamically. -""" -IOBuffer(size::Int) - -""" - IOBuffer(string) - -Create a read-only IOBuffer on the data underlying the given string. -""" -IOBuffer(::AbstractString) - -""" - IOBuffer([data,],[readable,writable,[maxsize]]) - -Create an IOBuffer, which may optionally operate on a pre-existing array. If the -readable/writable arguments are given, they restrict whether or not the buffer may be read -from or written to respectively. By default the buffer is readable but not writable. The -last argument optionally specifies a size beyond which the buffer may not be grown. -""" -IOBuffer(data=?) - -""" - tempname() - -Generate a unique temporary file path. -""" -tempname - -""" - poll_fd(fd, timeout_s::Real; readable=false, writable=false) - -Monitor a file descriptor `fd` for changes in the read or write availability, and with a -timeout given by `timeout_s` seconds. - -The keyword arguments determine which of read and/or write status should be monitored; at -least one of them must be set to `true`. - -The returned value is an object with boolean fields `readable`, `writable`, and `timedout`, -giving the result of the polling. -""" -poll_fd - -""" - prevpow2(n) - -The largest power of two not greater than `n`. Returns 0 for `n==0`, and returns -`-prevpow2(-n)` for negative arguments. -""" -prevpow2 - """ Mmap.sync!(array) @@ -4120,21 +1615,6 @@ Typically, any type that implements `hash` should also implement its own `==` (h """ hash -""" - atan2(y, x) - -Compute the inverse tangent of `y/x`, using the signs of both `x` and `y` to determine the -quadrant of the return value. -""" -atan2 - -""" - send(socket::UDPSocket, host::IPv4, port::Integer, msg) - -Send `msg` over `socket` to `host:port`. -""" -send - """ atanh(x) @@ -4149,33 +1629,6 @@ Read a single value of type `T` from `stream`, in canonical binary representatio """ read(stream, t) -""" - read(stream::IO, T, dims) - -Read a series of values of type `T` from `stream`, in canonical binary representation. -`dims` is either a tuple or a series of integer arguments specifying the size of the `Array{T}` -to return. -""" -read(stream, t, dims) - -""" - read(filename::AbstractString, args...) - -Open a file and read its contents. `args` is passed to `read`: this is equivalent to -`open(io->read(io, args...), filename)`. -""" -read(filename, args...) - -""" - isopen(object) -> Bool - -Determine whether an object - such as a stream, timer, or mmap -- is not yet closed. Once an -object is closed, it will never produce a new event. However, a closed stream may still have -data to read in its buffer, use `eof` to check for the ability to read data. Use `poll_fd` -to be notified when a stream might be writable or readable. -""" -isopen - """ shift!(collection) -> item @@ -4208,25 +1661,10 @@ shift! """ spawn(command) -Run a command object asynchronously, returning the resulting `Process` object. +Run a command object asynchronously, returning the resulting [`Process`](:obj:`Process`) object. """ spawn -""" - isposdef(A) -> Bool - -Test whether a matrix is positive definite. -""" -isposdef - -""" - nextind(str, i) - -Get the next valid string index after `i`. Returns a value greater than `endof(str)` at or -after the end of the string. -""" -nextind - """ eta(x) @@ -4253,27 +1691,20 @@ Compute the cotangent of `x`, where `x` is in degrees. """ cotd -""" - dec(n, [pad]) - -Convert an integer to a decimal string, optionally specifying a number of digits to pad to. -""" -dec - """ wait([x]) Block the current task until some event occurs, depending on the type of the argument: -* `RemoteChannel` : Wait for a value to become available on the specified remote channel. -* `Future` : Wait for a value to become available for the specified future. -* `Channel`: Wait for a value to be appended to the channel. -* `Condition`: Wait for `notify` on a condition. -* `Process`: Wait for a process or process chain to exit. The `exitcode` field of a process +* [`RemoteChannel`](:obj:`RemoteChannel`) : Wait for a value to become available on the specified remote channel. +* [`Future`](:obj:`Future`) : Wait for a value to become available for the specified future. +* [`Channel`](:obj:`Channel`): Wait for a value to be appended to the channel. +* [`Condition`](:obj:`Condition`): Wait for [`notify`](:func:`notify`) on a condition. +* [`Process`](:obj:`Process`): Wait for a process or process chain to exit. The `exitcode` field of a process can be used to determine success or failure. -* `Task`: Wait for a `Task` to finish, returning its result value. If the task fails with an +* [`Task`](:obj:`Task`): Wait for a `Task` to finish, returning its result value. If the task fails with an exception, the exception is propagated (re-thrown in the task that called `wait`). -* `RawFD`: Wait for changes on a file descriptor (see `poll_fd` for keyword arguments and return code) +* [`RawFD`](:obj:`RawFD`): Wait for changes on a file descriptor (see [`poll_fd`](:func:`poll_fd`) for keyword arguments and return code) If no argument is passed, the task blocks for an undefined period. A task can only be restarted by an explicit call to `schedule` or `yieldto`. @@ -4282,50 +1713,6 @@ Often `wait` is called within a `while` loop to ensure a waited-for condition is """ wait -""" - shuffle([rng,] v) - -Return a randomly permuted copy of `v`. The optional `rng` argument specifies a random -number generator (see [Random Numbers](:ref:`Random Numbers <random-numbers>`)). -To permute `v` in-place, see [`shuffle!`](:func:`shuffle!`). To obtain randomly permuted -indices, see [`randperm`](:func:`randperm`). -""" -shuffle - -""" - Dict([itr]) - -`Dict{K,V}()` constructs a hash table with keys of type `K` and values of type `V`. - -Given a single iterable argument, constructs a [`Dict`](:obj:`Dict`) whose key-value pairs -are taken from 2-tuples `(key,value)` generated by the argument. - -```jldoctest -julia> Dict([("A", 1), ("B", 2)]) -Dict{String,Int64} with 2 entries: - "B" => 2 - "A" => 1 -``` - -Alternatively, a sequence of pair arguments may be passed. - -```jldoctest -julia> Dict("A"=>1, "B"=>2) -Dict{String,Int64} with 2 entries: - "B" => 2 - "A" => 1 -``` -""" -Dict - -""" - sqrt(x) - -Return ``\\sqrt{x}``. Throws `DomainError` for negative `Real` arguments. Use complex -negative arguments instead. The prefix operator `√` is equivalent to `sqrt`. -""" -sqrt - """ atexit(f) @@ -4334,103 +1721,6 @@ called in last in first out (LIFO) order and run before object finalizers. """ atexit -""" - readchomp(x) - -Read the entirety of `x` as a string and remove a single trailing newline. Equivalent to `chomp(readstring(x))`. -""" -readchomp - -""" - pinv(M[, tol]) - -Computes the Moore-Penrose pseudoinverse. - -For matrices `M` with floating point elements, it is convenient to compute -the pseudoinverse by inverting only singular values above a given threshold, -`tol`. - -The optimal choice of `tol` varies both with the value of `M` and the intended application -of the pseudoinverse. The default value of `tol` is -`eps(real(float(one(eltype(M)))))*maximum(size(A))`, which is essentially machine epsilon -for the real part of a matrix element multiplied by the larger matrix dimension. For -inverting dense ill-conditioned matrices in a least-squares sense, -`tol = sqrt(eps(real(float(one(eltype(M))))))` is recommended. - -For more information, see [^issue8859], [^B96], [^S84], [^KY88]. - -[^issue8859]: Issue 8859, "Fix least squares", https://github.com/JuliaLang/julia/pull/8859 - -[^B96]: Åke Björck, "Numerical Methods for Least Squares Problems", SIAM Press, Philadelphia, 1996, "Other Titles in Applied Mathematics", Vol. 51. [doi:10.1137/1.9781611971484](http://epubs.siam.org/doi/book/10.1137/1.9781611971484) - -[^S84]: G. W. Stewart, "Rank Degeneracy", SIAM Journal on Scientific and Statistical Computing, 5(2), 1984, 403-413. [doi:10.1137/0905030](http://epubs.siam.org/doi/abs/10.1137/0905030) - -[^KY88]: Konstantinos Konstantinides and Kung Yao, "Statistical analysis of effective singular values in matrix rank determination", IEEE Transactions on Acoustics, Speech and Signal Processing, 36(5), 1988, 757-763. [doi:10.1109/29.1585](http://dx.doi.org/10.1109/29.1585) -""" -pinv - -""" - readbytes!(stream::IO, b::AbstractVector{UInt8}, nb=length(b); all=true) - -Read at most `nb` bytes from `stream` into `b`, returning the number of bytes read. -The size of `b` will be increased if needed (i.e. if `nb` is greater than `length(b)` -and enough bytes could be read), but it will never be decreased. - -See `read` for a description of the `all` option. -""" -readbytes! - -""" - basename(path::AbstractString) -> AbstractString - -Get the file name part of a path. -""" -basename - -""" - isdiag(A) -> Bool - -Test whether a matrix is diagonal. -""" -isdiag - -""" - !==(x, y) - ≢(x,y) - -Equivalent to `!is(x, y)`. -""" -Base.:(!==) - -""" - trailing_ones(x::Integer) -> Integer - -Number of ones trailing the binary representation of `x`. - -```jldoctest -julia> trailing_ones(3) -2 -``` -""" -trailing_ones - -""" - repeated(x[, n::Int]) - -An iterator that generates the value `x` forever. If `n` is specified, generates `x` that -many times (equivalent to `take(repeated(x), n)`). -""" -repeated - -""" - isnumber(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is numeric, or whether this is true for all elements of a string. -A character is classified as numeric if it belongs to the Unicode general category Number, -i.e. a character whose category code begins with 'N'. -""" -isnumber - """ copy(x) @@ -4463,68 +1753,6 @@ results to `r`. """ sumabs! -""" - Sys.set_process_title(title::AbstractString) - -Set the process title. No-op on some operating systems. (not exported) -""" -Sys.set_process_title - -""" - htol(x) - -Converts the endianness of a value from that used by the Host to Little-endian. -""" -htol - -""" - ctime(file) - -Equivalent to `stat(file).ctime` -""" -ctime - -""" - normpath(path::AbstractString) -> AbstractString - -Normalize a path, removing "." and ".." entries. -""" -normpath - -""" - unmark(s) - -Remove a mark from stream `s`. Returns `true` if the stream was marked, `false` otherwise. - -See also [`mark`](:func:`mark`), [`reset`](:func:`reset`), [`ismarked`](:func:`ismarked`). -""" -unmark - -""" - module_name(m::Module) -> Symbol - -Get the name of a `Module` as a `Symbol`. -""" -module_name - -""" - reset(s) - -Reset a stream `s` to a previously marked position, and remove the mark. Returns the -previously marked position. Throws an error if the stream is not marked. - -See also [`mark`](:func:`mark`), [`unmark`](:func:`unmark`), [`ismarked`](:func:`ismarked`). -""" -reset - -""" - modf(x) - -Return a tuple (fpart,ipart) of the fractional and integral parts of a number. Both parts -have the same sign as the argument. -""" -modf - """ hex2num(str) @@ -4532,20 +1760,6 @@ Convert a hexadecimal string to the floating point number it represents. """ hex2num -""" - ishermitian(A) -> Bool - -Test whether a matrix is Hermitian. -""" -ishermitian - -""" - min(x, y, ...) - -Return the minimum of the arguments. Operates elementwise over arrays. -""" -min - """ InexactError() @@ -4567,42 +1781,6 @@ Test whether all values along the given dimensions of an array are `true`. """ all(A::AbstractArray, dims) -""" - bind(socket::Union{UDPSocket, TCPSocket}, host::IPAddr, port::Integer; ipv6only=false) - -Bind `socket` to the given `host:port`. Note that `0.0.0.0` will listen on all devices. -`ipv6only` parameter disables dual stack mode. If it's `true`, only IPv6 stack is created. -""" -bind - -""" - cld(x, y) - -Smallest integer larger than or equal to `x/y`. -""" -cld - -""" - issetuid(path) -> Bool - -Returns `true` if `path` has the setuid flag set, `false` otherwise. -""" -issetuid - -""" - scale!(A, b) - scale!(b, A) - -Scale an array `A` by a scalar `b` overwriting `A` in-place. - -If `A` is a matrix and `b` is a vector, then `scale!(A,b)` scales each column `i` of `A` by -`b[i]` (similar to `A*Diagonal(b)`), while `scale!(b,A)` scales each row `i` of `A` by `b[i]` -(similar to `Diagonal(b)*A`), again operating in-place on `A`. An `InexactError` exception is -thrown if the scaling produces a number not representable by the element type of `A`, -e.g. for integer types. -""" -scale! - """ DomainError() @@ -4610,13 +1788,6 @@ The arguments to a function or constructor are outside the valid domain. """ DomainError -""" - issymmetric(A) -> Bool - -Test whether a matrix is symmetric. -""" -issymmetric - """ acosh(x) @@ -4651,34 +1822,6 @@ the topmost backend that does not throw a `MethodError`). """ pushdisplay -""" - prevind(str, i) - -Get the previous valid string index before `i`. Returns a value less than `1` at the -beginning of the string. -""" -prevind - -""" - setenv(command, env; dir=working_dir) - -Set environment variables to use when running the given `command`. `env` is either a -dictionary mapping strings to strings, an array of strings of the form `"var=val"`, or zero -or more `"var"=>val` pair arguments. In order to modify (rather than replace) the existing -environment, create `env` by `copy(ENV)` and then setting `env["var"]=val` as desired, or -use `withenv`. - -The `dir` keyword argument can be used to specify a working directory for the command. -""" -setenv - -""" - lowercase(string) - -Returns `string` with all characters converted to lowercase. -""" -lowercase - """ produce(value) @@ -4695,13 +1838,6 @@ recurses infinitely. """ StackOverflowError -""" - process_running(p::Process) - -Determine whether a process is currently running. -""" -process_running - """ BigInt(x) @@ -4714,44 +1850,6 @@ string literal. """ BigInt -""" - rsearch(string, chars, [start]) - -Similar to `search`, but returning the last occurrence of the given characters within the -given string, searching in reverse from `start`. -""" -rsearch - -""" - isdirpath(path::AbstractString) -> Bool - -Determines whether a path refers to a directory (for example, ends with a path separator). -""" -isdirpath - -""" - in(item, collection) -> Bool - ∈(item,collection) -> Bool - ∋(collection,item) -> Bool - ∉(item,collection) -> Bool - ∌(collection,item) -> Bool - -Determine whether an item is in the given collection, in the sense that it is `==` to one of -the values generated by iterating over the collection. Some collections need a slightly -different definition; for example [`Set`](:obj:`Set`)s check whether the item -[`isequal`](:func:`isequal`) to one of the elements. [`Dict`](:obj:`Dict`)s look for -`(key,value)` pairs, and the key is compared using [`isequal`](:func:`isequal`). To test for -the presence of a key in a dictionary, use [`haskey`](:func:`haskey`) or `k in keys(dict)`. -""" -Base.in - -""" - isblockdev(path) -> Bool - -Returns `true` if `path` is a block device, `false` otherwise. -""" -isblockdev - """ ==(x, y) @@ -4809,8 +1907,8 @@ Technically, the `MIME"mime"` macro defines a singleton type for the given `mime which allows us to exploit Julia's dispatch mechanisms in determining how to display objects of any given type. -The first argument to `show` can be an `IOContext` specifying output format properties. -See `IOContext` for details. +The first argument to `show` can be an [`IOContext`](:obj:`IOContext`) specifying output format properties. +See [`IOContext`](:obj:`IOContext`) for details. """ show(stream, mime, x) @@ -4821,19 +1919,6 @@ Compute the mean of `v` over the singleton dimensions of `r`, and write results """ mean! -""" - join(strings, delim, [last]) - -Join an array of `strings` into a single string, inserting the given delimiter between -adjacent strings. If `last` is given, it will be used instead of `delim` between the last -two strings. For example - - join(["apples", "bananas", "pineapples"], ", ", " and ") == "apples, bananas and pineapples" - -`strings` can be any iterable over elements `x` which are convertible to strings via `print(io::IOBuffer, x)`. -""" -join(strings, delim, last) - """ isless(x, y) @@ -4866,22 +1951,6 @@ Raise an `ErrorException` with the given message. """ error -""" - less(file::AbstractString, [line]) - -Show a file using the default pager, optionally providing a starting line number. Returns to -the `julia` prompt when you quit the pager. -""" -less(f::AbstractString, ?) - -""" - less(function, [types]) - -Show the definition of a function using the default pager, optionally specifying a tuple of -types to indicate which method to see. -""" -less(func, ?) - """ sqrtm(A) @@ -4899,13 +1968,6 @@ and then the complex square root of the triangular factor. """ sqrtm -""" - conv(u,v) - -Convolution of two vectors. Uses FFT algorithm. -""" -conv - """ unsafe_store!(p::Ptr{T}, x, [i::Integer=1]) @@ -4918,23 +1980,6 @@ program, in the same manner as C. """ unsafe_store! -""" - expm(A) - -Compute the matrix exponential of `A`, defined by - -```math -e^A = \\sum_{n=0}^{\\infty} \\frac{A^n}{n!}. -``` - -For symmetric or Hermitian `A`, an eigendecomposition ([`eigfact`](:func:`eigfact`)) is -used, otherwise the scaling and squaring algorithm (see [^H05]) is chosen. - -[^H05]: Nicholas J. Higham, "The squaring and scaling method for the matrix exponential revisited", SIAM Journal on Matrix Analysis and Applications, 26(4), 2005, 1179-1193. [doi:10.1137/090768539](http://dx.doi.org/10.1137/090768539) - -""" -expm - """ hessfact!(A) @@ -4943,13 +1988,6 @@ the input `A`, instead of creating a copy. """ hessfact! -""" - Sys.get_process_title() - -Get the process title. On some systems, will always return empty string. (not exported) -""" -Sys.get_process_title - """ readcsv(source, [T::Type]; options...) @@ -4957,14 +1995,6 @@ Equivalent to `readdlm` with `delim` set to comma. """ readcsv -""" - current_module() -> Module - -Get the *dynamically* current `Module`, which is the `Module` code is currently being read -from. In general, this is not the same as the module containing the call to this function. -""" -current_module - """ erfcx(x) @@ -4987,14 +2017,6 @@ Perform garbage collection. This should not generally be used. """ gc -""" - iscntrl(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is a control character, or whether this is true for all elements -of a string. Control characters are the non-printing characters of the Latin-1 subset of Unicode. -""" -iscntrl - """ minimum!(r, A) @@ -5002,13 +2024,6 @@ Compute the minimum value of `A` over the singleton dimensions of `r`, and write """ minimum! -""" - diagm(v[, k]) - -Construct a diagonal matrix and place `v` on the `k`th diagonal. -""" -diagm - """ .-(x, y) @@ -5016,13 +2031,6 @@ Element-wise subtraction operator. """ Base.:(.-) -""" - imag(z) - -Return the imaginary part of the complex number `z`. -""" -imag - """ unsafe_trunc(T, x) @@ -5040,36 +2048,6 @@ it is not a view. """ parent -""" - <(x, y) - -Less-than comparison operator. New numeric types should implement this function for two -arguments of the new type. Because of the behavior of floating-point NaN values, `<` -implements a partial order. Types with a canonical partial order should implement `<`, and -types with a canonical total order should implement `isless`. -""" -Base.:(<) - -""" - EnvHash() -> EnvHash - -A singleton of this type provides a hash table interface to environment variables. -""" -EnvHash - -""" - method_exists(f, Tuple type) -> Bool - -Determine whether the given generic function has a method matching the given -[`Tuple`](:obj:`Tuple`) of argument types. - -```jldoctest -julia> method_exists(length, Tuple{Array}) -true -``` -""" -method_exists - """ nextpow(a, x) @@ -5078,13 +2056,6 @@ greater than 1, and `x` must be greater than 0. """ nextpow -""" - rad2deg(x) - -Convert `x` from radians to degrees. -""" -rad2deg - """ gc_enable(on::Bool) @@ -5094,30 +2065,6 @@ used only with extreme caution, as it can cause memory use to grow without bound """ gc_enable -""" - sub2ind(dims, i, j, k...) -> index - -The inverse of `ind2sub`, returns the linear index corresponding to the provided subscripts. -""" -sub2ind - -""" - supertype(T::DataType) - -Return the supertype of DataType `T`. -""" -supertype - -""" - readline(stream::IO=STDIN) - readline(filename::AbstractString) - -Read a single line of text, including a trailing newline character (if one is reached before -the end of the input), from the given I/O stream or file (defaults to `STDIN`). -When reading from a file, the text is assumed to be encoded in UTF-8. -""" -readline - """ atan(x) @@ -5125,48 +2072,6 @@ Compute the inverse tangent of `x`, where the output is in radians. """ atan -""" - logabsdet(M) - -Log of absolute value of determinant of real matrix. Equivalent to `(log(abs(det(M))), sign(det(M)))`, -but may provide increased accuracy and/or speed. -""" -logabsdet - -""" - joinpath(parts...) -> AbstractString - -Join path components into a full path. If some argument is an absolute path, then prior -components are dropped. -""" -joinpath - -""" - precision(BigFloat) - -Get the precision (in bits) currently used for `BigFloat` arithmetic. -""" -precision(::Type{BigFloat}) - -""" - homedir() -> AbstractString - -Return the current user's home directory. -""" -homedir - -""" - count_zeros(x::Integer) -> Integer - -Number of zeros in the binary representation of `x`. - -```jldoctest -julia> count_zeros(Int32(2 ^ 16 - 1)) -16 -``` -""" -count_zeros - """ isinf(f) -> Bool @@ -5188,16 +2093,6 @@ The result of an expression is too large for the specified type and will cause a """ OverflowError -""" - ctranspose!(dest,src) - -Conjugate transpose array `src` and store the result in the preallocated array `dest`, which -should have a size corresponding to `(size(src,2),size(src,1))`. No in-place transposition -is supported and unexpected results will happen if `src` and `dest` have overlapping memory -regions. -""" -ctranspose! - """ object_id(x) @@ -5205,37 +2100,6 @@ Get a hash value for `x` based on object identity. `object_id(x)==object_id(y)` """ object_id -""" - norm(A, [p]) - -Compute the `p`-norm of a vector or the operator norm of a matrix `A`, defaulting to the `p=2`-norm. - -For vectors, `p` can assume any numeric value (even though not all values produce a -mathematically valid vector norm). In particular, `norm(A, Inf)` returns the largest value -in `abs(A)`, whereas `norm(A, -Inf)` returns the smallest. - -For matrices, the matrix norm induced by the vector `p`-norm is used, where valid values of -`p` are `1`, `2`, or `Inf`. (Note that for sparse matrices, `p=2` is currently not -implemented.) Use [`vecnorm`](:func:`vecnorm`) to compute the Frobenius norm. -""" -norm - -""" - unescape_string(io, s::AbstractString) - -General unescaping of traditional C and Unicode escape sequences. Reverse of [`escape_string`](:func:`escape_string`). -""" -unescape_string(io, s) - -""" - digits!(array, n, [base]) - -Fills an array of the digits of `n` in the given base. More significant digits are at higher -indexes. If the array length is insufficient, the least significant digits are filled up to -the array length. If the array length is excessive, the excess portion is filled with zeros. -""" -digits! - """ cat(dims, A...) @@ -5253,61 +2117,6 @@ diagonal. """ cat -""" - factorial(n) - -Factorial of `n`. If `n` is an [`Integer`](:obj:`Integer`), the factorial is computed as an -integer (promoted to at least 64 bits). Note that this may overflow if `n` is not small, -but you can use `factorial(big(n))` to compute the result exactly in arbitrary precision. -If `n` is not an `Integer`, `factorial(n)` is equivalent to [`gamma(n+1)`](:func:`gamma(n+1) <gamma>`). -""" -factorial(n) - -""" - bitrand([rng], [dims...]) - -Generate a `BitArray` of random boolean values. -""" -bitrand - -""" - randcycle([rng,] n) - -Construct a random cyclic permutation of length `n`. The optional `rng` -argument specifies a random number generator, see [Random Numbers](:ref:`Random Numbers <random-numbers>`). -""" -randcycle - -""" - leading_zeros(x::Integer) -> Integer - -Number of zeros leading the binary representation of `x`. - -```jldoctest -julia> leading_zeros(Int32(1)) -31 -``` -""" -leading_zeros - -""" - lexcmp(x, y) - -Compare `x` and `y` lexicographically and return -1, 0, or 1 depending on whether `x` is -less than, equal to, or greater than `y`, respectively. This function should be defined for -lexicographically comparable types, and `lexless` will call `lexcmp` by default. -""" -lexcmp - -""" - isupper(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is an uppercase letter, or whether this is true for all elements -of a string. A character is classified as uppercase if it belongs to Unicode category Lu, -Letter: Uppercase, or Lt, Letter: Titlecase. -""" -isupper - """ show(x) @@ -5317,7 +2126,6 @@ by `show` generally includes Julia-specific formatting and type information. """ show(x) - """ Array(dims) @@ -5334,16 +2142,6 @@ Test whether `x` or all its elements are numerically equal to some real number. """ isreal -""" - randsubseq(A, p) -> Vector - -Return a vector consisting of a random subsequence of the given array `A`, where each -element of `A` is included (in order) with independent probability `p`. (Complexity is -linear in `p*length(A)`, so this function is efficient even if `p` is small and `A` is -large.) Technically, this process is known as "Bernoulli sampling" of `A`. -""" -randsubseq - """ issubtype(type1, type2) @@ -5367,53 +2165,6 @@ Compute the hyperbolic cosecant of `x`. """ csch -""" - isequal(x, y) - -Similar to `==`, except treats all floating-point `NaN` values as equal to each other, and -treats `-0.0` as unequal to `0.0`. The default implementation of `isequal` calls `==`, so if -you have a type that doesn't have these floating-point subtleties then you probably only -need to define `==`. - -`isequal` is the comparison function used by hash tables (`Dict`). `isequal(x,y)` must imply -that `hash(x) == hash(y)`. - -This typically means that if you define your own `==` function then you must define a -corresponding `hash` (and vice versa). Collections typically implement `isequal` by calling -`isequal` recursively on all contents. - -Scalar types generally do not need to implement `isequal` separate from `==`, unless they -represent floating-point numbers amenable to a more efficient implementation than that -provided as a generic fallback (based on `isnan`, `signbit`, and `==`). -""" -isequal - -""" - lyap(A, C) - -Computes the solution `X` to the continuous Lyapunov equation `AX + XA' + C = 0`, where no -eigenvalue of `A` has a zero real part and no two eigenvalues are negative complex -conjugates of each other. -""" -lyap - -""" - condskeel(M, [x, p]) - -```math -\\kappa_S(M, p) & = \\left\\Vert \\left\\vert M \\right\\vert \\left\\vert M^{-1} \\right\\vert \\right\\Vert_p \\\\ -\\kappa_S(M, x, p) & = \\left\\Vert \\left\\vert M \\right\\vert \\left\\vert M^{-1} \\right\\vert \\left\\vert x \\right\\vert \\right\\Vert_p -``` - -Skeel condition number ``\\kappa_S`` of the matrix `M`, optionally with respect to the -vector `x`, as computed using the operator `p`-norm. `p` is `Inf` by default, if not -provided. Valid values for `p` are `1`, `2`, or `Inf`. - -This quantity is also known in the literature as the Bauer condition number, relative -condition number, or componentwise relative condition number. -""" -condskeel - """ sec(x) @@ -5421,20 +2172,6 @@ Compute the secant of `x`, where `x` is in radians. """ sec -""" - recv(socket::UDPSocket) - -Read a UDP packet from the specified socket, and return the bytes received. This call blocks. -""" -recv - -""" - det(M) - -Matrix determinant. -""" -det - """ TypeError(func::Symbol, context::AbstractString, expected::Type, got) @@ -5442,56 +2179,6 @@ A type assertion failure, or calling an intrinsic function with an incorrect arg """ TypeError -""" - A_rdiv_Bt(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``A / Bᵀ``. -""" -A_rdiv_Bt - -""" - pwd() -> AbstractString - -Get the current working directory. -""" -pwd - -""" - getipaddr() -> IPAddr - -Get the IP address of the local machine. -""" -getipaddr - -""" - uppercase(string) - -Returns `string` with all characters converted to uppercase. -""" -uppercase - -""" - cycle(iter) - -An iterator that cycles through `iter` forever. -""" -cycle - -""" - operm(file) - -Like uperm but gets the permissions for people who neither own the file nor are a member of -the group owning the file -""" -operm - -""" - rpad(string, n, p) - -Make a string at least `n` columns wide when printed, by padding on the right with copies of `p`. -""" -rpad - """ setfield!(value, name::Symbol, x) @@ -5503,19 +2190,12 @@ setfield! """ @printf([io::IOStream], "%Fmt", args...) -Print `args` using C `printf()` style format specification string. Optionally, an `IOStream` +Print `args` using C `printf()` style format specification string. +Optionally, an [`IOStream`](:obj:`IOStream`) may be passed as the first argument to redirect output. """ :@printf -""" - rstrip(string, [chars]) - -Return `string` with any trailing whitespace removed. If `chars` (a character, or vector or -set of characters) is provided, instead remove characters contained in it. -""" -rstrip - """ countlines(io,[eol::Char]) @@ -5525,15 +2205,6 @@ passing them as the second argument. """ countlines -""" -``` -*(A, B) -``` - -Matrix multiplication. -""" -Base.:(*)(::AbstractMatrix, ::AbstractMatrix) - """ .\\(x, y) @@ -5541,15 +2212,6 @@ Element-wise left division operator. """ Base.:(.\)(x,y) -""" - \\(x, y) - -Left division operator: multiplication of `y` by the inverse of `x` on the left. Gives -floating-point results for integer arguments. -""" -Base.:(\)(x::Number,y::Number) - - """ ``` *(x, y...) @@ -5559,33 +2221,12 @@ Multiplication operator. `x*y*z*...` calls this function with all arguments, i.e """ Base.:(*)(x, y...) -""" -``` -*(s, t) -``` - -Concatenate strings. The `*` operator is an alias to this function. - -```jldoctest -julia> "Hello " * "world" -"Hello world" -``` -""" -Base.:(*)(s::AbstractString, t::AbstractString) - """ time() Get the system time in seconds since the epoch, with fairly high (typically, microsecond) resolution. """ -time() - -""" - procs(S::SharedArray) - -Get the vector of processes that have mapped the shared array. -""" -procs(::SharedArray) +time() """ qr(A [,pivot=Val{false}][;thin=true]) -> Q, R, [p] @@ -5596,15 +2237,6 @@ extended with zeros when the full `Q` is requested. """ qr -""" - invmod(x,m) - -Take the inverse of `x` modulo `m`: `y` such that ``x y = 1 \\pmod m``, -with ``div(x,y) = 0``. This is undefined for ``m = 0``, or if -``gcd(x,m) \\neq 1``. -""" -invmod - """ TextDisplay(stream) @@ -5628,39 +2260,10 @@ Compute ``e^x``. """ exp -""" - searchindex(string, substring, [start]) - -Similar to `search`, but return only the start index at which the substring is found, or `0` if it is not. -""" -searchindex - -""" - listenany(port_hint) -> (UInt16,TCPServer) - -Create a `TCPServer` on any port, using hint as a starting point. Returns a tuple of the -actual port that the server was created on and the server itself. -""" -listenany - -""" - getpid() -> Int32 - -Get Julia's process ID. -""" -getpid - -""" - cbrt(x) - -Return ``x^{1/3}``. The prefix operator `∛` is equivalent to `cbrt`. -""" -cbrt - """ matchall(r::Regex, s::AbstractString[, overlap::Bool=false]) -> Vector{AbstractString} -Return a vector of the matching substrings from eachmatch. +Return a vector of the matching substrings from [`eachmatch`](:func:`eachmatch`). """ matchall @@ -5687,13 +2290,6 @@ This is intended to be called using `do` block syntax: """ get!(f::Function,collection,key) -""" - inv(M) - -Matrix inverse. -""" -inv - """ @assert cond [text] @@ -5702,61 +2298,17 @@ Message `text` is optionally displayed upon assertion failure. """ :@assert -""" - intersect!(s1, s2) - -Intersects sets `s1` and `s2` and overwrites the set `s1` with the result. If needed, `s1` -will be expanded to the size of `s2`. -""" -intersect! - -""" - listen([addr,]port) -> TCPServer - -Listen on port on the address specified by `addr`. By default this listens on localhost -only. To listen on all interfaces pass `IPv4(0)` or `IPv6(0)` as appropriate. -""" -listen(addr,port) - -""" - listen(path) -> PipeServer - -Create and listen on a named pipe / UNIX domain socket. -""" -listen(path) - -""" - leading_ones(x::Integer) -> Integer - -Number of ones leading the binary representation of `x`. - -```jldoctest -julia> leading_ones(UInt32(2 ^ 32 - 2)) -31 -``` -""" -leading_ones - """ deserialize(stream) -Read a value written by `serialize`. `deserialize` assumes the binary data read from -`stream` is correct and has been serialized by a compatible implementation of `serialize`. +Read a value written by [`serialize`](:func:`serialize`). `deserialize` assumes the binary data read from +`stream` is correct and has been serialized by a compatible implementation of [`serialize`](:func:`serialize`). It has been designed with simplicity and performance as a goal and does not validate the data read. Malformed data can result in process termination. The caller has to ensure the integrity and correctness of data read from `stream`. """ deserialize -""" - ismarked(s) - -Returns `true` if stream `s` is marked. - -See also [`mark`](:func:`mark`), [`unmark`](:func:`unmark`), [`reset`](:func:`reset`). -""" -ismarked - """ first(coll) @@ -5768,7 +2320,7 @@ first """ median!(v) -Like `median`, but may overwrite the input vector. +Like [`median`](:func:`median`), but may overwrite the input vector. """ median! @@ -5803,33 +2355,6 @@ in the requested `mime` format and simply returns `x`. """ reprmime -""" - rm(path::AbstractString; force=false, recursive=false) - -Delete the file, link, or empty directory at the given path. If `force=true` is passed, a -non-existing path is not treated as error. If `recursive=true` is passed and the path is a -directory, then all contents are removed recursively. -""" -rm - -""" - MersenneTwister([seed]) - -Create a `MersenneTwister` RNG object. Different RNG objects can have their own seeds, which -may be useful for generating different streams of random numbers. -""" -MersenneTwister - -""" - graphemes(s) -> iterator over substrings of s - -Returns an iterator over substrings of `s` that correspond to the extended graphemes in the -string, as defined by Unicode UAX #29. (Roughly, these are what users would perceive as -single characters, even though they may contain more than one codepoint; for example a -letter combined with an accent mark is a single grapheme.) -""" -graphemes - """ @__FILE__ -> AbstractString @@ -5839,44 +2364,6 @@ macro. Returns `nothing` if run from a REPL or an empty string if evaluated by """ :@__FILE__ -""" - charwidth(c) - -Gives the number of columns needed to print a character. -""" -charwidth - -""" - abspath(path::AbstractString) -> AbstractString - -Convert a path to an absolute path by adding the current directory if necessary. -""" -abspath - -""" - ispunct(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character belongs to the Unicode general category Punctuation, i.e. a -character whose category code begins with 'P'. For strings, tests whether this is true for -all elements of the string. -""" -ispunct - -""" - ismount(path) -> Bool - -Returns `true` if `path` is a mount point, `false` otherwise. -""" -ismount - -""" - endswith(string, suffix) - -Returns `true` if `string` ends with `suffix`. If `suffix` is a vector or set of -characters, tests whether the last character of `string` belongs to that set. -""" -endswith - """ !(x) @@ -5892,28 +2379,12 @@ is valid. For unordered collections, the number of elements. """ length(collection) -""" - length(s) - -The number of characters in string `s`. -""" -length(::AbstractString) - -""" - rand!([rng], A, [coll]) - -Populate the array `A` with random values. If the indexable collection `coll` is specified, -the values are picked randomly from `coll`. This is equivalent to `copy!(A, rand(rng, coll, size(A)))` -or `copy!(A, rand(rng, eltype(A), size(A)))` but without allocating a new array. -""" -rand! - """ bkfact(A) -> BunchKaufman Compute the Bunch-Kaufman [^Bunch1977] factorization of a real symmetric or complex Hermitian matrix `A` and return a `BunchKaufman` object. The following functions are available for -`BunchKaufman` objects: `size`, `\\`, `inv`, `issymmetric`, `ishermitian`. +`BunchKaufman` objects: [`size`](:func:`size`), `\\`, [`inv`](:func:`inv`), [`issymmetric`](:func:`issymmetric`), [`ishermitian`](:func:`ishermitian`). [^Bunch1977]: J R Bunch and L Kaufman, Some stable methods for calculating inertia and solving symmetric linear systems, Mathematics of Computation 31:137 (1977), 163-179. [url](http://www.ams.org/journals/mcom/1977-31-137/S0025-5718-1977-0428694-0). @@ -5949,17 +2420,10 @@ Test whether a floating point number is subnormal. """ issubnormal -""" - Ac_ldiv_B(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ`` \\ ``B``. -""" -Ac_ldiv_B - """ NullException() -An attempted access to a `Nullable` with no defined value. +An attempted access to a [`Nullable`](:obj:`Nullable`) with no defined value. """ NullException @@ -5989,78 +2453,32 @@ For example: """ cfunction -""" - recvfrom(socket::UDPSocket) -> (address, data) - -Read a UDP packet from the specified socket, returning a tuple of (address, data), where -address will be either IPv4 or IPv6 as appropriate. -""" -recvfrom - """ intersect(s1,s2...) ∩(s1,s2) -Construct the intersection of two or more sets. Maintains order and multiplicity of the -first argument for arrays and ranges. +Construct the intersection of two or more sets. +Maintains order and multiplicity of the first argument for arrays and ranges. """ intersect -""" - !=(x, y) - ≠(x,y) - -Not-equals comparison operator. Always gives the opposite answer as `==`. New types should -generally not implement this, and rely on the fallback definition `!=(x,y) = !(x==y)` instead. -""" -Base.:(!=) - """ @spawn Creates a closure around an expression and runs it on an automatically-chosen process, -returning a `Future` to the result. +returning a [`Future`](:obj:`Future`) to the result. """ :@spawn """ promote_rule(type1, type2) -Specifies what type should be used by `promote` when given values of types `type1` and +Specifies what type should be used by [`promote`](:func:`promote`) when given values of types `type1` and `type2`. This function should not be called directly, but should have definitions added to it for new types as appropriate. """ promote_rule -""" - mtime(file) - -Equivalent to `stat(file).mtime`. -""" -mtime - -""" - logspace(start, stop, n=50) - -Construct a vector of `n` logarithmically spaced numbers from `10^start` to `10^stop`. -""" -logspace - -""" - @gensym - -Generates a gensym symbol for a variable. For example, `@gensym x y` is transformed into -`x = gensym("x"); y = gensym("y")`. -""" -:@gensym - -""" - sumabs2(itr) - -Sum squared absolute values of all elements in a collection. This is equivalent to `sum(abs2(itr))` but faster. -""" -sumabs2(itr) - """ sumabs2(A, dims) @@ -6068,33 +2486,10 @@ Sum squared absolute values of elements of an array over the given dimensions. """ sumabs2(A,dims) -""" - uperm(file) - -Gets the permissions of the owner of the file as a bitfield of - -| Value | Description | -|:------|:-------------------| -| 01 | Execute Permission | -| 02 | Write Permission | -| 04 | Read Permission | - -For allowed arguments, see `stat`. -""" -uperm - -""" - run(command) - -Run a command object, constructed with backticks. Throws an error if anything goes wrong, -including the process exiting with a non-zero status. -""" -run - """ showall(x) -Similar to `show`, except shows all elements of arrays. +Similar to [`show`](:func:`show`), except shows all elements of arrays. """ showall @@ -6103,27 +2498,10 @@ showall Returns a boolean value indicating whether or not the object `x` can be written as the given `mime` type. (By default, this is determined automatically by the existence of the -corresponding `show` function for `typeof(x)`.) +corresponding [`show`](:func:`show`) function for `typeof(x)`.) """ mimewritable -""" - vecdot(x, y) - -For any iterable containers `x` and `y` (including arrays of any dimension) of numbers (or -any element type for which `dot` is defined), compute the Euclidean dot product (the sum of -`dot(x[i],y[i])`) as if they were vectors. -""" -vecdot - -""" - >(x, y) - -Greater-than comparison operator. Generally, new types should implement `<` instead of this -function, and rely on the fallback definition `>(x,y) = y<x`. -""" -Base.:(>) - """ match(r::Regex, s::AbstractString[, idx::Integer[, addopts]]) @@ -6134,13 +2512,6 @@ retrieved by accessing `m.match` and the captured sequences can be retrieved by """ match -""" - Ac_mul_B(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ⋅B``. -""" -Ac_mul_B - """ qrfact!(A [,pivot=Val{false}]) @@ -6151,13 +2522,6 @@ representable by the element type of `A`, e.g. for integer types. """ qrfact! -""" - At_rdiv_B(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ / B``. -""" -At_rdiv_B - """ coth(x) @@ -6165,20 +2529,6 @@ Compute the hyperbolic cotangent of `x`. """ coth -""" - conj(z) - -Compute the complex conjugate of a complex number `z`. -""" -conj - -""" - conj!(A) - -Convert an array to its complex conjugate in-place. -""" -conj! - """ start(iter) -> state @@ -6186,23 +2536,6 @@ Get initial iteration state for an iterable object. """ start -""" - div(x, y) - ÷(x, y) - -The quotient from Euclidean division. Computes `x/y`, truncated to an integer. -""" -div - -""" - relpath(path::AbstractString, startpath::AbstractString = ".") -> AbstractString - -Return a relative filepath to path either from the current directory or from an optional -start directory. This is a path computation: the filesystem is not accessed to confirm the -existence or nature of path or startpath. -""" -relpath - """ readavailable(stream) @@ -6211,14 +2544,6 @@ result is a `Vector{UInt8,1}`. """ readavailable -""" - slicedim(A, d, i) - -Return all the data of `A` where the index for dimension `d` equals `i`. Equivalent to -`A[:,:,...,i,:,:,...]` where `i` is in position `d`. -""" -slicedim - """ isa(x, type) -> Bool @@ -6226,14 +2551,6 @@ Determine whether `x` is of the given `type`. """ isa -""" - <=(x, y) - ≤(x,y) - -Less-than-or-equals comparison operator. -""" -Base.:(<=) - """ unsafe_load(p::Ptr{T}, [i::Integer=1]) @@ -6253,14 +2570,6 @@ Get the backtrace of the current exception, for use within `catch` blocks. """ catch_backtrace -""" - get_zero_subnormals() -> Bool - -Returns `false` if operations on subnormal floating-point values ("denormals") obey rules -for IEEE arithmetic, and `true` if they might be converted to zeros. -""" -get_zero_subnormals - """ cos(x) @@ -6268,34 +2577,6 @@ Compute cosine of `x`, where `x` is in radians. """ cos -""" - base64encode(writefunc, args...) - base64encode(args...) - -Given a `write`-like function `writefunc`, which takes an I/O stream as its first argument, -`base64encode(writefunc, args...)` calls `writefunc` to write `args...` to a base64-encoded -string, and returns the string. `base64encode(args...)` is equivalent to `base64encode(write, args...)`: -it converts its arguments into bytes using the standard `write` functions and returns the -base64-encoded string. -""" -base64encode - -""" - filt!(out, b, a, x, [si]) - -Same as [`filt`](:func:`filt`) but writes the result into the `out` argument, which may -alias the input `x` to modify it in-place. -""" -filt! - -""" - ascii(s::AbstractString) - -Convert a string to `String` type and check that it contains only ASCII data, otherwise -throwing an `ArgumentError` indicating the position of the first non-ASCII byte. -""" -ascii(s) - """ maxabs(A, dims) @@ -6325,7 +2606,7 @@ julia> convert(Int, 3.0) julia> convert(Int, 3.5) ERROR: InexactError() - in convert(::Type{Int64}, ::Float64) at ./int.jl:239 + in convert(::Type{Int64}, ::Float64) at ./int.jl:330 ... ``` @@ -6380,13 +2661,6 @@ true """ convert -""" - A_ldiv_Bt(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``A`` \\ ``Bᵀ``. -""" -A_ldiv_Bt - """ applicable(f, args...) -> Bool @@ -6406,19 +2680,12 @@ true """ applicable -""" - RandomDevice() - -Create a `RandomDevice` RNG object. Two such objects will always generate different streams of random numbers. -""" -RandomDevice - """ fma(x, y, z) Computes `x*y+z` without rounding the intermediate result `x*y`. On some systems this is significantly more expensive than `x*y+z`. `fma` is used to improve accuracy in certain -algorithms. See `muladd`. +algorithms. See [`muladd`](:func:`muladd`). """ fma @@ -6439,37 +2706,6 @@ options. """ eigvals -""" - A_ldiv_Bc(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``A`` \\ ``Bᴴ``. -""" -A_ldiv_Bc - -""" - escape_string(str::AbstractString) -> AbstractString - -General escaping of traditional C and Unicode escape sequences. -""" -escape_string(str) - -""" - significand(x) - -Extract the `significand(s)` (a.k.a. mantissa), in binary representation, of a -floating-point number or array. If `x` is a non-zero finite number, then the result will be -a number of the same type on the interval ``[1,2)``. Otherwise `x` is returned. - -```jldoctest -julia> significand(15.2)/15.2 -0.125 - -julia> significand(15.2)*8 -15.2 -``` -""" -significand - """ pointer_from_objref(object_instance) @@ -6479,55 +2715,6 @@ remains referenced for the whole time that the `Ptr` will be used. """ pointer_from_objref -""" - cumsum_kbn(A, [dim]) - -Cumulative sum along a dimension, using the Kahan-Babuska-Neumaier compensated summation -algorithm for additional accuracy. The dimension defaults to 1. -""" -cumsum_kbn - -""" - cmp(x,y) - -Return -1, 0, or 1 depending on whether `x` is less than, equal to, or greater than `y`, -respectively. Uses the total order implemented by `isless`. For floating-point numbers, uses `<` -but throws an error for unordered arguments. -""" -cmp - -""" - issorted(v, [by=<transform>,] [lt=<comparison>,] [rev=false]) - -Test whether a vector is in sorted order. The `by`, `lt` and `rev` keywords modify what -order is considered to be sorted just as they do for `sort`. -""" -issorted - -""" - isbits(T) - -Return `true` if `T` is a "plain data" type, meaning it is immutable and contains no -references to other values. Typical examples are numeric types such as `UInt8`, `Float64`, -and `Complex{Float64}`. - -```jldoctest -julia> isbits(Complex{Float64}) -true - -julia> isbits(Complex) -false -``` -""" -isbits - -""" - angle(z) - -Compute the phase angle in radians of a complex number `z`. -""" -angle - """ copy!(dest, src) @@ -6543,21 +2730,6 @@ offset `do`. Returns `dest`. """ copy!(dest,d,src,so,N) -""" - broadcast(f, As...) - -Broadcasts the arrays `As` to a common size by expanding singleton dimensions, and returns -an array of the results `f(as...)` for each position. -""" -broadcast - -""" - ntoh(x) - -Converts the endianness of a value from Network byte order (big-endian) to that used by the Host. -""" -ntoh - """ qrfact(A) -> SPQR.Factorization @@ -6569,33 +2741,11 @@ exported. qrfact(A) """ - +(x, y...) - -Addition operator. `x+y+z+...` calls this function with all arguments, i.e. `+(x, y, z, ...)`. -""" -+ - -""" - identity(x) - -The identity function. Returns its argument. -""" -identity - -""" - iseven(x::Integer) -> Bool - -Returns `true` is `x` is even (that is, divisible by 2), and `false` otherwise. - -```jldoctest -julia> iseven(9) -false + +(x, y...) -julia> iseven(10) -true -``` +Addition operator. `x+y+z+...` calls this function with all arguments, i.e. `+(x, y, z, ...)`. """ -iseven ++ """ setindex!(A, X, inds...) @@ -6615,20 +2765,12 @@ setindex!(collection,value,key...) """ signif(x, digits, [base]) -Rounds (in the sense of `round`) `x` so that there are `digits` significant digits, under a +Rounds (in the sense of [`round`](:func:`round`)) `x` so that there are `digits` significant digits, under a base `base` representation, default 10. E.g., `signif(123.456, 2)` is `120.0`, and `signif(357.913, 4, 2)` is `352.0`. """ signif -""" - nextpow2(n) - -The smallest power of two not less than `n`. Returns 0 for `n==0`, and returns -`-nextpow2(-n)` for negative arguments. -""" -nextpow2 - """ full(F) @@ -6649,28 +2791,6 @@ that spans all rows of `R` in its corresponding QR factorization. """ full(::LinAlg.QRCompactWYQ, ?) -""" - map(f, c...) -> collection - -Transform collection `c` by applying `f` to each element. For multiple collection arguments, -apply `f` elementwise. - -```jldoctest -julia> map((x) -> x * 2, [1, 2, 3]) -3-element Array{Int64,1}: - 2 - 4 - 6 - -julia> map(+, [1, 2, 3], [10, 20, 30]) -3-element Array{Int64,1}: - 11 - 22 - 33 -``` -""" -map - """ throw(e) @@ -6678,24 +2798,6 @@ Throw an object as an exception. """ throw -""" - isxdigit(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is a valid hexadecimal digit, or whether this is true for all elements of a string. -""" -isxdigit - -""" - fill(x, dims) - -Create an array filled with the value `x`. For example, `fill(1.0, (10,10))` returns a 10×10 -array of floats, with each element initialized to `1.0`. - -If `x` is an object reference, all elements will refer to the same object. `fill(Foo(), -dims)` will return an array filled with the result of evaluating `Foo()` once. -""" -fill - """ issubset(a, b) ⊆(a,b) -> Bool @@ -6714,36 +2816,6 @@ Return `true` if `A` is a subset of or equal to `S`. """ issubset -""" - istriu(A) -> Bool - -Test whether a matrix is upper triangular. -""" -istriu - -""" - map!(function, collection) - -In-place version of [`map`](:func:`map`). -""" -map!(f,collection) - -""" - map!(function, destination, collection...) - -Like [`map`](:func:`map`), but stores the result in `destination` rather than a new -collection. `destination` must be at least as large as the first collection. -""" -map!(f,destination,collection...) - -""" - unescape_string(s::AbstractString) -> AbstractString - -General unescaping of traditional C and Unicode escape sequences. Reverse of -[`escape_string`](:func:`escape_string`). See also [`unescape_string`](:func:`unescape_string`). -""" -unescape_string(s) - """ print_with_color(color::Symbol, [io], strings...) @@ -6757,17 +2829,10 @@ print_with_color stringmime(mime, x) Returns an `AbstractString` containing the representation of `x` in the requested `mime` -type. This is similar to `reprmime` except that binary data is base64-encoded as an ASCII string. +type. This is similar to [`reprmime`](:func:`reprmime`) except that binary data is base64-encoded as an ASCII string. """ stringmime -""" - ischardev(path) -> Bool - -Returns `true` if `path` is a character device, `false` otherwise. -""" -ischardev - """ zero(x) @@ -6782,31 +2847,11 @@ Test whether any values along the given dimensions of an array are `true`. """ any(::AbstractArray,dims) -""" - getkey(collection, key, default) - -Return the key matching argument `key` if one exists in `collection`, otherwise return `default`. -""" -getkey - -""" - At_ldiv_Bt(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ`` \\ ``Bᵀ``. -""" -At_ldiv_Bt - -""" - Ac_mul_Bc(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ Bᴴ``. -""" -Ac_mul_Bc - """ zeros(type, dims) -Create an array of all zeros of specified type. The type defaults to Float64 if not specified. +Create an array of all zeros of specified type. +The type defaults to Float64 if not specified. """ zeros(t,dims) @@ -6831,24 +2876,6 @@ Riemann zeta function ``\\zeta(s)``. """ zeta(s) -""" - A_mul_Bt(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``A⋅Bᵀ``. -""" -A_mul_Bt - -""" - vecnorm(A, [p]) - -For any iterable container `A` (including arrays of any dimension) of numbers (or any -element type for which `norm` is defined), compute the `p`-norm (defaulting to `p=2`) as if -`A` were a vector of the corresponding length. - -For example, if `A` is a matrix and `p=2`, then this is equivalent to the Frobenius norm. -""" -vecnorm - """ isvalid(value) -> Bool @@ -6866,108 +2893,6 @@ Values for `String` can be of that type, or `Vector{UInt8}`. """ isvalid(T,value) -""" - isvalid(str, i) - -Tells whether index `i` is valid for the given string. -""" -isvalid(::AbstractString,i) - -""" - esc(e::ANY) - -Only valid in the context of an `Expr` returned from a macro. Prevents the macro hygiene -pass from turning embedded variables into gensym variables. See the [macro](:ref:`man-macros`) -section of the Metaprogramming chapter of the manual for more details and examples. -""" -esc - -""" - bitbroadcast(f, As...) - -Like `broadcast`, but allocates a `BitArray` to store the result, rather then an `Array`. -""" -bitbroadcast - -""" - set_zero_subnormals(yes::Bool) -> Bool - -If `yes` is `false`, subsequent floating-point operations follow rules for IEEE arithmetic -on subnormal values ("denormals"). Otherwise, floating-point operations are permitted (but -not required) to convert subnormal inputs or outputs to zero. Returns `true` unless -`yes==true` but the hardware does not support zeroing of subnormal numbers. - -`set_zero_subnormals(true)` can speed up some computations on some hardware. However, it can -break identities such as `(x-y==0) == (x==y)`. -""" -set_zero_subnormals - -""" - frexp(val) - -Return `(x,exp)` such that `x` has a magnitude in the interval ``[1/2, 1)`` or 0, and val = -``x \\times 2^{exp}``. -""" -frexp - -""" - sortcols(A, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) - -Sort the columns of matrix `A` lexicographically. -""" -sortcols - -""" - rsplit(string, [chars]; limit=0, keep=true) - -Similar to `split`, but starting from the end of the string. -""" -rsplit - -""" - trace(M) - -Matrix trace. -""" -trace - -""" - runtests([tests=["all"] [, numcores=ceil(Integer, Sys.CPU_CORES / 2) ]]) - -Run the Julia unit tests listed in `tests`, which can be either a string or an array of -strings, using `numcores` processors. (not exported) -""" -runtests - -""" - time_ns() - -Get the time in nanoseconds. The time corresponding to 0 is undefined, and wraps every 5.8 years. -""" -time_ns - -""" - exponent(x) -> Int - -Get the exponent of a normalized floating-point number. -""" -exponent - -""" - rsearchindex(string, substring, [start]) - -Similar to `rsearch`, but return only the start index at which the substring is found, or `0` if it is not. -""" -rsearchindex - -""" - muladd(x, y, z) - -Combined multiply-add, computes `x*y+z` in an efficient manner. This may on some systems be -equivalent to `x*y+z`, or to `fma(x,y,z)`. `muladd` is used to improve performance. See `fma`. -""" -muladd - """ unsigned(x) -> Unsigned @@ -6976,14 +2901,6 @@ unsigned without checking for negative values. """ unsigned -""" - mkdir(path, [mode]) - -Make a new directory with name `path` and permissions `mode`. `mode` defaults to `0o777`, -modified by the current file creation mask. -""" -mkdir - """ midpoints(e) @@ -7016,48 +2933,6 @@ smallest suitable `AbstractFloat` type is used. Converts strings to `Float64`. """ float -""" - include_dependency(path::AbstractString) - -In a module, declare that the file specified by `path` (relative or absolute) is a -dependency for precompilation; that is, the module will need to be recompiled if this file -changes. - -This is only needed if your module depends on a file that is not used via `include`. It has -no effect outside of compilation. -""" -include_dependency - -""" - ldexp(x, n) - -Compute ``x \\times 2^n``. -""" -ldexp - -""" - islower(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is a lowercase letter, or whether this is true for all elements of -a string. A character is classified as lowercase if it belongs to Unicode category Ll, -Letter: Lowercase. -""" -islower - -""" - exp2(x) - -Compute ``2^x``. -""" -exp2 - -""" - gcd(x,y) - -Greatest common (positive) divisor (or zero if `x` and `y` are both zero). -""" -gcd - """ signbit(x) @@ -7075,18 +2950,11 @@ cscd """ tryparse(type, str, [base]) -Like `parse`, but returns a `Nullable` of the requested type. The result will be null if the +Like [`parse`](:func:`parse`), but returns a [`Nullable`](:obj:`Nullable`) of the requested type. The result will be null if the string does not contain a valid number. """ tryparse -""" - lexless(x, y) - -Determine whether `x` is lexicographically less than `y`. -""" -lexless - """ all!(r, A) @@ -7094,13 +2962,6 @@ Test whether all values in `A` along the singleton dimensions of `r` are `true`, """ all! -""" - is_assigned_char(c) -> Bool - -Returns `true` if the given char or integer is an assigned Unicode code point. -""" -is_assigned_char - """ exit([code]) @@ -7116,20 +2977,6 @@ Determine whether a MIME type is text data. """ istextmime -""" - merge!(collection, others...) - -Update collection with pairs from the other collections. -""" -merge! - -""" - realpath(path::AbstractString) -> AbstractString - -Canonicalize a path by expanding symbolic links and removing "." and ".." entries. -""" -realpath - """ skipchars(stream, predicate; linecomment::Char) @@ -7154,29 +3001,6 @@ Union each element of `iterable` into set `s` in-place. """ union! -""" - At_ldiv_B(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ`` \\ ``B``. -""" -At_ldiv_B - -""" - dot(x, y) - ⋅(x,y) - -Compute the dot product. For complex vectors, the first vector is conjugated. -""" -dot - -""" - cond(M, [p]) - -Condition number of the matrix `M`, computed using the operator `p`-norm. Valid values for -`p` are `1`, `2` (default), or `Inf`. -""" -cond - """ deepcopy(x) @@ -7215,22 +3039,6 @@ julia> widen(1.5f0) """ widen -""" - @eval - -Evaluate an expression and return the value. -""" -:@eval - -""" - eval([m::Module], expr::Expr) - -Evaluate an expression in the given module and return the result. Every `Module` (except -those defined with `baremodule`) has its own 1-argument definition of `eval`, which -evaluates expressions in that module. -""" -eval - """ Set([itr]) @@ -7248,35 +3056,6 @@ for arbitrary complex `x`. """ erf -""" - lcm(x,y) - -Least common (non-negative) multiple. -""" -lcm - -""" - isprint(c::Union{Char,AbstractString}) -> Bool - -Tests whether a character is printable, including spaces, but not a control character. For -strings, tests whether this is true for all elements of the string. -""" -isprint - -""" - splitdir(path::AbstractString) -> (AbstractString,AbstractString) - -Split a path into a tuple of the directory name and file name. -""" -splitdir - -""" - sign(x) - -Return zero if `x==0` and ``x/|x|`` otherwise (i.e., ±1 for real `x`). -""" -sign - """ signed(x) @@ -7295,13 +3074,6 @@ having to test the boolean value at runtime. """ Val -""" - iswritable(io) -> Bool - -Returns `true` if the specified IO object is writable (if that can be determined). -""" -iswritable - """ |(x, y) @@ -7309,22 +3081,6 @@ Bitwise or. """ Base.:(|) -""" - readandwrite(command) - -Starts running a command asynchronously, and returns a tuple (stdout,stdin,process) of the -output stream and input stream of the process, and the process object itself. -""" -readandwrite - -""" - splitdrive(path::AbstractString) -> (AbstractString,AbstractString) - -On Windows, split a path into the drive letter part and the path part. On Unix systems, the -first component is always the empty string. -""" -splitdrive - """ pop!(collection, key[, default]) @@ -7362,16 +3118,6 @@ julia> A """ pop!(collection) -""" - randperm([rng,] n) - -Construct a random permutation of length `n`. The optional `rng` argument specifies a random -number generator (see [Random Numbers](:ref:`Random Numbers <random-numbers>`)). -To randomly permute a arbitrary vector, see [`shuffle`](:func:`shuffle`) -or [`shuffle!`](:func:`shuffle!`). -""" -randperm - """ seekend(s) @@ -7386,39 +3132,6 @@ Integer division was attempted with a denominator value of 0. """ DivideError -""" - Ac_ldiv_Bc(A, B) - -For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ`` \\ ``Bᴴ``. -""" -Ac_ldiv_Bc - -""" - @enum EnumName EnumValue1[=x] EnumValue2[=y] - -Create an [`Enum`](:obj:`Enum`) type with name `EnumName` and enum member values of -`EnumValue1` and `EnumValue2` with optional assigned values of `x` and `y`, respectively. -`EnumName` can be used just like other types and enum member values as regular values, such as - -```jldoctest -julia> @enum FRUIT apple=1 orange=2 kiwi=3 - -julia> f(x::FRUIT) = "I'm a FRUIT with value: \$(Int(x))" -f (generic function with 1 method) - -julia> f(apple) -"I'm a FRUIT with value: 1" -``` -""" -:@enum - -""" - widemul(x, y) - -Multiply `x` and `y`, giving the result as a larger type. -""" -widemul - """ unsafe_pointer_to_objref(p::Ptr) @@ -7428,21 +3141,6 @@ considered "unsafe" and should be used with care. """ unsafe_pointer_to_objref -""" - chomp(string) - -Remove a single trailing newline from a string. -""" -chomp - -""" - >=(x, y) - ≥(x,y) - -Greater-than-or-equals comparison operator. -""" -Base.:(>=) - """ dawson(x) @@ -7451,38 +3149,9 @@ Compute the Dawson function (scaled imaginary error function) of `x`, defined by """ dawson -""" - randjump(r::MersenneTwister, jumps, [jumppoly]) -> Vector{MersenneTwister} - -Create an array of the size `jumps` of initialized `MersenneTwister` RNG objects where the -first RNG object given as a parameter and following `MersenneTwister` RNGs in the array -initialized such that a state of the RNG object in the array would be moved forward (without -generating numbers) from a previous RNG object array element on a particular number of steps -encoded by the jump polynomial `jumppoly`. - -Default jump polynomial moves forward `MersenneTwister` RNG state by 10^20 steps. -""" -randjump - -""" - :(start, [step], stop) - -Range operator. `a:b` constructs a range from `a` to `b` with a step size of 1, and `a:s:b` -is similar but uses a step size of `s`. These syntaxes call the function `colon`. The colon -is also used in indexing to select whole dimensions. -""" -colon(start, step, stop) - """ \$(x, y) Bitwise exclusive or. """ Base.:$(x, y) - -""" - getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr,UInt16) - -Get the IP address and the port that the given TCP socket is connected to (or bound to, in the case of TCPServer). -""" -getsockname diff --git a/base/dsp.jl b/base/dsp.jl index 43ec69440cbe6..8341e6fd0914e 100644 --- a/base/dsp.jl +++ b/base/dsp.jl @@ -8,6 +8,12 @@ export filt, filt!, deconv, conv, conv2, xcorr _zerosi(b,a,T) = zeros(promote_type(eltype(b), eltype(a), T), max(length(a), length(b))-1) +""" + filt(b, a, x, [si]) + +Apply filter described by vectors `a` and `b` to vector `x`, with an optional initial filter +state vector `si` (defaults to zeros). +""" function filt{T,S}(b::Union{AbstractVector, Number}, a::Union{AbstractVector, Number}, x::AbstractArray{T}, si::AbstractArray{S}=_zerosi(b,a,T)) filt!(Array{promote_type(eltype(b), eltype(a), T, S)}(size(x)), b, a, x, si) @@ -15,6 +21,13 @@ end # in-place filtering: returns results in the out argument, which may shadow x # (and does so by default) + +""" + filt!(out, b, a, x, [si]) + +Same as [`filt`](:func:`filt`) but writes the result into the `out` argument, which may +alias the input `x` to modify it in-place. +""" function filt!{T,S,N}(out::AbstractArray, b::Union{AbstractVector, Number}, a::Union{AbstractVector, Number}, x::AbstractArray{T}, si::AbstractArray{S,N}=_zerosi(b,a,T)) isempty(b) && throw(ArgumentError("filter vector b must be non-empty")) @@ -89,6 +102,12 @@ function _filt_fir!(out, b, x, si, col) end end +""" + deconv(b,a) -> c + +Construct vector `c` such that `b = conv(a,c) + r`. +Equivalent to polynomial division. +""" function deconv{T}(b::StridedVector{T}, a::StridedVector{T}) lb = size(b,1) la = size(a,1) @@ -101,6 +120,11 @@ function deconv{T}(b::StridedVector{T}, a::StridedVector{T}) filt(b, a, x) end +""" + conv(u,v) + +Convolution of two vectors. Uses FFT algorithm. +""" function conv{T<:Base.LinAlg.BlasFloat}(u::StridedVector{T}, v::StridedVector{T}) nu = length(u) nv = length(v) @@ -121,6 +145,13 @@ conv{T<:Integer}(u::StridedVector{T}, v::StridedVector{T}) = round(Int,conv(floa conv{T<:Integer, S<:Base.LinAlg.BlasFloat}(u::StridedVector{T}, v::StridedVector{S}) = conv(float(u), v) conv{T<:Integer, S<:Base.LinAlg.BlasFloat}(u::StridedVector{S}, v::StridedVector{T}) = conv(u, float(v)) +""" + conv2(u,v,A) + +2-D convolution of the matrix `A` with the 2-D separable kernel generated by +the vectors `u` and `v`. +Uses 2-D FFT algorithm. +""" function conv2{T}(u::StridedVector{T}, v::StridedVector{T}, A::StridedMatrix{T}) m = length(u)+size(A,1)-1 n = length(v)+size(A,2)-1 @@ -135,6 +166,11 @@ function conv2{T}(u::StridedVector{T}, v::StridedVector{T}, A::StridedMatrix{T}) return C end +""" + conv2(B,A) + +2-D convolution of the matrix `B` with the matrix `A`. Uses 2-D FFT algorithm. +""" function conv2{T}(A::StridedMatrix{T}, B::StridedMatrix{T}) sa, sb = size(A), size(B) At = zeros(T, sa[1]+sb[1]-1, sa[2]+sb[2]-1) @@ -151,6 +187,11 @@ end conv2{T<:Integer}(A::StridedMatrix{T}, B::StridedMatrix{T}) = round(Int,conv2(float(A), float(B))) conv2{T<:Integer}(u::StridedVector{T}, v::StridedVector{T}, A::StridedMatrix{T}) = round(Int,conv2(float(u), float(v), float(A))) +""" + xcorr(u,v) + +Compute the cross-correlation of two vectors. +""" function xcorr(u, v) su = size(u,1); sv = size(v,1) if su < sv diff --git a/base/env.jl b/base/env.jl index 6482d7737c56e..cca18315941bc 100644 --- a/base/env.jl +++ b/base/env.jl @@ -58,6 +58,11 @@ end # os test ## ENV: hash interface ## +""" + EnvHash() -> EnvHash + +A singleton of this type provides a hash table interface to environment variables. +""" type EnvHash <: Associative{String,String}; end """ diff --git a/base/essentials.jl b/base/essentials.jl index 38880994435cf..75fa67251fa56 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -132,6 +132,13 @@ function precompile(argt::Type) ccall(:jl_compile_hint, Cint, (Any,), argt) != 0 end +""" + esc(e::ANY) + +Only valid in the context of an `Expr` returned from a macro. Prevents the macro hygiene +pass from turning embedded variables into gensym variables. See the [macro](:ref:`man-macros`) +section of the Metaprogramming chapter of the manual for more details and examples. +""" esc(e::ANY) = Expr(:escape, e) macro boundscheck(blk) diff --git a/base/expr.jl b/base/expr.jl index 8a6f5a91d39d9..0b7a6cd7f4459 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -2,6 +2,11 @@ ## symbols ## +""" + gensym([tag]) + +Generates a symbol which will not conflict with other variable names. +""" gensym() = ccall(:jl_gensym, Ref{Symbol}, ()) gensym(s::String) = gensym(s.data) @@ -11,6 +16,12 @@ gensym(ss::String...) = map(gensym, ss) gensym(s::Symbol) = ccall(:jl_tagged_gensym, Ref{Symbol}, (Ptr{UInt8}, Int32), s, ccall(:strlen, Csize_t, (Ptr{UInt8},), s)) +""" + @gensym + +Generates a gensym symbol for a variable. For example, `@gensym x y` is transformed into +`x = gensym("x"); y = gensym("y")`. +""" macro gensym(names...) blk = Expr(:block) for name in names @@ -35,11 +46,36 @@ copy_exprargs(x::Array{Any,1}) = Any[copy_exprs(a) for a in x] ==(x::Expr, y::Expr) = x.head === y.head && isequal(x.args, y.args) ==(x::QuoteNode, y::QuoteNode) = isequal(x.value, y.value) +""" + expand(x) + +Takes the expression `x` and returns an equivalent expression in lowered form. +""" expand(x::ANY) = ccall(:jl_expand, Any, (Any,), x) + +""" + macroexpand(x) + +Takes the expression `x` and returns an equivalent expression with all macros removed (expanded). +""" macroexpand(x::ANY) = ccall(:jl_macroexpand, Any, (Any,), x) ## misc syntax ## +""" + eval([m::Module], expr::Expr) + +Evaluate an expression in the given module and return the result. Every `Module` (except +those defined with `baremodule`) has its own 1-argument definition of `eval`, which +evaluates expressions in that module. +""" +eval + +""" + @eval + +Evaluate an expression and return the value. +""" macro eval(x) :($(esc(:eval))($(Expr(:quote,x)))) end @@ -57,7 +93,7 @@ macro pure(ex) end """ - @propagate_inbounds(ex) + @propagate_inbounds Tells the compiler to inline a function while retaining the caller's inbounds context. """ @@ -72,6 +108,8 @@ macro propagate_inbounds(ex) end """ + @polly + Tells the compiler to apply the polyhedral optimizer Polly to a function. """ macro polly(ex) diff --git a/base/file.jl b/base/file.jl index 396ba739f2cd4..e3c69f2ff0aca 100644 --- a/base/file.jl +++ b/base/file.jl @@ -28,6 +28,11 @@ export # get and set current directory +""" + pwd() -> AbstractString + +Get the current working directory. +""" function pwd() b = Array{UInt8}(1024) len = Ref{Csize_t}(length(b)) @@ -35,6 +40,11 @@ function pwd() String(b[1:len[]]) end +""" + cd(dir::AbstractString=homedir()) + +Set the current working directory. +""" function cd(dir::AbstractString) uv_error("chdir $dir", ccall(:uv_chdir, Cint, (Cstring,), dir)) end @@ -63,8 +73,19 @@ else end end end +""" + cd(f::Function, dir::AbstractString=homedir()) + +Temporarily changes the current working directory and applies function `f` before returning. +""" cd(f::Function) = cd(f, homedir()) +""" + mkdir(path::AbstractString, mode::Unsigned=0o777) + +Make a new directory with name `path` and permissions `mode`. `mode` defaults to `0o777`, +modified by the current file creation mask. +""" function mkdir(path::AbstractString, mode::Unsigned=0o777) @static if is_windows() ret = ccall(:_wmkdir, Int32, (Cwstring,), path) @@ -74,6 +95,12 @@ function mkdir(path::AbstractString, mode::Unsigned=0o777) systemerror(:mkdir, ret != 0; extrainfo=path) end +""" + mkpath(path::AbstractString, mode::Unsigned=0o777) + +Create all directories in the given `path`, with permissions `mode`. `mode` defaults to +`0o777`, modified by the current file creation mask. +""" function mkpath(path::AbstractString, mode::Unsigned=0o777) isdirpath(path) && (path = dirname(path)) dir = dirname(path) @@ -95,6 +122,13 @@ end mkdir(path::AbstractString, mode::Signed) = throw(ArgumentError("mode must be an unsigned integer; try 0o$mode")) mkpath(path::AbstractString, mode::Signed) = throw(ArgumentError("mode must be an unsigned integer; try 0o$mode")) +""" + rm(path::AbstractString; force::Bool=false, recursive::Bool=false) + +Delete the file, link, or empty directory at the given path. If `force=true` is passed, a +non-existing path is not treated as error. If `recursive=true` is passed and the path is a +directory, then all contents are removed recursively. +""" function rm(path::AbstractString; force::Bool=false, recursive::Bool=false) if islink(path) || !isdir(path) try @@ -168,6 +202,16 @@ function cptree(src::AbstractString, dst::AbstractString; remove_destination::Bo end end +""" + cp(src::AbstractString, dst::AbstractString; remove_destination::Bool=false, follow_symlinks::Bool=false) + +Copy the file, link, or directory from *src* to *dest*. `remove_destination=true` will first +remove an existing `dst`. + +If `follow_symlinks=false`, and `src` is a symbolic link, `dst` will be created as a +symbolic link. If `follow_symlinks=true` and `src` is a symbolic link, `dst` will be a copy +of the file or directory `src` refers to. +""" function cp(src::AbstractString, dst::AbstractString; remove_destination::Bool=false, follow_symlinks::Bool=false) checkfor_mv_cp_cptree(src, dst, "copying"; remove_destination=remove_destination) @@ -180,11 +224,22 @@ function cp(src::AbstractString, dst::AbstractString; remove_destination::Bool=f end end +""" + mv(src::AbstractString, dst::AbstractString; remove_destination::Bool=false) + +Move the file, link, or directory from `src` to `dst`. +`remove_destination=true` will first remove an existing `dst`. +""" function mv(src::AbstractString, dst::AbstractString; remove_destination::Bool=false) checkfor_mv_cp_cptree(src, dst, "moving"; remove_destination=remove_destination) rename(src, dst) end +""" + touch(path::AbstractString) + +Update the last-modified timestamp on a file to the current time. +""" function touch(path::AbstractString) f = open(path, JL_O_WRONLY | JL_O_CREAT, 0o0666) try @@ -196,6 +251,7 @@ function touch(path::AbstractString) end if is_windows() + function tempdir() temppath = Array{UInt16}(32767) lentemppath = ccall(:GetTempPathW,stdcall,UInt32,(UInt32,Ptr{UInt16}),length(temppath),temppath) @@ -218,10 +274,12 @@ function tempname(temppath::AbstractString,uunique::UInt32) resize!(tname,lentname) return transcode(String, tname) end + function mktemp(parent=tempdir()) filename = tempname(parent, UInt32(0)) return (filename, Base.open(filename, "r+")) end + function mktempdir(parent=tempdir()) seed::UInt32 = rand(UInt32) while true @@ -270,6 +328,42 @@ end end # os-test + +""" + tempdir() + +Obtain the path of a temporary directory (possibly shared with other processes). +""" +tempdir() + +""" + tempname() + +Generate a unique temporary file path. +""" +tempname() + +""" + mktemp(parent=tempdir()) + +Returns `(path, io)`, where `path` is the path of a new temporary file in `parent` and `io` +is an open file object for this path. +""" +mktemp(parent) + +""" + mktempdir(parent=tempdir()) + +Create a temporary directory in the `parent` directory and return its path. +""" +mktempdir(parent) + + +""" + mktemp(f::Function, parent=tempdir()) + +Apply the function `f` to the result of `mktemp(parent)` and remove the temporary file upon completion. +""" function mktemp(fn::Function, parent=tempdir()) (tmp_path, tmp_io) = mktemp(parent) try @@ -280,6 +374,12 @@ function mktemp(fn::Function, parent=tempdir()) end end +""" + mktempdir(f::Function, parent=tempdir()) + +Apply the function `f` to the result of `mktempdir(parent)` and remove the temporary +directory upon completion. +""" function mktempdir(fn::Function, parent=tempdir()) tmpdir = mktempdir(parent) try @@ -293,6 +393,12 @@ immutable uv_dirent_t name::Ptr{UInt8} typ::Cint end + +""" + readdir(dir::AbstractString=".") -> Vector{String} + +Returns the files and directories in the directory `dir` (or the current working directory if not given). +""" function readdir(path::AbstractString) # Allocate space for uv_fs_t struct uv_readdir_req = zeros(UInt8, ccall(:jl_sizeof_uv_fs_t, Int32, ())) @@ -321,10 +427,13 @@ readdir() = readdir(".") """ walkdir(dir; topdown=true, follow_symlinks=false, onerror=throw) -The walkdir method return an iterator that walks the directory tree of a directory. The iterator returns a tuple containing -`(rootpath, dirs, files)`. The directory tree can be traversed top-down or bottom-up. If walkdir encounters a SystemError -it will raise the error. A custom error handling function can be provided through `onerror` keyword argument, the function -is called with a SystemError as argument. +The `walkdir` method returns an iterator that walks the directory tree of a directory. +The iterator returns a tuple containing `(rootpath, dirs, files)`. +The directory tree can be traversed top-down or bottom-up. +If `walkdir` encounters a [`SystemError`](:obj:`SystemError`) +it will raise the error by default. +A custom error handling function can be provided through `onerror` keyword argument. +`onerror` is called with a `SystemError` as argument. for (root, dirs, files) in walkdir(".") println("Directories in \$root") @@ -424,7 +533,7 @@ if is_windows() end """ - symlink(target, link) + symlink(target::AbstractString, link::AbstractString) Creates a symbolic link to `target` with the name `link`. @@ -454,6 +563,11 @@ function symlink(p::AbstractString, np::AbstractString) uv_error("symlink",err) end +""" + readlink(path::AbstractString) -> AbstractString + +Returns the value of a symbolic link `path`. +""" function readlink(path::AbstractString) req = Libc.malloc(_sizeof_uv_fs) try @@ -473,6 +587,13 @@ function readlink(path::AbstractString) end end +""" + chmod(path::AbstractString, mode::Integer; recursive::Bool=false) + +Change the permissions mode of `path` to `mode`. Only integer `mode`s (e.g. `0o777`) are +currently supported. If `recursive=true` and the path is a directory all permissions in +that directory will be recursively changed. +""" function chmod(path::AbstractString, mode::Integer; recursive::Bool=false) err = ccall(:jl_fs_chmod, Int32, (Cstring, Cint), path, mode) uv_error("chmod", err) @@ -486,6 +607,12 @@ function chmod(path::AbstractString, mode::Integer; recursive::Bool=false) nothing end +""" + chown(path::AbstractString, owner::Integer, group::Integer=-1) + +Change the owner and/or group of `path` to `owner` and/or `group`. If the value entered for `owner` or `group` +is `-1` the corresponding ID will not change. Only integer `owner`s and `group`s are currently supported. +""" function chown(path::AbstractString, owner::Integer, group::Integer=-1) err = ccall(:jl_fs_chown, Int32, (Cstring, Cint, Cint), path, owner, group) uv_error("chown",err) diff --git a/base/float.jl b/base/float.jl index dae5ad233f39f..aec7926920feb 100644 --- a/base/float.jl +++ b/base/float.jl @@ -352,6 +352,11 @@ end abs(x::Float64) = box(Float64,abs_float(unbox(Float64,x))) abs(x::Float32) = box(Float32,abs_float(unbox(Float32,x))) +""" + isnan(f) -> Bool + +Test whether a floating point number is not a number (NaN). +""" isnan(x::AbstractFloat) = x != x isnan(x::Real) = false diff --git a/base/int.jl b/base/int.jl index 6b244499f4b0b..cfb6320d724ac 100644 --- a/base/int.jl +++ b/base/int.jl @@ -35,7 +35,34 @@ typealias BitUnsigned64T Union{Type{UInt8},Type{UInt16},Type{UInt32},Type{UInt64 /(x::Integer, y::Integer) = float(x)/float(y) inv(x::Integer) = float(one(x))/float(x) +""" + isodd(x::Integer) -> Bool + +Returns `true` if `x` is odd (that is, not divisible by 2), and `false` otherwise. + +```jldoctest +julia> isodd(9) +true + +julia> isodd(10) +false +``` +""" isodd(n::Integer) = rem(n,2) != 0 + +""" + iseven(x::Integer) -> Bool + +Returns `true` is `x` is even (that is, divisible by 2), and `false` otherwise. + +```jldoctest +julia> iseven(9) +false + +julia> iseven(10) +true +``` +""" iseven(n::Integer) = !isodd(n) signbit(x::Integer) = x < 0 @@ -146,12 +173,76 @@ bswap{T<:Union{Int8,UInt8}}(x::T) = x bswap{T<:Union{Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128}}(x::T) = box(T,bswap_int(unbox(T,x))) +""" + count_ones(x::Integer) -> Integer + +Number of ones in the binary representation of `x`. + +```jldoctest +julia> count_ones(7) +3 +``` +""" count_ones{T<:BitInteger}(x::T) = Int(box(T,ctpop_int(unbox(T,x)))) + +""" + leading_zeros(x::Integer) -> Integer + +Number of zeros leading the binary representation of `x`. + +```jldoctest +julia> leading_zeros(Int32(1)) +31 +``` +""" leading_zeros{T<:BitInteger}(x::T) = Int(box(T,ctlz_int(unbox(T,x)))) + +""" + trailing_zeros(x::Integer) -> Integer + +Number of zeros trailing the binary representation of `x`. + +```jldoctest +julia> trailing_zeros(2) +1 +``` +""" trailing_zeros{T<:BitInteger}(x::T) = Int(box(T,cttz_int(unbox(T,x)))) +""" + count_zeros(x::Integer) -> Integer + +Number of zeros in the binary representation of `x`. + +```jldoctest +julia> count_zeros(Int32(2 ^ 16 - 1)) +16 +``` +""" count_zeros( x::Integer) = count_ones(~x) + +""" + leading_ones(x::Integer) -> Integer + +Number of ones leading the binary representation of `x`. + +```jldoctest +julia> leading_ones(UInt32(2 ^ 32 - 2)) +31 +``` +""" leading_ones( x::Integer) = leading_zeros(~x) + +""" + trailing_ones(x::Integer) -> Integer + +Number of ones trailing the binary representation of `x`. + +```jldoctest +julia> trailing_ones(3) +2 +``` +""" trailing_ones(x::Integer) = trailing_zeros(~x) ## integer comparisons ## diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 24b5f49f727a6..91ad7d981d36f 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -23,6 +23,12 @@ function editor() return args end +""" + edit(path::AbstractString, line::Integer=0) + +Edit a file or directory optionally providing a line number to edit the file at. +Returns to the `julia` prompt when you quit the editor. +""" function edit(path::AbstractString, line::Integer=0) command = editor() name = basename(first(command)) @@ -64,6 +70,12 @@ function edit(path::AbstractString, line::Integer=0) nothing end +""" + edit(function, [types]) + +Edit the definition of a function, optionally specifying a tuple of types to +indicate which method to edit. +""" edit(f) = edit(functionloc(f)...) edit(f, t::ANY) = edit(functionloc(f,t)...) edit(file, line::Integer) = error("could not find source file for function") @@ -83,7 +95,20 @@ else end end +""" + less(file::AbstractString, [line::Integer]) + +Show a file using the default pager, optionally providing a starting line number. Returns to +the `julia` prompt when you quit the pager. +""" less(file::AbstractString) = less(file, 1) + +""" + less(function, [types]) + +Show the definition of a function using the default pager, optionally specifying a tuple of +types to indicate which method to see. +""" less(f) = less(functionloc(f)...) less(f, t::ANY) = less(functionloc(f,t)...) less(file, line::Integer) = error("could not find source file for function") @@ -146,7 +171,6 @@ elseif is_windows() ccall((:CloseClipboard, "user32"), stdcall, Void, ()) end clipboard(x) = clipboard(sprint(io->print(io,x))::String) - function clipboard() systemerror(:OpenClipboard, 0==ccall((:OpenClipboard, "user32"), stdcall, Cint, (Ptr{Void},), C_NULL)) pdata = ccall((:GetClipboardData, "user32"), stdcall, Ptr{UInt16}, (UInt32,), 13) @@ -167,6 +191,22 @@ else clipboard(x="") = error("`clipboard` function not implemented for $(Sys.KERNEL)") end + +""" + clipboard(x) + +Send a printed form of `x` to the operating system clipboard ("copy"). +""" +clipboard(x) + +""" + clipboard() -> AbstractString + +Return a string with the contents of the operating system clipboard ("paste"). +""" +clipboard() + + # system information # used by sysinfo.jl @@ -195,6 +235,13 @@ function _show_cpuinfo(io::IO, info::Sys.CPUinfo, header::Bool=true, prefix::Abs end end + +""" + versioninfo(io::IO=STDOUT, verbose::Bool=false) + +Print information about the version of Julia in use. If the `verbose` argument is `true`, +detailed system information is shown as well. +""" function versioninfo(io::IO=STDOUT, verbose::Bool=false) println(io, "Julia Version $VERSION") if !isempty(GIT_VERSION_INFO.commit_short) @@ -258,7 +305,7 @@ versioninfo(verbose::Bool) = versioninfo(STDOUT,verbose) """ - code_warntype([io], f, types) + code_warntype([io::IO], f, types) Prints lowered and type-inferred ASTs for the methods matching the given generic function and type signature to `io` which defaults to `STDOUT`. The ASTs are annotated in such a way @@ -451,7 +498,7 @@ end # `methodswith` -- shows a list of methods using the type given """ - methodswith(typ[, module or function][, showparents]) + methodswith(typ[, module or function][, showparents::Bool=false, meths=Method[]]) Return an array of methods with an argument of type `typ`. @@ -542,8 +589,28 @@ function download(url::AbstractString) download(url, filename) end +""" + download(url::AbstractString, [localfile::AbstractString]) + +Download a file from the given url, optionally renaming it to the given local file name. +Note that this function relies on the availability of external tools such as `curl`, `wget` +or `fetch` to download the file and is provided for convenience. For production use or +situations in which more options are needed, please use a package that provides the desired +functionality instead. +""" +download(url, filename) + # workspace management +""" + workspace() + +Replace the top-level module (`Main`) with a new one, providing a clean workspace. The +previous `Main` module is made available as `LastMain`. A previously-loaded package can be +accessed using a statement such as `using LastMain.Package`. + +This function should only be used interactively. +""" function workspace() last = Core.Main b = last.Base @@ -560,6 +627,12 @@ end # testing +""" + runtests([tests=["all"] [, numcores=ceil(Int, Sys.CPU_CORES / 2) ]]) + +Run the Julia unit tests listed in `tests`, which can be either a string or an array of +strings, using `numcores` processors. (not exported) +""" function runtests(tests = ["all"], numcores = ceil(Int, Sys.CPU_CORES / 2)) if isa(tests,AbstractString) tests = split(tests) @@ -581,7 +654,7 @@ end """ - whos([io,] [Module,] [pattern::Regex]) + whos(io::IO=STDOUT, m::Module=current_module(), pattern::Regex=r"") Print information about exported global variables in a module, optionally restricted to those matching `pattern`. diff --git a/base/intfuncs.jl b/base/intfuncs.jl index a46a3046be5a8..8d508b2793bf3 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -2,6 +2,19 @@ ## number-theoretic functions ## +""" + gcd(x,y) + +Greatest common (positive) divisor (or zero if `x` and `y` are both zero). + +```jldoctest +julia> gcd(6,9) +3 + +julia> gcd(6,-9) +3 +``` +""" function gcd{T<:Integer}(a::T, b::T) while b != 0 t = b @@ -35,6 +48,19 @@ function gcd{T<:Union{Int64,UInt64,Int128,UInt128}}(a::T, b::T) end # explicit a==0 test is to handle case of lcm(0,0) correctly + +""" + lcm(x,y) + +Least common (non-negative) multiple. +```jldoctest +julia> lcm(2,3) +6 + +julia> lcm(-2,3) +6 +``` +""" lcm{T<:Integer}(a::T, b::T) = a == 0 ? a : checked_abs(a * div(b, gcd(b,a))) gcd(a::Integer) = a @@ -92,6 +118,25 @@ end gcdx(a::Integer, b::Integer) = gcdx(promote(a,b)...) # multiplicative inverse of n mod m, error if none + +""" + invmod(x,m) + +Take the inverse of `x` modulo `m`: `y` such that ``x y = 1 \\pmod m``, +with ``div(x,y) = 0``. This is undefined for ``m = 0``, or if +``gcd(x,m) \\neq 1``. + +```jldoctest +julia> invmod(2,5) +3 + +julia> invmod(2,3) +2 + +julia> invmod(5,6) +5 +``` +""" function invmod{T<:Integer}(n::T, m::T) g, x, y = gcdx(n, m) (g != 1 || m == 0) && throw(DomainError()) @@ -144,6 +189,12 @@ end ^(x, p::Integer) = power_by_squaring(x,p) # b^p mod m + +""" + powermod(x::Integer, p::Integer, m) + +Compute ``x^p \\pmod m``. +""" function powermod{T<:Integer}(x::Integer, p::Integer, m::T) p < 0 && return powermod(invmod(x, m), -p, m) p == 0 && return mod(one(m),m) @@ -169,12 +220,51 @@ end powermod(x::Integer, p::Integer, m::Union{Int128,UInt128}) = oftype(m, powermod(x, p, big(m))) # smallest power of 2 >= x + +""" + nextpow2(n::Integer) + +The smallest power of two not less than `n`. Returns 0 for `n==0`, and returns +`-nextpow2(-n)` for negative arguments. + +```jldoctest +julia> nextpow2(16) +16 + +julia> nextpow2(17) +32 +``` +""" nextpow2(x::Unsigned) = one(x)<<((sizeof(x)<<3)-leading_zeros(x-one(x))) nextpow2(x::Integer) = reinterpret(typeof(x),x < 0 ? -nextpow2(unsigned(-x)) : nextpow2(unsigned(x))) +""" + prevpow2(n::Integer) + +The largest power of two not greater than `n`. Returns 0 for `n==0`, and returns +`-prevpow2(-n)` for negative arguments. + +```jldoctest +julia> prevpow2(5) +4 +``` +""" prevpow2(x::Unsigned) = one(x) << unsigned((sizeof(x)<<3)-leading_zeros(x)-1) prevpow2(x::Integer) = reinterpret(typeof(x),x < 0 ? -prevpow2(unsigned(-x)) : prevpow2(unsigned(x))) +""" + ispow2(n::Integer) -> Bool + +Test whether `n` is a power of two. + +```jldoctest +julia> ispow2(4) +true + +julia> ispow2(5) +false +``` +""" ispow2(x::Integer) = x > 0 && count_ones(x) == 1 # smallest a^n >= x, with integer n @@ -256,6 +346,11 @@ ndigitsnb(x::Integer, b::Integer) = x==0 ? 1 : ndigits0znb(x, b) ndigits(x::Unsigned, b::Integer) = x==0 ? 1 : ndigits0z(x,Int(b)) ndigits(x::Unsigned) = x==0 ? 1 : ndigits0z(x) +""" + ndigits(n::Integer, b::Integer=10) + +Compute the number of digits in integer `n` written in base `b`. +""" ndigits(x::Integer, b::Integer) = b >= 0 ? ndigits(unsigned(abs(x)),Int(b)) : ndigitsnb(x, b) ndigits(x::Integer) = ndigits(unsigned(abs(x))) @@ -330,6 +425,21 @@ function base(b::Int, x::Unsigned, pad::Int, neg::Bool) if neg; a[1]='-'; end String(a) end + +""" + base(base::Integer, n::Integer, pad::Integer=1) + +Convert an integer `n` to a string in the given `base`, +optionally specifying a number of digits to pad to. + +```jldoctest +julia> base(13,5,4) +"0005" + +julia> base(5,13,4) +"0023" +``` +""" base(b::Integer, n::Integer, pad::Integer=1) = base(Int(b), unsigned(abs(n)), pad, n<0) for sym in (:bin, :oct, :dec, :hex) @@ -343,12 +453,55 @@ for sym in (:bin, :oct, :dec, :hex) end end +""" + bin(n, pad::Int=1) + +Convert an integer to a binary string, optionally specifying a number of digits to pad to. + +```jldoctest +julia> bin(10,2) +"1010" + +julia> bin(10,8) +"00001010" +``` +""" +bin + +""" + hex(n, pad::Int=1) + +Convert an integer to a hexadecimal string, optionally specifying a number of digits to pad to. +""" +hex + +""" + oct(n, pad::Int=1) + +Convert an integer to an octal string, optionally specifying a number of digits to pad to. +""" +oct + +""" + dec(n, pad::Int=1) + +Convert an integer to a decimal string, optionally specifying a number of digits to pad to. +""" +dec + bits(x::Union{Bool,Int8,UInt8}) = bin(reinterpret(UInt8,x),8) bits(x::Union{Int16,UInt16,Float16}) = bin(reinterpret(UInt16,x),16) bits(x::Union{Char,Int32,UInt32,Float32}) = bin(reinterpret(UInt32,x),32) bits(x::Union{Int64,UInt64,Float64}) = bin(reinterpret(UInt64,x),64) bits(x::Union{Int128,UInt128}) = bin(reinterpret(UInt128,x),128) +""" + digits([T<:Integer], n::Integer, base::T=10, pad::Integer=1) + +Returns an array with element type `T` (default `Int`) of the digits of `n` in the given +base, optionally padded with zeros to a specified size. More significant digits are at +higher indexes, such that `n == sum([digits[k]*base^(k-1) for k=1:length(digits)])`. +""" digits{T<:Integer}(n::Integer, base::T=10, pad::Integer=1) = digits(T, n, base, pad) function digits{T<:Integer}(::Type{T}, n::Integer, base::Integer=10, pad::Integer=1) @@ -356,6 +509,13 @@ function digits{T<:Integer}(::Type{T}, n::Integer, base::Integer=10, pad::Intege digits!(zeros(T, max(pad, ndigits0z(n,base))), n, base) end +""" + digits!(array, n::Integer, base::Integer=10) + +Fills an array of the digits of `n` in the given base. More significant digits are at higher +indexes. If the array length is insufficient, the least significant digits are filled up to +the array length. If the array length is excessive, the excess portion is filled with zeros. +""" function digits!{T<:Integer}(a::AbstractArray{T,1}, n::Integer, base::Integer=10) 2 <= base || throw(ArgumentError("base must be ≥ 2, got $base")) base - 1 <= typemax(T) || throw(ArgumentError("type $T too small for base $base")) @@ -366,6 +526,16 @@ function digits!{T<:Integer}(a::AbstractArray{T,1}, n::Integer, base::Integer=10 return a end +""" + isqrt(n::Integer) + +Integer square root: the largest integer `m` such that `m*m <= n`. + +```jldoctest +julia> isqrt(5) +2 +``` +""" isqrt(x::Integer) = oftype(x, trunc(sqrt(x))) function isqrt(x::Union{Int64,UInt64,Int128,UInt128}) @@ -387,6 +557,11 @@ function factorial(n::Integer) return f end +""" + binomial(n,k) + +Number of ways to choose `k` out of `n` items. +""" function binomial{T<:Integer}(n::T, k::T) k < 0 && return zero(T) sgn = one(T) diff --git a/base/intset.jl b/base/intset.jl index 89f862d90f9da..eeefab7a3b379 100644 --- a/base/intset.jl +++ b/base/intset.jl @@ -243,6 +243,12 @@ union(s1::IntSet) = copy(s1) union(s1::IntSet, s2::IntSet) = (s1.limit >= s2.limit ? union!(copy(s1), s2) : union!(copy(s2), s1)) union(s1::IntSet, ss::IntSet...) = union(s1, union(ss...)) +""" + intersect!(s1::IntSet, s2::IntSet) + +Intersects sets `s1` and `s2` and overwrites the set `s1` with the result. If needed, `s1` +will be expanded to the size of `s2`. +""" function intersect!(s::IntSet, s2::IntSet) if s2.limit > s.limit sizehint!(s, s2.limit) diff --git a/base/io.jl b/base/io.jl index 9d65ae5e5e1d4..6622ea84ce337 100644 --- a/base/io.jl +++ b/base/io.jl @@ -9,7 +9,21 @@ reseteof(x::IO) = nothing const SZ_UNBUFFERED_IO = 65536 buffer_writes(x::IO, bufsize=SZ_UNBUFFERED_IO) = nothing +""" + isopen(object) -> Bool + +Determine whether an object - such as a stream, timer, or mmap -- is not yet closed. Once an +object is closed, it will never produce a new event. However, a closed stream may still have +data to read in its buffer, use [`eof`](:func:`eof`) to check for the ability to read data. +Use [`poll_fd`](:func:`poll_fd`) to be notified when a stream might be writable or readable. +""" function isopen end + +""" + close(stream) + +Close an I/O stream. Performs a [`flush`](:func:`flush`) first. +""" function close end function flush end function wait_connected end @@ -18,7 +32,19 @@ function wait_readbyte end function wait_close end function nb_available end function readavailable end + +""" + isreadable(io) -> Bool + +Returns `true` if the specified IO object is readable (if that can be determined). +""" function isreadable end + +""" + iswritable(io) -> Bool + +Returns `true` if the specified IO object is writable (if that can be determined). +""" function iswritable end function copy end function eof end @@ -41,7 +67,7 @@ read(s::IO, ::Type{UInt8}) = error(typeof(s)," does not support byte I/O") write(s::IO, x::UInt8) = error(typeof(s)," does not support byte I/O") """ - unsafe_write(io, ref, nbytes) + unsafe_write(io::IO, ref, nbytes::UInt) Copy nbytes from ref (converted to a pointer) into the IO stream object. @@ -58,9 +84,9 @@ function unsafe_write(s::IO, p::Ptr{UInt8}, n::UInt) end """ - unsafe_read(io, ref, nbytes) + unsafe_read(io::IO, ref, nbytes::UInt) -Copy nbytes from the IO stream object into ref (converted to a pointer). +Copy nbytes from the `IO` stream object into `ref` (converted to a pointer). It is recommended that subtypes `T<:IO` override the following method signature to provide more efficient implementations: @@ -97,7 +123,23 @@ close(io::AbstractPipe) = (close(pipe_writer(io)); close(pipe_reader(io))) wait_readnb(io::AbstractPipe, nb::Int) = wait_readnb(pipe_reader(io), nb) wait_readbyte(io::AbstractPipe, byte::UInt8) = wait_readbyte(pipe_reader(io), byte) wait_close(io::AbstractPipe) = (wait_close(pipe_writer(io)); wait_close(pipe_reader(io))) + +""" + nb_available(stream) + +Returns the number of bytes available for reading before a read from this stream or buffer will block. +""" nb_available(io::AbstractPipe) = nb_available(pipe_reader(io)) + +""" + eof(stream) -> Bool + +Tests whether an I/O stream is at end-of-file. If the stream is not yet exhausted, this +function will block to wait for more data if necessary, and then return `false`. Therefore +it is always safe to read one byte after seeing `eof` return `false`. `eof` will return +`false` as long as buffered data is still available, even if the remote end of a connection +is closed. +""" eof(io::AbstractPipe) = eof(pipe_reader(io)) reseteof(io::AbstractPipe) = reseteof(pipe_reader(io)) @@ -106,11 +148,42 @@ reseteof(io::AbstractPipe) = reseteof(pipe_reader(io)) write(filename::AbstractString, args...) = open(io->write(io, args...), filename, "w") +""" + read(filename::AbstractString, args...) + +Open a file and read its contents. `args` is passed to `read`: this is equivalent to +`open(io->read(io, args...), filename)`. +""" read(filename::AbstractString, args...) = open(io->read(io, args...), filename) read!(filename::AbstractString, a) = open(io->read!(io, a), filename) readstring(filename::AbstractString) = open(readstring, filename) + +""" + readuntil(stream::IO, delim) + readuntil(filename::AbstractString, delim) + +Read a string from an I/O stream or a file, up to and including the given delimiter byte. +The text is assumed to be encoded in UTF-8. +""" readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename) + +""" + readline(stream::IO=STDIN) + readline(filename::AbstractString) + +Read a single line of text, including a trailing newline character (if one is reached before +the end of the input), from the given I/O stream or file (defaults to `STDIN`). +When reading from a file, the text is assumed to be encoded in UTF-8. +""" readline(filename::AbstractString) = open(readline, filename) + +""" + readlines(stream::IO) + readlines(filename::AbstractString) + +Read all lines of an I/O stream or a file as a vector of strings. +The text is assumed to be encoded in UTF-8. +""" readlines(filename::AbstractString) = open(readlines, filename) @@ -139,6 +212,41 @@ else error("seriously? what is this machine?") end + +""" + ntoh(x) + +Converts the endianness of a value from Network byte order (big-endian) to that used by the Host. +""" +ntoh(x) + +""" + hton(x) + +Converts the endianness of a value from that used by the Host to Network byte order (big-endian). +""" +hton(x) + +""" + ltoh(x) + +Converts the endianness of a value from Little-endian to that used by the Host. +""" +ltoh(x) + +""" + htol(x) + +Converts the endianness of a value from that used by the Host to Little-endian. +""" +htol(x) + + +""" + isreadonly(stream) -> Bool + +Determine whether a stream is read-only. +""" isreadonly(s) = isreadable(s) && !iswritable(s) ## binary I/O ## @@ -236,6 +344,13 @@ read{T}(s::IO, t::Type{T}, d1::Int, dims::Int...) = read(s, t, tuple(d1,dims...) read{T}(s::IO, t::Type{T}, d1::Integer, dims::Integer...) = read(s, t, convert(Tuple{Vararg{Int}},tuple(d1,dims...))) +""" + read(stream::IO, T, dims) + +Read a series of values of type `T` from `stream`, in canonical binary representation. +`dims` is either a tuple or a series of integer arguments specifying the size of the `Array{T}` +to return. +""" read{T}(s::IO, ::Type{T}, dims::Dims) = read!(s, Array{T}(dims)) @noinline function read!(s::IO, a::Array{UInt8}) # mark noinline to ensure the array is gc-rooted somewhere (by the caller) @@ -335,9 +450,24 @@ end readline() = readline(STDIN) readline(s::IO) = readuntil(s, '\n') + +""" + readchomp(x) + +Read the entirety of `x` as a string and remove a single trailing newline. +Equivalent to `chomp!(readstring(x))`. +""" readchomp(x) = chomp!(readstring(x)) # read up to nb bytes into nb, returning # bytes read + +""" + readbytes!(stream::IO, b::AbstractVector{UInt8}, nb=length(b)) + +Read at most `nb` bytes from `stream` into `b`, returning the number of bytes read. +The size of `b` will be increased if needed (i.e. if `nb` is greater than `length(b)` +and enough bytes could be read), but it will never be decreased. +""" function readbytes!(s::IO, b::AbstractArray{UInt8}, nb=length(b)) olb = lb = length(b) nr = 0 @@ -405,16 +535,38 @@ iteratorsize(::Type{EachLine}) = SizeUnknown() # the concrete IO type. This may not be true for IO types # not in base. +""" + mark(s) + +Add a mark at the current position of stream `s`. Returns the marked position. + +See also [`unmark`](:func:`unmark`), [`reset`](:func:`reset`), [`ismarked`](:func:`ismarked`). +""" function mark(io::IO) io.mark = position(io) end +""" + unmark(s) + +Remove a mark from stream `s`. Returns `true` if the stream was marked, `false` otherwise. + +See also [`mark`](:func:`mark`), [`reset`](:func:`reset`), [`ismarked`](:func:`ismarked`). +""" function unmark(io::IO) !ismarked(io) && return false io.mark = -1 return true end +""" + reset(s) + +Reset a stream `s` to a previously marked position, and remove the mark. Returns the +previously marked position. Throws an error if the stream is not marked. + +See also [`mark`](:func:`mark`), [`unmark`](:func:`unmark`), [`ismarked`](:func:`ismarked`). +""" function reset{T<:IO}(io::T) ismarked(io) || throw(ArgumentError("$(T) not marked")) m = io.mark @@ -423,8 +575,21 @@ function reset{T<:IO}(io::T) return m end +""" + ismarked(s) + +Returns `true` if stream `s` is marked. + +See also [`mark`](:func:`mark`), [`unmark`](:func:`unmark`), [`reset`](:func:`reset`). +""" ismarked(io::IO) = io.mark >= 0 # Make sure all IO streams support flush, even if only as a no-op, # to make it easier to write generic I/O code. + +""" + flush(stream) + +Commit all currently buffered writes to the given stream. +""" flush(io::IO) = nothing diff --git a/base/iobuffer.jl b/base/iobuffer.jl index e644a7e3af497..d55b2aa7e2b2a 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -23,13 +23,44 @@ AbstractIOBuffer{T<:AbstractVector{UInt8}}(data::T, readable::Bool, writable::Bo AbstractIOBuffer{T}(data, readable, writable, seekable, append, maxsize) # IOBuffers behave like Files. They are typically readable and writable. They are seekable. (They can be appendable). + +""" + IOBuffer([data,],[readable::Bool=true, writable::Bool=true, [maxsize::Int=typemax(Int)]]) + +Create an `IOBuffer`, which may optionally operate on a pre-existing array. If the +readable/writable arguments are given, they restrict whether or not the buffer may be read +from or written to respectively. By default the buffer is readable but not writable. The +last argument optionally specifies a size beyond which the buffer may not be grown. +""" IOBuffer(data::AbstractVector{UInt8}, readable::Bool=true, writable::Bool=false, maxsize::Int=typemax(Int)) = AbstractIOBuffer(data, readable, writable, true, false, maxsize) IOBuffer(readable::Bool, writable::Bool) = IOBuffer(UInt8[], readable, writable) + +""" + IOBuffer() -> IOBuffer + +Create an in-memory I/O stream. +""" IOBuffer() = IOBuffer(true, true) + +""" + IOBuffer(size::Int) + +Create a fixed size IOBuffer. The buffer will not grow dynamically. +""" IOBuffer(maxsize::Int) = (x=IOBuffer(Array{UInt8}(maxsize), true, true, maxsize); x.size=0; x) # PipeBuffers behave like Unix Pipes. They are typically readable and writable, they act appendable, and are not seekable. + +""" + PipeBuffer(data::Vector{UInt8}=UInt8[],[maxsize::Int=typemax(Int)]) + +An [`IOBuffer`](:obj:`IOBuffer`) that allows reading and performs writes by appending. +Seeking and truncating are not supported. +See [`IOBuffer`](:obj:`IOBuffer`) for the available constructors. +If `data` is given, creates a `PipeBuffer` to operate on a data vector, +optionally specifying a size beyond which the underlying `Array` may not be grown. +""" PipeBuffer(data::Vector{UInt8}=UInt8[], maxsize::Int=typemax(Int)) = AbstractIOBuffer(data,true,true,false,true,maxsize) PipeBuffer(maxsize::Int) = (x = PipeBuffer(Array{UInt8}(maxsize),maxsize); x.size=0; x) @@ -259,6 +290,13 @@ function takebuf_array(io::IOBuffer) end return data end + +""" + takebuf_string(b::IOBuffer) + +Obtain the contents of an `IOBuffer` as a string, without copying. +Afterwards, the `IOBuffer` is reset to its initial state. +""" takebuf_string(io::AbstractIOBuffer) = String(takebuf_array(io)) function write(to::AbstractIOBuffer, from::AbstractIOBuffer) diff --git a/base/iostream.jl b/base/iostream.jl index d355bd259c3a6..7c1cde05eaf2f 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -76,6 +76,14 @@ eof(s::IOStream) = ccall(:ios_eof_blocking, Cint, (Ptr{Void},), s.ios)!=0 ## constructing and opening streams ## # "own" means the descriptor will be closed with the IOStream + +""" + fdio([name::AbstractString, ]fd::Integer[, own::Bool=false]) -> IOStream + +Create an `IOStream` object from an integer file descriptor. If `own` is `true`, closing +this object will close the underlying descriptor. By default, an `IOStream` is closed when +it is garbage collected. `name` allows you to associate the descriptor with a named file. +""" function fdio(name::AbstractString, fd::Integer, own::Bool=false) s = IOStream(name) ccall(:ios_fd, Ptr{Void}, (Ptr{Void}, Clong, Cint, Cint), @@ -84,6 +92,13 @@ function fdio(name::AbstractString, fd::Integer, own::Bool=false) end fdio(fd::Integer, own::Bool=false) = fdio(string("<fd ",fd,">"), fd, own) + +""" + open(filename::AbstractString, [read::Bool, write::Bool, create::Bool, truncate::Bool, append::Bool]) -> IOStream + +Open a file in a mode specified by five boolean arguments. The default is to open files for +reading only. Returns a stream for accessing the file. +""" function open(fname::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool) s = IOStream(string("<file ",fname,">")) systemerror("opening file $fname", @@ -97,6 +112,22 @@ function open(fname::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff: end open(fname::AbstractString) = open(fname, true, false, false, false, false) +""" + open(filename::AbstractString, [mode::AbstractString]) -> IOStream + +Alternate syntax for open, where a string-based mode specifier is used instead of the five +booleans. The values of `mode` correspond to those from `fopen(3)` or Perl `open`, and are +equivalent to setting the following boolean groups: + +| Mode | Description | +|:-----|:------------------------------| +| r | read | +| r+ | read, write | +| w | write, create, truncate | +| w+ | read, write, create, truncate | +| a | write, create, append | +| a+ | read, write, create, append | +""" function open(fname::AbstractString, mode::AbstractString) mode == "r" ? open(fname, true , false, false, false, false) : mode == "r+" ? open(fname, true , true , false, false, false) : @@ -107,6 +138,14 @@ function open(fname::AbstractString, mode::AbstractString) throw(ArgumentError("invalid open mode: $mode")) end +""" + open(f::Function, args...) + +Apply the function `f` to the result of `open(args...)` and close the resulting file +descriptor upon completion. + +**Example**: `open(readstring, "file.txt")` +""" function open(f::Function, args...) io = open(args...) try @@ -226,6 +265,15 @@ function readbytes_some!(s::IOStream, b::Array{UInt8}, nb) return nr end +""" + readbytes!(stream::IOStream, b::AbstractVector{UInt8}, nb=length(b); all::Bool=true) + +Read at most `nb` bytes from `stream` into `b`, returning the number of bytes read. +The size of `b` will be increased if needed (i.e. if `nb` is greater than `length(b)` +and enough bytes could be read), but it will never be decreased. + +See [`read`](:func:`read`) for a description of the `all` option. +""" function readbytes!(s::IOStream, b::Array{UInt8}, nb=length(b); all::Bool=true) return all ? readbytes_all!(s, b, nb) : readbytes_some!(s, b, nb) end diff --git a/base/iterator.jl b/base/iterator.jl index 715ff753bb0fa..3491555911c86 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -226,6 +226,12 @@ immutable Rest{I,S} itr::I st::S end + +""" + rest(iter, state) + +An iterator that yields the same elements as `iter`, but starting at the given `state`. +""" rest(itr,state) = Rest(itr,state) start(i::Rest) = i.st @@ -244,6 +250,24 @@ iteratorsize{I,S}(::Type{Rest{I,S}}) = rest_iteratorsize(iteratorsize(I)) Returns `head`: the first `n` elements of `c`; and `tail`: an iterator over the remaining elements. + +```jldoctest +julia> a = 1:10 +1:10 + +julia> b, c = Base.head_and_tail(a, 3) +([1,2,3],Base.Rest{UnitRange{Int64},Int64}(1:10,4)) + +julia> collect(c) +7-element Array{Any,1}: + 4 + 5 + 6 + 7 + 8 + 9 + 10 +``` """ function head_and_tail(c, n) head = Vector{eltype(c)}(n) @@ -263,6 +287,12 @@ immutable Count{S<:Number} start::S step::S end + +""" + countfrom(start=1, step=1) + +An iterator that counts forever, starting at `start` and incrementing by `step`. +""" countfrom(start::Number, step::Number) = Count(promote(start, step)...) countfrom(start::Number) = Count(start, one(start)) countfrom() = Count(1, 1) @@ -390,6 +420,12 @@ done(it::Drop, state) = done(it.xs, state) immutable Cycle{I} xs::I end + +""" + cycle(iter) + +An iterator that cycles through `iter` forever. +""" cycle(xs) = Cycle(xs) eltype{I}(::Type{Cycle{I}}) = eltype(I) @@ -419,6 +455,24 @@ immutable Repeated{O} x::O end repeated(x) = Repeated(x) + +""" + repeated(x[, n::Int]) + +An iterator that generates the value `x` forever. If `n` is specified, generates `x` that +many times (equivalent to `take(repeated(x), n)`). + +```jldoctest +julia> a = repeated([1 2], 4); + +julia> collect(a) +4-element Array{Array{Int64,2},1}: + [1 2] + [1 2] + [1 2] + [1 2] +``` +""" repeated(x, n::Int) = take(repeated(x), n) eltype{O}(::Type{Repeated{O}}) = O @@ -497,14 +551,16 @@ Returns an iterator over the product of several iterators. Each generated elemen a tuple whose `i`th element comes from the `i`th argument iterator. The first iterator changes the fastest. Example: - julia> collect(product(1:2,3:5)) - 6-element Array{Tuple{Int64,Int64},1}: - (1,3) - (2,3) - (1,4) - (2,4) - (1,5) - (2,5) +```jldoctest +julia> collect(product(1:2,3:5)) +6-element Array{Tuple{Int64,Int64},1}: + (1,3) + (2,3) + (1,4) + (2,4) + (1,5) + (2,5) +``` """ product(a, b) = Prod2(a, b) @@ -576,12 +632,14 @@ Given an iterator that yields iterators, return an iterator that yields the elements of those iterators. Put differently, the elements of the argument iterator are concatenated. Example: - julia> collect(flatten((1:2, 8:9))) - 4-element Array{Int64,1}: - 1 - 2 - 8 - 9 +```jldoctest +julia> collect(flatten((1:2, 8:9))) +4-element Array{Int64,1}: + 1 + 2 + 8 + 9 +``` """ flatten(itr) = Flatten(itr) diff --git a/base/libc.jl b/base/libc.jl index 8020147d42ce1..c3c67b57e8b71 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -207,10 +207,20 @@ time() = ccall(:jl_clock_now, Float64, ()) ## process-related functions ## +""" + getpid() -> Int32 + +Get Julia's process ID. +""" getpid() = ccall(:jl_getpid, Int32, ()) ## network functions ## +""" + gethostname() -> AbstractString + +Get the local machine's host name. +""" function gethostname() hn = Array{UInt8}(256) err = @static if is_windows() diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index 47f74e3b6666d..6957a1e579aa3 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -31,9 +31,21 @@ end #Test whether a matrix is positive-definite isposdef!{T<:BlasFloat}(A::StridedMatrix{T}, UL::Symbol) = LAPACK.potrf!(char_uplo(UL), A)[2] == 0 + +""" + isposdef!(A) -> Bool + +Test whether a matrix is positive definite, overwriting `A` in the processes. +""" isposdef!(A::StridedMatrix) = ishermitian(A) && isposdef!(A, :U) isposdef{T}(A::AbstractMatrix{T}, UL::Symbol) = (S = typeof(sqrt(one(T))); isposdef!(S == T ? copy(A) : convert(AbstractMatrix{S}, A), UL)) + +""" + isposdef(A) -> Bool + +Test whether a matrix is positive definite. +""" isposdef{T}(A::AbstractMatrix{T}) = (S = typeof(sqrt(one(T))); isposdef!(S == T ? copy(A) : convert(AbstractMatrix{S}, A))) isposdef(x::Number) = imag(x)==0 && real(x) > 0 @@ -53,6 +65,12 @@ vecnorm1{T<:BlasReal}(x::Union{Array{T},StridedVector{T}}) = vecnorm2{T<:BlasFloat}(x::Union{Array{T},StridedVector{T}}) = length(x) < NRM2_CUTOFF ? generic_vecnorm2(x) : BLAS.nrm2(x) +""" + triu!(M, k::Integer) + +Returns the upper triangle of `M` starting from the `k`th superdiagonal, +overwriting `M` in the process. +""" function triu!(M::AbstractMatrix, k::Integer) m, n = size(M) if (k > 0 && k > n) || (k < 0 && -k > m) @@ -71,6 +89,12 @@ end triu(M::Matrix, k::Integer) = triu!(copy(M), k) +""" + tril!(M, k::Integer) + +Returns the lower triangle of `M` starting from the `k`th superdiagonal, overwriting `M` in +the process. +""" function tril!(M::AbstractMatrix, k::Integer) m, n = size(M) if (k > 0 && k > n) || (k < 0 && -k > m) @@ -86,7 +110,6 @@ function tril!(M::AbstractMatrix, k::Integer) end M end - tril(M::Matrix, k::Integer) = tril!(copy(M), k) function gradient(F::Vector, h::Vector) @@ -113,10 +136,53 @@ function diagind(m::Integer, n::Integer, k::Integer=0) k <= 0 ? range(1-k, m+1, min(m+k, n)) : range(k*m+1, m+1, min(m, n-k)) end +""" + diagind(M, k::Integer=0) + +A [`Range`](:class:`Range`) giving the indices of the `k`th diagonal of the matrix `M`. + +```jldoctest +julia> A = [1 2 3; 4 5 6; 7 8 9] +3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + +julia> diagind(A,-1) +2:4:6 +``` +""" diagind(A::AbstractMatrix, k::Integer=0) = diagind(size(A,1), size(A,2), k) +""" + diag(M, k::Integer=0) + +The `k`th diagonal of a matrix, as a vector. +Use [`diagm`](:func:`diagm`) to construct a diagonal matrix. + +```jldoctest +julia> diag(A,1) +2-element Array{Int64,1}: + 2 + 6 +``` +""" diag(A::AbstractMatrix, k::Integer=0) = A[diagind(A,k)] +""" + diagm(v, k::Integer=0) + +Construct a diagonal matrix and place `v` on the `k`th diagonal. + +```jldoctest +julia> diagm([1,2,3],1) +4×4 Array{Int64,2}: + 0 1 0 0 + 0 0 2 0 + 0 0 0 3 + 0 0 0 0 +``` +""" function diagm{T}(v::AbstractVector{T}, k::Integer=0) n = length(v) + abs(k) A = zeros(T,n,n) @@ -135,6 +201,11 @@ function trace{T}(A::Matrix{T}) t end +""" + kron(A, B) + +Kronecker tensor product of two vectors or two matrices. +""" function kron{T,S}(a::AbstractMatrix{T}, b::AbstractMatrix{S}) R = Array{promote_type(T,S)}(size(a,1)*size(b,1), size(a,2)*size(b,2)) m = 1 @@ -201,6 +272,22 @@ function ^{T}(A::AbstractMatrix{T}, p::Real) end # Matrix exponential + +""" + expm(A) + +Compute the matrix exponential of `A`, defined by + +```math +e^A = \\sum_{n=0}^{\\infty} \\frac{A^n}{n!}. +``` + +For symmetric or Hermitian `A`, an eigendecomposition ([`eigfact`](:func:`eigfact`)) is +used, otherwise the scaling and squaring algorithm (see [^H05]) is chosen. + +[^H05]: Nicholas J. Higham, "The squaring and scaling method for the matrix exponential revisited", SIAM Journal on Matrix Analysis and Applications, 26(4), 2005, 1179-1193. [doi:10.1137/090768539](http://dx.doi.org/10.1137/090768539) + +""" expm{T<:BlasFloat}(A::StridedMatrix{T}) = expm!(copy(A)) expm{T<:Integer}(A::StridedMatrix{T}) = expm!(float(A)) expm(x::Number) = exp(x) @@ -511,6 +598,33 @@ function factorize{T}(A::StridedMatrix{T}) end ## Moore-Penrose pseudoinverse + +""" + pinv(M[, tol::Real]) + +Computes the Moore-Penrose pseudoinverse. + +For matrices `M` with floating point elements, it is convenient to compute +the pseudoinverse by inverting only singular values above a given threshold, +`tol`. + +The optimal choice of `tol` varies both with the value of `M` and the intended application +of the pseudoinverse. The default value of `tol` is +`eps(real(float(one(eltype(M)))))*maximum(size(A))`, which is essentially machine epsilon +for the real part of a matrix element multiplied by the larger matrix dimension. For +inverting dense ill-conditioned matrices in a least-squares sense, +`tol = sqrt(eps(real(float(one(eltype(M))))))` is recommended. + +For more information, see [^issue8859], [^B96], [^S84], [^KY88]. + +[^issue8859]: Issue 8859, "Fix least squares", https://github.com/JuliaLang/julia/pull/8859 + +[^B96]: Åke Björck, "Numerical Methods for Least Squares Problems", SIAM Press, Philadelphia, 1996, "Other Titles in Applied Mathematics", Vol. 51. [doi:10.1137/1.9781611971484](http://epubs.siam.org/doi/book/10.1137/1.9781611971484) + +[^S84]: G. W. Stewart, "Rank Degeneracy", SIAM Journal on Scientific and Statistical Computing, 5(2), 1984, 403-413. [doi:10.1137/0905030](http://epubs.siam.org/doi/abs/10.1137/0905030) + +[^KY88]: Konstantinos Konstantinides and Kung Yao, "Statistical analysis of effective singular values in matrix rank determination", IEEE Transactions on Acoustics, Speech and Signal Processing, 36(5), 1988, 757-763. [doi:10.1109/29.1585](http://dx.doi.org/10.1109/29.1585) +""" function pinv{T}(A::StridedMatrix{T}, tol::Real) m, n = size(A) Tout = typeof(zero(T)/sqrt(one(T) + one(T))) @@ -551,6 +665,12 @@ function pinv(x::Number) end ## Basis for null space + +""" + nullspace(M) + +Basis for nullspace of `M`. +""" function nullspace{T}(A::StridedMatrix{T}) m, n = size(A) (m == 0 || n == 0) && return eye(T, n) @@ -560,6 +680,12 @@ function nullspace{T}(A::StridedMatrix{T}) end nullspace(a::StridedVector) = nullspace(reshape(a, length(a), 1)) +""" + cond(M, p::Real=2) + +Condition number of the matrix `M`, computed using the operator `p`-norm. Valid values for +`p` are `1`, `2` (default), or `Inf`. +""" function cond(A::AbstractMatrix, p::Real=2) if p == 2 v = svdvals(A) @@ -575,6 +701,13 @@ end ## Lyapunov and Sylvester equation # AX + XB + C = 0 + +""" + sylvester(A, B, C) + +Computes the solution `X` to the Sylvester equation `AX + XB + C = 0`, where `A`, `B` and +`C` have compatible dimensions and `A` and `-B` have no eigenvalues with equal real part. +""" function sylvester{T<:BlasFloat}(A::StridedMatrix{T},B::StridedMatrix{T},C::StridedMatrix{T}) RA, QA = schur(A) RB, QB = schur(B) @@ -586,6 +719,14 @@ end sylvester{T<:Integer}(A::StridedMatrix{T},B::StridedMatrix{T},C::StridedMatrix{T}) = sylvester(float(A), float(B), float(C)) # AX + XA' + C = 0 + +""" + lyap(A, C) + +Computes the solution `X` to the continuous Lyapunov equation `AX + XA' + C = 0`, where no +eigenvalue of `A` has a zero real part and no two eigenvalues are negative complex +conjugates of each other. +""" function lyap{T<:BlasFloat}(A::StridedMatrix{T},C::StridedMatrix{T}) R, Q = schur(A) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 261bed72b8c39..ff77e554fbe92 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -41,22 +41,219 @@ end scale!(C::AbstractArray, s::Number, X::AbstractArray) = generic_scale!(C, X, s) scale!(C::AbstractArray, X::AbstractArray, s::Number) = generic_scale!(C, s, X) + +""" + scale!(A, b) + scale!(b, A) + +Scale an array `A` by a scalar `b` overwriting `A` in-place. + +If `A` is a matrix and `b` is a vector, then `scale!(A,b)` scales each column `i` of `A` by +`b[i]` (similar to `A*Diagonal(b)`), while `scale!(b,A)` scales each row `i` of `A` by `b[i]` +(similar to `Diagonal(b)*A`), again operating in-place on `A`. An `InexactError` exception is +thrown if the scaling produces a number not representable by the element type of `A`, +e.g. for integer types. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> b = [1; 2] +2-element Array{Int64,1}: + 1 + 2 + +julia> scale!(a,b) +2×2 Array{Int64,2}: + 1 4 + 3 8 + +julia> a = [1 2; 3 4]; + +julia> b = [1; 2]; + +julia> scale!(b,a) +2×2 Array{Int64,2}: + 1 2 + 6 8 +``` +""" scale!(X::AbstractArray, s::Number) = generic_scale!(X, s) scale!(s::Number, X::AbstractArray) = generic_scale!(s, X) +""" + cross(x, y) + ×(x,y) + +Compute the cross product of two 3-vectors. + +```jldoctest +julia> a = [0;1;0] +3-element Array{Int64,1}: + 0 + 1 + 0 + +julia> b = [0;0;1] +3-element Array{Int64,1}: + 0 + 0 + 1 + +julia> cross(a,b) +3-element Array{Int64,1}: + 1 + 0 + 0 +``` +""" cross(a::AbstractVector, b::AbstractVector) = [a[2]*b[3]-a[3]*b[2], a[3]*b[1]-a[1]*b[3], a[1]*b[2]-a[2]*b[1]] +""" + triu(M) + +Upper triangle of a matrix. + +```jldoctest +julia> a = ones(4,4) +4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + +julia> triu(a) +4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 0.0 1.0 1.0 1.0 + 0.0 0.0 1.0 1.0 + 0.0 0.0 0.0 1.0 +``` +""" triu(M::AbstractMatrix) = triu!(copy(M)) + +""" + tril(M) + +Lower triangle of a matrix. + +```jldoctest +julia> a = ones(4,4) +4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + +julia> tril(a) +4×4 Array{Float64,2}: + 1.0 0.0 0.0 0.0 + 1.0 1.0 0.0 0.0 + 1.0 1.0 1.0 0.0 + 1.0 1.0 1.0 1.0 +``` +""" tril(M::AbstractMatrix) = tril!(copy(M)) + +""" + triu(M, k::Integer) + +Returns the upper triangle of `M` starting from the `k`th superdiagonal. + +```jldoctest +julia> a = ones(4,4) +4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + +julia> triu(a,3) +4×4 Array{Float64,2}: + 0.0 0.0 0.0 1.0 + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + +julia> triu(a,-3) +4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 +``` +""" triu(M::AbstractMatrix,k::Integer) = triu!(copy(M),k) + +""" + tril(M, k::Integer) + +Returns the lower triangle of `M` starting from the `k`th superdiagonal. + +```jldoctest +julia> a = ones(4,4) +4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + +julia> tril(a,3) +4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + +julia> tril(a,-3) +4×4 Array{Float64,2}: + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + 1.0 0.0 0.0 0.0 +``` +""" tril(M::AbstractMatrix,k::Integer) = tril!(copy(M),k) + +""" + triu!(M) + +Upper triangle of a matrix, overwriting `M` in the process. +See also [`triu`](:func:`triu`). +""" triu!(M::AbstractMatrix) = triu!(M,0) + +""" + tril!(M) + +Lower triangle of a matrix, overwriting `M` in the process. +See also [`tril`](:func:`tril`). +""" tril!(M::AbstractMatrix) = tril!(M,0) -diff(a::AbstractMatrix) = diff(a, 1) diff(a::AbstractVector) = [ a[i+1] - a[i] for i=1:length(a)-1 ] -function diff(A::AbstractMatrix, dim::Integer) +""" + diff(A, [dim::Integer=1]) + +Finite difference operator of matrix or vector `A`. If `A` is a matrix, +compute the finite difference over a dimension `dim` (default `1`). + +```jldoctest +julia> a = [2 4; 6 16] +2×2 Array{Int64,2}: + 2 4 + 6 16 + +julia> diff(a,2) +2×1 Array{Int64,2}: + 2 + 10 +``` +""" +function diff(A::AbstractMatrix, dim::Integer=1) if dim == 1 [A[i+1,j] - A[i,j] for i=1:size(A,1)-1, j=1:size(A,2)] elseif dim == 2 @@ -68,6 +265,24 @@ end gradient(F::AbstractVector) = gradient(F, [1:length(F);]) + +""" + gradient(F::AbstractVector, [h::Real]) + +Compute differences along vector `F`, using `h` as the spacing between points. The default +spacing is one. + +```jldoctest +julia> a = [2,4,6,8]; + +julia> gradient(a) +4-element Array{Float64,1}: + 2.0 + 2.0 + 2.0 + 2.0 +``` +""" gradient(F::AbstractVector, h::Real) = gradient(F, [h*(1:length(F));]) diag(A::AbstractVector) = throw(ArgumentError("use diagm instead of diag to construct a diagonal matrix")) @@ -179,6 +394,15 @@ vecnorm1(x) = generic_vecnorm1(x) vecnorm2(x) = generic_vecnorm2(x) vecnormp(x, p) = generic_vecnormp(x, p) +""" + vecnorm(A, [p::Real=2]) + +For any iterable container `A` (including arrays of any dimension) of numbers (or any +element type for which `norm` is defined), compute the `p`-norm (defaulting to `p=2`) as if +`A` were a vector of the corresponding length. + +For example, if `A` is a matrix and `p=2`, then this is equivalent to the Frobenius norm. +""" function vecnorm(itr, p::Real=2) isempty(itr) && return float(real(zero(eltype(itr)))) if p == 2 @@ -238,6 +462,20 @@ function normInf{T}(A::AbstractMatrix{T}) end return convert(Tnorm, nrm) end + +""" + norm(A, [p::Real=2]) + +Compute the `p`-norm of a vector or the operator norm of a matrix `A`, defaulting to the `p=2`-norm. + +For vectors, `p` can assume any numeric value (even though not all values produce a +mathematically valid vector norm). In particular, `norm(A, Inf)` returns the largest value +in `abs(A)`, whereas `norm(A, -Inf)` returns the smallest. + +For matrices, the matrix norm induced by the vector `p`-norm is used, where valid values of +`p` are `1`, `2`, or `Inf`. (Note that for sparse matrices, `p=2` is currently not +implemented.) Use [`vecnorm`](:func:`vecnorm`) to compute the Frobenius norm. +""" function norm{T}(A::AbstractMatrix{T}, p::Real=2) if p == 2 return norm2(A) @@ -267,6 +505,13 @@ function vecdot(x::AbstractArray, y::AbstractArray) s end +""" + vecdot(x, y) + +For any iterable containers `x` and `y` (including arrays of any dimension) of numbers (or +any element type for which `dot` is defined), compute the Euclidean dot product (the sum of +`dot(x[i],y[i])`) as if they were vectors. +""" function vecdot(x, y) # arbitrary iterables ix = start(x) if done(x, ix) @@ -299,10 +544,22 @@ end vecdot(x::Number, y::Number) = conj(x) * y dot(x::Number, y::Number) = vecdot(x, y) + +""" + dot(x, y) + ⋅(x,y) + +Compute the dot product. For complex vectors, the first vector is conjugated. +""" dot(x::AbstractVector, y::AbstractVector) = vecdot(x, y) ########################################################################################### +""" + rank(M[, tol::Real]) + +Compute the rank of a matrix. +""" rank(A::AbstractMatrix, tol::Real) = sum(svdvals(A) .> tol) function rank(A::AbstractMatrix) m,n = size(A) @@ -312,6 +569,11 @@ function rank(A::AbstractMatrix) end rank(x::Number) = x==0 ? 0 : 1 +""" + trace(M) + +Matrix trace. +""" function trace(A::AbstractMatrix) checksquare(A) sum(diag(A)) @@ -324,6 +586,12 @@ trace(x::Number) = x #det(a::AbstractMatrix) inv(a::StridedMatrix) = throw(ArgumentError("argument must be a square matrix")) + +""" + inv(M) + +Matrix inverse. +""" function inv{T}(A::AbstractMatrix{T}) S = typeof(zero(T)/one(T)) A_ldiv_B!(factorize(convert(AbstractMatrix{S}, A)), eye(S, checksquare(A))) @@ -374,9 +642,48 @@ cond(x::Number, p) = cond(x) #Skeel condition numbers condskeel(A::AbstractMatrix, p::Real=Inf) = norm(abs(inv(A))*abs(A), p) condskeel{T<:Integer}(A::AbstractMatrix{T}, p::Real=Inf) = norm(abs(inv(float(A)))*abs(A), p) + +""" + condskeel(M, [x, p::Real=Inf]) + +```math +\\kappa_S(M, p) & = \\left\\Vert \\left\\vert M \\right\\vert \\left\\vert M^{-1} \\right\\vert \\right\\Vert_p \\\\ +\\kappa_S(M, x, p) & = \\left\\Vert \\left\\vert M \\right\\vert \\left\\vert M^{-1} \\right\\vert \\left\\vert x \\right\\vert \\right\\Vert_p +``` + +Skeel condition number ``\\kappa_S`` of the matrix `M`, optionally with respect to the +vector `x`, as computed using the operator `p`-norm. +`p` is `Inf` by default, if not provided. Valid values for `p` are `1`, `2`, or `Inf`. + +This quantity is also known in the literature as the Bauer condition number, relative +condition number, or componentwise relative condition number. +""" condskeel(A::AbstractMatrix, x::AbstractVector, p::Real=Inf) = norm(abs(inv(A))*abs(A)*abs(x), p) condskeel{T<:Integer}(A::AbstractMatrix{T}, x::AbstractVector, p::Real=Inf) = norm(abs(inv(float(A)))*abs(A)*abs(x), p) +""" + issymmetric(A) -> Bool + +Test whether a matrix is symmetric. + +```jldoctest +julia> a = [1 2; 2 -1] +2×2 Array{Int64,2}: + 1 2 + 2 -1 + +julia> issymmetric(a) +true + +julia> b = [1 im; -im 1] +2×2 Array{Complex{Int64},2}: + 1+0im 0+1im + 0-1im 1+0im + +julia> issymmetric(b) +false +``` +""" function issymmetric(A::AbstractMatrix) indsm, indsn = indices(A) if indsm != indsn @@ -392,6 +699,29 @@ end issymmetric(x::Number) = true +""" + ishermitian(A) -> Bool + +Test whether a matrix is Hermitian. + +```jldoctest +julia> a = [1 2; 2 -1] +2×2 Array{Int64,2}: + 1 2 + 2 -1 + +julia> ishermitian(a) +true + +julia> b = [1 im; -im 1] +2×2 Array{Complex{Int64},2}: + 1+0im 0+1im + 0-1im 1+0im + +julia> ishermitian(b) +true +``` +""" function ishermitian(A::AbstractMatrix) indsm, indsn = indices(A) if indsm != indsn @@ -407,6 +737,29 @@ end ishermitian(x::Number) = (x == conj(x)) +""" + istriu(A) -> Bool + +Test whether a matrix is upper triangular. + +```jldoctest +julia> a = [1 2; 2 -1] +2×2 Array{Int64,2}: + 1 2 + 2 -1 + +julia> istriu(a) +false + +julia> b = [1 im; 0 -1] +2×2 Array{Complex{Int64},2}: + 1+0im 0+1im + 0+0im -1+0im + +julia> istriu(b) +true +``` +""" function istriu(A::AbstractMatrix) m, n = size(A) for j = 1:min(n,m-1), i = j+1:m @@ -417,6 +770,29 @@ function istriu(A::AbstractMatrix) return true end +""" + istril(A) -> Bool + +Test whether a matrix is lower triangular. + +```jldoctest +julia> a = [1 2; 2 -1] +2×2 Array{Int64,2}: + 1 2 + 2 -1 + +julia> istril(a) +false + +julia> b = [1 0; -im -1] +2×2 Array{Complex{Int64},2}: + 1+0im 0+0im + 0-1im -1+0im + +julia> istril(b) +true +``` +""" function istril(A::AbstractMatrix) m, n = size(A) for j = 2:n, i = 1:min(j-1,m) @@ -427,6 +803,29 @@ function istril(A::AbstractMatrix) return true end +""" + isdiag(A) -> Bool + +Test whether a matrix is diagonal. + +```jldoctest +julia> a = [1 2; 2 -1] +2×2 Array{Int64,2}: + 1 2 + 2 -1 + +julia> isdiag(a) +false + +julia> b = [im 0; 0 -im] +2×2 Array{Complex{Int64},2}: + 0+1im 0+0im + 0+0im 0-1im + +julia> isdiag(b) +true +``` +""" isdiag(A::AbstractMatrix) = istril(A) && istriu(A) istriu(x::Number) = true @@ -440,7 +839,7 @@ Perform simple linear regression using Ordinary Least Squares. Returns `a` and ` that `a + b*x` is the closest straight line to the given points `(x, y)`, i.e., such that the squared error between `y` and `a + b*x` is minimized. -Examples: +**Examples:** using PyPlot x = 1.0:12.0 @@ -451,7 +850,7 @@ Examples: See also: -`\\`, `cov`, `std`, `mean` +`\\`, [`cov`](:func:`cov`), [`std`](:func:`std`), [`mean`](:func:`mean`). """ function linreg(x::AbstractVector, y::AbstractVector) @@ -485,6 +884,20 @@ scale!(b::AbstractVector, A::AbstractMatrix) = scale!(A,b,A) #findmax(a::AbstractArray) #findmin(a::AbstractArray) +""" + peakflops(n::Integer=2000; parallel::Bool=false) + +`peakflops` computes the peak flop rate of the computer by using double precision +[`gemm!`](:func:`Base.LinAlg.BLAS.gemm!`). By default, if no arguments are specified, it +multiplies a matrix of size `n x n`, where `n = 2000`. If the underlying BLAS is using +multiple threads, higher flop rates are realized. The number of BLAS threads can be set with +[`BLAS.set_num_threads(n)`](:func:`Base.LinAlg.BLAS.set_num_threads`). + +If the keyword argument `parallel` is set to `true`, `peakflops` is run in parallel on all +the worker processors. The flop rate of the entire parallel computer is returned. When +running in parallel, only 1 BLAS thread is used. The argument `n` still refers to the size +of the problem that is solved on each processor. +""" function peakflops(n::Integer=2000; parallel::Bool=false) a = ones(Float64,100,100) t = @elapsed a2 = a*a @@ -570,6 +983,11 @@ end return A end +""" + det(M) + +Matrix determinant. +""" function det{T}(A::AbstractMatrix{T}) if istriu(A) || istril(A) S = typeof((one(T)*zero(T) + zero(T))/one(T)) @@ -579,7 +997,20 @@ function det{T}(A::AbstractMatrix{T}) end det(x::Number) = x +""" + logabsdet(M) + +Log of absolute value of determinant of real matrix. Equivalent to `(log(abs(det(M))), sign(det(M)))`, +but may provide increased accuracy and/or speed. +""" logabsdet(A::AbstractMatrix) = logabsdet(lufact(A)) + +""" + logdet(M) + +Log of matrix determinant. Equivalent to `log(det(M))`, but may provide increased accuracy +and/or speed. +""" function logdet(A::AbstractMatrix) d,s = logabsdet(A) return d + log(s) @@ -597,24 +1028,10 @@ function isapprox{T<:Number,S<:Number}(x::AbstractArray{T}, y::AbstractArray{S}; end """ - normalize!(v, [p=2]) + normalize!(v, [p::Real=2]) Normalize the vector `v` in-place with respect to the `p`-norm. - -Inputs: - -- `v::AbstractVector` - vector to be normalized -- `p::Real` - The `p`-norm to normalize with respect to. Default: 2 - -Output: - -- `v` - A unit vector being the input vector, rescaled to have norm 1. - The input vector is modified in-place. - -See also: - -`normalize`, `qr` - +See also [`vecnorm`](:func:`vecnorm`) and [`normalize`](:func:`normalize`). """ function normalize!(v::AbstractVector, p::Real=2) nrm = norm(v, p) @@ -640,22 +1057,26 @@ end end """ - normalize(v, [p=2]) + normalize(v, [p::Real=2]) Normalize the vector `v` with respect to the `p`-norm. - -Inputs: - -- `v::AbstractVector` - vector to be normalized -- `p::Real` - The `p`-norm to normalize with respect to. Default: 2 - -Output: - -- `v` - A unit vector being a copy of the input vector, scaled to have norm 1 - -See also: - -`normalize!`, `qr` +See also [`normalize!`](:func:`normalize!`) and [`vecnorm`](:func:`vecnorm`). + +```jldoctest +julia> a = [1,2,4]; + +julia> normalize(a) +3-element Array{Float64,1}: + 0.218218 + 0.436436 + 0.872872 + +julia> normalize(a,1) +3-element Array{Float64,1}: + 0.142857 + 0.285714 + 0.571429 +``` """ function normalize(v::AbstractVector, p::Real = 2) nrm = norm(v, p) diff --git a/base/linalg/matmul.jl b/base/linalg/matmul.jl index aa713321a42bb..8e05724dbbfa2 100644 --- a/base/linalg/matmul.jl +++ b/base/linalg/matmul.jl @@ -124,6 +124,13 @@ Ac_mul_B!(y::AbstractVector, A::AbstractVecOrMat, x::AbstractVector) = generic_m # Matrix-matrix multiplication +""" +``` +*(A::AbstractMatrix, B::AbstractMatrix) +``` + +Matrix multiplication. +""" function (*){T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_B!(similar(B, TS, (size(A,1), size(B,2))), A, B) @@ -139,6 +146,23 @@ for elty in (Float32,Float64) end end end + +""" + A_mul_B!(Y, A, B) -> Y + +Calculates the matrix-matrix or matrix-vector product ``A⋅B`` and stores the result in `Y`, +overwriting the existing value of `Y`. Note that `Y` must not be aliased with either `A` or +`B`. + +```jldoctest +julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; Y = similar(B); A_mul_B!(Y, A, B); + +julia> Y +2×2 Array{Float64,2}: + 3.0 3.0 + 7.0 7.0 +``` +""" A_mul_B!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'N', 'N', A, B) function At_mul_B{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) diff --git a/base/loading.jl b/base/loading.jl index ddca6283d848f..087d73e2e2d40 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -252,6 +252,17 @@ function _include_dependency(_path::AbstractString) end return path, prev end + +""" + include_dependency(path::AbstractString) + +In a module, declare that the file specified by `path` (relative or absolute) is a +dependency for precompilation; that is, the module will need to be recompiled if this file +changes. + +This is only needed if your module depends on a file that is not used via `include`. It has +no effect outside of compilation. +""" function include_dependency(path::AbstractString) _include_dependency(path) return nothing @@ -339,6 +350,26 @@ end # require always works in Main scope and loads files from node 1 toplevel_load = true + +""" + require(module::Symbol) + +This function is part of the implementation of `using` / `import`, if a module is not +already defined in `Main`. It can also be called directly to force reloading a module, +regardless of whether it has been loaded before (for example, when interactively developing +libraries). + +Loads a source files, in the context of the `Main` module, on every active node, searching +standard locations for files. `require` is considered a top-level operation, so it sets the +current `include` path but does not use it to search for files (see help for `include`). +This function is typically used to load library code, and is implicitly called by `using` to +load packages. + +When searching for files, `require` first looks for package code under `Pkg.dir()`, +then tries paths in the global array `LOAD_PATH`. `require` is case-sensitive on +all platforms, including those with case-insensitive filesystems like macOS and +Windows. +""" function require(mod::Symbol) # dependency-tracking is only used for one top-level include(path), # and is not applied recursively to imported modules: @@ -431,6 +462,12 @@ end # remote/parallel load +""" + include_string(code::AbstractString, filename::AbstractString="string") + +Like `include`, except reads code from the given string rather than from a file. Since there +is no file path involved, no path processing or fetching from node 1 is done. +""" include_string(txt::String, fname::String) = ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Cstring), txt, sizeof(txt), fname) @@ -493,6 +530,12 @@ function include_from_node1(_path::String) result end +""" + evalfile(path::AbstractString, args::Vector{String}=String[]) + +Load the file using [`include`](:func:`include`), evaluate all expressions, +and return the value of the last one. +""" function evalfile(path::AbstractString, args::Vector{String}=String[]) return eval(Module(:__anon__), Expr(:toplevel, @@ -549,6 +592,16 @@ function create_expr_cache(input::String, output::String, concrete_deps::Vector{ end compilecache(mod::Symbol) = compilecache(string(mod)) + +""" + Base.compilecache(module::String) + +Creates a precompiled cache file for module (see help for [`require`](:func:`require`)) and all of its +dependencies. This can be used to reduce package load times. Cache files are stored in +`LOAD_CACHE_PATH[1]`, which defaults to `~/.julia/lib/VERSION`. See +[Module initialization and precompilation](:ref:`Module initialization and precompilation <man-modules-initialization-precompilation>`) +for important notes. +""" function compilecache(name::String) myid() == 1 || error("can only precompile from node 1") # decide where to get the source file from diff --git a/base/math.jl b/base/math.jl index 6c96f5ab67319..37d194ef546fc 100644 --- a/base/math.jl +++ b/base/math.jl @@ -87,6 +87,15 @@ end # Horner's method if z is real, but for complex z it uses a more # efficient algorithm described in Knuth, TAOCP vol. 2, section 4.6.4, # equation (3). + +""" + @evalpoly(z, c...) + +Evaluate the polynomial ``\\sum_k c[k] z^{k-1}`` for the coefficients `c[1]`, `c[2]`, ...; +that is, the coefficients are given in ascending order by power of `z`. This macro expands +to efficient inline code that uses either Horner's method or, for complex `z`, a more +efficient Goertzel-like algorithm. +""" macro evalpoly(z, p...) a = :($(esc(p[end]))) b = :($(esc(p[end-1]))) @@ -112,7 +121,28 @@ macro evalpoly(z, p...) end) end +""" + rad2deg(x) + +Convert `x` from radians to degrees. + +```jldoctest +julia> rad2deg(pi) +180.0 +``` +""" rad2deg(z::AbstractFloat) = z * (180 / oftype(z, pi)) + +""" + deg2rad(x) + +Convert `x` from degrees to radians. + +```jldoctest +julia> deg2rad(90) +1.5707963267948966 +``` +""" deg2rad(z::AbstractFloat) = z * (oftype(z, pi) / 180) rad2deg(z::Real) = rad2deg(float(z)) deg2rad(z::Real) = deg2rad(float(z)) @@ -120,6 +150,20 @@ deg2rad(z::Real) = deg2rad(float(z)) @vectorize_1arg Real deg2rad log{T<:Number}(b::T, x::T) = log(x)/log(b) + +""" + log(b,x) + +Compute the base `b` logarithm of `x`. Throws `DomainError` for negative `Real` arguments. + +```jldoctest +julia> log(4,8) +1.5 + +julia> log(4,2) +0.5 +``` +""" log(b::Number, x::Number) = log(promote(b,x)...) @vectorize_2arg Number log @@ -139,7 +183,29 @@ for f in (:cbrt, :sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :exp2, : end # fallback definitions to prevent infinite loop from $f(x::Real) def above + +""" + cbrt(x) + +Return ``x^{1/3}``. The prefix operator `∛` is equivalent to `cbrt`. + +```jldoctest +julia> cbrt(big(27)) +3.000000000000000000000000000000000000000000000000000000000000000000000000000000 +``` +""" cbrt(x::AbstractFloat) = x^(1//3) + +""" + exp2(x) + +Compute ``2^x``. + +```jldoctest +julia> exp2(5) +32.0 +``` +""" exp2(x::AbstractFloat) = 2^x for f in (:sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :expm1) @eval ($f)(x::AbstractFloat) = error("not implemented for ", typeof(x)) @@ -182,6 +248,13 @@ end sqrt(x::Float64) = box(Float64,sqrt_llvm(unbox(Float64,x))) sqrt(x::Float32) = box(Float32,sqrt_llvm(unbox(Float32,x))) + +""" + sqrt(x) + +Return ``\\sqrt{x}``. Throws `DomainError` for negative `Real` arguments. Use complex +negative arguments instead. The prefix operator `√` is equivalent to `sqrt`. +""" sqrt(x::Real) = sqrt(float(x)) @vectorize_1arg Number sqrt @@ -224,6 +297,12 @@ Compute the hypotenuse ``\\sqrt{\\sum x_i^2}`` avoiding overflow and underflow. """ hypot(x::Number...) = vecnorm(x) +""" + atan2(y, x) + +Compute the inverse tangent of `y/x`, using the signs of both `x` and `y` to determine the +quadrant of the return value. +""" atan2(y::Real, x::Real) = atan2(promote(float(y),float(x))...) atan2{T<:AbstractFloat}(y::T, x::T) = Base.no_op_err("atan2", T) @@ -245,10 +324,21 @@ minmax{T<:AbstractFloat}(x::T, y::T) = ifelse(isnan(x-y), ifelse(isnan(x), (y, y ifelse((y > x) | (signbit(y) < signbit(x)), (x, y), ifelse(x == x, (x, x), (y, y))))) + +""" + ldexp(x, n) + +Compute ``x \\times 2^n``. +""" ldexp(x::Float64,e::Integer) = ccall((:scalbn,libm), Float64, (Float64,Int32), x, Int32(e)) ldexp(x::Float32,e::Integer) = ccall((:scalbnf,libm), Float32, (Float32,Int32), x, Int32(e)) # TODO: vectorize ldexp +""" + exponent(x) -> Int + +Get the exponent of a normalized floating-point number. +""" function exponent{T<:AbstractFloat}(x::T) xu = reinterpret(Unsigned,x) xe = xu & exponent_mask(T) @@ -265,6 +355,21 @@ function exponent{T<:AbstractFloat}(x::T) end @vectorize_1arg Real exponent +""" + significand(x) + +Extract the `significand(s)` (a.k.a. mantissa), in binary representation, of a +floating-point number or array. If `x` is a non-zero finite number, then the result will be +a number of the same type on the interval ``[1,2)``. Otherwise `x` is returned. + +```jldoctest +julia> significand(15.2)/15.2 +0.125 + +julia> significand(15.2)*8 +15.2 +``` +""" function significand{T<:AbstractFloat}(x::T) xu = reinterpret(Unsigned,x) xe = xu & exponent_mask(T) @@ -283,6 +388,12 @@ function significand{T<:AbstractFloat}(x::T) end @vectorize_1arg Real significand +""" + frexp(val) + +Return `(x,exp)` such that `x` has a magnitude in the interval ``[1/2, 1)`` or 0, +and `val` is equal to ``x \\times 2^{exp}``. +""" function frexp{T<:AbstractFloat}(x::T) xu = reinterpret(Unsigned,x) xe = xu & exponent_mask(T) @@ -312,6 +423,17 @@ function frexp{T<:AbstractFloat}(A::Array{T}) return (F, E) end +""" + modf(x) + +Return a tuple (fpart,ipart) of the fractional and integral parts of a number. Both parts +have the same sign as the argument. + +```jldoctest +julia> modf(3.5) +(0.5,3.0) +``` +""" modf(x) = rem(x,one(x)), trunc(x) const _modff_temp = Ref{Float32}() @@ -397,6 +519,11 @@ Modulus after division by `2π`, returning in the range ``[0,2π)``. This function computes a floating point representation of the modulus after division by numerically exact `2π`, and is therefore not exactly the same as `mod(x,2π)`, which would compute the modulus of `x` relative to division by the floating-point number `2π`. + +```jldoctest +julia> mod2pi(9*pi/4) +0.7853981633974481 +``` """ function mod2pi(x::Float64) # or modtau(x) # with r = mod2pi(x) @@ -443,6 +570,14 @@ function mod2pi(x::Int64) end # generic fallback; for number types, promotion.jl does promotion + +""" + muladd(x, y, z) + +Combined multiply-add, computes `x*y+z` in an efficient manner. This may on some systems be +equivalent to `x*y+z`, or to `fma(x,y,z)`. `muladd` is used to improve performance. +See [`fma`](:func:`fma`). +""" muladd(x,y,z) = x*y+z # Float16 definitions diff --git a/base/mpfr.jl b/base/mpfr.jl index afd4dda83e485..f4a2e2543b524 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -42,6 +42,25 @@ const DEFAULT_PRECISION = [256] # Basic type and initialization definitions +""" + BigFloat(x) + +Create an arbitrary precision floating point number. `x` may be an `Integer`, a `Float64` or +a `BigInt`. The usual mathematical operators are defined for this type, and results are +promoted to a `BigFloat`. + +Note that because decimal literals are converted to floating point numbers when parsed, +`BigFloat(2.1)` may not yield what you expect. You may instead prefer to initialize +constants from strings via [`parse`](:func:`parse`), or using the `big` string literal. + +```jldoctest +julia> BigFloat(2.1) +2.100000000000000088817841970012523233890533447265625000000000000000000000000000 + +julia> big"2.1" +2.099999999999999999999999999999999999999999999999999999999999999999999999999986 +``` +""" type BigFloat <: AbstractFloat prec::Clong sign::Cint @@ -681,6 +700,11 @@ cmp(x::CdoubleMax, y::BigFloat) = -cmp(y,x) signbit(x::BigFloat) = ccall((:mpfr_signbit, :libmpfr), Int32, (Ptr{BigFloat},), &x) != 0 +""" + precision(BigFloat) + +Get the precision (in bits) currently used for `BigFloat` arithmetic. +""" function precision(x::BigFloat) # precision of an object of type BigFloat return ccall((:mpfr_get_prec, :libmpfr), Clong, (Ptr{BigFloat},), &x) end diff --git a/base/number.jl b/base/number.jl index 42ffb708736ee..b98655c5cd5e3 100644 --- a/base/number.jl +++ b/base/number.jl @@ -31,9 +31,35 @@ first(x::Number) = x last(x::Number) = x copy(x::Number) = x # some code treats numbers as collection-like +""" + divrem(x, y) + +The quotient and remainder from Euclidean division. Equivalent to `(div(x,y), rem(x,y))` or +`(x÷y, x%y)`. + +```jldoctest +julia> divrem(3,7) +(0,3) + +julia> divrem(7,3) +(2,1) +``` +""" divrem(x,y) = (div(x,y),rem(x,y)) + +""" + fldmod(x, y) + +The floored quotient and modulus after division. Equivalent to `(fld(x,y), mod(x,y))`. +""" fldmod(x,y) = (fld(x,y),mod(x,y)) signbit(x::Real) = x < 0 + +""" + sign(x) + +Return zero if `x==0` and ``x/|x|`` otherwise (i.e., ±1 for real `x`). +""" sign(x::Number) = x == 0 ? x/abs(one(x)) : x/abs(x) sign(x::Real) = ifelse(x < 0, oftype(x,-1), ifelse(x > 0, one(x), x)) sign(x::Unsigned) = ifelse(x > 0, one(x), x) @@ -48,6 +74,16 @@ ctranspose(x::Number) = conj(x) inv(x::Number) = one(x)/x angle(z::Real) = atan2(zero(z), z) +""" + widemul(x, y) + +Multiply `x` and `y`, giving the result as a larger type. + +```jldoctest +julia> widemul(Float32(3.), 4.) +1.200000000000000000000000000000000000000000000000000000000000000000000000000000e+01 +``` +""" widemul(x::Number, y::Number) = widen(x)*widen(y) start(x::Number) = false @@ -63,4 +99,12 @@ zero{T<:Number}(::Type{T}) = convert(T,0) one(x::Number) = oftype(x,1) one{T<:Number}(::Type{T}) = convert(T,1) +""" + factorial(n) + +Factorial of `n`. If `n` is an [`Integer`](:obj:`Integer`), the factorial is computed as an +integer (promoted to at least 64 bits). Note that this may overflow if `n` is not small, +but you can use `factorial(big(n))` to compute the result exactly in arbitrary precision. +If `n` is not an `Integer`, `factorial(n)` is equivalent to [`gamma(n+1)`](:func:`gamma(n+1) <gamma>`). +""" factorial(x::Number) = gamma(x + 1) # fallback for x not Integer diff --git a/base/operators.jl b/base/operators.jl index 8127281eed78b..5f37a0fb1e51a 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -6,13 +6,48 @@ typealias Dims{N} NTuple{N,Int} typealias DimsInteger{N} NTuple{N,Integer} typealias Indices{N} NTuple{N,AbstractUnitRange} +""" + <:(T1, T2) + +Subtype operator, equivalent to `issubtype(T1,T2)`. +""" const (<:) = issubtype +""" + supertype(T::DataType) + +Return the supertype of DataType `T`. + +```jldoctest +julia> supertype(Int32) +Signed +``` +""" supertype(T::DataType) = T.super ## generic comparison ## ==(x, y) = x === y + +""" + isequal(x, y) + +Similar to `==`, except treats all floating-point `NaN` values as equal to each other, and +treats `-0.0` as unequal to `0.0`. The default implementation of `isequal` calls `==`, so if +you have a type that doesn't have these floating-point subtleties then you probably only +need to define `==`. + +`isequal` is the comparison function used by hash tables (`Dict`). `isequal(x,y)` must imply +that `hash(x) == hash(y)`. + +This typically means that if you define your own `==` function then you must define a +corresponding `hash` (and vice versa). Collections typically implement `isequal` by calling +`isequal` recursively on all contents. + +Scalar types generally do not need to implement `isequal` separate from `==`, unless they +represent floating-point numbers amenable to a more efficient implementation than that +provided as a generic fallback (based on `isnan`, `signbit`, and `==`). +""" isequal(x, y) = x == y # TODO: these can be deleted once the deprecations of ==(x::Char, y::Integer) and @@ -54,19 +89,85 @@ end ## comparison fallbacks ## +""" + !=(x, y) + ≠(x,y) + +Not-equals comparison operator. Always gives the opposite answer as `==`. New types should +generally not implement this, and rely on the fallback definition `!=(x,y) = !(x==y)` instead. +""" !=(x,y) = !(x==y) const ≠ = != + +""" + is(x, y) -> Bool + ===(x,y) -> Bool + ≡(x,y) -> Bool + +Determine whether `x` and `y` are identical, in the sense that no program could distinguish +them. Compares mutable objects by address in memory, and compares immutable objects (such as +numbers) by contents at the bit level. This function is sometimes called `egal`. +""" const ≡ = is + +""" + !==(x, y) + ≢(x,y) + +Equivalent to `!is(x, y)`. +""" !==(x,y) = !is(x,y) const ≢ = !== +""" + <(x, y) + +Less-than comparison operator. New numeric types should implement this function for two +arguments of the new type. Because of the behavior of floating-point NaN values, `<` +implements a partial order. Types with a canonical partial order should implement `<`, and +types with a canonical total order should implement `isless`. +""" <(x,y) = isless(x,y) + +""" + >(x, y) + +Greater-than comparison operator. Generally, new types should implement `<` instead of this +function, and rely on the fallback definition `>(x,y) = y<x`. +""" >(x,y) = y < x + +""" + <=(x, y) + ≤(x,y) + +Less-than-or-equals comparison operator. +""" <=(x,y) = !(y < x) const ≤ = <= + +""" + >=(x, y) + ≥(x,y) + +Greater-than-or-equals comparison operator. +""" >=(x,y) = (y <= x) const ≥ = >= + +""" + .>(x, y) + +Element-wise greater-than comparison operator. +""" .>(x,y) = y .< x + +""" + .>=(x, y) + .≥(x,y) + +Element-wise greater-than-or-equals comparison operator. +""" .>=(x,y) = y .<= x const .≥ = .>= @@ -75,17 +176,58 @@ const .≥ = .>= isless(x::Real, y::Real) = x<y lexcmp(x::Real, y::Real) = isless(x,y) ? -1 : ifelse(isless(y,x), 1, 0) +""" + ifelse(condition::Bool, x, y) + +Return `x` if `condition` is `true`, otherwise return `y`. This differs from `?` or `if` in +that it is an ordinary function, so all the arguments are evaluated first. In some cases, +using `ifelse` instead of an `if` statement can eliminate the branch in generated code and +provide higher performance in tight loops. +""" ifelse(c::Bool, x, y) = select_value(c, x, y) +""" + cmp(x,y) + +Return -1, 0, or 1 depending on whether `x` is less than, equal to, or greater than `y`, +respectively. Uses the total order implemented by `isless`. For floating-point numbers, uses `<` +but throws an error for unordered arguments. +""" cmp(x,y) = isless(x,y) ? -1 : ifelse(isless(y,x), 1, 0) + +""" + lexcmp(x, y) + +Compare `x` and `y` lexicographically and return -1, 0, or 1 depending on whether `x` is +less than, equal to, or greater than `y`, respectively. This function should be defined for +lexicographically comparable types, and `lexless` will call `lexcmp` by default. +""" lexcmp(x,y) = cmp(x,y) + +""" + lexless(x, y) + +Determine whether `x` is lexicographically less than `y`. +""" lexless(x,y) = lexcmp(x,y)<0 # cmp returns -1, 0, +1 indicating ordering cmp(x::Integer, y::Integer) = ifelse(isless(x,y), -1, ifelse(isless(y,x), 1, 0)) +""" + max(x, y, ...) + +Return the maximum of the arguments. Operates elementwise over arrays. +""" max(x,y) = ifelse(y < x, x, y) + +""" + min(x, y, ...) + +Return the minimum of the arguments. Operates elementwise over arrays. +""" min(x,y) = ifelse(y < x, y, x) + """ minmax(x, y) @@ -110,6 +252,11 @@ scalarmin(x::AbstractArray, y ) = throw(ArgumentError("ordering is ## definitions providing basic traits of arithmetic operators ## +""" + identity(x) + +The identity function. Returns its argument. +""" identity(x) = x +(x::Number) = x @@ -142,6 +289,12 @@ for op in (:+, :*, :&, :|, :$, :min, :max, :kron) end end +""" + \\(x, y) + +Left division operator: multiplication of `y` by the inverse of `x` on the left. Gives +floating-point results for integer arguments. +""" \(x,y) = (y'/x')' # .<op> defaults to <op> @@ -158,7 +311,21 @@ end .!=(x::Number,y::Number) = x != y .<( x::Real,y::Real) = x < y .<=(x::Real,y::Real) = x <= y + +""" + .<=(x, y) + .≤(x,y) + +Element-wise less-than-or-equals comparison operator. +""" const .≤ = .<= + +""" + .!=(x, y) + .≠(x,y) + +Element-wise not-equals comparison operator. +""" const .≠ = .!= # Core <<, >>, and >>> take either Int or UInt as second arg. Signed shift @@ -264,15 +431,55 @@ See also [`>>`](:func:`>>`), [`<<`](:func:`<<`). # NOTE: C89 fmod() and x87 FPREM implicitly provide truncating float division, # so it is used here as the basis of float div(). div{T<:Real}(x::T, y::T) = convert(T,round((x-rem(x,y))/y)) + +""" + fld(x, y) + +Largest integer less than or equal to `x/y`. + +```jldoctest +julia> fld(7.3,5.5) +1.0 +``` +""" fld{T<:Real}(x::T, y::T) = convert(T,round((x-mod(x,y))/y)) + +""" + cld(x, y) + +Smallest integer larger than or equal to `x/y`. +```jldoctest +julia> cld(5.5,2.2) +3.0 +``` +""" cld{T<:Real}(x::T, y::T) = convert(T,round((x-modCeil(x,y))/y)) #rem{T<:Real}(x::T, y::T) = convert(T,x-y*trunc(x/y)) #mod{T<:Real}(x::T, y::T) = convert(T,x-y*floor(x/y)) modCeil{T<:Real}(x::T, y::T) = convert(T,x-y*ceil(x/y)) # operator alias + +""" + rem(x, y) + %(x, y) + +Remainder from Euclidean division, returning a value of the same sign as `x`, and smaller in +magnitude than `y`. This value is always exact. + +```julia +x == div(x,y)*y + rem(x,y) +``` +""" const % = rem .%(x::Real, y::Real) = x%y + +""" + div(x, y) + ÷(x, y) + +The quotient from Euclidean division. Computes `x/y`, truncated to an integer. +""" const ÷ = div .÷(x::Real, y::Real) = x÷y @@ -312,41 +519,180 @@ fldmod1{T<:Real}(x::T, y::T) = (fld1(x,y), mod1(x,y)) fldmod1{T<:Integer}(x::T, y::T) = (fld1(x,y), mod1(x,y)) # transpose + +""" + ctranspose(A) + +The conjugate transposition operator (`'`). +""" ctranspose(x) = conj(transpose(x)) conj(x) = x # transposed multiply + +""" + Ac_mul_B(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ⋅B``. +""" Ac_mul_B(a,b) = ctranspose(a)*b + +""" + A_mul_Bc(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``A⋅Bᴴ``. +""" A_mul_Bc(a,b) = a*ctranspose(b) + +""" + Ac_mul_Bc(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ Bᴴ``. +""" Ac_mul_Bc(a,b) = ctranspose(a)*ctranspose(b) + +""" + At_mul_B(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ⋅B``. +""" At_mul_B(a,b) = transpose(a)*b + +""" + A_mul_Bt(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``A⋅Bᵀ``. +""" A_mul_Bt(a,b) = a*transpose(b) + +""" + At_mul_Bt(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ⋅Bᵀ``. +""" At_mul_Bt(a,b) = transpose(a)*transpose(b) # transposed divide + +""" + Ac_rdiv_B(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ / B``. +""" Ac_rdiv_B(a,b) = ctranspose(a)/b + +""" + A_rdiv_Bc(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``A / Bᴴ``. +""" A_rdiv_Bc(a,b) = a/ctranspose(b) + +""" + Ac_rdiv_Bc(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ / Bᴴ``. +""" Ac_rdiv_Bc(a,b) = ctranspose(a)/ctranspose(b) + +""" + At_rdiv_B(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ / B``. +""" At_rdiv_B(a,b) = transpose(a)/b + +""" + A_rdiv_Bt(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``A / Bᵀ``. +""" A_rdiv_Bt(a,b) = a/transpose(b) + +""" + At_rdiv_Bt(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ / Bᵀ``. +""" At_rdiv_Bt(a,b) = transpose(a)/transpose(b) +""" + Ac_ldiv_B(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ`` \\ ``B``. +""" Ac_ldiv_B(a,b) = ctranspose(a)\b + +""" + A_ldiv_Bc(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``A`` \\ ``Bᴴ``. +""" A_ldiv_Bc(a,b) = a\ctranspose(b) + +""" + Ac_ldiv_Bc(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ`` \\ ``Bᴴ``. +""" Ac_ldiv_Bc(a,b) = ctranspose(a)\ctranspose(b) + +""" + At_ldiv_B(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ`` \\ ``B``. +""" At_ldiv_B(a,b) = transpose(a)\b + +""" + A_ldiv_Bt(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``A`` \\ ``Bᵀ``. +""" A_ldiv_Bt(a,b) = a\transpose(b) + +""" + At_ldiv_Bt(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᵀ`` \\ ``Bᵀ``. +""" At_ldiv_Bt(a,b) = At_ldiv_B(a,transpose(b)) + +""" + Ac_ldiv_Bt(A, B) + +For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ`` \\ ``Bᵀ``. +""" Ac_ldiv_Bt(a,b) = Ac_ldiv_B(a,transpose(b)) widen{T<:Number}(x::T) = convert(widen(T), x) +""" + eltype(type) + +Determine the type of the elements generated by iterating a collection of the given `type`. +For associative collection types, this will be a `Pair{KeyType,ValType}`. The definition +`eltype(x) = eltype(typeof(x))` is provided for convenience so that instances can be passed +instead of types. However the form that accepts a type argument should be defined for new +types. +""" eltype(::Type) = Any eltype(::Type{Any}) = Any eltype(t::DataType) = eltype(supertype(t)) eltype(x) = eltype(typeof(x)) # function pipelining + +""" + |>(x, f) + +Applies a function to the preceding argument. This allows for easy function chaining. + +```jldoctest +julia> [1:5;] |> x->x.^2 |> sum |> inv +0.01818181818181818 +``` +""" |>(x, f) = f(x) # array shape rules @@ -376,6 +722,24 @@ function promote_shape(a::Tuple{Int, Int}, b::Tuple{Int, Int}) return a end +""" + promote_shape(s1, s2) + +Check two array shapes for compatibility, allowing trailing singleton dimensions, and return +whichever shape has more dimensions. + +```jldoctest +julia> a = ones(3,4,1,1,1); + +julia> b = ones(3,4); + +julia> promote_shape(a,b) +(Base.OneTo(3),Base.OneTo(4),Base.OneTo(1),Base.OneTo(1),Base.OneTo(1)) + +julia> promote_shape((2,3,1,4), (2,3,1,4,1)) +(2,3,1,4,1) +``` +""" function promote_shape(a::Dims, b::Dims) if length(a) < length(b) return promote_shape(b, a) diff --git a/base/path.jl b/base/path.jl index d19779734913a..cb1c4121b895d 100644 --- a/base/path.jl +++ b/base/path.jl @@ -43,9 +43,63 @@ else error("path primitives for this OS need to be defined") end + +""" + splitdrive(path::AbstractString) -> (AbstractString, AbstractString) + +On Windows, split a path into the drive letter part and the path part. On Unix systems, the +first component is always the empty string. +""" +splitdrive(path::AbstractString) + +""" + homedir() -> AbstractString + +Return the current user's home directory. +""" +homedir() + + +""" + isabspath(path::AbstractString) -> Bool + +Determines whether a path is absolute (begins at the root directory). + +```jldoctest +julia> isabspath("/home") +true + +julia> isabspath("home") +false +``` +""" isabspath(path::String) = ismatch(path_absolute_re, path) + +""" + isdirpath(path::AbstractString) -> Bool + +Determines whether a path refers to a directory (for example, ends with a path separator). + +```jldoctest +julia> isdirpath("/home") +false + +julia> isdirpath("/home/") +true +``` +""" isdirpath(path::String) = ismatch(path_directory_re, splitdrive(path)[2]) +""" + splitdir(path::AbstractString) -> (AbstractString, AbstractString) + +Split a path into a tuple of the directory name and file name. + +```jldoctest +julia> splitdir("/home/myuser") +("/home","myuser") +``` +""" function splitdir(path::String) a, b = splitdrive(path) m = match(path_dir_splitter,b) @@ -54,9 +108,45 @@ function splitdir(path::String) a, String(m.captures[3]) end +""" + dirname(path::AbstractString) -> AbstractString + +Get the directory part of a path. + +```jldoctest +julia> dirname("/home/myuser") +"/home" +``` +""" dirname(path::AbstractString) = splitdir(path)[1] + +""" + basename(path::AbstractString) -> AbstractString + +Get the file name part of a path. + + ```jldoctest +julia> basename("/home/myuser/example.jl") +"example.jl" +``` +""" basename(path::AbstractString) = splitdir(path)[2] +""" + splitext(path::AbstractString) -> (AbstractString, AbstractString) + +If the last component of a path contains a dot, split the path into everything before the +dot and everything including and after the dot. Otherwise, return a tuple of the argument +unmodified and the empty string. + +```jldoctest +julia> splitext("/home/myuser/example.jl") +("/home/myuser/example",".jl") + +julia> splitext("/home/myuser/example") +("/home/myuser/example","") +``` +""" function splitext(path::String) a, b = splitdrive(path) m = match(path_ext_splitter, b) @@ -73,6 +163,18 @@ function pathsep(paths::AbstractString...) end joinpath(a::AbstractString) = a + +""" + joinpath(parts...) -> AbstractString + +Join path components into a full path. If some argument is an absolute path, then prior +components are dropped. + +```jldoctest +julia> joinpath("/home/myuser","example.jl") +"/home/myuser/example.jl" +``` +""" joinpath(a::AbstractString, b::AbstractString, c::AbstractString...) = joinpath(joinpath(a,b), c...) function joinpath(a::String, b::String) @@ -87,6 +189,16 @@ function joinpath(a::String, b::String) end joinpath(a::AbstractString, b::AbstractString) = joinpath(String(a), String(b)) +""" + normpath(path::AbstractString) -> AbstractString + +Normalize a path, removing "." and ".." entries. + +```jldoctest +julia> normpath("/home/myuser/../example.jl") +"/home/example.jl" +``` +""" function normpath(path::String) isabs = isabspath(path) isdir = isdirpath(path) @@ -122,6 +234,11 @@ function normpath(path::String) end normpath(a::AbstractString, b::AbstractString...) = normpath(joinpath(a,b...)) +""" + abspath(path::AbstractString) -> AbstractString + +Convert a path to an absolute path by adding the current directory if necessary. +""" abspath(a::String) = normpath(isabspath(a) ? a : joinpath(pwd(),a)) abspath(a::AbstractString, b::AbstractString...) = abspath(joinpath(a,b...)) @@ -162,6 +279,15 @@ function realpath(path::AbstractString) end end # os-test + +""" + realpath(path::AbstractString) -> AbstractString + +Canonicalize a path by expanding symbolic links and removing "." and ".." entries. +""" +realpath(path::AbstractString) + + if is_windows() expanduser(path::AbstractString) = path # on windows, ~ means "temporary file" else @@ -176,6 +302,22 @@ function expanduser(path::AbstractString) end end + +""" + expanduser(path::AbstractString) -> AbstractString + +On Unix systems, replace a tilde character at the start of a path with the current user's home directory. +""" +expanduser(path::AbstractString) + + +""" + relpath(path::AbstractString, startpath::AbstractString = ".") -> AbstractString + +Return a relative filepath to `path` either from the current directory or from an optional +start directory. This is a path computation: the filesystem is not accessed to confirm the +existence or nature of `path` or `startpath`. +""" function relpath(path::String, startpath::String = ".") isempty(path) && throw(ArgumentError("`path` must be specified")) isempty(startpath) && throw(ArgumentError("`startpath` must be specified")) diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index f940bd87e864b..2f2ff12fde1d8 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -43,11 +43,27 @@ _genperm(out, I) = out @inline _genperm(out, I, p, perm...) = _genperm((out..., I[p]), I, perm...) @inline genperm(I, perm::AbstractVector{Int}) = genperm(I, (perm...,)) +""" + permutedims(A, perm) + +Permute the dimensions of array `A`. `perm` is a vector specifying a permutation of length +`ndims(A)`. This is a generalization of transpose for multi-dimensional arrays. Transpose is +equivalent to `permutedims(A, [2,1])`. +""" function Base.permutedims{T,N}(A::AbstractArray{T,N}, perm) dest = similar(A, genperm(indices(A), perm)) permutedims!(dest, A, perm) end +""" + permutedims!(dest, src, perm) + +Permute the dimensions of array `src` and store the result in the array `dest`. `perm` is a +vector specifying a permutation of length `ndims(src)`. The preallocated array `dest` should +have `size(dest) == size(src)[perm]` and is completely overwritten. No in-place permutation +is supported and unexpected results will happen if `src` and `dest` have overlapping memory +regions. +""" function Base.permutedims!(dest, src::AbstractArray, perm) Base.checkdims_perm(dest, src, perm) P = PermutedDimsArray(dest, invperm(perm)) diff --git a/base/poll.jl b/base/poll.jl index cce9a9af06f9e..a4ec19d6964a1 100644 --- a/base/poll.jl +++ b/base/poll.jl @@ -365,6 +365,18 @@ function wait(m::FileMonitor) return filename, events end +""" + poll_fd(fd, timeout_s::Real=-1; readable=false, writable=false) + +Monitor a file descriptor `fd` for changes in the read or write availability, and with a +timeout given by `timeout_s` seconds. + +The keyword arguments determine which of read and/or write status should be monitored; at +least one of them must be set to `true`. + +The returned value is an object with boolean fields `readable`, `writable`, and `timedout`, +giving the result of the polling. +""" function poll_fd(s::Union{RawFD, is_windows() ? WindowsRawSocket : Union{}}, timeout_s::Real=-1; readable=false, writable=false) wt = Condition() fdw = _FDWatcher(s, readable, writable) @@ -385,6 +397,18 @@ function poll_fd(s::Union{RawFD, is_windows() ? WindowsRawSocket : Union{}}, tim end end +""" + watch_file(path::AbstractString, timeout_s::Real=-1) + +Watch file or directory `path` for changes until a change occurs or `timeout_s` seconds have +elapsed. + +The returned value is an object with boolean fields `changed`, `renamed`, and `timedout`, +giving the result of watching the file. + +This behavior of this function varies slightly across platforms. See +<https://nodejs.org/api/fs.html#fs_caveats> for more detailed information. +""" function watch_file(s::AbstractString, timeout_s::Real=-1) wt = Condition() fm = FileMonitor(s) @@ -405,6 +429,19 @@ function watch_file(s::AbstractString, timeout_s::Real=-1) end end +""" + poll_file(path::AbstractString, interval_s::Real=5.007, timeout_s::Real=-1) -> (previous::StatStruct, current::StatStruct) + +Monitor a file for changes by polling every `interval_s` seconds until a change occurs or +`timeout_s` seconds have elapsed. The `interval_s` should be a long period; the default is +5.007 seconds. + +Returns a pair of `StatStruct` objects `(previous, current)` when a change is detected. + +To determine when a file was modified, compare `mtime(prev) != mtime(current)` to detect +notification of changes. However, using [`watch_file`](:func:`watch_file`) for this operation is preferred, since +it is more reliable and efficient, although in some situations it may not be available. +""" function poll_file(s::AbstractString, interval_seconds::Real=5.007, timeout_s::Real=-1) wt = Condition() pfw = PollingFileWatcher(s, Float64(interval_seconds)) diff --git a/base/process.jl b/base/process.jl index be39453865c4b..2b311332bb13f 100644 --- a/base/process.jl +++ b/base/process.jl @@ -215,6 +215,17 @@ byteenv(env::Void) = nothing byteenv{T<:AbstractString}(env::Union{AbstractVector{Pair{T}}, Tuple{Vararg{Pair{T}}}}) = String[cstr(k*"="*string(v)) for (k,v) in env] +""" + setenv(command::Cmd, env; dir="") + +Set environment variables to use when running the given `command`. `env` is either a +dictionary mapping strings to strings, an array of strings of the form `"var=val"`, or zero +or more `"var"=>val` pair arguments. In order to modify (rather than replace) the existing +environment, create `env` by `copy(ENV)` and then setting `env["var"]=val` as desired, or +use `withenv`. + +The `dir` keyword argument can be used to specify a working directory for the command. +""" setenv(cmd::Cmd, env; dir="") = Cmd(cmd; env=byteenv(env), dir=dir) setenv{T<:AbstractString}(cmd::Cmd, env::Pair{T}...; dir="") = setenv(cmd, env; dir=dir) @@ -238,6 +249,22 @@ redir_err(src::AbstractCmd, dest::AbstractString) = CmdRedirect(src, FileRedirec redir_out_append(src::AbstractCmd, dest::AbstractString) = CmdRedirect(src, FileRedirect(dest, true), STDOUT_NO) redir_err_append(src::AbstractCmd, dest::AbstractString) = CmdRedirect(src, FileRedirect(dest, true), STDERR_NO) +""" + pipeline(command; stdin, stdout, stderr, append=false) + +Redirect I/O to or from the given `command`. Keyword arguments specify which of the +command's streams should be redirected. `append` controls whether file output appends to the +file. This is a more general version of the 2-argument `pipeline` function. +`pipeline(from, to)` is equivalent to `pipeline(from, stdout=to)` when `from` is a command, +and to `pipeline(to, stdin=from)` when `from` is another kind of data source. + +**Examples**: + +```julia +run(pipeline(`dothings`, stdout="out.txt", stderr="errs.txt")) +run(pipeline(`update`, stdout="log.txt", append=true)) +``` +""" function pipeline(cmd::AbstractCmd; stdin=nothing, stdout=nothing, stderr=nothing, append::Bool=false) if append && stdout === nothing && stderr === nothing error("append set to true, but no output redirections specified") @@ -257,6 +284,24 @@ end pipeline(cmd::AbstractCmd, dest) = pipeline(cmd, stdout=dest) pipeline(src::Union{Redirectable,AbstractString}, cmd::AbstractCmd) = pipeline(cmd, stdin=src) +""" + pipeline(from, to, ...) + +Create a pipeline from a data source to a destination. The source and destination can be +commands, I/O streams, strings, or results of other `pipeline` calls. At least one argument +must be a command. Strings refer to filenames. When called with more than two arguments, +they are chained together from left to right. For example `pipeline(a,b,c)` is equivalent to +`pipeline(pipeline(a,b),c)`. This provides a more concise way to specify multi-stage +pipelines. + +**Examples**: + +```julia +run(pipeline(`ls`, `grep xyz`)) +run(pipeline(`ls`, "out.txt")) +run(pipeline("out.txt", `grep xyz`)) +``` +""" pipeline(a, b, c, d...) = pipeline(pipeline(a,b), c, d...) type Process <: AbstractPipe @@ -517,6 +562,15 @@ end eachline(cmd::AbstractCmd) = eachline(cmd, DevNull) # return a Process object to read-to/write-from the pipeline +""" + open(command, mode::AbstractString="r", stdio=DevNull) + +Start running `command` asynchronously, and return a tuple `(stream,process)`. If `mode` is +`"r"`, then `stream` reads from the process's standard output and `stdio` optionally +specifies the process's standard input stream. If `mode` is `"w"`, then `stream` writes to +the process's standard input and `stdio` optionally specifies the process's standard output +stream. +""" function open(cmds::AbstractCmd, mode::AbstractString="r", other::Redirectable=DevNull) if mode == "r" in = other @@ -534,6 +588,13 @@ function open(cmds::AbstractCmd, mode::AbstractString="r", other::Redirectable=D return (io, processes) end +""" + open(f::Function, command, mode::AbstractString="r", stdio=DevNull) + +Similar to `open(command, mode, stdio)`, but calls `f(stream)` on the resulting read or +write stream, then closes the stream and waits for the process to complete. Returns the +value returned by `f`. +""" function open(f::Function, cmds::AbstractCmd, args...) io, P = open(cmds, args...) ret = try @@ -549,6 +610,13 @@ function open(f::Function, cmds::AbstractCmd, args...) end # TODO: deprecate this + +""" + readandwrite(command) + +Starts running a command asynchronously, and returns a tuple (stdout,stdin,process) of the +output stream and input stream of the process, and the process object itself. +""" function readandwrite(cmds::AbstractCmd) in = Pipe() out, processes = open(cmds, "r", in) @@ -572,6 +640,12 @@ function writeall(cmd::AbstractCmd, stdin::AbstractString, stdout::Redirectable= end end +""" + run(command, args...) + +Run a command object, constructed with backticks. Throws an error if anything goes wrong, +including the process exiting with a non-zero status. +""" function run(cmds::AbstractCmd, args...) ps = spawn(cmds, spawn_opts_inherit(args...)...) success(ps) ? nothing : pipeline_error(ps) @@ -594,6 +668,13 @@ function success(x::Process) end success(procs::Vector{Process}) = mapreduce(success, &, procs) success(procs::ProcessChain) = success(procs.processes) + +""" + success(command) + +Run a command object, constructed with backticks, and tell whether it was successful (exited +with a code of 0). An exception is raised if the process cannot be started. +""" success(cmd::AbstractCmd) = success(spawn(cmd)) function pipeline_error(proc::Process) @@ -620,6 +701,12 @@ function pipeline_error(procs::ProcessChain) end _jl_kill(p::Process, signum::Integer) = ccall(:uv_process_kill, Int32, (Ptr{Void},Int32), p.handle, signum) + +""" + kill(p::Process, signum=SIGTERM) + +Send a signal to a process. The default is to terminate the process. +""" function kill(p::Process, signum::Integer) if process_running(p) @assert p.handle != C_NULL @@ -637,10 +724,21 @@ function _contains_newline(bufptr::Ptr{Void}, len::Int32) end ## process status ## + +""" + process_running(p::Process) + +Determine whether a process is currently running. +""" process_running(s::Process) = s.exitcode == typemin(fieldtype(Process, :exitcode)) process_running(s::Vector{Process}) = any(process_running, s) process_running(s::ProcessChain) = process_running(s.processes) +""" + process_exited(p::Process) + +Determine whether a process has exited. +""" process_exited(s::Process) = !process_running(s) process_exited(s::Vector{Process}) = all(process_exited, s) process_exited(s::ProcessChain) = process_exited(s.processes) diff --git a/base/profile.jl b/base/profile.jl index e3e2154ece3f8..25b81bb61ecee 100644 --- a/base/profile.jl +++ b/base/profile.jl @@ -6,6 +6,12 @@ import Base.StackTraces: lookup, UNKNOWN, show_spec_linfo export @profile +""" + @profile + +`@profile <expression>` runs your expression while taking periodic backtraces. These are +appended to an internal buffer of backtraces. +""" macro profile(ex) quote try diff --git a/base/random.jl b/base/random.jl index 8ec17996cfe0d..06b1eae0a3bd3 100644 --- a/base/random.jl +++ b/base/random.jl @@ -29,6 +29,7 @@ type Close1Open2 <: FloatInterval end ## RandomDevice if is_windows() + immutable RandomDevice <: AbstractRNG buffer::Vector{UInt128} @@ -53,6 +54,15 @@ else # !windows rand!{T<:Union{Bool, Base.BitInteger}}(rd::RandomDevice, A::Array{T}) = read!(rd.file, A) end # os-test + +""" + RandomDevice() + +Create a `RandomDevice` RNG object. Two such objects will always generate different streams of random numbers. +""" +RandomDevice + + rand(rng::RandomDevice, ::Type{Close1Open2}) = reinterpret(Float64, 0x3ff0000000000000 | rand(rng, UInt64) & 0x000fffffffffffff) @@ -78,6 +88,12 @@ end MersenneTwister(seed::Vector{UInt32}, state::DSFMT_state) = MersenneTwister(seed, state, zeros(Float64, MTCacheLength), MTCacheLength) +""" + MersenneTwister(seed=0) + +Create a `MersenneTwister` RNG object. Different RNG objects can have their own seeds, which +may be useful for generating different streams of random numbers. +""" MersenneTwister(seed=0) = srand(MersenneTwister(Vector{UInt32}(), DSFMT_state()), seed) function copy!(dst::MersenneTwister, src::MersenneTwister) @@ -133,6 +149,18 @@ function srand(r::MersenneTwister, seed::Vector{UInt32}) end # MersenneTwister jump + +""" + randjump(r::MersenneTwister, jumps::Integer, [jumppoly::AbstractString=dSFMT.JPOLY1e21]) -> Vector{MersenneTwister} + +Create an array of the size `jumps` of initialized `MersenneTwister` RNG objects where the +first RNG object given as a parameter and following `MersenneTwister` RNGs in the array +initialized such that a state of the RNG object in the array would be moved forward (without +generating numbers) from a previous RNG object array element on a particular number of steps +encoded by the jump polynomial `jumppoly`. + +Default jump polynomial moves forward `MersenneTwister` RNG state by 10^20 steps. +""" function randjump(mt::MersenneTwister, jumps::Integer, jumppoly::AbstractString) mts = MersenneTwister[] push!(mts, mt) @@ -191,6 +219,15 @@ end ## srand() +""" + srand([rng=GLOBAL_RNG], [seed]) + +Reseed the random number generator. If a `seed` is provided, the RNG will give a +reproducible sequence of numbers, otherwise Julia will get entropy from the system. For +`MersenneTwister`, the `seed` may be a non-negative integer, a vector of `UInt32` integers +or a filename, in which case the seed is read from a file. `RandomDevice` does not support +seeding. +""" srand(r::MersenneTwister) = srand(r, make_seed()) srand(r::MersenneTwister, n::Integer) = srand(r, make_seed(n)) srand(r::MersenneTwister, filename::AbstractString, n::Integer=4) = srand(r, make_seed(filename, n)) @@ -224,6 +261,17 @@ globalRNG() = GLOBAL_RNG # rand: a non-specified RNG defaults to GLOBAL_RNG +""" + rand([rng=GLOBAL_RNG], [S], [dims...]) + +Pick a random element or array of random elements from the set of values specified by `S`; `S` can be + +* an indexable collection (for example `1:n` or `['x','y','z']`), or +* a type: the set of values to pick from is then equivalent to `typemin(S):typemax(S)` for + integers (this is not applicable to `BigInt`), and to ``[0, 1)`` for floating point numbers; + +`S` defaults to `Float64`. +""" @inline rand() = rand(GLOBAL_RNG, CloseOpen) @inline rand(T::Type) = rand(GLOBAL_RNG, T) rand(dims::Dims) = rand(GLOBAL_RNG, dims) @@ -233,6 +281,14 @@ rand(T::Type, d1::Integer, dims::Integer...) = rand(T, tuple(Int(d1), convert(Tu rand!(A::AbstractArray) = rand!(GLOBAL_RNG, A) rand(r::AbstractArray) = rand(GLOBAL_RNG, r) + +""" + rand!([rng=GLOBAL_RNG], A, [coll]) + +Populate the array `A` with random values. If the indexable collection `coll` is specified, +the values are picked randomly from `coll`. This is equivalent to `copy!(A, rand(rng, coll, size(A)))` +or `copy!(A, rand(rng, eltype(A), size(A)))` but without allocating a new array. +""" rand!(A::AbstractArray, r::AbstractArray) = rand!(GLOBAL_RNG, A, r) rand(r::AbstractArray, dims::Dims) = rand(GLOBAL_RNG, r, dims) @@ -630,6 +686,11 @@ function rand!(rng::AbstractRNG, B::BitArray) return B end +""" + bitrand([rng=GLOBAL_RNG], [dims...]) + +Generate a `BitArray` of random boolean values. +""" bitrand(r::AbstractRNG, dims::Dims) = rand!(r, BitArray(dims)) bitrand(r::AbstractRNG, dims::Int...) = rand!(r, BitArray(dims)) @@ -1131,7 +1192,7 @@ const ziggurat_nor_inv_r = inv(ziggurat_nor_r) const ziggurat_exp_r = 7.6971174701310497140446280481 """ - randn([rng], [T=Float64], [dims...]) + randn([rng=GLOBAL_RNG], [T=Float64], [dims...]) Generate a normally-distributed random number of type `T` with mean 0 and standard deviation 1. Optionally generate an array of normally-distributed random numbers. @@ -1165,7 +1226,7 @@ function randn_unlikely(rng, idx, rabs, x) end """ - randexp([rng], [T=Float64], [dims...]) + randexp([rng=GLOBAL_RNG], [T=Float64], [dims...]) Generate a random number of type `T` according to the exponential distribution with scale 1. Optionally generate an array of such random numbers. @@ -1193,15 +1254,15 @@ function randexp_unlikely(rng, idx, x) end """ - randn!([rng], A::AbstractArray) -> A + randn!([rng=GLOBAL_RNG], A::AbstractArray) -> A Fill the array `A` with normally-distributed (mean 0, standard deviation 1) random numbers. -Also see the `rand` function. +Also see the [`rand`](:func:`rand`) function. """ function randn! end """ - randexp!([rng], A::AbstractArray) -> A + randexp!([rng=GLOBAL_RNG], A::AbstractArray) -> A Fill the array `A` with random numbers following the exponential distribution (with scale 1). """ @@ -1248,7 +1309,7 @@ immutable UUID end """ - uuid1([rng::AbstractRNG]) -> UUID + uuid1([rng::AbstractRNG=GLOBAL_RNG]) -> UUID Generates a version 1 (time-based) universally unique identifier (UUID), as specified by RFC 4122. Note that the Node ID is randomly generated (does not identify the host) @@ -1278,7 +1339,7 @@ function uuid1(rng::AbstractRNG=GLOBAL_RNG) end """ - uuid4([rng::AbstractRNG]) -> UUID + uuid4([rng::AbstractRNG=GLOBAL_RNG]) -> UUID Generates a version 4 (random or pseudo-random) universally unique identifier (UUID), as specified by RFC 4122. @@ -1383,6 +1444,15 @@ end randsubseq!(S::AbstractArray, A::AbstractArray, p::Real) = randsubseq!(GLOBAL_RNG, S, A, p) randsubseq{T}(r::AbstractRNG, A::AbstractArray{T}, p::Real) = randsubseq!(r, T[], A, p) + +""" + randsubseq(A, p) -> Vector + +Return a vector consisting of a random subsequence of the given array `A`, where each +element of `A` is included (in order) with independent probability `p`. (Complexity is +linear in `p*length(A)`, so this function is efficient even if `p` is small and `A` is +large.) Technically, this process is known as "Bernoulli sampling" of `A`. +""" randsubseq(A::AbstractArray, p::Real) = randsubseq(GLOBAL_RNG, A, p) "Return a random `Int` (masked with `mask`) in ``[0, n)``, when `n <= 2^52`." @@ -1395,6 +1465,12 @@ randsubseq(A::AbstractArray, p::Real) = randsubseq(GLOBAL_RNG, A, p) end end +""" + shuffle!([rng=GLOBAL_RNG,] v) + +In-place version of [`shuffle`](:func:`shuffle`): randomly permute the array `v` in-place, +optionally supplying the random-number generator `rng`. +""" function shuffle!(r::AbstractRNG, a::AbstractVector) n = length(a) @assert n <= Int64(2)^52 @@ -1409,9 +1485,25 @@ end shuffle!(a::AbstractVector) = shuffle!(GLOBAL_RNG, a) +""" + shuffle([rng=GLOBAL_RNG,] v) + +Return a randomly permuted copy of `v`. The optional `rng` argument specifies a random +number generator (see [Random Numbers](:ref:`Random Numbers <random-numbers>`)). +To permute `v` in-place, see [`shuffle!`](:func:`shuffle!`). To obtain randomly permuted +indices, see [`randperm`](:func:`randperm`). +""" shuffle(r::AbstractRNG, a::AbstractVector) = shuffle!(r, copymutable(a)) shuffle(a::AbstractVector) = shuffle(GLOBAL_RNG, a) +""" + randperm([rng=GLOBAL_RNG,] n::Integer) + +Construct a random permutation of length `n`. The optional `rng` argument specifies a random +number generator (see [Random Numbers](:ref:`Random Numbers <random-numbers>`)). +To randomly permute a arbitrary vector, see [`shuffle`](:func:`shuffle`) +or [`shuffle!`](:func:`shuffle!`). +""" function randperm(r::AbstractRNG, n::Integer) a = Array{typeof(n)}(n) @assert n <= Int64(2)^52 @@ -1432,6 +1524,12 @@ function randperm(r::AbstractRNG, n::Integer) end randperm(n::Integer) = randperm(GLOBAL_RNG, n) +""" + randcycle([rng=GLOBAL_RNG,] n::Integer) + +Construct a random cyclic permutation of length `n`. The optional `rng` +argument specifies a random number generator, see [Random Numbers](:ref:`Random Numbers <random-numbers>`). +""" function randcycle(r::AbstractRNG, n::Integer) a = Array{typeof(n)}(n) n == 0 && return a diff --git a/base/range.jl b/base/range.jl index 00b2b217c1f6a..0254718af8d82 100644 --- a/base/range.jl +++ b/base/range.jl @@ -111,12 +111,30 @@ range{T}(a::T, len::Integer) = # this is for non-numeric ranges where step can be quite different colon{A<:Real,C<:Real}(a::A, b, c::C) = colon(convert(promote_type(A,C),a), b, convert(promote_type(A,C),c)) +""" + colon(start, [step], stop) + +Called by `:` syntax for constructing ranges. +""" colon{T<:Real}(start::T, step, stop::T) = StepRange(start, step, stop) + +""" + :(start, [step], stop) + +Range operator. `a:b` constructs a range from `a` to `b` with a step size of 1, and `a:s:b` +is similar but uses a step size of `s`. These syntaxes call the function `colon`. The colon +is also used in indexing to select whole dimensions. +""" colon{T<:Real}(start::T, step::T, stop::T) = StepRange(start, step, stop) colon{T<:Real}(start::T, step::Real, stop::T) = StepRange(promote(start, step, stop)...) colon{T}(start::T, step, stop::T) = StepRange(start, step, stop) +""" + range(start, [step], length) + +Construct a range by length, given a starting value and optional step (defaults to 1). +""" range{T,S}(a::T, step::S, len::Integer) = StepRange{T,S}(a, step, convert(T, a+step*(len-1))) ## floating point ranges @@ -250,6 +268,18 @@ function linspace{T<:AbstractFloat}(start::T, stop::T, len::Real) T_len == len || throw(InexactError()) linspace(start, stop, T_len) end + +""" + linspace(start::Real, stop::Real, n::Real=50) + +Construct a range of `n` linearly spaced elements from `start` to `stop`. + +```jldoctest +julia> linspace(1.3,2.9,9) +9-element LinSpace{Float64}: + 1.3,1.5,1.7,1.9,2.1,2.3,2.5,2.7,2.9 +``` +""" linspace(start::Real, stop::Real, len::Real=50) = linspace(promote(AbstractFloat(start), AbstractFloat(stop))..., len) @@ -320,6 +350,21 @@ function print_range(io::IO, r::Range, end end +""" + logspace(start::Real, stop::Real, n::Integer=50) + +Construct a vector of `n` logarithmically spaced numbers from `10^start` to `10^stop`. + +```jldoctest +julia> logspace(1.,10.,5) +5-element Array{Float64,1}: + 10.0 + 1778.28 + 3.16228e5 + 5.62341e7 + 1.0e10 +``` +""" logspace(start::Real, stop::Real, n::Integer=50) = 10.^linspace(start, stop, n) ## interface implementations diff --git a/base/reduce.jl b/base/reduce.jl index c0ba39c7310fd..2bffbe215c753 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -52,8 +52,20 @@ function mapfoldl_impl(f, op, v0, itr, i) end end +""" + mapfoldl(f, op, v0, itr) + +Like [`mapreduce`](:func:`mapreduce`), but with guaranteed left associativity. `v0` will be +used exactly once. +""" mapfoldl(f, op, v0, itr) = mapfoldl_impl(f, op, v0, itr, start(itr)) +""" + mapfoldl(f, op, itr) + +Like `mapfoldl(f, op, v0, itr)`, but using the first element of `itr` as `v0`. In general, +this cannot be used with empty collections (see `reduce(op, itr)`). +""" function mapfoldl(f, op, itr) i = start(itr) if done(itr, i) @@ -64,7 +76,20 @@ function mapfoldl(f, op, itr) mapfoldl_impl(f, op, v0, itr, i) end +""" + foldl(op, v0, itr) + +Like [`reduce`](:func:`reduce`), but with guaranteed left associativity. `v0` will be used +exactly once. +""" foldl(op, v0, itr) = mapfoldl(identity, op, v0, itr) + +""" + foldl(op, itr) + +Like `foldl(op, v0, itr)`, but using the first element of `itr` as `v0`. In general, this +cannot be used with empty collections (see `reduce(op, itr)`). +""" foldl(op, itr) = mapfoldl(identity, op, itr) ## foldr & mapfoldr @@ -85,10 +110,36 @@ function mapfoldr_impl(f, op, v0, itr, i::Integer) end end +""" + mapfoldr(f, op, v0, itr) + +Like [`mapreduce`](:func:`mapreduce`), but with guaranteed right associativity. `v0` will be +used exactly once. +""" mapfoldr(f, op, v0, itr) = mapfoldr_impl(f, op, v0, itr, endof(itr)) + +""" + mapfoldr(f, op, itr) + +Like `mapfoldr(f, op, v0, itr)`, but using the first element of `itr` as `v0`. In general, +this cannot be used with empty collections (see `reduce(op, itr)`). +""" mapfoldr(f, op, itr) = (i = endof(itr); mapfoldr_impl(f, op, f(itr[i]), itr, i-1)) +""" + foldr(op, v0, itr) + +Like [`reduce`](:func:`reduce`), but with guaranteed right associativity. `v0` will be used +exactly once. +""" foldr(op, v0, itr) = mapfoldr(identity, op, v0, itr) + +""" + foldr(op, itr) + +Like `foldr(op, v0, itr)`, but using the last element of `itr` as `v0`. In general, this +cannot be used with empty collections (see `reduce(op, itr)`). +""" foldr(op, itr) = mapfoldr(identity, op, itr) ## reduce & mapreduce @@ -113,7 +164,35 @@ function mapreduce_impl(f, op, A::AbstractArray, ifirst::Integer, ilast::Integer end end +""" + mapreduce(f, op, itr) + +Like `mapreduce(f, op, v0, itr)`. In general, this cannot be used with empty collections +(see `reduce(op, itr)`). +""" mapreduce(f, op, itr) = mapfoldl(f, op, itr) + +""" + mapreduce(f, op, v0, itr) + +Apply function `f` to each element in `itr`, and then reduce the result using the binary +function `op`. `v0` must be a neutral element for `op` that will be returned for empty +collections. It is unspecified whether `v0` is used for non-empty collections. + +[`mapreduce`](:func:`mapreduce`) is functionally equivalent to calling `reduce(op, v0, +map(f, itr))`, but will in general execute faster since no intermediate collection needs to +be created. See documentation for [`reduce`](:func:`reduce`) and [`map`](:func:`map`). + +```jldoctest +julia> mapreduce(x->x^2, +, [1:3;]) # == 1 + 4 + 9 +14 +``` + +The associativity of the reduction is implementation-dependent. Additionally, some +implementations may reuse the return value of `f` for elements that appear multiple times in +`itr`. Use [`mapfoldl`](:func:`mapfoldl`) or [`mapfoldr`](:func:`mapfoldr`) instead for +guaranteed left or right associativity and invocation of `f` for every value. +""" mapreduce(f, op, v0, itr) = mapfoldl(f, op, v0, itr) # Note: sum_seq usually uses four or more accumulators after partial @@ -169,7 +248,35 @@ _mapreduce{T}(f, op, ::LinearSlow, A::AbstractArray{T}) = mapfoldl(f, op, A) mapreduce(f, op, A::AbstractArray) = _mapreduce(f, op, linearindexing(A), A) mapreduce(f, op, a::Number) = f(a) +""" + reduce(op, v0, itr) + +Reduce the given collection `ìtr` with the given binary operator `op`. `v0` must be a +neutral element for `op` that will be returned for empty collections. It is unspecified +whether `v0` is used for non-empty collections. + +Reductions for certain commonly-used operators have special implementations which should be +used instead: `maximum(itr)`, `minimum(itr)`, `sum(itr)`, `prod(itr)`, `any(itr)`, +`all(itr)`. + +The associativity of the reduction is implementation dependent. This means that you can't +use non-associative operations like `-` because it is undefined whether `reduce(-,[1,2,3])` +should be evaluated as `(1-2)-3` or `1-(2-3)`. Use [`foldl`](:func:`foldl`) or +[`foldr`](:func:`foldr`) instead for guaranteed left or right associativity. + +Some operations accumulate error, and parallelism will also be easier if the reduction can +be executed in groups. Future versions of Julia might change the algorithm. Note that the +elements are not reordered if you use an ordered collection. +""" reduce(op, v0, itr) = mapreduce(identity, op, v0, itr) + +""" + reduce(op, itr) + +Like `reduce(op, v0, itr)`. This cannot be used with empty collections, except for some +special cases (e.g. when `op` is one of `+`, `*`, `max`, `min`, `&`, `|`) when Julia can +determine the neutral element of `op`. +""" reduce(op, itr) = mapreduce(identity, op, itr) reduce(op, a::Number) = a @@ -225,14 +332,45 @@ mapreduce(f, op::ShortCircuiting, itr::Any) = mapreduce_sc(f,op,itr) ## sum +""" + sum(f, itr) + +Sum the results of calling function `f` on each element of `itr`. +""" sum(f::Callable, a) = mapreduce(f, +, a) + +""" + sum(itr) + +Returns the sum of all elements in a collection. +""" sum(a) = mapreduce(identity, +, a) sum(a::AbstractArray{Bool}) = countnz(a) + +""" + sumabs(itr) + +Sum absolute values of all elements in a collection. This is equivalent to `sum(abs(itr))` but faster. +""" sumabs(a) = mapreduce(abs, +, a) + +""" + sumabs2(itr) + +Sum squared absolute values of all elements in a collection. +This is equivalent to `sum(abs2(itr))` but faster. +""" sumabs2(a) = mapreduce(abs2, +, a) # Kahan (compensated) summation: O(1) error growth, at the expense # of a considerable increase in computational expense. + +""" + sum_kbn(A) + +Returns the sum of all array elements, using the Kahan-Babuska-Neumaier compensated +summation algorithm for additional accuracy. +""" function sum_kbn{T<:AbstractFloat}(A::AbstractArray{T}) c = r_promote(+, zero(T)::T) if isempty(A) @@ -257,6 +395,12 @@ end ## prod prod(f::Callable, a) = mapreduce(f, *, a) + +""" + prod(itr) + +Returns the product of all elements of a collection. +""" prod(a) = mapreduce(identity, *, a) ## maximum & minimum @@ -422,6 +566,18 @@ end any(itr) -> Bool Test whether any elements of a boolean collection are `true`. + +```jldoctest +julia> a = [true,false,false,true] +4-element Array{Bool,1}: + true + false + false + true + +julia> any(a) +true +``` """ any(itr) = any(identity, itr) @@ -429,6 +585,18 @@ any(itr) = any(identity, itr) all(itr) -> Bool Test whether all elements of a boolean collection are `true`. + +```jldoctest +julia> a = [true,false,false,true] +4-element Array{Bool,1}: + true + false + false + true + +julia> all(a) +false +``` """ all(itr) = all(identity, itr) @@ -480,6 +648,31 @@ all(f::typeof(identity), itr) = in(x, itr) = any(Predicate(y -> y == x), itr) +""" + in(item, collection) -> Bool + ∈(item,collection) -> Bool + ∋(collection,item) -> Bool + ∉(item,collection) -> Bool + ∌(collection,item) -> Bool + +Determine whether an item is in the given collection, in the sense that it is `==` to one of +the values generated by iterating over the collection. Some collections need a slightly +different definition; for example [`Set`](:obj:`Set`)s check whether the item +[`isequal`](:func:`isequal`) to one of the elements. [`Dict`](:obj:`Dict`)s look for +`(key,value)` pairs, and the key is compared using [`isequal`](:func:`isequal`). To test for +the presence of a key in a dictionary, use [`haskey`](:func:`haskey`) or `k in keys(dict)`. + +```jldoctest +julia> a = 1:3:20 +1:3:19 + +julia> 4 in a +true + +julia> 5 in a +false +``` +""" const ∈ = in ∉(x, itr)=!∈(x, itr) ∋(itr, x)= ∈(x, itr) @@ -517,6 +710,6 @@ end countnz(A) Counts the number of nonzero values in array `A` (dense or sparse). Note that this is not a constant-time operation. -For sparse matrices, one should usually use `nnz`, which returns the number of stored values. +For sparse matrices, one should usually use [`nnz`](:func:`nnz`), which returns the number of stored values. """ countnz(a) = count(x -> x != 0, a) diff --git a/base/reflection.jl b/base/reflection.jl index 80d29f822cfff..862c381b2abaa 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1,10 +1,54 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # name and module reflection + +""" + module_name(m::Module) -> Symbol + +Get the name of a `Module` as a `Symbol`. + +```jldoctest +julia> module_name(Base.LinAlg) +:LinAlg +``` +""" module_name(m::Module) = ccall(:jl_module_name, Ref{Symbol}, (Any,), m) + +""" + module_parent(m::Module) -> Module + +Get a module's enclosing `Module`. `Main` is its own parent, as is `LastMain` after `workspace()`. +```jldoctest +julia> module_parent(Main) +Main + +julia> module_parent(Base.LinAlg.BLAS) +Base.LinAlg +``` +""" module_parent(m::Module) = ccall(:jl_module_parent, Ref{Module}, (Any,), m) + +""" + current_module() -> Module + +Get the *dynamically* current `Module`, which is the `Module` code is currently being read +from. In general, this is not the same as the module containing the call to this function. +""" current_module() = ccall(:jl_get_current_module, Ref{Module}, ()) +""" + fullname(m::Module) + +Get the fully-qualified name of a module as a tuple of symbols. For example, + +```jldoctest +julia> fullname(Base.Pkg) +(:Base,:Pkg) + +julia> fullname(Main) +() +``` +""" function fullname(m::Module) m === Main && return () m === Base && return (:Base,) # issue #10653 @@ -27,6 +71,12 @@ function fullname(m::Module) return tuple(fullname(mp)..., mn) end +""" + names(x::Module, all::Bool=false, imported::Bool=false) + +Get an array of the names exported by a `Module`, with optionally more `Module` globals +according to the additional parameters. +""" names(m::Module, all::Bool=false, imported::Bool=false) = sort!(ccall(:jl_module_names, Array{Symbol,1}, (Any,Int32,Int32), m, all, imported)) isexported(m::Module, s::Symbol) = ccall(:jl_module_exports_p, Cint, (Any, Any), m, s) != 0 @@ -48,9 +98,17 @@ function resolve(g::GlobalRef; force::Bool=false) end """ - fieldname(x::DataType, i) + fieldname(x::DataType, i::Integer) Get the name of field `i` of a `DataType`. + +```jldoctest +julia> fieldname(SparseMatrixCSC,1) +:m + +julia> fieldname(SparseMatrixCSC,5) +:nzval +``` """ fieldname(t::DataType, i::Integer) = t.name.names[i]::Symbol fieldname{T<:Tuple}(t::Type{T}, i::Integer) = i < 1 || i > nfields(t) ? throw(BoundsError(t, i)) : Int(i) @@ -59,6 +117,13 @@ fieldname{T<:Tuple}(t::Type{T}, i::Integer) = i < 1 || i > nfields(t) ? throw(Bo fieldnames(x::DataType) Get an array of the fields of a `DataType`. + +```jldoctest +julia> fieldnames(Hermitian) +2-element Array{Symbol,1}: + :data + :uplo +``` """ function fieldnames(v) t = typeof(v) @@ -86,6 +151,12 @@ datatype_module(t::DataType) = t.name.module isconst(s::Symbol) = ccall(:jl_is_const, Int32, (Ptr{Void}, Any), C_NULL, s) != 0 +""" + isconst([m::Module], s::Symbol) -> Bool + +Determine whether a global is declared `const` in a given `Module`. The default `Module` +argument is [`current_module()`](:func:`current_module`). +""" isconst(m::Module, s::Symbol) = ccall(:jl_is_const, Int32, (Any, Any), m, s) != 0 @@ -115,14 +186,50 @@ datatype_pointerfree(dt::DataType) = dt.layout == C_NULL ? throw(UndefRefError() datatype_fielddesc_type(dt::DataType) = dt.layout == C_NULL ? throw(UndefRefError()) : (unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).alignment >> 30) & 3 +""" + isimmutable(v) + +Return `true` iff value `v` is immutable. See [manual](:ref:`man-immutable-composite-types`) +for a discussion of immutability. Note that this function works on values, so if you give it +a type, it will tell you that a value of `DataType` is mutable. +""" isimmutable(x::ANY) = (@_pure_meta; (isa(x,Tuple) || !typeof(x).mutable)) isstructtype(t::DataType) = (@_pure_meta; nfields(t) != 0 || (t.size==0 && !t.abstract)) isstructtype(x) = (@_pure_meta; false) + +""" + isbits(T) + +Return `true` if `T` is a "plain data" type, meaning it is immutable and contains no +references to other values. Typical examples are numeric types such as `UInt8`, `Float64`, +and `Complex{Float64}`. + +```jldoctest +julia> isbits(Complex{Float64}) +true + +julia> isbits(Complex) +false +``` +""" isbits(t::DataType) = (@_pure_meta; !t.mutable & (t.layout != C_NULL) && datatype_pointerfree(t)) isbits(t::Type) = (@_pure_meta; false) isbits(x) = (@_pure_meta; isbits(typeof(x))) + +""" + isleaftype(T) + +Determine whether `T` is a concrete type that can have instances, meaning its only subtypes +are itself and `Union{}` (but `T` itself is not `Union{}`). +""" isleaftype(t::ANY) = (@_pure_meta; isa(t, DataType) && t.isleaftype) +""" + typeintersect(T, S) + +Compute a type that contains the intersection of `T` and `S`. Usually this will be the +smallest such type or one close to it. +""" typeintersect(a::ANY,b::ANY) = (@_pure_meta; ccall(:jl_type_intersection, Any, (Any,Any), a, b)) typeseq(a::ANY,b::ANY) = (@_pure_meta; a<:b && b<:a) @@ -163,6 +270,13 @@ fieldtype type_alignment(x::DataType) = (@_pure_meta; ccall(:jl_get_alignment, Csize_t, (Any,), x)) # return all instances, for types that can be enumerated + +""" + instances(T::Type) + +Return a collection of all instances of the given type, if applicable. Mostly used for +enumerated types (see `@enum`). +""" function instances end # subtypes @@ -182,6 +296,22 @@ function _subtypes(m::Module, x::DataType, sts=Set(), visited=Set()) return sts end subtypes(m::Module, x::DataType) = sort(collect(_subtypes(m, x)), by=string) + +""" + subtypes(T::DataType) + +Return a list of immediate subtypes of DataType `T`. Note that all currently loaded subtypes +are included, including those not visible in the current module. + +```jldoctest +julia> subtypes(Integer) +4-element Array{Any,1}: + BigInt + Bool + Signed + Unsigned +``` +""" subtypes(x::DataType) = subtypes(Main, x) function to_tuple_type(t::ANY) @@ -274,6 +404,13 @@ function MethodList(mt::MethodTable) MethodList(ms, mt) end +""" + methods(f, [types]) + +Returns the method table for `f`. + +If `types` is specified, returns an array of methods whose types match. +""" function methods(f::ANY, t::ANY) if isa(f, Core.Builtin) throw(ArgumentError("argument is not a generic function")) @@ -290,7 +427,6 @@ function methods_including_ambiguous(f::ANY, t::ANY) ms = ccall(:jl_matching_methods, Any, (Any,Cint,Cint), tt, -1, 1)::Array{Any,1} return MethodList(Method[m[3] for m in ms], typeof(f).name.mt) end - function methods(f::ANY) # return all matches return methods(f, Tuple{Vararg{Any}}) @@ -456,6 +592,13 @@ function return_types(f::ANY, types::ANY=Tuple) rt end +""" + which(f, types) + +Returns the method of `f` (a `Method` object) that would be called for arguments of the given `types`. + +If `types` is an abstract type, then the method that would be called by `invoke` is returned. +""" function which(f::ANY, t::ANY) if isa(f, Core.Builtin) throw(ArgumentError("argument is not a generic function")) @@ -481,6 +624,11 @@ function which(f::ANY, t::ANY) end end +""" + which(symbol) + +Return the module in which the binding for the variable referenced by `symbol` was created. +""" which(s::Symbol) = which_module(current_module(), s) # TODO: making this a method of which() causes a strange error function which_module(m::Module, s::Symbol) @@ -499,6 +647,12 @@ Get the name of a generic `Function` as a symbol, or `:anonymous`. function_name(f::Function) = typeof(f).name.mt.name functionloc(m::LambdaInfo) = functionloc(m.def) + +""" + functionloc(m::Method) + +Returns a tuple `(filename,line)` giving the location of a `Method` definition. +""" function functionloc(m::Method) ln = m.line if ln <= 0 @@ -507,6 +661,11 @@ function functionloc(m::Method) (find_source_file(string(m.file)), ln) end +""" + functionloc(f::Function, types) + +Returns a tuple `(filename,line)` giving the location of a generic `Function` definition. +""" functionloc(f::ANY, types::ANY) = functionloc(which(f,types)) function functionloc(f) @@ -545,6 +704,17 @@ function function_module(f, types::ANY) first(m).module end +""" + method_exists(f, Tuple type) -> Bool + +Determine whether the given generic function has a method matching the given +[`Tuple`](:obj:`Tuple`) of argument types. + +```jldoctest +julia> method_exists(length, Tuple{Array}) +true +``` +""" function method_exists(f::ANY, t::ANY) t = to_tuple_type(t) t = Tuple{isa(f,Type) ? Type{f} : typeof(f), t.parameters...} diff --git a/base/rounding.jl b/base/rounding.jl index cbb740af82889..c6b26b85dd739 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -206,7 +206,25 @@ function _convert_rounding{T<:AbstractFloat}(::Type{T},x::Real,r::RoundingMode{: end end +""" + set_zero_subnormals(yes::Bool) -> Bool + +If `yes` is `false`, subsequent floating-point operations follow rules for IEEE arithmetic +on subnormal values ("denormals"). Otherwise, floating-point operations are permitted (but +not required) to convert subnormal inputs or outputs to zero. Returns `true` unless +`yes==true` but the hardware does not support zeroing of subnormal numbers. + +`set_zero_subnormals(true)` can speed up some computations on some hardware. However, it can +break identities such as `(x-y==0) == (x==y)`. +""" set_zero_subnormals(yes::Bool) = ccall(:jl_set_zero_subnormals,Int32,(Int8,),yes)==0 + +""" + get_zero_subnormals() -> Bool + +Returns `false` if operations on subnormal floating-point values ("denormals") obey rules +for IEEE arithmetic, and `true` if they might be converted to zeros. +""" get_zero_subnormals() = ccall(:jl_get_zero_subnormals,Int32,())!=0 end #module diff --git a/base/sharedarray.jl b/base/sharedarray.jl index 9b527f9a1b4e6..a2488b171468f 100644 --- a/base/sharedarray.jl +++ b/base/sharedarray.jl @@ -251,9 +251,26 @@ function reshape{T,N}(a::SharedArray{T}, dims::NTuple{N,Int}) A end +""" + procs(S::SharedArray) + +Get the vector of processes that have mapped the shared array. +""" procs(S::SharedArray) = S.pids + +""" + indexpids(S::SharedArray) + +Returns the index of the current worker into the `pids` vector, i.e., the list of workers +mapping the SharedArray +""" indexpids(S::SharedArray) = S.pidx +""" + sdata(S::SharedArray) + +Returns the actual `Array` object backing `S`. +""" sdata(S::SharedArray) = S.s sdata(A::AbstractArray) = A diff --git a/base/socket.jl b/base/socket.jl index 52b2dd0178062..8e6961b6f97fb 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -21,6 +21,11 @@ immutable IPv4 <: IPAddr end end +""" + IPv4(host::Integer) -> IPv4 + +Returns an IPv4 object from ip address `host` formatted as an `Integer`. +""" function IPv4(host::Integer) if host < 0 throw(ArgumentError("IPv4 address must be positive")) @@ -63,6 +68,11 @@ immutable IPv6 <: IPAddr end end +""" + IPv6(host::Integer) -> IPv6 + +Returns an IPv6 object from ip address `host` formatted as an `Integer`. +""" function IPv6(host::Integer) if host < 0 throw(ArgumentError("IPv6 address must be positive")) @@ -393,6 +403,15 @@ _bind(sock::UDPSocket, host::IPv4, port::UInt16, flags::UInt32 = UInt32(0)) = cc _bind(sock::UDPSocket, host::IPv6, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_udp_bind6, Int32, (Ptr{Void}, UInt16, Ptr{UInt128}, UInt32), sock.handle, hton(port), Ref(hton(host.host)), flags) +""" + bind(socket::Union{UDPSocket, TCPSocket}, host::IPAddr, port::Integer; ipv6only=false, reuseaddr=false, kws...) + +Bind `socket` to the given `host:port`. Note that `0.0.0.0` will listen on all devices. + +* The `ipv6only` parameter disables dual stack mode. If `ipv6only=true`, only an IPv6 stack is created. +* If `reuseaddr=true`, multiple threads or processes can bind to the same address without error + if they all set `reuseaddr=true`, but only the last to bind will receive any traffic. +""" function bind(sock::Union{TCPServer, UDPSocket}, host::IPAddr, port::Integer; ipv6only = false, reuseaddr = false, kws...) if sock.status != StatusInit error("$(typeof(sock)) is not in initialization state") @@ -420,6 +439,17 @@ end bind(sock::TCPServer, addr::InetAddr) = bind(sock, addr.host, addr.port) +""" + setopt(sock::UDPSocket; multicast_loop = nothing, multicast_ttl=nothing, enable_broadcast=nothing, ttl=nothing) + +Set UDP socket options. + +* `multicast_loop`: loopback for multicast packets (default: `true`). +* `multicast_ttl`: TTL for multicast packets (default: `nothing`). +* `enable_broadcast`: flag must be set to `true` if socket will be used for broadcast messages, + or else the UDP system will return an access error (default: `false`). +* `ttl`: Time-to-live of packets sent on the socket (default: `nothing`). +""" function setopt(sock::UDPSocket; multicast_loop = nothing, multicast_ttl=nothing, enable_broadcast=nothing, ttl=nothing) if sock.status == StatusUninit error("Cannot set options on uninitialized socket") @@ -449,11 +479,22 @@ end _recv_stop(sock::UDPSocket) = uv_error("recv_stop", ccall(:uv_udp_recv_stop, Cint, (Ptr{Void}, ), sock.handle)) +""" + recv(socket::UDPSocket) + +Read a UDP packet from the specified socket, and return the bytes received. This call blocks. +""" function recv(sock::UDPSocket) addr, data = recvfrom(sock) data end +""" + recvfrom(socket::UDPSocket) -> (address, data) + +Read a UDP packet from the specified socket, returning a tuple of (address, data), where +address will be either IPv4 or IPv6 as appropriate. +""" function recvfrom(sock::UDPSocket) # If the socket has not been bound, it will be bound implicitly to ::0 and a random port if sock.status != StatusInit && sock.status != StatusOpen @@ -499,6 +540,11 @@ function _send(sock::UDPSocket, ipaddr::IPv6, port::UInt16, buf) sock.handle, hton(port), &hton(ipaddr.host), buf, sizeof(buf), uv_jl_sendcb::Ptr{Void}) end +""" + send(socket::UDPSocket, host, port::Integer, msg) + +Send `msg` over `socket` to `host:port`. +""" function send(sock::UDPSocket,ipaddr,port,msg) # If the socket has not been bound, it will be bound implicitly to ::0 and a random port if sock.status != StatusInit && sock.status != StatusOpen @@ -577,6 +623,11 @@ function getaddrinfo(cb::Function, host::String) end getaddrinfo(cb::Function, host::AbstractString) = getaddrinfo(cb, String(host)) +""" + getaddrinfo(host::AbstractString) -> IPAddr + +Gets the IP address of the `host` (may have to do a DNS lookup) +""" function getaddrinfo(host::String) c = Condition() getaddrinfo(host) do IP @@ -600,6 +651,11 @@ getaddrinfo(host::AbstractString) = getaddrinfo(String(host)) const _sizeof_uv_interface_address = ccall(:jl_uv_sizeof_interface_address,Int32,()) +""" + getipaddr() -> IPAddr + +Get the IP address of the local machine. +""" function getipaddr() addr = Array{Ptr{UInt8}}(1) addr[1] = C_NULL @@ -664,6 +720,12 @@ end connect!(sock::TCPSocket, addr::InetAddr) = connect!(sock, addr.host, addr.port) # Default Host to localhost + +""" + connect([host], port::Integer) -> TCPSocket + +Connect to the host `host` on port `port`. +""" connect(sock::TCPSocket, port::Integer) = connect(sock,IPv4(127,0,0,1), port) connect(port::Integer) = connect(IPv4(127,0,0,1), port) @@ -687,6 +749,12 @@ end ## +""" + listen([addr, ]port::Integer; backlog::Integer=BACKLOG_DEFAULT) -> TCPServer + +Listen on port on the address specified by `addr`. By default this listens on localhost +only. To listen on all interfaces pass `IPv4(0)` or `IPv6(0)` as appropriate. +""" function listen(addr; backlog::Integer=BACKLOG_DEFAULT) sock = TCPServer() !bind(sock, addr) && error("cannot bind to port; may already be in use or access denied") @@ -735,6 +803,12 @@ end ## Utility functions +""" + listenany([host::IPAddr,] port_hint) -> (UInt16, TCPServer) + +Create a `TCPServer` on any port, using hint as a starting point. Returns a tuple of the +actual port that the server was created on and the server itself. +""" function listenany(host::IPAddr, default_port) addr = InetAddr(host, default_port) while true @@ -752,6 +826,12 @@ end listenany(default_port) = listenany(IPv4(UInt32(0)), default_port) +""" + getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr, UInt16) + +Get the IP address and the port that the given `TCPSocket` is connected to +(or bound to, in the case of `TCPServer`). +""" function getsockname(sock::Union{TCPServer,TCPSocket}) rport = Ref{Cushort}(0) raddress = zeros(UInt8, 16) diff --git a/base/sort.jl b/base/sort.jl index 91e752202c57b..9c4d6e34a3744 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -54,6 +54,13 @@ function issorted(itr, order::Ordering) end return true end + +""" + issorted(v, by=identity, rev:Bool=false, order::Ordering=Forward) + +Test whether a vector is in sorted order. The `by`, `lt` and `rev` keywords modify what +order is considered to be sorted just as they do for [`sort`](:func:`sort`). +""" issorted(itr; lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) = issorted(itr, ord(lt,by,rev,order)) @@ -405,6 +412,18 @@ function sort!(v::AbstractVector, alg::Algorithm, order::Ordering) sort!(v,first(inds),last(inds),alg,order) end +""" + sort!(v; alg::Algorithm=defalg(v), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) + +Sort the vector `v` in place. `QuickSort` is used by default for numeric arrays while +`MergeSort` is used for other arrays. You can specify an algorithm to use via the `alg` +keyword (see Sorting Algorithms for available algorithms). The `by` keyword lets you provide +a function that will be applied to each element before comparison; the `lt` keyword allows +providing a custom "less than" function; use `rev=true` to reverse the sorting order. These +options are independent and can be used together in all possible combinations: if both `by` +and `lt` are specified, the `lt` function is applied to the result of the `by` function; +`rev=true` reverses whatever ordering specified via the `by` and `lt` keywords. +""" function sort!(v::AbstractVector; alg::Algorithm=defalg(v), lt=isless, @@ -414,6 +433,11 @@ function sort!(v::AbstractVector; sort!(v, alg, ord(lt,by,rev,order)) end +""" + sort(v; alg::Algorithm=defalg(v), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) + +Variant of [`sort!`](:func:`sort!`) that returns a sorted copy of `v` leaving `v` itself unmodified. +""" sort(v::AbstractVector; kws...) = sort!(copymutable(v); kws...) @@ -442,6 +466,19 @@ end ## sortperm: the permutation to sort an array ## +""" + sortperm(v; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) + +Return a permutation vector of indices of `v` that puts it in sorted order. Specify `alg` to +choose a particular sorting algorithm (see Sorting Algorithms). `MergeSort` is used by +default, and since it is stable, the resulting permutation will be the lexicographically +first one that puts the input array into sorted order – i.e. indices of equal elements +appear in ascending order. If you choose a non-stable sorting algorithm such as `QuickSort`, +a different permutation that puts the array into order may be returned. The order is +specified using the same keywords as `sort!`. + +See also [`sortperm!`](:func:`sortperm!`). +""" function sortperm(v::AbstractVector; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, @@ -455,6 +492,13 @@ function sortperm(v::AbstractVector; sort!(p, alg, Perm(ord(lt,by,rev,order),v)) end + +""" + sortperm!(ix, v; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward, initialized::Bool=false) + +Like [`sortperm`](:func:`sortperm`), but accepts a preallocated index vector `ix`. If `initialized` is `false` +(the default), ix is initialized to contain the values `1:length(v)`. +""" function sortperm!{I<:Integer}(x::AbstractVector{I}, v::AbstractVector; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, @@ -475,6 +519,11 @@ end ## sorting multi-dimensional arrays ## +""" + sort(A, dim::Integer; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward, initialized::Bool=false) + +Sort a multidimensional array `A` along the given dimension. `lt` defines the comparison to use. +""" function sort(A::AbstractArray, dim::Integer; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, @@ -505,6 +554,14 @@ end Av end + +""" + sortrows(A; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) + +Sort the rows of matrix `A` lexicographically. +See [`sort`](:func:`sort`) for a description of possible +keyword arguments. +""" function sortrows(A::AbstractMatrix; kws...) inds = indices(A,1) T = slicetypeof(A, inds, :) @@ -516,6 +573,13 @@ function sortrows(A::AbstractMatrix; kws...) A[p,:] end +""" + sortcols(A; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) + +Sort the columns of matrix `A` lexicographically. +See [`sort`](:func:`sort`) for a description of possible +keyword arguments. +""" function sortcols(A::AbstractMatrix; kws...) inds = indices(A,2) T = slicetypeof(A, :, inds) diff --git a/base/stat.jl b/base/stat.jl index 9b1b5391edf99..073efd3d0b954 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -80,35 +80,179 @@ stat(fd::Integer) = @stat_call jl_fstat Int32 fd stat(path::AbstractString) = @stat_call jl_stat Cstring path lstat(path::AbstractString) = @stat_call jl_lstat Cstring path +""" + stat(file) + +Returns a structure whose fields contain information about the file. +The fields of the structure are: + +| Name | Description | +|:--------|:-------------------------------------------------------------------| +| size | The size (in bytes) of the file | +| device | ID of the device that contains the file | +| inode | The inode number of the file | +| mode | The protection mode of the file | +| nlink | The number of hard links to the file | +| uid | The user id of the owner of the file | +| gid | The group id of the file owner | +| rdev | If this file refers to a device, the ID of the device it refers to | +| blksize | The file-system preferred block size for the file | +| blocks | The number of such blocks allocated | +| mtime | Unix timestamp of when the file was last modified | +| ctime | Unix timestamp of when the file was created | + +""" stat(path...) = stat(joinpath(path...)) + +""" + lstat(file) + +Like [`stat`](:func:`stat`), but for symbolic links gets the info for the link +itself rather than the file it refers to. +This function must be called on a file path rather than a file object or a file +descriptor. +""" lstat(path...) = lstat(joinpath(path...)) # some convenience functions +""" + filemode(file) + +Equivalent to `stat(file).mode` +""" filemode(st::StatStruct) = st.mode + +""" + filesize(path...) + +Equivalent to `stat(file).size`. +""" filesize(st::StatStruct) = st.size + +""" + mtime(file) + +Equivalent to `stat(file).mtime`. +""" mtime(st::StatStruct) = st.mtime + +""" + ctime(file) + +Equivalent to `stat(file).ctime` +""" ctime(st::StatStruct) = st.ctime # mode type predicates +""" + ispath(path) -> Bool + +Returns `true` if `path` is a valid filesystem path, `false` otherwise. +""" ispath(st::StatStruct) = filemode(st) & 0xf000 != 0x0000 + +""" + isfifo(path) -> Bool + +Returns `true` if `path` is a FIFO, `false` otherwise. +""" isfifo(st::StatStruct) = filemode(st) & 0xf000 == 0x1000 + +""" + ischardev(path) -> Bool + +Returns `true` if `path` is a character device, `false` otherwise. +""" ischardev(st::StatStruct) = filemode(st) & 0xf000 == 0x2000 + + """ + isdir(path) -> Bool + + Returns `true` if `path` is a directory, `false` otherwise. + """ isdir(st::StatStruct) = filemode(st) & 0xf000 == 0x4000 + + """ + isblockdev(path) -> Bool + + Returns `true` if `path` is a block device, `false` otherwise. + """ isblockdev(st::StatStruct) = filemode(st) & 0xf000 == 0x6000 + +""" + isfile(path) -> Bool + +Returns `true` if `path` is a regular file, `false` otherwise. +""" isfile(st::StatStruct) = filemode(st) & 0xf000 == 0x8000 + +""" + islink(path) -> Bool + +Returns `true` if `path` is a symbolic link, `false` otherwise. +""" islink(st::StatStruct) = filemode(st) & 0xf000 == 0xa000 + +""" + issocket(path) -> Bool + +Returns `true` if `path` is a socket, `false` otherwise. +""" issocket(st::StatStruct) = filemode(st) & 0xf000 == 0xc000 # mode permission predicates +""" + issetuid(path) -> Bool + +Returns `true` if `path` has the setuid flag set, `false` otherwise. +""" issetuid(st::StatStruct) = (filemode(st) & 0o4000) > 0 + +""" + issetgid(path) -> Bool + +Returns `true` if `path` has the setgid flag set, `false` otherwise. +""" issetgid(st::StatStruct) = (filemode(st) & 0o2000) > 0 + +""" + issticky(path) -> Bool + +Returns `true` if `path` has the sticky bit set, `false` otherwise. +""" issticky(st::StatStruct) = (filemode(st) & 0o1000) > 0 +""" + uperm(file) + +Gets the permissions of the owner of the file as a bitfield of + +| Value | Description | +|:------|:-------------------| +| 01 | Execute Permission | +| 02 | Write Permission | +| 04 | Read Permission | + +For allowed arguments, see [`stat`](:func:`stat`). +""" uperm(st::StatStruct) = UInt8((filemode(st) >> 6) & 0x7) + +""" + gperm(file) + +Like [`uperm`](:func:`uperm`) but gets the permissions of the group owning the file. +""" gperm(st::StatStruct) = UInt8((filemode(st) >> 3) & 0x7) + +""" + operm(file) + +Like [`uperm`](:func:`uperm`) but gets the permissions for people who neither own the file nor are a member of +the group owning the file +""" operm(st::StatStruct) = UInt8((filemode(st) ) & 0x7) # mode predicate methods for file names @@ -149,6 +293,11 @@ function samefile(a::AbstractString, b::AbstractString) end end +""" + ismount(path) -> Bool + +Returns `true` if `path` is a mount point, `false` otherwise. +""" function ismount(path...) path = joinpath(path...) isdir(path) || return false diff --git a/base/statistics.jl b/base/statistics.jl index 37ef2afe459c5..864d23f63001f 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -177,6 +177,22 @@ var{T}(A::AbstractArray{T}; corrected::Bool=true, mean=nothing) = convert(real(momenttype(T)), varm(A, mean === nothing ? Base.mean(A) : mean; corrected=corrected)) +""" + var(v[, region]; corrected::Bool=true, mean=nothing) + +Compute the sample variance of a vector or array `v`, optionally along dimensions in +`region`. The algorithm will return an estimator of the generative distribution's variance +under the assumption that each entry of `v` is an IID drawn from that generative +distribution. This computation is equivalent to calculating `sumabs2(v - mean(v)) / +(length(v) - 1)`. If `corrected` is `true`, then the sum is scaled with `n-1`, +whereas the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. +The mean `m` over the region may be provided. + +!!! note + Julia does not ignore `NaN` values in the computation. For + applications requiring the handling of missing data, the + `DataArrays.jl` package is recommended. +""" var(A::AbstractArray, region; corrected::Bool=true, mean=nothing) = varm(A, mean === nothing ? Base.mean(A, region) : mean, region; corrected=corrected) diff --git a/base/stream.jl b/base/stream.jl index f42af7b82c2ec..081d5e5c1297e 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -889,7 +889,11 @@ function bind(server::PipeServer, name::AbstractString) return true end +""" + listen(path::AbstractString) -> PipeServer +Create and listen on a named pipe / UNIX domain socket. +""" function listen(path::AbstractString) sock = PipeServer() bind(sock, path) || throw(ArgumentError("could not listen on path $path")) @@ -913,6 +917,12 @@ end # Libuv will internally reset read/writability, which is uses to # mark that this is an invalid pipe. + +""" + connect(path::AbstractString) -> PipeEndpoint + +Connect to the named pipe / UNIX domain socket at `path`. +""" connect(path::AbstractString) = connect(init_pipe!(PipeEndpoint(); readable=false, writable=false, julia_only=true),path) _fd(x::IOStream) = RawFD(fd(x)) @@ -956,35 +966,31 @@ for (x, writable, unix_fd, c_symbol) in end """ - redirect_stdout() - -Create a pipe to which all C and Julia level `STDOUT` output will be redirected. Returns a -tuple `(rd,wr)` representing the pipe ends. Data written to `STDOUT` may now be read from -the rd end of the pipe. The wr end is given for convenience in case the old `STDOUT` object -was cached by the user and needs to be replaced elsewhere. + redirect_stdout([stream]) -> (rd, wr) + +Create a pipe to which all C and Julia level [`STDOUT`](:obj:`STDOUT`) output +will be redirected. +Returns a tuple `(rd, wr)` representing the pipe ends. +Data written to [`STDOUT`](:obj:`STDOUT`) may now be read from the `rd` end of +the pipe. The `wr` end is given for convenience in case the old +[`STDOUT`](:obj:`STDOUT`) object was cached by the user and needs to be replaced +elsewhere. """ redirect_stdout """ - redirect_stdout(stream) - -Replace `STDOUT` by stream for all C and Julia level output to `STDOUT`. Note that `stream` -must be a TTY, a `Pipe` or a `TCPSocket`. -""" -redirect_stdout(stream) - -""" - redirect_stderr([stream]) + redirect_stderr([stream]) -> (rd, wr) -Like `redirect_stdout`, but for `STDERR`. +Like [`redirect_stdout`](:func:`redirect_stdout`), but for [`STDERR`](:obj:`STDERR`). """ redirect_stderr """ - redirect_stdin([stream]) + redirect_stdin([stream]) -> (rd, wr) -Like redirect_stdout, but for STDIN. Note that the order of the return tuple is still -(rd,wr), i.e. data to be read from STDIN, may be written to wr. +Like [`redirect_stdout`](:func:`redirect_stdout`), but for [`STDIN`](:obj:`STDIN`). +Note that the order of the return tuple is still `(rd, wr)`, +i.e. data to be read from [`STDIN`](:obj:`STDIN`) may be written to `wr`. """ redirect_stdin diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 8e2eedca19399..04a0332e691a3 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -77,15 +77,43 @@ getindex(s::AbstractString, v::AbstractVector) = Symbol(s::AbstractString) = Symbol(String(s)) +""" + sizeof(s::AbstractString) + +The number of bytes in string `s`. + +```jldoctest +julia> sizeof("❤") +3 +``` +""" sizeof(s::AbstractString) = error("type $(typeof(s)) has no canonical binary representation") eltype{T<:AbstractString}(::Type{T}) = Char +""" +``` +*(s::AbstractString, t::AbstractString) +``` + +Concatenate strings. The `*` operator is an alias to this function. + +```jldoctest +julia> "Hello " * "world" +"Hello world" +``` +""" (*)(s1::AbstractString, ss::AbstractString...) = string(s1, ss...) (.*){T<:AbstractString}(v::Vector{T},s::AbstractString) = [i*s for i in v] (.*){T<:AbstractString}(s::AbstractString,v::Vector{T}) = [s*i for i in v] length(s::DirectIndexString) = endof(s) + +""" + length(s::AbstractString) + +The number of characters in string `s`. +""" function length(s::AbstractString) i = start(s) if done(s,i) @@ -139,6 +167,12 @@ isless(a::Symbol, b::Symbol) = cmp(a,b) < 0 ## Generic validation functions ## isvalid(s::DirectIndexString, i::Integer) = (start(s) <= i <= endof(s)) + +""" + isvalid(str::AbstractString, i::Integer) + +Tells whether index `i` is valid for the given string. +""" function isvalid(s::AbstractString, i::Integer) i < 1 && return false done(s,i) && return false @@ -183,6 +217,12 @@ function nextind(s::String, i::Integer) j end +""" + prevind(str::AbstractString, i::Integer) + +Get the previous valid string index before `i`. +Returns a value less than `1` at the beginning of the string. +""" function prevind(s::AbstractString, i::Integer) e = endof(s) if i > e @@ -198,6 +238,12 @@ function prevind(s::AbstractString, i::Integer) return 0 # out of range end +""" + nextind(str::AbstractString, i::Integer) + +Get the next valid string index after `i`. +Returns a value greater than `endof(str)` at or after the end of the string. +""" function nextind(s::AbstractString, i::Integer) e = endof(s) if i < 1 @@ -223,6 +269,12 @@ checkbounds{T<:Integer}(s::AbstractString, I::AbstractArray{T}) = all(i -> check ind2chr(s::DirectIndexString, i::Integer) = begin checkbounds(s,i); i end chr2ind(s::DirectIndexString, i::Integer) = begin checkbounds(s,i); i end + +""" + ind2chr(s::AbstractString, i::Integer) + +Convert a byte index `i` to a character index. +""" function ind2chr(s::AbstractString, i::Integer) s[i] # throws error if invalid j = 1 @@ -237,6 +289,11 @@ function ind2chr(s::AbstractString, i::Integer) end end +""" + chr2ind(s::AbstractString, i::Integer) + +Convert a character index `i` to a byte index. +""" function chr2ind(s::AbstractString, i::Integer) i < start(s) && throw(BoundsError(s, i)) j = 1 @@ -268,8 +325,24 @@ typealias ByteArray Union{Vector{UInt8},Vector{Int8}} ## character column width function ## +""" + strwidth(s::AbstractString) + +Gives the number of columns needed to print a string. + +```jldoctest +julia> strwidth("March") +5 +``` +""" strwidth(s::AbstractString) = (w=0; for c in s; w += charwidth(c); end; w) +""" + isascii(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character belongs to the ASCII character set, or whether this is true for +all elements of a string. +""" isascii(c::Char) = c < Char(0x80) isascii(s::AbstractString) = all(isascii, s) @@ -277,6 +350,19 @@ isascii(s::AbstractString) = all(isascii, s) promote_rule{S<:AbstractString,T<:AbstractString}(::Type{S}, ::Type{T}) = String +""" + isxdigit(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is a valid hexadecimal digit, or whether this is true for all elements of a string. + +```jldoctest +julia> isxdigit("abc") +true + +julia> isxdigit("0x9") +false +``` +""" isxdigit(c::Char) = '0'<=c<='9' || 'a'<=c<='f' || 'A'<=c<='F' isxdigit(s::AbstractString) = all(isxdigit, s) @@ -293,12 +379,55 @@ isvalid(::Type{String}, s::Union{Vector{UInt8},String}) = byte_string_classify(s isvalid(s::String) = isvalid(String, s) ## uppercase and lowercase transformations ## + +""" + uppercase(s::AbstractString) + +Returns `s` with all characters converted to uppercase. + +```jldoctest +julia> uppercase("Julia") +"JULIA" +``` +""" uppercase(s::AbstractString) = map(uppercase, s) + +""" + lowercase(s::AbstractString) + +Returns `s` with all characters converted to lowercase. + +```jldoctest +julia> lowercase("STRINGS AND THINGS") +"strings and things" +``` +""" lowercase(s::AbstractString) = map(lowercase, s) +""" + ucfirst(s::AbstractString) + +Returns `string` with the first character converted to uppercase. + +```jldoctest +julia> ucfirst("python") +"Python" +``` +""" function ucfirst(s::AbstractString) isempty(s) || isupper(s[1]) ? s : string(uppercase(s[1]),s[nextind(s,1):end]) end + +""" + lcfirst(s::AbstractString) + +Returns `string` with the first character converted to lowercase. + +```jldoctest +julia> lcfirst("Julia") +"julia" +``` +""" function lcfirst(s::AbstractString) isempty(s) || islower(s[1]) ? s : string(lowercase(s[1]),s[nextind(s,1):end]) end diff --git a/base/strings/io.jl b/base/strings/io.jl index 4dc87ae72c2b5..35b27426bdffd 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -2,6 +2,15 @@ ## core text I/O ## + +""" + print(io::IO, x) + +Write (to the default output stream) a canonical (un-decorated) text +representation of a value if there is one, otherwise call [`show`](:func:`show`). +The representation used by `print` includes minimal formatting and tries to +avoid Julia-specific details. +""" function print(io::IO, x) lock(io) try @@ -24,6 +33,12 @@ function print(io::IO, xs...) return nothing end +""" + println(io::IO, xs...) + +Print (using [`print`](:func:`print`)) `xs` followed by a newline. +If `io` is not supplied, prints to [`STDOUT`](:obj:`STDOUT`). +""" println(io::IO, xs...) = print(io, xs..., '\n') ## conversion of general objects to strings ## @@ -38,6 +53,18 @@ function sprint(size::Integer, f::Function, args...; env=nothing) end String(resize!(s.data, s.size)) end + +""" + sprint(f::Function, args...) + +Call the given function with an I/O stream and the supplied extra arguments. +Everything written to this I/O stream is returned as a string. + +```jldoctest +julia> sprint(showcompact, 66.66666) +"66.6667" +``` +""" sprint(f::Function, args...) = sprint(0, f, args...) tostr_sizehint(x) = 0 @@ -65,6 +92,12 @@ function print_to_string(xs...; env=nothing) end string_with_env(env, xs...) = print_to_string(xs...; env=env) + +""" + string(xs...) + +Create a string from any values using the [`print`](:func:`print`) function. +""" string(xs...) = print_to_string(xs...) print(io::IO, s::AbstractString) = (write(io, s); nothing) @@ -84,6 +117,11 @@ function print_quoted_literal(io, s::AbstractString) print(io, '"') end +""" + repr(x) + +Create a string from any value using the [`showall`](:func:`showall`) function. +""" function repr(x) s = IOBuffer() showall(s, x) @@ -91,10 +129,32 @@ function repr(x) end # IOBuffer views of a (byte)string: + +""" + IOBuffer(string::String) + +Create a read-only `IOBuffer` on the data underlying the given string. +""" IOBuffer(str::String) = IOBuffer(str.data) IOBuffer(s::SubString{String}) = IOBuffer(view(s.string.data, s.offset + 1 : s.offset + sizeof(s))) # join is implemented using IO + +""" + join(io::IO, strings, delim, [last]) + +Join an array of `strings` into a single string, inserting the given delimiter between +adjacent strings. If `last` is given, it will be used instead of `delim` between the last +two strings. For example, + +```jldoctest +julia> join(["apples", "bananas", "pineapples"], ", ", " and ") +"apples, bananas and pineapples" +``` + +`strings` can be any iterable over elements `x` which are convertible to strings +via `print(io::IOBuffer, x)`. +""" function join(io::IO, strings, delim, last) i = start(strings) if done(strings,i) @@ -124,7 +184,6 @@ function join(io::IO, strings, delim) end end join(io::IO, strings) = join(io, strings, "") - join(args...) = sprint(join, args...) ## string escaping & unescaping ## @@ -134,6 +193,13 @@ need_full_hex(s::AbstractString, i::Int) = !done(s,i) && isxdigit(next(s,i)[1]) escape_nul(s::AbstractString, i::Int) = !done(s,i) && '0' <= next(s,i)[1] <= '7' ? "\\x00" : "\\0" +""" + escape_string([io,] str::AbstractString[, esc::AbstractString]) -> AbstractString + +General escaping of traditional C and Unicode escape sequences. +Any characters in `esc` are also escaped (with a backslash). +See also [`unescape_string`](:func:`unescape_string`). +""" function escape_string(io, s::AbstractString, esc::AbstractString) i = start(s) while !done(s,i) @@ -180,6 +246,12 @@ unescape_chars(s::AbstractString, esc::AbstractString) = # general unescaping of traditional C and Unicode escape sequences +""" + unescape_string([io,] s::AbstractString) -> AbstractString + +General unescaping of traditional C and Unicode escape sequences. Reverse of +[`escape_string`](:func:`escape_string`). +""" function unescape_string(io, s::AbstractString) i = start(s) while !done(s,i) diff --git a/base/strings/search.jl b/base/strings/search.jl index eb6f7ebd4e2f8..3158380697343 100644 --- a/base/strings/search.jl +++ b/base/strings/search.jl @@ -1,5 +1,27 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +""" + search(string::AbstractString, chars::Chars, [start::Integer]) + +Search for the first occurrence of the given characters within the given string. The second +argument may be a single character, a vector or a set of characters, a string, or a regular +expression (though regular expressions are only allowed on contiguous strings, such as ASCII +or UTF-8 strings). The third argument optionally specifies a starting index. The return +value is a range of indexes where the matching sequence is found, such that `s[search(s,x)] == x`: + +`search(string, "substring")` = `start:end` such that `string[start:end] == "substring"`, or +`0:-1` if unmatched. + +`search(string, 'c')` = `index` such that `string[index] == 'c'`, or `0` if unmatched. + +```jldoctest +julia> search("Hello to the world", "z") +0:-1 + +julia> search("JuliaLang","Julia") +1:5 +``` +""" function search(s::AbstractString, c::Chars, i::Integer) if isempty(c) return 1 <= i <= nextind(s,endof(s)) ? i : @@ -118,6 +140,13 @@ function _searchindex(s::Array, t::Array, i) end searchindex(s::ByteArray, t::ByteArray, i) = _searchindex(s,t,i) + +""" + searchindex(s::AbstractString, substring, [start::Integer]) + +Similar to [`search`](:func:`search`), but return only the start index at which +the substring is found, or `0` if it is not. +""" searchindex(s::AbstractString, t::AbstractString, i::Integer) = _searchindex(s,t,i) searchindex(s::AbstractString, t::AbstractString) = searchindex(s,t,start(s)) searchindex(s::AbstractString, c::Char, i::Integer) = _searchindex(s,c,i) @@ -157,6 +186,17 @@ function rsearch(s::AbstractString, c::Chars) endof(s)-j+1 end +""" + rsearch(s::AbstractString, chars::Chars, [start::Integer]) + +Similar to [`search`](:func:`search`), but returning the last occurrence of the given characters within the +given string, searching in reverse from `start`. + +```jldoctest +julia> rsearch("aaabbb","b") +6:6 +``` +""" function rsearch(s::AbstractString, c::Chars, i::Integer) e = endof(s) j = search(RevString(s), c, e-i+1) @@ -260,6 +300,12 @@ function _rsearchindex(s::Array, t::Array, k) end rsearchindex(s::ByteArray,t::ByteArray,i) = _rsearchindex(s,t,i) + +""" + rsearchindex(s::AbstractString, substring, [start::Integer]) + +Similar to [`rsearch`](:func:`rsearch`), but return only the start index at which the substring is found, or `0` if it is not. +""" rsearchindex(s::AbstractString, t::AbstractString, i::Integer) = _rsearchindex(s,t,i) rsearchindex(s::AbstractString, t::AbstractString) = (isempty(s) && isempty(t)) ? 1 : rsearchindex(s,t,endof(s)) @@ -307,6 +353,16 @@ function rsearch(s::AbstractString, t::AbstractString, i::Integer=endof(s)) end end +""" + contains(haystack::AbstractString, needle::AbstractString) + +Determine whether the second argument is a substring of the first. + +```jldoctest +julia> contains("JuliaLang is pretty cool!", "Julia") +true +``` +""" contains(haystack::AbstractString, needle::AbstractString) = searchindex(haystack,needle)!=0 in(::AbstractString, ::AbstractString) = error("use contains(x,y) for string containment") @@ -356,5 +412,3 @@ function rsearch(a::ByteArray, b::Char, i::Integer) end end rsearch(a::ByteArray, b::Union{Int8,UInt8,Char}) = rsearch(a,b,length(a)) - - diff --git a/base/strings/types.jl b/base/strings/types.jl index b98ea229f57ab..beb64916c0ea6 100644 --- a/base/strings/types.jl +++ b/base/strings/types.jl @@ -113,6 +113,15 @@ function next(s::RevString, i::Int) (s.string[j], n-prevind(s.string,j)+1) end +""" + reverse(s::AbstractString) -> AbstractString + +Reverses a string. +```jldoctest +julia> reverse("JuliaLang") +"gnaLailuJ" +``` +""" reverse(s::AbstractString) = RevString(s) reverse(s::RevString) = s.string @@ -175,6 +184,17 @@ function repeat(s::String, r::Integer) convert(typeof(s), out) end +""" + ^(s::AbstractString, n::Integer) + +Repeat `n` times the string `s`. +The [`repeat`](:func:`repeat`) function is an alias to this operator. + +```jldoctest +julia> "Test "^3 +"Test Test Test " +``` +""" (^)(s::AbstractString, r::Integer) = repeat(s,r) pointer(x::SubString{String}) = pointer(x.string.data) + x.offset diff --git a/base/strings/utf8proc.jl b/base/strings/utf8proc.jl index aadcf97ba83da..ce08d32014b3e 100644 --- a/base/strings/utf8proc.jl +++ b/base/strings/utf8proc.jl @@ -104,6 +104,40 @@ function normalize_string(s::AbstractString; stable::Bool=false, compat::Bool=fa utf8proc_map(s, flags) end +""" + normalize_string(s::AbstractString, normalform::Symbol) + +Normalize the string `s` according to one of the four "normal forms" of the Unicode +standard: `normalform` can be `:NFC`, `:NFD`, `:NFKC`, or `:NFKD`. Normal forms C +(canonical composition) and D (canonical decomposition) convert different visually identical +representations of the same abstract string into a single canonical form, with form C being +more compact. Normal forms KC and KD additionally canonicalize "compatibility equivalents": +they convert characters that are abstractly similar but visually distinct into a single +canonical choice (e.g. they expand ligatures into the individual characters), with form KC +being more compact. + +Alternatively, finer control and additional transformations may be be obtained by calling +`normalize_string(s; keywords...)`, where any number of the following boolean keywords +options (which all default to `false` except for `compose`) are specified: + +* `compose=false`: do not perform canonical composition +* `decompose=true`: do canonical decomposition instead of canonical composition (`compose=true` + is ignored if present) +* `compat=true`: compatibility equivalents are canonicalized +* `casefold=true`: perform Unicode case folding, e.g. for case-insensitive string comparison +* `newline2lf=true`, `newline2ls=true`, or `newline2ps=true`: convert various newline sequences + (LF, CRLF, CR, NEL) into a linefeed (LF), line-separation (LS), or paragraph-separation (PS) + character, respectively +* `stripmark=true`: strip diacritical marks (e.g. accents) +* `stripignore=true`: strip Unicode's "default ignorable" characters (e.g. the soft hyphen + or the left-to-right marker) +* `stripcc=true`: strip control characters; horizontal tabs and form feeds are converted to + spaces; newlines are also converted to spaces unless a newline-conversion flag was specified +* `rejectna=true`: throw an error if unassigned code points are found +* `stable=true`: enforce Unicode Versioning Stability + +For example, NFKC corresponds to the options `compose=true, compat=true, stable=true`. +""" function normalize_string(s::AbstractString, nf::Symbol) utf8proc_map(s, nf == :NFC ? (UTF8PROC_STABLE | UTF8PROC_COMPOSE) : nf == :NFD ? (UTF8PROC_STABLE | UTF8PROC_DECOMPOSE) : @@ -116,6 +150,11 @@ end ############################################################################ +""" + charwidth(c) + +Gives the number of columns needed to print a character. +""" charwidth(c::Char) = Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), c)) lowercase(c::Char) = isascii(c) ? ('A' <= c <= 'Z' ? c + 0x20 : c) : Char(ccall(:utf8proc_tolower, UInt32, (UInt32,), c)) @@ -128,22 +167,71 @@ function category_code(c) return ccall(:utf8proc_category, Cint, (UInt32,), c) end +""" + is_assigned_char(c) -> Bool + +Returns `true` if the given char or integer is an assigned Unicode code point. +""" is_assigned_char(c) = category_code(c) != UTF8PROC_CATEGORY_CN ## libc character class predicates ## +""" + islower(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is a lowercase letter, or whether this is true for all elements of +a string. A character is classified as lowercase if it belongs to Unicode category Ll, +Letter: Lowercase. +""" islower(c::Char) = (category_code(c) == UTF8PROC_CATEGORY_LL) # true for Unicode upper and mixed case + +""" + isupper(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is an uppercase letter, or whether this is true for all elements +of a string. A character is classified as uppercase if it belongs to Unicode category Lu, +Letter: Uppercase, or Lt, Letter: Titlecase. +""" function isupper(c::Char) ccode = category_code(c) return ccode == UTF8PROC_CATEGORY_LU || ccode == UTF8PROC_CATEGORY_LT end +""" + isdigit(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is a numeric digit (0-9), or whether this is true for all elements +of a string. +""" isdigit(c::Char) = ('0' <= c <= '9') + +""" + isalpha(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is alphabetic, or whether this is true for all elements of a +string. A character is classified as alphabetic if it belongs to the Unicode general +category Letter, i.e. a character whose category code begins with 'L'. +""" isalpha(c::Char) = (UTF8PROC_CATEGORY_LU <= category_code(c) <= UTF8PROC_CATEGORY_LO) + +""" + isnumber(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is numeric, or whether this is true for all elements of a string. +A character is classified as numeric if it belongs to the Unicode general category Number, +i.e. a character whose category code begins with 'N'. +""" isnumber(c::Char) = (UTF8PROC_CATEGORY_ND <= category_code(c) <= UTF8PROC_CATEGORY_NO) +""" + isalnum(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is alphanumeric, or whether this is true for all elements of a +string. A character is classified as alphabetic if it belongs to the Unicode general +category Letter or Number, i.e. a character whose category code begins with 'L' or 'N'. +""" function isalnum(c::Char) ccode = category_code(c) return (UTF8PROC_CATEGORY_LU <= ccode <= UTF8PROC_CATEGORY_LO) || @@ -151,16 +239,52 @@ function isalnum(c::Char) end # following C++ only control characters from the Latin-1 subset return true + +""" + iscntrl(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is a control character, or whether this is true for all elements +of a string. Control characters are the non-printing characters of the Latin-1 subset of Unicode. +""" iscntrl(c::Char) = (c <= Char(0x1f) || Char(0x7f) <= c <= Char(0x9f)) +""" + ispunct(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character belongs to the Unicode general category Punctuation, i.e. a +character whose category code begins with 'P'. For strings, tests whether this is true for +all elements of the string. +""" ispunct(c::Char) = (UTF8PROC_CATEGORY_PC <= category_code(c) <= UTF8PROC_CATEGORY_PO) # \u85 is the Unicode Next Line (NEL) character + +""" + isspace(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is any whitespace character. Includes ASCII characters '\\t', +'\\n', '\\v', '\\f', '\\r', and ' ', Latin-1 character U+0085, and characters in Unicode +category Zs. For strings, tests whether this is true for all elements of the string. +""" @inline isspace(c::Char) = c == ' ' || '\t' <= c <='\r' || c == '\u85' || '\ua0' <= c && category_code(c) == UTF8PROC_CATEGORY_ZS +""" + isprint(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is printable, including spaces, but not a control character. For +strings, tests whether this is true for all elements of the string. +""" isprint(c::Char) = (UTF8PROC_CATEGORY_LU <= category_code(c) <= UTF8PROC_CATEGORY_ZS) # true in principal if a printer would use ink + +""" + isgraph(c::Union{Char,AbstractString}) -> Bool + +Tests whether a character is printable, and not a space, or whether this is true for all +elements of a string. Any character that would cause a printer to use ink should be +classified with `isgraph(c)==true`. +""" isgraph(c::Char) = (UTF8PROC_CATEGORY_LU <= category_code(c) <= UTF8PROC_CATEGORY_SO) for name = ("alnum", "alpha", "cntrl", "digit", "number", "graph", @@ -193,6 +317,15 @@ isgraphemebreak!(state::Ref{Int32}, c1::Char, c2::Char) = immutable GraphemeIterator{S<:AbstractString} s::S # original string (for generation of SubStrings) end + +""" + graphemes(s::AbstractString) -> GraphemeIterator + +Returns an iterator over substrings of `s` that correspond to the extended graphemes in the +string, as defined by Unicode UAX #29. (Roughly, these are what users would perceive as +single characters, even though they may contain more than one codepoint; for example a +letter combined with an accent mark is a single grapheme.) +""" graphemes(s::AbstractString) = GraphemeIterator{typeof(s)}(s) eltype{S}(::Type{GraphemeIterator{S}}) = SubString{S} diff --git a/base/strings/util.jl b/base/strings/util.jl index 009b1c7eedfa9..f0cb04c75ce9e 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -2,6 +2,17 @@ # starts with and ends with predicates +""" + startswith(s::AbstractString, prefix::AbstractString) + +Returns `true` if `s` starts with `prefix`. If `prefix` is a vector or set +of characters, tests whether the first character of `s` belongs to that set. + +```jldoctest +julia> startswith("JuliaLang", "Julia") +true +``` +""" function startswith(a::AbstractString, b::AbstractString) i = start(a) j = start(b) @@ -14,6 +25,17 @@ function startswith(a::AbstractString, b::AbstractString) end startswith(str::AbstractString, chars::Chars) = !isempty(str) && first(str) in chars +""" + endswith(s::AbstractString, suffix::AbstractString) + +Returns `true` if `s` ends with `suffix`. If `suffix` is a vector or set of +characters, tests whether the last character of `s` belongs to that set. + +```jldoctest +julia> endswith("Sunday", "day") +true +``` +""" function endswith(a::AbstractString, b::AbstractString) i = endof(a) j = endof(b) @@ -36,8 +58,27 @@ startswith(a::Vector{UInt8}, b::Vector{UInt8}) = # TODO: fast endswith + +""" + chop(s::AbstractString) + +Remove the last character from a string. + +```jldoctest +julia> a = string("March") +"March" + +julia> chop(a) +"Marc" +``` +""" chop(s::AbstractString) = s[1:end-1] +""" + chomp(s::AbstractString) + +Remove a single trailing newline from a string. +""" function chomp(s::AbstractString) i = endof(s) if (i < 1 || s[i] != '\n') return s end @@ -61,6 +102,13 @@ chomp!(s::AbstractString) = chomp(s) # copying fallback for other string types const _default_delims = [' ','\t','\n','\v','\f','\r'] +""" + lstrip(s::AbstractString[, chars::Chars]) + +Return `s` with any leading whitespace and delimiters removed. +If `chars` (a character, or vector or set of characters) is provided, +instead remove characters contained in it. +""" function lstrip(s::AbstractString, chars::Chars=_default_delims) i = start(s) while !done(s,i) @@ -73,6 +121,21 @@ function lstrip(s::AbstractString, chars::Chars=_default_delims) s[end+1:end] end +""" + rstrip(s::AbstractString[, chars::Chars]) + +Return `s` with any trailing whitespace and delimiters removed. +If `chars` (a character, or vector or set of characters) is provided, +instead remove characters contained in it. + +```jldoctest +julia> a = rpad("March",20) +"March " + +julia> rstrip(a) +"March" +``` +""" function rstrip(s::AbstractString, chars::Chars=_default_delims) r = RevString(s) i = start(r) @@ -86,6 +149,13 @@ function rstrip(s::AbstractString, chars::Chars=_default_delims) s[1:0] end +""" + strip(s::AbstractString, [chars::Chars]) + +Return `s` with any leading and trailing whitespace removed. +If `chars` (a character, or vector or set of characters) is provided, +instead remove characters contained in it. +""" strip(s::AbstractString) = lstrip(rstrip(s)) strip(s::AbstractString, chars::Chars) = lstrip(rstrip(s, chars), chars) @@ -117,13 +187,58 @@ function rpad(s::AbstractString, n::Integer, p::AbstractString=" ") string(s, p^q, p[1:i]) end +""" + lpad(s, n::Integer, p::AbstractString=" ") + +Make a string at least `n` columns wide when printed by padding `s` on the left +with copies of `p`. + +```jldoctest +julia> lpad("March",10) +" March" +``` +""" lpad(s, n::Integer, p=" ") = lpad(string(s),n,string(p)) + +""" + rpad(s, n::Integer, p::AbstractString=" ") + +Make a string at least `n` columns wide when printed by padding `s` on the right +with copies of `p`. + +```jldoctest +julia> rpad("March",20) +"March " +``` +""" rpad(s, n::Integer, p=" ") = rpad(string(s),n,string(p)) cpad(s, n::Integer, p=" ") = rpad(lpad(s,div(n+strwidth(s),2),p),n,p) # splitter can be a Char, Vector{Char}, AbstractString, Regex, ... # any splitter that provides search(s::AbstractString, splitter) split{T<:SubString}(str::T, splitter; limit::Integer=0, keep::Bool=true) = _split(str, splitter, limit, keep, T[]) + +""" + split(s::AbstractString, [chars]; limit::Integer=0, keep::Bool=true) + +Return an array of substrings by splitting the given string on occurrences of the given +character delimiters, which may be specified in any of the formats allowed by `search`'s +second argument (i.e. a single character, collection of characters, string, or regular +expression). If `chars` is omitted, it defaults to the set of all space characters, and +`keep` is taken to be `false`. The two keyword arguments are optional: they are a +maximum size for the result and a flag determining whether empty fields should be kept in +the result. + +```jldoctest +julia> a = "Ma.rch" +"Ma.rch" + +julia> split(a,".") +2-element Array{SubString{String},1}: + "Ma" + "rch" +``` +""" split{T<:AbstractString}(str::T, splitter; limit::Integer=0, keep::Bool=true) = _split(str, splitter, limit, keep, SubString{T}[]) function _split{T<:AbstractString,U<:Array}(str::T, splitter, limit::Integer, keep_empty::Bool, strs::U) i = start(str) @@ -151,6 +266,34 @@ end split(str::AbstractString) = split(str, _default_delims; limit=0, keep=false) rsplit{T<:SubString}(str::T, splitter; limit::Integer=0, keep::Bool=true) = _rsplit(str, splitter, limit, keep, T[]) + +""" + rsplit(s::AbstractString, [chars]; limit::Integer=0, keep::Bool=true) + +Similar to [`split`](:func:`split`), but starting from the end of the string. + +```jldoctest +julia> a = "M.a.r.c.h" +"M.a.r.c.h" + +julia> rsplit(a,".") +5-element Array{SubString{String},1}: + "M" + "a" + "r" + "c" + "h" + +julia> rsplit(a,".";limit=1) +1-element Array{SubString{String},1}: + "M.a.r.c.h" + +julia> rsplit(a,".";limit=2) +2-element Array{SubString{String},1}: + "M.a.r.c" + "h" +``` +""" rsplit{T<:AbstractString}(str::T, splitter ; limit::Integer=0, keep::Bool=true) = _rsplit(str, splitter, limit, keep, SubString{T}[]) function _rsplit{T<:AbstractString,U<:Array}(str::T, splitter, limit::Integer, keep_empty::Bool, strs::U) i = start(str) @@ -207,11 +350,38 @@ function replace(str::String, pattern, repl, limit::Integer) write(out, SubString(str,i)) takebuf_string(out) end + +""" + replace(string::AbstractString, pat, r[, n::Integer=0]) + +Search for the given pattern `pat`, and replace each occurrence with `r`. If `n` is +provided, replace at most `n` occurrences. As with search, the second argument may be a +single character, a vector or a set of characters, a string, or a regular expression. If `r` +is a function, each occurrence is replaced with `r(s)` where `s` is the matched substring. +If `pat` is a regular expression and `r` is a `SubstitutionString`, then capture group +references in `r` are replaced with the corresponding matched text. +""" replace(s::AbstractString, pat, f, n::Integer) = replace(String(s), pat, f, n) replace(s::AbstractString, pat, r) = replace(s, pat, r, 0) # hex <-> bytes conversion +""" + hex2bytes(s::AbstractString) + +Convert an arbitrarily long hexadecimal string to its binary representation. Returns an +`Array{UInt8,1}`, i.e. an array of bytes. + +```jldoctest +julia> a = hex(12345) +"3039" + +julia> hex2bytes(a) +2-element Array{UInt8,1}: + 0x30 + 0x39 +``` +""" function hex2bytes(s::AbstractString) a = zeros(UInt8, div(endof(s), 2)) i, j = start(s), 0 @@ -234,6 +404,25 @@ function hex2bytes(s::AbstractString) return a end +""" + bytes2hex(bin_arr::Array{UInt8, 1}) -> String + +Convert an array of bytes to its hexadecimal representation. +All characters are in lower-case. + +```jldoctest +julia> a = hex(12345) +"3039" + +julia> b = hex2bytes(a) +2-element Array{UInt8,1}: + 0x30 + 0x39 + +julia> bytes2hex(b) +"3039" +``` +""" function bytes2hex(a::AbstractArray{UInt8}) b = Vector{UInt8}(2*length(a)) i = 0 @@ -252,4 +441,11 @@ function ascii(s::String) end return s end + +""" + ascii(s::AbstractString) + +Convert a string to `String` type and check that it contains only ASCII data, otherwise +throwing an `ArgumentError` indicating the position of the first non-ASCII byte. +""" ascii(x::AbstractString) = ascii(convert(String, x)) diff --git a/base/sysinfo.jl b/base/sysinfo.jl index 040163fa80a5b..6554ec6283bca 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -154,6 +154,11 @@ end free_memory() = ccall(:uv_get_free_memory, UInt64, ()) total_memory() = ccall(:uv_get_total_memory, UInt64, ()) +""" + Sys.get_process_title() + +Get the process title. On some systems, will always return empty string. (not exported) +""" function get_process_title() buf = zeros(UInt8, 512) err = ccall(:uv_get_process_title, Cint, (Ptr{UInt8}, Cint), buf, 512) @@ -161,6 +166,11 @@ function get_process_title() return unsafe_string(pointer(buf)) end +""" + Sys.set_process_title(title::AbstractString) + +Set the process title. No-op on some operating systems. (not exported) +""" function set_process_title(title::AbstractString) err = ccall(:uv_set_process_title, Cint, (Cstring,), title) Base.uv_error("set_process_title", err) diff --git a/base/tuple.jl b/base/tuple.jl index fe4c3fca4e80f..04e6e1f6eb131 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -4,7 +4,8 @@ """ NTuple{N, T} -A compact way of representing the type for a tuple of length `N` where all elements are of type `T`.""" +A compact way of representing the type for a tuple of length `N` where all elements are of type `T`. +""" NTuple ## indexing ## @@ -69,6 +70,12 @@ end ## mapping ## +""" + ntuple(f::Function, n::Integer) + +Create a tuple of length `n`, computing each element as `f(i)`, +where `i` is the index of the element. +""" ntuple(f::Function, n::Integer) = n <= 0 ? () : n == 1 ? (f(1),) : diff --git a/base/util.jl b/base/util.jl index 0c2013b22b7d6..7101958d5d24e 100644 --- a/base/util.jl +++ b/base/util.jl @@ -5,6 +5,12 @@ # time() in libc.jl # high-resolution relative time, in nanoseconds + +""" + time_ns() + +Get the time in nanoseconds. The time corresponding to 0 is undefined, and wraps every 5.8 years. +""" time_ns() = ccall(:jl_hrtime, UInt64, ()) # This type must be kept in sync with the C struct in src/gc.h @@ -314,6 +320,11 @@ println_with_color(color::Symbol, msg::AbstractString...) = ## warnings and messages ## +""" + info(msg...; prefix="INFO: ") + +Display an informational message. Argument `msg` is a string describing the information to be displayed. +""" function info(io::IO, msg...; prefix="INFO: ") println_with_color(info_color(), io, prefix, chomp(string(msg...))) end @@ -347,6 +358,12 @@ function warn(io::IO, msg...; println(io) return end + +""" + warn(msg) + +Display a warning. Argument `msg` is a string describing the warning to be displayed. +""" warn(msg...; kw...) = warn(STDERR, msg...; kw...) warn(io::IO, err::Exception; prefix="ERROR: ", kw...) = diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index ef400cd751bc2..4c1fd9013514a 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -517,7 +517,7 @@ the name of the function to vectorize. Here is a simple example: julia> methods(square) # 2 methods for generic function "square": - square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:550 + square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:914 square(x) at none:1 julia> square([1 2 4; 5 6 7]) diff --git a/doc/manual/complex-and-rational-numbers.rst b/doc/manual/complex-and-rational-numbers.rst index ae30e44e838eb..5cca714bd043f 100644 --- a/doc/manual/complex-and-rational-numbers.rst +++ b/doc/manual/complex-and-rational-numbers.rst @@ -159,7 +159,7 @@ versus ``-1 + 0im`` even though ``-1 == -1 + 0im``: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:185 + in sqrt(::Int64) at ./math.jl:258 ... julia> sqrt(-1 + 0im) @@ -345,4 +345,3 @@ types effortless: julia> 1//3 - 0.33 0.0033333333333332993 - diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 4168cb0118ce0..0c6a396e3c8b1 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -666,7 +666,7 @@ negative real value: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:185 + in sqrt(::Int64) at ./math.jl:258 ... You may define your own exceptions in the following way: diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index 36fba25e8c46f..be6c9768f68e5 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -427,7 +427,7 @@ The following examples show the different forms. julia> round(Int8,127.6) ERROR: InexactError() - in trunc(::Type{Int8}, ::Float64) at ./float.jl:458 + in trunc(::Type{Int8}, ::Float64) at ./float.jl:463 in round(::Type{Int8}, ::Float64) at ./float.jl:211 ... diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index a581d4155906a..b31d1bed2ede5 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -36,7 +36,7 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example in eval_user_input(::Any, ::Bool) at client.jl:117 in eval(::Module, ::Any) at boot.jl:234 in eval_user_input(::Any, ::Bool) at client.jl:117 - in _start() at client.jl:355 + in _start() at client.jl:363 julia> @noinline child() = stacktrace() child (generic function with 1 method) diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index de17e181bdc21..a827b73a192e7 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -554,8 +554,8 @@ contained in a string: julia> contains("Xylophon", 'o') ERROR: MethodError: no method matching contains(::String, ::Char) Closest candidates are: - contains(!Matched::Function, ::Any, !Matched::Any) at reduce.jl:489 - contains(::AbstractString, !Matched::AbstractString) at strings/search.jl:310 + contains(!Matched::Function, ::Any, !Matched::Any) at reduce.jl:682 + contains(::AbstractString, !Matched::AbstractString) at strings/search.jl:366 ... The last error is because ``'o'`` is a character literal, and :func:`contains` diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 4fc6612761c2f..18e01cfd1d72b 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -44,12 +44,26 @@ Basic functions Returns the tuple of valid indices for array ``A``\ . + .. doctest:: + + julia> A = ones(5,6,7); + + julia> indices(A) + (Base.OneTo(5),Base.OneTo(6),Base.OneTo(7)) + .. function:: indices(A, d) .. Docstring generated from Julia source Returns the valid range of indices for array ``A`` along dimension ``d``\ . + .. doctest:: + + julia> A = ones(5,6,7); + + julia> indices(A,2) + Base.OneTo(6) + .. function:: length(A::AbstractArray) -> Integer .. Docstring generated from Julia source @@ -96,7 +110,7 @@ Basic functions (iter.I[1],iter.I[2]) = (2,3) A[iter] = 0 - If you supply more than one ``AbstractArray`` argument, ``eachindex`` will create an iterable object that is fast for all arguments (a ``UnitRange`` if all inputs have fast linear indexing, a CartesianRange otherwise). If the arrays have different sizes and/or dimensionalities, ``eachindex`` returns an iterable that spans the largest range along each dimension. + If you supply more than one ``AbstractArray`` argument, ``eachindex`` will create an iterable object that is fast for all arguments (a ``UnitRange`` if all inputs have fast linear indexing, a ```CartesianRange`` <:obj`CartesianRange`>`_ otherwise). If the arrays have different sizes and/or dimensionalities, ``eachindex`` returns an iterable that spans the largest range along each dimension. .. function:: linearindices(A) @@ -106,6 +120,15 @@ Basic functions Calling this function is the "safe" way to write algorithms that exploit linear indexing. + .. doctest:: + + julia> A = ones(5,6,7); + + julia> b = linearindices(A); + + julia> extrema(b) + (1,210) + .. function:: Base.linearindexing(A) .. Docstring generated from Julia source @@ -122,13 +145,15 @@ Basic functions .. Docstring generated from Julia source - Counts the number of nonzero values in array ``A`` (dense or sparse). Note that this is not a constant-time operation. For sparse matrices, one should usually use ``nnz``\ , which returns the number of stored values. + Counts the number of nonzero values in array ``A`` (dense or sparse). Note that this is not a constant-time operation. For sparse matrices, one should usually use :func:`nnz`\ , which returns the number of stored values. .. function:: conj!(A) .. Docstring generated from Julia source - Convert an array to its complex conjugate in-place. + Transform an array to its complex conjugate in-place. + + See also :func:`conj`\ . .. function:: stride(A, k::Integer) @@ -173,17 +198,46 @@ Basic functions provides the indices of the maximum element. + .. doctest:: + + julia> ind2sub((3,4),2) + (2,1) + + julia> ind2sub((3,4),3) + (3,1) + + julia> ind2sub((3,4),4) + (1,2) + .. function:: ind2sub(a, index) -> subscripts .. Docstring generated from Julia source Returns a tuple of subscripts into array ``a`` corresponding to the linear index ``index``\ . + .. doctest:: + + julia> A = ones(5,6,7); + + julia> ind2sub(A,35) + (5,1,2) + + julia> ind2sub(A,70) + (5,2,3) + .. function:: sub2ind(dims, i, j, k...) -> index .. Docstring generated from Julia source - The inverse of ``ind2sub``\ , returns the linear index corresponding to the provided subscripts. + The inverse of :func:`ind2sub`\ , returns the linear index corresponding to the provided subscripts. + + .. doctest:: + + julia> sub2ind((5,6,7),1,2,3) + 66 + + julia> sub2ind((5,6,7),1,6,3) + 86 .. function:: LinAlg.checksquare(A) @@ -258,7 +312,17 @@ Constructors .. Docstring generated from Julia source - Create an array filled with the value ``x``\ . For example, ``fill(1.0, (10,10))`` returns a 10×10 array of floats, with each element initialized to ``1.0``\ . + Create an array filled with the value ``x``\ . For example, ``fill(1.0, (5,5))`` returns a 10×10 array of floats, with each element initialized to ``1.0``\ . + + .. doctest:: + + julia> fill(1.0, (5,5)) + 5×5 Array{Float64,2}: + 1.0 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 1.0 If ``x`` is an object reference, all elements will refer to the same object. ``fill(Foo(), dims)`` will return an array filled with the result of evaluating ``Foo()`` once. @@ -374,18 +438,34 @@ Constructors Note the difference from :func:`ones`\ . -.. function:: linspace(start, stop, n=50) +.. function:: linspace(start::Real, stop::Real, n::Real=50) .. Docstring generated from Julia source Construct a range of ``n`` linearly spaced elements from ``start`` to ``stop``\ . -.. function:: logspace(start, stop, n=50) + .. doctest:: + + julia> linspace(1.3,2.9,9) + 9-element LinSpace{Float64}: + 1.3,1.5,1.7,1.9,2.1,2.3,2.5,2.7,2.9 + +.. function:: logspace(start::Real, stop::Real, n::Integer=50) .. Docstring generated from Julia source Construct a vector of ``n`` logarithmically spaced numbers from ``10^start`` to ``10^stop``\ . + .. doctest:: + + julia> logspace(1.,10.,5) + 5-element Array{Float64,1}: + 10.0 + 1778.28 + 3.16228e5 + 5.62341e7 + 1.0e10 + Mathematical operators and functions ------------------------------------ @@ -397,17 +477,53 @@ All mathematical operations and functions are supported for arrays Broadcasts the arrays ``As`` to a common size by expanding singleton dimensions, and returns an array of the results ``f(as...)`` for each position. + .. doctest:: + + julia> A = [1, 2, 3, 4, 5] + 5-element Array{Int64,1}: + 1 + 2 + 3 + 4 + 5 + + julia> B = [1 2; 3 4; 5 6; 7 8; 9 10] + 5×2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 + 7 8 + 9 10 + + julia> broadcast(+, A, B) + 5×2 Array{Int64,2}: + 2 3 + 5 6 + 8 9 + 11 12 + 14 15 + .. function:: broadcast!(f, dest, As...) .. Docstring generated from Julia source - Like ``broadcast``\ , but store the result of ``broadcast(f, As...)`` in the ``dest`` array. Note that ``dest`` is only used to store the result, and does not supply arguments to ``f`` unless it is also listed in the ``As``\ , as in ``broadcast!(f, A, A, B)`` to perform ``A[:] = broadcast(f, A, B)``\ . + Like :func:`broadcast`\ , but store the result of ``broadcast(f, As...)`` in the ``dest`` array. Note that ``dest`` is only used to store the result, and does not supply arguments to ``f`` unless it is also listed in the ``As``\ , as in ``broadcast!(f, A, A, B)`` to perform ``A[:] = broadcast(f, A, B)``\ . .. function:: bitbroadcast(f, As...) .. Docstring generated from Julia source - Like ``broadcast``\ , but allocates a ``BitArray`` to store the result, rather then an ``Array``\ . + Like :func:`broadcast`\ , but allocates a ``BitArray`` to store the result, rather then an ``Array``\ . + + .. doctest:: + + julia> bitbroadcast(isodd,[1,2,3,4,5]) + 5-element BitArray{1}: + true + false + true + false + true Indexing, Assignment, and Concatenation --------------------------------------- @@ -442,12 +558,24 @@ Indexing, Assignment, and Concatenation From an array view ``A``\ , returns the corresponding indexes in the parent. -.. function:: slicedim(A, d, i) +.. function:: slicedim(A, d::Integer, i) .. Docstring generated from Julia source Return all the data of ``A`` where the index for dimension ``d`` equals ``i``\ . Equivalent to ``A[:,:,...,i,:,:,...]`` where ``i`` is in position ``d``\ . + .. doctest:: + + julia> A = [1 2 3 4; 5 6 7 8] + 2×4 Array{Int64,2}: + 1 2 3 4 + 5 6 7 8 + + julia> slicedim(A,2,3) + 2-element Array{Int64,1}: + 3 + 7 + .. function:: setindex!(A, X, inds...) .. Docstring generated from Julia source @@ -458,13 +586,45 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Broadcasts the ``inds`` arrays to a common size like ``broadcast``\ , and returns an array of the results ``A[ks...]``\ , where ``ks`` goes over the positions in the broadcast. + Broadcasts the ``inds`` arrays to a common size like :func:`broadcast` and returns an array of the results ``A[ks...]``\ , where ``ks`` goes over the positions in the broadcast result ``A``\ . + + .. doctest:: + + julia> A = [1, 2, 3, 4, 5] + 5-element Array{Int64,1}: + 1 + 2 + 3 + 4 + 5 + + julia> B = [1 2; 3 4; 5 6; 7 8; 9 10] + 5×2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 + 7 8 + 9 10 + + julia> C = broadcast(+,A,B) + 5×2 Array{Int64,2}: + 2 3 + 5 6 + 8 9 + 11 12 + 14 15 + + julia> broadcast_getindex(C,[1,2,10]) + 3-element Array{Int64,1}: + 2 + 5 + 15 .. function:: broadcast_setindex!(A, X, inds...) .. Docstring generated from Julia source - Broadcasts the ``X`` and ``inds`` arrays to a common size and stores the value from each position in ``X`` at the indices given by the same positions in ``inds``\ . + Broadcasts the ``X`` and ``inds`` arrays to a common size and stores the value from each position in ``X`` at the indices in ``A`` given by the same positions in ``inds``\ . .. function:: isassigned(array, i) -> Bool @@ -588,7 +748,7 @@ Indexing, Assignment, and Concatenation If the first argument is a single integer ``n``\ , then all block rows are assumed to have ``n`` block columns. -.. function:: flipdim(A, d) +.. function:: flipdim(A, d::Integer) .. Docstring generated from Julia source @@ -635,7 +795,7 @@ Indexing, Assignment, and Concatenation 4 8 12 16 1 5 9 13 - See also ``circshift!``\ . + See also :func:`circshift!`\ . .. function:: circshift!(dest, src, shifts) @@ -1051,6 +1211,18 @@ Indexing, Assignment, and Concatenation Check two array shapes for compatibility, allowing trailing singleton dimensions, and return whichever shape has more dimensions. + .. doctest:: + + julia> a = ones(3,4,1,1,1); + + julia> b = ones(3,4); + + julia> promote_shape(a,b) + (Base.OneTo(3),Base.OneTo(4),Base.OneTo(1),Base.OneTo(1),Base.OneTo(1)) + + julia> promote_shape((2,3,1,4), (2,3,1,4,1)) + (2,3,1,4,1) + .. function:: checkbounds(A, I...) .. Docstring generated from Julia source @@ -1061,9 +1233,9 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Return ``true`` if the specified indices ``I`` are in bounds for the given array ``A``\ . Subtypes of ``AbstractArray`` should specialize this method if they need to provide custom bounds checking behaviors; however, in many cases one can rely on ``A``\ 's indices and ``checkindex``\ . + Return ``true`` if the specified indices ``I`` are in bounds for the given array ``A``\ . Subtypes of ``AbstractArray`` should specialize this method if they need to provide custom bounds checking behaviors; however, in many cases one can rely on ``A``\ 's indices and :func:`checkindex`\ . - See also ``checkindex``\ . + See also :func:`checkindex`\ . .. function:: checkindex(Bool, inds::AbstractUnitRange, index) @@ -1071,6 +1243,14 @@ Indexing, Assignment, and Concatenation Return ``true`` if the given ``index`` is within the bounds of ``inds``\ . Custom types that would like to behave as indices for all arrays can extend this method in order to provide a specialized bounds checking implementation. + .. doctest:: + + julia> checkindex(Bool,1:20,8) + true + + julia> checkindex(Bool,1:20,21) + false + .. function:: randsubseq(A, p) -> Vector .. Docstring generated from Julia source @@ -1081,7 +1261,7 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Like ``randsubseq``\ , but the results are stored in ``S`` (which is resized as needed). + Like :func:`randsubseq`\ , but the results are stored in ``S`` (which is resized as needed). Array functions --------------- @@ -1144,7 +1324,7 @@ Array functions Cumulative sum of ``A`` along a dimension, storing the result in ``B``\ . The dimension defaults to 1. -.. function:: cumsum_kbn(A, [dim]) +.. function:: cumsum_kbn(A, [dim::Integer=1]) .. Docstring generated from Julia source @@ -1162,18 +1342,41 @@ Array functions Cumulative maximum along a dimension. The dimension defaults to 1. -.. function:: diff(A, [dim]) +.. function:: diff(A, [dim::Integer=1]) .. Docstring generated from Julia source - Finite difference operator of matrix or vector. + Finite difference operator of matrix or vector ``A``\ . If ``A`` is a matrix, compute the finite difference over a dimension ``dim`` (default ``1``\ ). -.. function:: gradient(F, [h]) + .. doctest:: + + julia> a = [2 4; 6 16] + 2×2 Array{Int64,2}: + 2 4 + 6 16 + + julia> diff(a,2) + 2×1 Array{Int64,2}: + 2 + 10 + +.. function:: gradient(F::AbstractVector, [h::Real]) .. Docstring generated from Julia source Compute differences along vector ``F``\ , using ``h`` as the spacing between points. The default spacing is one. + .. doctest:: + + julia> a = [2,4,6,8]; + + julia> gradient(a) + 4-element Array{Float64,1}: + 2.0 + 2.0 + 2.0 + 2.0 + .. function:: rot180(A) .. Docstring generated from Julia source @@ -1417,7 +1620,7 @@ Array functions Combinatorics ------------- -.. function:: randperm([rng,] n) +.. function:: randperm([rng=GLOBAL_RNG,] n::Integer) .. Docstring generated from Julia source @@ -1449,19 +1652,19 @@ Combinatorics Like ``permute!``\ , but the inverse of the given permutation is applied. -.. function:: randcycle([rng,] n) +.. function:: randcycle([rng=GLOBAL_RNG,] n::Integer) .. Docstring generated from Julia source Construct a random cyclic permutation of length ``n``\ . The optional ``rng`` argument specifies a random number generator, see :ref:`Random Numbers <random-numbers>`\ . -.. function:: shuffle([rng,] v) +.. function:: shuffle([rng=GLOBAL_RNG,] v) .. Docstring generated from Julia source Return a randomly permuted copy of ``v``\ . The optional ``rng`` argument specifies a random number generator (see :ref:`Random Numbers <random-numbers>`\ ). To permute ``v`` in-place, see :func:`shuffle!`\ . To obtain randomly permuted indices, see :func:`randperm`\ . -.. function:: shuffle!([rng,] v) +.. function:: shuffle!([rng=GLOBAL_RNG,] v) .. Docstring generated from Julia source diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 79f275f7651f5..837c0244e441c 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -50,7 +50,7 @@ Getting Around Determine whether Julia is running an interactive session. -.. function:: whos([io,] [Module,] [pattern::Regex]) +.. function:: whos(io::IO=STDOUT, m::Module=current_module(), pattern::Regex=r"") .. Docstring generated from Julia source @@ -64,7 +64,7 @@ Getting Around Compute the amount of memory used by all unique objects reachable from the argument. Keyword argument ``exclude`` specifies a type of objects to exclude from the traversal. -.. function:: edit(path::AbstractString, [line]) +.. function:: edit(path::AbstractString, line::Integer=0) .. Docstring generated from Julia source @@ -82,7 +82,7 @@ Getting Around Evaluates the arguments to the function or macro call, determines their types, and calls the ``edit`` function on the resulting expression. -.. function:: less(file::AbstractString, [line]) +.. function:: less(file::AbstractString, [line::Integer]) .. Docstring generated from Julia source @@ -132,7 +132,7 @@ Getting Around .. Docstring generated from Julia source - Creates a precompiled cache file for module (see help for ``require``\ ) and all of its dependencies. This can be used to reduce package load times. Cache files are stored in ``LOAD_CACHE_PATH[1]``\ , which defaults to ``~/.julia/lib/VERSION``\ . See :ref:`Module initialization and precompilation <man-modules-initialization-precompilation>` for important notes. + Creates a precompiled cache file for module (see help for :func:`require`\ ) and all of its dependencies. This can be used to reduce package load times. Cache files are stored in ``LOAD_CACHE_PATH[1]``\ , which defaults to ``~/.julia/lib/VERSION``\ . See :ref:`Module initialization and precompilation <man-modules-initialization-precompilation>` for important notes. .. function:: __precompile__(isprecompilable::Bool=true) @@ -150,7 +150,7 @@ Getting Around Evaluate the contents of a source file in the current context. During including, a task-local include path is set to the directory containing the file. Nested calls to ``include`` will search relative to that path. All paths refer to files on node 1 when running in parallel, and files will be fetched from node 1. This function is typically used to load source interactively, or to combine files in packages that are broken into multiple source files. -.. function:: include_string(code::AbstractString, [filename]) +.. function:: include_string(code::AbstractString, filename::AbstractString="string") .. Docstring generated from Julia source @@ -198,7 +198,7 @@ Getting Around If ``types`` is specified, returns an array of methods whose types match. -.. function:: methodswith(typ[, module or function][, showparents]) +.. function:: methodswith(typ[, module or function][, showparents::Bool=false, meths=Method[]]) .. Docstring generated from Julia source @@ -214,7 +214,7 @@ Getting Around Show an expression and result, returning the result. -.. function:: versioninfo([verbose::Bool]) +.. function:: versioninfo(io::IO=STDOUT, verbose::Bool=false) .. Docstring generated from Julia source @@ -299,7 +299,7 @@ All Objects Construct a tuple of the given objects. -.. function:: ntuple(f::Function, n) +.. function:: ntuple(f::Function, n::Integer) .. Docstring generated from Julia source @@ -370,7 +370,7 @@ All Objects julia> convert(Int, 3.5) ERROR: InexactError() - in convert(::Type{Int64}, ::Float64) at ./int.jl:239 + in convert(::Type{Int64}, ::Float64) at ./int.jl:330 ... If ``T`` is a :obj:`AbstractFloat` or :obj:`Rational` type, then it will return the closest value to ``x`` representable by ``T``\ . @@ -463,6 +463,11 @@ Types Return the supertype of DataType ``T``\ . + .. doctest:: + + julia> supertype(Int32) + Signed + .. function:: issubtype(type1, type2) .. Docstring generated from Julia source @@ -481,6 +486,15 @@ Types Return a list of immediate subtypes of DataType ``T``\ . Note that all currently loaded subtypes are included, including those not visible in the current module. + .. doctest:: + + julia> subtypes(Integer) + 4-element Array{Any,1}: + BigInt + Bool + Signed + Unsigned + .. function:: typemin(T) .. Docstring generated from Julia source @@ -545,7 +559,7 @@ Types .. Docstring generated from Julia source - Specifies what type should be used by ``promote`` when given values of types ``type1`` and ``type2``\ . This function should not be called directly, but should have definitions added to it for new types as appropriate. + Specifies what type should be used by :func:`promote` when given values of types ``type1`` and ``type2``\ . This function should not be called directly, but should have definitions added to it for new types as appropriate. .. function:: getfield(value, name::Symbol) @@ -720,11 +734,11 @@ Syntax Evaluate an expression and return the value. -.. function:: evalfile(path::AbstractString) +.. function:: evalfile(path::AbstractString, args::Vector{String}=String[]) .. Docstring generated from Julia source - Load the file using ``include``\ , evaluate all expressions, and return the value of the last one. + Load the file using :func:`include`\ , evaluate all expressions, and return the value of the last one. .. function:: esc(e::ANY) @@ -744,6 +758,12 @@ Syntax Generates a gensym symbol for a variable. For example, ``@gensym x y`` is transformed into ``x = gensym("x"); y = gensym("y")``\ . +.. function:: @polly + + .. Docstring generated from Julia source + + Tells the compiler to apply the polyhedral optimizer Polly to a function. + .. function:: parse(str, start; greedy=true, raise=true) .. Docstring generated from Julia source @@ -780,7 +800,7 @@ Nullables System ------ -.. function:: run(command) +.. function:: run(command, args...) .. Docstring generated from Julia source @@ -790,7 +810,7 @@ System .. Docstring generated from Julia source - Run a command object asynchronously, returning the resulting ``Process`` object. + Run a command object asynchronously, returning the resulting :obj:`Process` object. .. data:: DevNull @@ -876,7 +896,7 @@ System Cmd(`echo "Hello world"`, ignorestatus=true, detach=false) -.. function:: setenv(command, env; dir=working_dir) +.. function:: setenv(command::Cmd, env; dir="") .. Docstring generated from Julia source @@ -1098,7 +1118,7 @@ Errors .. Docstring generated from Julia source - Throw an ``AssertionError`` if ``cond`` is ``false``\ . Also available as the macro ``@assert expr``\ . + Throw an :obj:`AssertionError` if ``cond`` is ``false``\ . Also available as the macro ``@assert expr``\ . .. function:: @assert cond [text] @@ -1188,7 +1208,7 @@ Errors .. Docstring generated from Julia source - An attempted access to a ``Nullable`` with no defined value. + An attempted access to a :obj:`Nullable` with no defined value. .. function:: OutOfMemoryError() @@ -1307,12 +1327,25 @@ Reflection Get the name of a ``Module`` as a ``Symbol``\ . + .. doctest:: + + julia> module_name(Base.LinAlg) + :LinAlg + .. function:: module_parent(m::Module) -> Module .. Docstring generated from Julia source Get a module's enclosing ``Module``\ . ``Main`` is its own parent, as is ``LastMain`` after ``workspace()``\ . + .. doctest:: + + julia> module_parent(Main) + Main + + julia> module_parent(Base.LinAlg.BLAS) + Base.LinAlg + .. function:: current_module() -> Module .. Docstring generated from Julia source @@ -1323,9 +1356,17 @@ Reflection .. Docstring generated from Julia source - Get the fully-qualified name of a module as a tuple of symbols. For example, ``fullname(Base.Pkg)`` gives ``(:Base,:Pkg)``\ , and ``fullname(Main)`` gives ``()``\ . + Get the fully-qualified name of a module as a tuple of symbols. For example, + + .. doctest:: + + julia> fullname(Base.Pkg) + (:Base,:Pkg) + + julia> fullname(Main) + () -.. function:: names(x::Module[, all=false[, imported=false]]) +.. function:: names(x::Module, all::Bool=false, imported::Bool=false) .. Docstring generated from Julia source @@ -1343,12 +1384,27 @@ Reflection Get an array of the fields of a ``DataType``\ . -.. function:: fieldname(x::DataType, i) + .. doctest:: + + julia> fieldnames(Hermitian) + 2-element Array{Symbol,1}: + :data + :uplo + +.. function:: fieldname(x::DataType, i::Integer) .. Docstring generated from Julia source Get the name of field ``i`` of a ``DataType``\ . + .. doctest:: + + julia> fieldname(SparseMatrixCSC,1) + :m + + julia> fieldname(SparseMatrixCSC,5) + :nzval + .. function:: Base.datatype_module(t::DataType) -> Module .. Docstring generated from Julia source @@ -1365,7 +1421,7 @@ Reflection .. Docstring generated from Julia source - Determine whether a global is declared ``const`` in a given ``Module``\ . The default ``Module`` argument is ``current_module()``\ . + Determine whether a global is declared ``const`` in a given ``Module``\ . The default ``Module`` argument is :func:`current_module`\ . .. function:: Base.function_name(f::Function) -> Symbol @@ -1454,7 +1510,7 @@ Internals Evaluates the arguments to the function or macro call, determines their types, and calls :func:`code_typed` on the resulting expression. -.. function:: code_warntype([io], f, types) +.. function:: code_warntype([io::IO], f, types) .. Docstring generated from Julia source diff --git a/doc/stdlib/c.rst b/doc/stdlib/c.rst index b90f1fed69da7..b6e2793faf684 100644 --- a/doc/stdlib/c.rst +++ b/doc/stdlib/c.rst @@ -18,7 +18,7 @@ .. Docstring generated from Julia source - Obtain a pointer to a global variable in a C-exported shared library, specified exactly as in ``ccall``\ . Returns a ``Ptr{Type}``\ , defaulting to ``Ptr{Void}`` if no Type argument is supplied. The values can be read or written by ``unsafe_load`` or ``unsafe_store!``\ , respectively. + Obtain a pointer to a global variable in a C-exported shared library, specified exactly as in :func:`ccall`\ . Returns a ``Ptr{Type}``\ , defaulting to ``Ptr{Void}`` if no ``Type`` argument is supplied. The values can be read or written by :func:`unsafe_load` or :func:`unsafe_store!`\ , respectively. .. function:: cfunction(function::Function, ReturnType::Type, (ArgumentTypes...)) @@ -44,7 +44,7 @@ Convert ``x`` to a value of type ``T`` - In cases where ``convert`` would need to take a Julia object and turn it into a ``Ptr``\ , this function should be used to define and perform that conversion. + In cases where :func:`convert` would need to take a Julia object and turn it into a ``Ptr``\ , this function should be used to define and perform that conversion. Be careful to ensure that a Julia reference to ``x`` exists as long as the result of this function will be used. Accordingly, the argument ``x`` to this function should never be an expression, only a variable name or field reference. For example, ``x=a.b.c`` is acceptable, but ``x=[a,b,c]`` is not. @@ -56,7 +56,7 @@ Convert ``x`` to a value of type ``T``\ , typically by calling ``convert(T,x)`` - In cases where ``x`` cannot be safely converted to ``T``\ , unlike ``convert``\ , ``cconvert`` may return an object of a type different from ``T``\ , which however is suitable for ``unsafe_convert`` to handle. + In cases where ``x`` cannot be safely converted to ``T``\ , unlike :func:`convert`\ , ``cconvert`` may return an object of a type different from ``T``\ , which however is suitable for :func:`unsafe_convert` to handle. Neither ``convert`` nor ``cconvert`` should take a Julia object and turn it into a ``Ptr``\ . diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 3c2eac2380a99..e2cb44eea95bc 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -169,6 +169,17 @@ type. An iterator that generates the value ``x`` forever. If ``n`` is specified, generates ``x`` that many times (equivalent to ``take(repeated(x), n)``\ ). + .. doctest:: + + julia> a = repeated([1 2], 4); + + julia> collect(a) + 4-element Array{Array{Int64,2},1}: + [1 2] + [1 2] + [1 2] + [1 2] + .. function:: iteratorsize(itertype::Type) -> IteratorSize .. Docstring generated from Julia source @@ -279,6 +290,17 @@ Iterable Collections Determine whether an item is in the given collection, in the sense that it is ``==`` to one of the values generated by iterating over the collection. Some collections need a slightly different definition; for example :obj:`Set`\ s check whether the item :func:`isequal` to one of the elements. :obj:`Dict`\ s look for ``(key,value)`` pairs, and the key is compared using :func:`isequal`\ . To test for the presence of a key in a dictionary, use :func:`haskey` or ``k in keys(dict)``\ . + .. doctest:: + + julia> a = 1:3:20 + 1:3:19 + + julia> 4 in a + true + + julia> 5 in a + false + .. function:: eltype(type) .. Docstring generated from Julia source @@ -370,7 +392,7 @@ Iterable Collections Reductions for certain commonly-used operators have special implementations which should be used instead: ``maximum(itr)``\ , ``minimum(itr)``\ , ``sum(itr)``\ , ``prod(itr)``\ , ``any(itr)``\ , ``all(itr)``\ . - The associativity of the reduction is implementation dependent. This means that you can't use non-associative operations like ``-`` because it is undefined whether ``reduce(-,[1,2,3])`` should be evaluated as ``(1-2)-3`` or ``1-(2-3)``\ . Use ``foldl`` or ``foldr`` instead for guaranteed left or right associativity. + The associativity of the reduction is implementation dependent. This means that you can't use non-associative operations like ``-`` because it is undefined whether ``reduce(-,[1,2,3])`` should be evaluated as ``(1-2)-3`` or ``1-(2-3)``\ . Use :func:`foldl` or :func:`foldr` instead for guaranteed left or right associativity. Some operations accumulate error, and parallelism will also be easier if the reduction can be executed in groups. Future versions of Julia might change the algorithm. Note that the elements are not reordered if you use an ordered collection. @@ -674,6 +696,18 @@ Iterable Collections Test whether any elements of a boolean collection are ``true``\ . + .. doctest:: + + julia> a = [true,false,false,true] + 4-element Array{Bool,1}: + true + false + false + true + + julia> any(a) + true + .. function:: any(A, dims) .. Docstring generated from Julia source @@ -692,6 +726,18 @@ Iterable Collections Test whether all elements of a boolean collection are ``true``\ . + .. doctest:: + + julia> a = [true,false,false,true] + 4-element Array{Bool,1}: + true + false + false + true + + julia> all(a) + false + .. function:: all(A, dims) .. Docstring generated from Julia source @@ -867,6 +913,18 @@ Iterable Collections Return an ``Array`` of all items in a collection or iterator. For associative collections, returns ``Pair{KeyType, ValType}``\ . If the argument is array-like or is an iterator with the ``HasShape()`` trait, the result will have the same shape and number of dimensions as the argument. + .. doctest:: + + julia> collect(1:2:13) + 7-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + 13 + .. function:: collect(element_type, collection) .. Docstring generated from Julia source @@ -907,6 +965,16 @@ Iterable Collections Update ``collection``\ , removing elements for which ``function`` is ``false``\ . For associative collections, the function is passed two arguments (key and value). + .. doctest:: + + julia> filter!(isodd, collect(1:10)) + 5-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + Indexable Collections --------------------- @@ -987,6 +1055,19 @@ Given a dictionary ``D``, the syntax ``D[x]`` returns the value of key ``x`` (if Determine whether a collection has a mapping for a given key. + .. doctest:: + + julia> a = Dict('a'=>2, 'b'=>3) + Dict{Char,Int64} with 2 entries: + 'b' => 3 + 'a' => 2 + + julia> haskey(a,'a') + true + + julia> haskey(a,'c') + false + .. function:: get(collection, key, default) .. Docstring generated from Julia source @@ -1035,6 +1116,19 @@ Given a dictionary ``D``, the syntax ``D[x]`` returns the value of key ``x`` (if Return the key matching argument ``key`` if one exists in ``collection``\ , otherwise return ``default``\ . + .. doctest:: + + julia> a = Dict('a'=>2, 'b'=>3) + Dict{Char,Int64} with 2 entries: + 'b' => 3 + 'a' => 2 + + julia> getkey(a,'a',1) + 'a' + + julia> getkey(a,'d','a') + 'a' + .. function:: delete!(collection, key) .. Docstring generated from Julia source @@ -1047,19 +1141,43 @@ Given a dictionary ``D``, the syntax ``D[x]`` returns the value of key ``x`` (if Delete and return the mapping for ``key`` if it exists in ``collection``\ , otherwise return ``default``\ , or throw an error if default is not specified. -.. function:: keys(collection) +.. function:: keys(a::Associative) .. Docstring generated from Julia source - Return an iterator over all keys in a collection. ``collect(keys(d))`` returns an array of keys. + Return an iterator over all keys in a collection. ``collect(keys(d))`` returns an array of keys. Since the keys are stored internally in a hash table, the order in which they are returned may vary. + + .. doctest:: + + julia> a = Dict('a'=>2, 'b'=>3) + Dict{Char,Int64} with 2 entries: + 'b' => 3 + 'a' => 2 -.. function:: values(collection) + julia> collect(keys(a)) + 2-element Array{Char,1}: + 'b' + 'a' + +.. function:: values(a::Associative) .. Docstring generated from Julia source Return an iterator over all values in a collection. ``collect(values(d))`` returns an array of values. -.. function:: merge(collection, others...) + .. doctest:: + + julia> a = Dict('a'=>2, 'b'=>3) + Dict{Char,Int64} with 2 entries: + 'b' => 3 + 'a' => 2 + + julia> collect(values(a)) + 2-element Array{Int64,1}: + 3 + 2 + +.. function:: merge(d::Associative, others::Associative...) .. Docstring generated from Julia source @@ -1089,11 +1207,11 @@ Given a dictionary ``D``, the syntax ``D[x]`` returns the value of key ``x`` (if "baz" => 17.0 "foo" => 0.0 -.. function:: merge!(collection, others...) +.. function:: merge!(d::Associative, others::Associative...) .. Docstring generated from Julia source - Update collection with pairs from the other collections. + Update collection with pairs from the other collections. See also :func:`merge`\ . .. function:: sizehint!(s, n) @@ -1105,13 +1223,13 @@ Given a dictionary ``D``, the syntax ``D[x]`` returns the value of key ``x`` (if .. Docstring generated from Julia source - Get the key type of an associative collection type. Behaves similarly to ``eltype``\ . + Get the key type of an associative collection type. Behaves similarly to :func:`eltype`\ . .. function:: valtype(type) .. Docstring generated from Julia source - Get the value type of an associative collection type. Behaves similarly to ``eltype``\ . + Get the value type of an associative collection type. Behaves similarly to :func:`eltype`\ . Fully implemented by: @@ -1213,7 +1331,7 @@ Set-Like Collections Construct the symmetric difference of sets ``s1`` and ``s2``\ , storing the result in ``s1``\ . -.. function:: intersect!(s1, s2) +.. function:: intersect!(s1::IntSet, s2::IntSet) .. Docstring generated from Julia source @@ -1330,11 +1448,11 @@ Dequeues 5 6 -.. function:: insert!(collection, index, item) +.. function:: insert!(a::Vector, index::Integer, item) .. Docstring generated from Julia source - Insert an ``item`` into ``collection`` at the given ``index``\ . ``index`` is the index of ``item`` in the resulting ``collection``\ . + Insert an ``item`` into ``a`` at the given ``index``\ . ``index`` is the index of ``item`` in the resulting ``a``\ . .. doctest:: @@ -1379,14 +1497,14 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:611 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:724 ... -.. function:: splice!(collection, index, [replacement]) -> item +.. function:: splice!(a::Vector, index::Integer, [replacement]) -> item .. Docstring generated from Julia source - Remove the item at the given index, and return the removed item. Subsequent items are shifted down to fill the resulting gap. If specified, replacement values from an ordered collection will be spliced in place of the removed item. + Remove the item at the given index, and return the removed item. Subsequent items are shifted left to fill the resulting gap. If specified, replacement values from an ordered collection will be spliced in place of the removed item. .. doctest:: @@ -1427,11 +1545,11 @@ Dequeues To insert ``replacement`` before an index ``n`` without removing any items, use ``splice!(collection, n:n-1, replacement)``\ . -.. function:: splice!(collection, range, [replacement]) -> items +.. function:: splice!(a::Vector, range, [replacement]) -> items .. Docstring generated from Julia source - Remove items in the specified index range, and return a collection containing the removed items. Subsequent items are shifted down to fill the resulting gap. If specified, replacement values from an ordered collection will be spliced in place of the removed items. + Remove items in the specified index range, and return a collection containing the removed items. Subsequent items are shifted left to fill the resulting gap. If specified, replacement values from an ordered collection will be spliced in place of the removed items. To insert ``replacement`` before an index ``n`` without removing any items, use ``splice!(collection, n:n-1, replacement)``\ . @@ -1451,11 +1569,11 @@ Dequeues 3 -1 -.. function:: resize!(collection, n) -> collection +.. function:: resize!(a::Vector, n::Integer) -> Vector .. Docstring generated from Julia source - Resize ``collection`` to contain ``n`` elements. If ``n`` is smaller than the current collection length, the first ``n`` elements will be retained. If ``n`` is larger, the new elements are not guaranteed to be initialized. + Resize ``a`` to contain ``n`` elements. If ``n`` is smaller than the current collection length, the first ``n`` elements will be retained. If ``n`` is larger, the new elements are not guaranteed to be initialized. .. doctest:: @@ -1505,11 +1623,11 @@ Dequeues Use :func:`push!` to add individual items to ``collection`` which are not already themselves in another collection. The result is of the preceding example is equivalent to ``push!([1, 2, 3], 4, 5, 6)``\ . -.. function:: prepend!(collection, items) -> collection +.. function:: prepend!(a::Vector, items) -> collection .. Docstring generated from Julia source - Insert the elements of ``items`` to the beginning of ``collection``\ . + Insert the elements of ``items`` to the beginning of ``a``\ . .. doctest:: diff --git a/doc/stdlib/file.rst b/doc/stdlib/file.rst index b10ca641e2935..d26758e48ac99 100644 --- a/doc/stdlib/file.rst +++ b/doc/stdlib/file.rst @@ -10,19 +10,19 @@ Get the current working directory. -.. function:: cd([dir::AbstractString=homedir()]) +.. function:: cd(dir::AbstractString=homedir()) .. Docstring generated from Julia source Set the current working directory. -.. function:: cd(f, [dir=homedir()]) +.. function:: cd(f::Function, dir::AbstractString=homedir()) .. Docstring generated from Julia source Temporarily changes the current working directory and applies function ``f`` before returning. -.. function:: readdir([dir]) -> Vector{String} +.. function:: readdir(dir::AbstractString=".") -> Vector{String} .. Docstring generated from Julia source @@ -32,7 +32,7 @@ .. Docstring generated from Julia source - The walkdir method return an iterator that walks the directory tree of a directory. The iterator returns a tuple containing ``(rootpath, dirs, files)``\ . The directory tree can be traversed top-down or bottom-up. If walkdir encounters a SystemError it will raise the error. A custom error handling function can be provided through ``onerror`` keyword argument, the function is called with a SystemError as argument. + The ``walkdir`` method returns an iterator that walks the directory tree of a directory. The iterator returns a tuple containing ``(rootpath, dirs, files)``\ . The directory tree can be traversed top-down or bottom-up. If ``walkdir`` encounters a :obj:`SystemError` it will raise the error by default. A custom error handling function can be provided through ``onerror`` keyword argument. ``onerror`` is called with a ``SystemError`` as argument. .. code-block:: julia @@ -47,19 +47,19 @@ end end -.. function:: mkdir(path, [mode]) +.. function:: mkdir(path::AbstractString, mode::Unsigned=0o777) .. Docstring generated from Julia source Make a new directory with name ``path`` and permissions ``mode``\ . ``mode`` defaults to ``0o777``\ , modified by the current file creation mask. -.. function:: mkpath(path, [mode]) +.. function:: mkpath(path::AbstractString, mode::Unsigned=0o777) .. Docstring generated from Julia source Create all directories in the given ``path``\ , with permissions ``mode``\ . ``mode`` defaults to ``0o777``\ , modified by the current file creation mask. -.. function:: symlink(target, link) +.. function:: symlink(target::AbstractString, link::AbstractString) .. Docstring generated from Julia source @@ -69,19 +69,19 @@ This function raises an error under operating systems that do not support soft symbolic links, such as Windows XP. -.. function:: readlink(path) -> AbstractString +.. function:: readlink(path::AbstractString) -> AbstractString .. Docstring generated from Julia source Returns the value of a symbolic link ``path``\ . -.. function:: chmod(path, mode; recursive=false) +.. function:: chmod(path::AbstractString, mode::Integer; recursive::Bool=false) .. Docstring generated from Julia source Change the permissions mode of ``path`` to ``mode``\ . Only integer ``mode``\ s (e.g. ``0o777``\ ) are currently supported. If ``recursive=true`` and the path is a directory all permissions in that directory will be recursively changed. -.. function:: chown(path, owner, group=-1) +.. function:: chown(path::AbstractString, owner::Integer, group::Integer=-1) .. Docstring generated from Julia source @@ -125,7 +125,7 @@ .. Docstring generated from Julia source - Like stat, but for symbolic links gets the info for the link itself rather than the file it refers to. This function must be called on a file path rather than a file object or a file descriptor. + Like :func:`stat`\ , but for symbolic links gets the info for the link itself rather than the file it refers to. This function must be called on a file path rather than a file object or a file descriptor. .. function:: ctime(file) @@ -167,19 +167,19 @@ | 04 | Read Permission | +-------+--------------------+ - For allowed arguments, see ``stat``\ . + For allowed arguments, see :func:`stat`\ . .. function:: gperm(file) .. Docstring generated from Julia source - Like uperm but gets the permissions of the group owning the file. + Like :func:`uperm` but gets the permissions of the group owning the file. .. function:: operm(file) .. Docstring generated from Julia source - Like uperm but gets the permissions for people who neither own the file nor are a member of the group owning the file + Like :func:`uperm` but gets the permissions for people who neither own the file nor are a member of the group owning the file .. function:: cp(src::AbstractString, dst::AbstractString; remove_destination::Bool=false, follow_symlinks::Bool=false) @@ -189,19 +189,19 @@ If ``follow_symlinks=false``\ , and ``src`` is a symbolic link, ``dst`` will be created as a symbolic link. If ``follow_symlinks=true`` and ``src`` is a symbolic link, ``dst`` will be a copy of the file or directory ``src`` refers to. -.. function:: download(url,[localfile]) +.. function:: download(url::AbstractString, [localfile::AbstractString]) .. Docstring generated from Julia source Download a file from the given url, optionally renaming it to the given local file name. Note that this function relies on the availability of external tools such as ``curl``\ , ``wget`` or ``fetch`` to download the file and is provided for convenience. For production use or situations in which more options are needed, please use a package that provides the desired functionality instead. -.. function:: mv(src::AbstractString,dst::AbstractString; remove_destination::Bool=false) +.. function:: mv(src::AbstractString, dst::AbstractString; remove_destination::Bool=false) .. Docstring generated from Julia source Move the file, link, or directory from ``src`` to ``dst``\ . ``remove_destination=true`` will first remove an existing ``dst``\ . -.. function:: rm(path::AbstractString; force=false, recursive=false) +.. function:: rm(path::AbstractString; force::Bool=false, recursive::Bool=false) .. Docstring generated from Julia source @@ -225,25 +225,25 @@ Obtain the path of a temporary directory (possibly shared with other processes). -.. function:: mktemp([parent=tempdir()]) +.. function:: mktemp(parent=tempdir()) .. Docstring generated from Julia source Returns ``(path, io)``\ , where ``path`` is the path of a new temporary file in ``parent`` and ``io`` is an open file object for this path. -.. function:: mktemp(f::Function, [parent=tempdir()]) +.. function:: mktemp(f::Function, parent=tempdir()) .. Docstring generated from Julia source Apply the function ``f`` to the result of ``mktemp(parent)`` and remove the temporary file upon completion. -.. function:: mktempdir([parent=tempdir()]) +.. function:: mktempdir(parent=tempdir()) .. Docstring generated from Julia source Create a temporary directory in the ``parent`` directory and return its path. -.. function:: mktempdir(f::Function, [parent=tempdir()]) +.. function:: mktempdir(f::Function, parent=tempdir()) .. Docstring generated from Julia source @@ -333,12 +333,22 @@ Get the directory part of a path. + .. doctest:: + + julia> dirname("/home/myuser") + "/home" + .. function:: basename(path::AbstractString) -> AbstractString .. Docstring generated from Julia source Get the file name part of a path. + .. doctest:: + + julia> basename("/home/myuser/example.jl") + "example.jl" + .. function:: @__FILE__ -> AbstractString .. Docstring generated from Julia source @@ -357,18 +367,39 @@ Determines whether a path is absolute (begins at the root directory). + .. doctest:: + + julia> isabspath("/home") + true + + julia> isabspath("home") + false + .. function:: isdirpath(path::AbstractString) -> Bool .. Docstring generated from Julia source Determines whether a path refers to a directory (for example, ends with a path separator). + .. doctest:: + + julia> isdirpath("/home") + false + + julia> isdirpath("/home/") + true + .. function:: joinpath(parts...) -> AbstractString .. Docstring generated from Julia source Join path components into a full path. If some argument is an absolute path, then prior components are dropped. + .. doctest:: + + julia> joinpath("/home/myuser","example.jl") + "/home/myuser/example.jl" + .. function:: abspath(path::AbstractString) -> AbstractString .. Docstring generated from Julia source @@ -381,6 +412,11 @@ Normalize a path, removing "." and ".." entries. + .. doctest:: + + julia> normpath("/home/myuser/../example.jl") + "/home/example.jl" + .. function:: realpath(path::AbstractString) -> AbstractString .. Docstring generated from Julia source @@ -391,7 +427,7 @@ .. Docstring generated from Julia source - Return a relative filepath to path either from the current directory or from an optional start directory. This is a path computation: the filesystem is not accessed to confirm the existence or nature of path or startpath. + Return a relative filepath to ``path`` either from the current directory or from an optional start directory. This is a path computation: the filesystem is not accessed to confirm the existence or nature of ``path`` or ``startpath``\ . .. function:: expanduser(path::AbstractString) -> AbstractString @@ -399,21 +435,34 @@ On Unix systems, replace a tilde character at the start of a path with the current user's home directory. -.. function:: splitdir(path::AbstractString) -> (AbstractString,AbstractString) +.. function:: splitdir(path::AbstractString) -> (AbstractString, AbstractString) .. Docstring generated from Julia source Split a path into a tuple of the directory name and file name. -.. function:: splitdrive(path::AbstractString) -> (AbstractString,AbstractString) + .. doctest:: + + julia> splitdir("/home/myuser") + ("/home","myuser") + +.. function:: splitdrive(path::AbstractString) -> (AbstractString, AbstractString) .. Docstring generated from Julia source On Windows, split a path into the drive letter part and the path part. On Unix systems, the first component is always the empty string. -.. function:: splitext(path::AbstractString) -> (AbstractString,AbstractString) +.. function:: splitext(path::AbstractString) -> (AbstractString, AbstractString) .. Docstring generated from Julia source If the last component of a path contains a dot, split the path into everything before the dot and everything including and after the dot. Otherwise, return a tuple of the argument unmodified and the empty string. + .. doctest:: + + julia> splitext("/home/myuser/example.jl") + ("/home/myuser/example",".jl") + + julia> splitext("/home/myuser/example") + ("/home/myuser/example","") + diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index b78e2d1e22b17..cb648d2e7e597 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -25,13 +25,13 @@ General I/O Global variable referring to the standard input stream. -.. function:: open(filename, [read, write, create, truncate, append]) -> IOStream +.. function:: open(filename::AbstractString, [read::Bool, write::Bool, create::Bool, truncate::Bool, append::Bool]) -> IOStream .. Docstring generated from Julia source Open a file in a mode specified by five boolean arguments. The default is to open files for reading only. Returns a stream for accessing the file. -.. function:: open(filename, [mode]) -> IOStream +.. function:: open(filename::AbstractString, [mode::AbstractString]) -> IOStream .. Docstring generated from Julia source @@ -85,17 +85,17 @@ General I/O Create a fixed size IOBuffer. The buffer will not grow dynamically. -.. function:: IOBuffer(string) +.. function:: IOBuffer(string::String) .. Docstring generated from Julia source - Create a read-only IOBuffer on the data underlying the given string. + Create a read-only ``IOBuffer`` on the data underlying the given string. -.. function:: IOBuffer([data,],[readable,writable,[maxsize]]) +.. function:: IOBuffer([data,],[readable::Bool=true, writable::Bool=true, [maxsize::Int=typemax(Int)]]) .. Docstring generated from Julia source - Create an IOBuffer, which may optionally operate on a pre-existing array. If the readable/writable arguments are given, they restrict whether or not the buffer may be read from or written to respectively. By default the buffer is readable but not writable. The last argument optionally specifies a size beyond which the buffer may not be grown. + Create an ``IOBuffer``\ , which may optionally operate on a pre-existing array. If the readable/writable arguments are given, they restrict whether or not the buffer may be read from or written to respectively. By default the buffer is readable but not writable. The last argument optionally specifies a size beyond which the buffer may not be grown. .. function:: takebuf_array(b::IOBuffer) @@ -107,9 +107,9 @@ General I/O .. Docstring generated from Julia source - Obtain the contents of an ``IOBuffer`` as a string, without copying. Afterwards, the IOBuffer is reset to its initial state. + Obtain the contents of an ``IOBuffer`` as a string, without copying. Afterwards, the ``IOBuffer`` is reset to its initial state. -.. function:: fdio([name::AbstractString, ]fd::Integer[, own::Bool]) -> IOStream +.. function:: fdio([name::AbstractString, ]fd::Integer[, own::Bool=false]) -> IOStream .. Docstring generated from Julia source @@ -125,7 +125,7 @@ General I/O .. Docstring generated from Julia source - Close an I/O stream. Performs a ``flush`` first. + Close an I/O stream. Performs a :func:`flush` first. .. function:: write(stream::IO, x) write(filename::AbstractString, x) @@ -160,14 +160,12 @@ General I/O Read binary data from an I/O stream or file, filling in ``array``\ . -.. function:: readbytes!(stream::IO, b::AbstractVector{UInt8}, nb=length(b); all=true) +.. function:: readbytes!(stream::IO, b::AbstractVector{UInt8}, nb=length(b)) .. Docstring generated from Julia source Read at most ``nb`` bytes from ``stream`` into ``b``\ , returning the number of bytes read. The size of ``b`` will be increased if needed (i.e. if ``nb`` is greater than ``length(b)`` and enough bytes could be read), but it will never be decreased. - See ``read`` for a description of the ``all`` option. - .. function:: read(s::IO, nb=typemax(Int)) .. Docstring generated from Julia source @@ -188,15 +186,15 @@ General I/O Open a file and read its contents. ``args`` is passed to ``read``\ : this is equivalent to ``open(io->read(io, args...), filename)``\ . -.. function:: unsafe_read(io, ref, nbytes) +.. function:: unsafe_read(io::IO, ref, nbytes::UInt) .. Docstring generated from Julia source - Copy nbytes from the IO stream object into ref (converted to a pointer). + Copy nbytes from the ``IO`` stream object into ``ref`` (converted to a pointer). It is recommended that subtypes ``T<:IO`` override the following method signature to provide more efficient implementations: ``unsafe_read(s::T, p::Ptr{UInt8}, n::UInt)`` -.. function:: unsafe_write(io, ref, nbytes) +.. function:: unsafe_write(io::IO, ref, nbytes::UInt) .. Docstring generated from Julia source @@ -294,37 +292,19 @@ General I/O .. Docstring generated from Julia source - Determine whether an object - such as a stream, timer, or mmap – is not yet closed. Once an object is closed, it will never produce a new event. However, a closed stream may still have data to read in its buffer, use ``eof`` to check for the ability to read data. Use ``poll_fd`` to be notified when a stream might be writable or readable. + Determine whether an object - such as a stream, timer, or mmap – is not yet closed. Once an object is closed, it will never produce a new event. However, a closed stream may still have data to read in its buffer, use :func:`eof` to check for the ability to read data. Use :func:`poll_fd` to be notified when a stream might be writable or readable. .. function:: serialize(stream, value) .. Docstring generated from Julia source - Write an arbitrary value to a stream in an opaque format, such that it can be read back by ``deserialize``\ . The read-back value will be as identical as possible to the original. In general, this process will not work if the reading and writing are done by different versions of Julia, or an instance of Julia with a different system image. ``Ptr`` values are serialized as all-zero bit patterns (``NULL``\ ). + Write an arbitrary value to a stream in an opaque format, such that it can be read back by :func:`deserialize`\ . The read-back value will be as identical as possible to the original. In general, this process will not work if the reading and writing are done by different versions of Julia, or an instance of Julia with a different system image. ``Ptr`` values are serialized as all-zero bit patterns (``NULL``\ ). .. function:: deserialize(stream) .. Docstring generated from Julia source - Read a value written by ``serialize``\ . ``deserialize`` assumes the binary data read from ``stream`` is correct and has been serialized by a compatible implementation of ``serialize``\ . It has been designed with simplicity and performance as a goal and does not validate the data read. Malformed data can result in process termination. The caller has to ensure the integrity and correctness of data read from ``stream``\ . - -.. function:: escape_string(io, str::AbstractString, esc::AbstractString) - - .. Docstring generated from Julia source - - General escaping of traditional C and Unicode escape sequences, plus any characters in esc are also escaped (with a backslash). - -.. function:: unescape_string(io, s::AbstractString) - - .. Docstring generated from Julia source - - General unescaping of traditional C and Unicode escape sequences. Reverse of :func:`escape_string`\ . - -.. function:: join(io, items, delim, [last]) - - .. Docstring generated from Julia source - - Print elements of ``items`` to ``io`` with ``delim`` between them. If ``last`` is specified, it is used as the final delimiter instead of ``delim``\ . + Read a value written by :func:`serialize`\ . ``deserialize`` assumes the binary data read from ``stream`` is correct and has been serialized by a compatible implementation of :func:`serialize`\ . It has been designed with simplicity and performance as a goal and does not validate the data read. Malformed data can result in process termination. The caller has to ensure the integrity and correctness of data read from ``stream``\ . .. function:: print_shortest(io, x) @@ -338,17 +318,11 @@ General I/O Returns the file descriptor backing the stream or file. Note that this function only applies to synchronous ``File``\ 's and ``IOStream``\ 's not to any of the asynchronous streams. -.. function:: redirect_stdout() +.. function:: redirect_stdout([stream]) -> (rd, wr) .. Docstring generated from Julia source - Create a pipe to which all C and Julia level ``STDOUT`` output will be redirected. Returns a tuple ``(rd,wr)`` representing the pipe ends. Data written to ``STDOUT`` may now be read from the rd end of the pipe. The wr end is given for convenience in case the old ``STDOUT`` object was cached by the user and needs to be replaced elsewhere. - -.. function:: redirect_stdout(stream) - - .. Docstring generated from Julia source - - Replace ``STDOUT`` by stream for all C and Julia level output to ``STDOUT``\ . Note that ``stream`` must be a TTY, a ``Pipe`` or a ``TCPSocket``\ . + Create a pipe to which all C and Julia level :obj:`STDOUT` output will be redirected. Returns a tuple ``(rd, wr)`` representing the pipe ends. Data written to :obj:`STDOUT` may now be read from the ``rd`` end of the pipe. The ``wr`` end is given for convenience in case the old :obj:`STDOUT` object was cached by the user and needs to be replaced elsewhere. .. function:: redirect_stdout(f::Function, stream) @@ -356,11 +330,11 @@ General I/O Run the function ``f`` while redirecting ``STDOUT`` to ``stream``\ . Upon completion, ``STDOUT`` is restored to its prior setting. -.. function:: redirect_stderr([stream]) +.. function:: redirect_stderr([stream]) -> (rd, wr) .. Docstring generated from Julia source - Like ``redirect_stdout``\ , but for ``STDERR``\ . + Like :func:`redirect_stdout`\ , but for :obj:`STDERR`\ . .. function:: redirect_stderr(f::Function, stream) @@ -368,11 +342,11 @@ General I/O Run the function ``f`` while redirecting ``STDERR`` to ``stream``\ . Upon completion, ``STDERR`` is restored to its prior setting. -.. function:: redirect_stdin([stream]) +.. function:: redirect_stdin([stream]) -> (rd, wr) .. Docstring generated from Julia source - Like redirect_stdout, but for STDIN. Note that the order of the return tuple is still (rd,wr), i.e. data to be read from STDIN, may be written to wr. + Like :func:`redirect_stdout`\ , but for :obj:`STDIN`\ . Note that the order of the return tuple is still ``(rd, wr)``\ , i.e. data to be read from :obj:`STDIN` may be written to ``wr``\ . .. function:: redirect_stdin(f::Function, stream) @@ -384,7 +358,7 @@ General I/O .. Docstring generated from Julia source - Read the entirety of ``x`` as a string and remove a single trailing newline. Equivalent to ``chomp(readstring(x))``\ . + Read the entirety of ``x`` as a string and remove a single trailing newline. Equivalent to ``chomp!(readstring(x))``\ . .. function:: truncate(file,n) @@ -404,17 +378,11 @@ General I/O Read ``io`` until the end of the stream/file and count the number of lines. To specify a file pass the filename as the first argument. EOL markers other than '\\n' are supported by passing them as the second argument. -.. function:: PipeBuffer() - - .. Docstring generated from Julia source - - An IOBuffer that allows reading and performs writes by appending. Seeking and truncating are not supported. See IOBuffer for the available constructors. - -.. function:: PipeBuffer(data::Vector{UInt8},[maxsize]) +.. function:: PipeBuffer(data::Vector{UInt8}=UInt8[],[maxsize::Int=typemax(Int)]) .. Docstring generated from Julia source - Create a PipeBuffer to operate on a data vector, optionally specifying a size beyond which the underlying Array may not be grown. + An :obj:`IOBuffer` that allows reading and performs writes by appending. Seeking and truncating are not supported. See :obj:`IOBuffer` for the available constructors. If ``data`` is given, creates a ``PipeBuffer`` to operate on a data vector, optionally specifying a size beyond which the underlying ``Array`` may not be grown. .. function:: readavailable(stream) @@ -480,7 +448,7 @@ Text I/O .. Docstring generated from Julia source - Similar to ``show``\ , except shows all elements of arrays. + Similar to :func:`show`\ , except shows all elements of arrays. .. function:: summary(x) @@ -490,17 +458,17 @@ Text I/O For arrays, returns a string of size and type info, e.g. ``10-element Array{Int64,1}``\ . -.. function:: print(x) +.. function:: print(io::IO, x) .. Docstring generated from Julia source - Write (to the default output stream) a canonical (un-decorated) text representation of a value if there is one, otherwise call ``show``\ . The representation used by ``print`` includes minimal formatting and tries to avoid Julia-specific details. + Write (to the default output stream) a canonical (un-decorated) text representation of a value if there is one, otherwise call :func:`show`\ . The representation used by ``print`` includes minimal formatting and tries to avoid Julia-specific details. -.. function:: println(x) +.. function:: println(io::IO, xs...) .. Docstring generated from Julia source - Print (using :func:`print`\ ) ``x`` followed by a newline. + Print (using :func:`print`\ ) ``xs`` followed by a newline. If ``io`` is not supplied, prints to :obj:`STDOUT`\ . .. function:: print_with_color(color::Symbol, [io], strings...) @@ -510,7 +478,7 @@ Text I/O ``color`` may take any of the values ``:normal``\ , ``:bold``\ , ``:black``\ , ``:blue``\ , ``:cyan``\ , ``:green``\ , ``:magenta``\ , ``:red``\ , ``:white``\ , or ``:yellow``\ . -.. function:: info(msg) +.. function:: info(msg...; prefix="INFO: ") .. Docstring generated from Julia source @@ -526,7 +494,7 @@ Text I/O .. Docstring generated from Julia source - Print ``args`` using C ``printf()`` style format specification string. Optionally, an ``IOStream`` may be passed as the first argument to redirect output. + Print ``args`` using C ``printf()`` style format specification string. Optionally, an :obj:`IOStream` may be passed as the first argument to redirect output. .. function:: @sprintf("%Fmt", args...) @@ -547,6 +515,11 @@ Text I/O Call the given function with an I/O stream and the supplied extra arguments. Everything written to this I/O stream is returned as a string. + .. doctest:: + + julia> sprint(showcompact, 66.66666) + "66.6667" + .. function:: showerror(io, e) .. Docstring generated from Julia source @@ -721,7 +694,7 @@ Julia environments (such as the IPython-based IJulia notebook). .. Docstring generated from Julia source - Display ``x`` using the topmost applicable display in the display stack, typically using the richest supported multimedia output for ``x``\ , with plain-text ``STDOUT`` output as a fallback. The ``display(d, x)`` variant attempts to display ``x`` on the given display ``d`` only, throwing a ``MethodError`` if ``d`` cannot display objects of this type. + Display ``x`` using the topmost applicable display in the display stack, typically using the richest supported multimedia output for ``x``\ , with plain-text :obj:`STDOUT` output as a fallback. The ``display(d, x)`` variant attempts to display ``x`` on the given display ``d`` only, throwing a ``MethodError`` if ``d`` cannot display objects of this type. There are also two variants with a ``mime`` argument (a MIME type string, such as ``"image/png"``\ ), which attempt to display ``x`` using the requested MIME type *only*, throwing a ``MethodError`` if this type is not supported by either the display(s) or by ``x``\ . With these variants, one can also supply the "raw" data in the requested MIME type by passing ``x::AbstractString`` (for MIME types with text-based storage, such as text/html or application/postscript) or ``x::Vector{UInt8}`` (for binary MIME types). @@ -732,7 +705,7 @@ Julia environments (such as the IPython-based IJulia notebook). .. Docstring generated from Julia source - By default, the ``redisplay`` functions simply call ``display``\ . However, some display backends may override ``redisplay`` to modify an existing display of ``x`` (if any). Using ``redisplay`` is also a hint to the backend that ``x`` may be redisplayed several times, and the backend may choose to defer the display until (for example) the next interactive prompt. + By default, the ``redisplay`` functions simply call :func:`display`\ . However, some display backends may override ``redisplay`` to modify an existing display of ``x`` (if any). Using ``redisplay`` is also a hint to the backend that ``x`` may be redisplayed several times, and the backend may choose to defer the display until (for example) the next interactive prompt. .. function:: displayable(mime) -> Bool displayable(d::Display, mime) -> Bool @@ -753,13 +726,13 @@ Julia environments (such as the IPython-based IJulia notebook). Technically, the ``MIME"mime"`` macro defines a singleton type for the given ``mime`` string, which allows us to exploit Julia's dispatch mechanisms in determining how to display objects of any given type. - The first argument to ``show`` can be an ``IOContext`` specifying output format properties. See ``IOContext`` for details. + The first argument to ``show`` can be an :obj:`IOContext` specifying output format properties. See :obj:`IOContext` for details. .. function:: mimewritable(mime, x) .. Docstring generated from Julia source - Returns a boolean value indicating whether or not the object ``x`` can be written as the given ``mime`` type. (By default, this is determined automatically by the existence of the corresponding ``show`` function for ``typeof(x)``\ .) + Returns a boolean value indicating whether or not the object ``x`` can be written as the given ``mime`` type. (By default, this is determined automatically by the existence of the corresponding :func:`show` function for ``typeof(x)``\ .) .. function:: reprmime(mime, x) @@ -773,7 +746,7 @@ Julia environments (such as the IPython-based IJulia notebook). .. Docstring generated from Julia source - Returns an ``AbstractString`` containing the representation of ``x`` in the requested ``mime`` type. This is similar to ``reprmime`` except that binary data is base64-encoded as an ASCII string. + Returns an ``AbstractString`` containing the representation of ``x`` in the requested ``mime`` type. This is similar to :func:`reprmime` except that binary data is base64-encoded as an ASCII string. As mentioned above, one can also define new display backends. For example, a module that can display PNG images in a window can register @@ -900,53 +873,53 @@ Memory-mapped I/O Network I/O ----------- -.. function:: connect([host],port) -> TCPSocket +.. function:: connect([host], port::Integer) -> TCPSocket .. Docstring generated from Julia source Connect to the host ``host`` on port ``port``\ . -.. function:: connect(path) -> PipeEndpoint +.. function:: connect(path::AbstractString) -> PipeEndpoint .. Docstring generated from Julia source Connect to the named pipe / UNIX domain socket at ``path``\ . -.. function:: listen([addr,]port) -> TCPServer +.. function:: listen([addr, ]port::Integer; backlog::Integer=BACKLOG_DEFAULT) -> TCPServer .. Docstring generated from Julia source Listen on port on the address specified by ``addr``\ . By default this listens on localhost only. To listen on all interfaces pass ``IPv4(0)`` or ``IPv6(0)`` as appropriate. -.. function:: listen(path) -> PipeServer +.. function:: listen(path::AbstractString) -> PipeServer .. Docstring generated from Julia source Create and listen on a named pipe / UNIX domain socket. -.. function:: getaddrinfo(host) +.. function:: getaddrinfo(host::AbstractString) -> IPAddr .. Docstring generated from Julia source Gets the IP address of the ``host`` (may have to do a DNS lookup) -.. function:: getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr,UInt16) +.. function:: getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr, UInt16) .. Docstring generated from Julia source - Get the IP address and the port that the given TCP socket is connected to (or bound to, in the case of TCPServer). + Get the IP address and the port that the given ``TCPSocket`` is connected to (or bound to, in the case of ``TCPServer``\ ). .. function:: IPv4(host::Integer) -> IPv4 .. Docstring generated from Julia source - Returns IPv4 object from ip address formatted as Integer. + Returns an IPv4 object from ip address ``host`` formatted as an ``Integer``\ . .. function:: IPv6(host::Integer) -> IPv6 .. Docstring generated from Julia source - Returns IPv6 object from ip address formatted as Integer + Returns an IPv6 object from ip address ``host`` formatted as an ``Integer``\ . .. function:: nb_available(stream) @@ -960,13 +933,13 @@ Network I/O Accepts a connection on the given server and returns a connection to the client. An uninitialized client stream may be provided, in which case it will be used instead of creating a new stream. -.. function:: listenany(port_hint) -> (UInt16,TCPServer) +.. function:: listenany([host::IPAddr,] port_hint) -> (UInt16, TCPServer) .. Docstring generated from Julia source Create a ``TCPServer`` on any port, using hint as a starting point. Returns a tuple of the actual port that the server was created on and the server itself. -.. function:: poll_fd(fd, timeout_s::Real; readable=false, writable=false) +.. function:: poll_fd(fd, timeout_s::Real=-1; readable=false, writable=false) .. Docstring generated from Julia source @@ -976,7 +949,7 @@ Network I/O The returned value is an object with boolean fields ``readable``\ , ``writable``\ , and ``timedout``\ , giving the result of the polling. -.. function:: poll_file(path, interval_s::Real, timeout_s::Real) -> (previous::StatStruct, current::StatStruct) +.. function:: poll_file(path::AbstractString, interval_s::Real=5.007, timeout_s::Real=-1) -> (previous::StatStruct, current::StatStruct) .. Docstring generated from Julia source @@ -984,9 +957,9 @@ Network I/O Returns a pair of ``StatStruct`` objects ``(previous, current)`` when a change is detected. - To determine when a file was modified, compare ``mtime(prev) != mtime(current)`` to detect notification of changes. However, using ``watch_file`` for this operation is preferred, since it is more reliable and efficient, although in some situations it may not be available. + To determine when a file was modified, compare ``mtime(prev) != mtime(current)`` to detect notification of changes. However, using :func:`watch_file` for this operation is preferred, since it is more reliable and efficient, although in some situations it may not be available. -.. function:: watch_file(path, timeout_s::Real) +.. function:: watch_file(path::AbstractString, timeout_s::Real=-1) .. Docstring generated from Julia source @@ -996,13 +969,16 @@ Network I/O This behavior of this function varies slightly across platforms. See <https://nodejs.org/api/fs.html#fs_caveats> for more detailed information. -.. function:: bind(socket::Union{UDPSocket, TCPSocket}, host::IPAddr, port::Integer; ipv6only=false) +.. function:: bind(socket::Union{UDPSocket, TCPSocket}, host::IPAddr, port::Integer; ipv6only=false, reuseaddr=false, kws...) .. Docstring generated from Julia source - Bind ``socket`` to the given ``host:port``\ . Note that ``0.0.0.0`` will listen on all devices. ``ipv6only`` parameter disables dual stack mode. If it's ``true``\ , only IPv6 stack is created. + Bind ``socket`` to the given ``host:port``\ . Note that ``0.0.0.0`` will listen on all devices. + + * The ``ipv6only`` parameter disables dual stack mode. If ``ipv6only=true``\ , only an IPv6 stack is created. + * If ``reuseaddr=true``\ , multiple threads or processes can bind to the same address without error if they all set ``reuseaddr=true``\ , but only the last to bind will receive any traffic. -.. function:: send(socket::UDPSocket, host::IPv4, port::Integer, msg) +.. function:: send(socket::UDPSocket, host, port::Integer, msg) .. Docstring generated from Julia source @@ -1024,7 +1000,12 @@ Network I/O .. Docstring generated from Julia source - Set UDP socket options. ``multicast_loop``\ : loopback for multicast packets (default: ``true``\ ). ``multicast_ttl``\ : TTL for multicast packets. ``enable_broadcast``\ : flag must be set to ``true`` if socket will be used for broadcast messages, or else the UDP system will return an access error (default: ``false``\ ). ``ttl``\ : Time-to-live of packets sent on the socket. + Set UDP socket options. + + * ``multicast_loop``\ : loopback for multicast packets (default: ``true``\ ). + * ``multicast_ttl``\ : TTL for multicast packets (default: ``nothing``\ ). + * ``enable_broadcast``\ : flag must be set to ``true`` if socket will be used for broadcast messages, or else the UDP system will return an access error (default: ``false``\ ). + * ``ttl``\ : Time-to-live of packets sent on the socket (default: ``nothing``\ ). .. function:: ntoh(x) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index ff1a4808cd8d5..1b324599d2152 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -13,7 +13,7 @@ Standard Functions Linear algebra functions in Julia are largely implemented by calling functions from `LAPACK <http://www.netlib.org/lapack/>`_. Sparse factorizations call functions from `SuiteSparse <http://faculty.cse.tamu.edu/davis/suitesparse.html>`_. -.. function:: *(A, B) +.. function:: *(A::AbstractMatrix, B::AbstractMatrix) .. Docstring generated from Julia source @@ -49,6 +49,26 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the cross product of two 3-vectors. + .. doctest:: + + julia> a = [0;1;0] + 3-element Array{Int64,1}: + 0 + 1 + 0 + + julia> b = [0;0;1] + 3-element Array{Int64,1}: + 0 + 0 + 1 + + julia> cross(a,b) + 3-element Array{Int64,1}: + 1 + 0 + 0 + .. function:: factorize(A) .. Docstring generated from Julia source @@ -566,7 +586,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the Bunch-Kaufman [Bunch1977]_ factorization of a real symmetric or complex Hermitian matrix ``A`` and return a ``BunchKaufman`` object. The following functions are available for ``BunchKaufman`` objects: ``size``\ , ``\``\ , ``inv``\ , ``issymmetric``\ , ``ishermitian``\ . + Compute the Bunch-Kaufman [Bunch1977]_ factorization of a real symmetric or complex Hermitian matrix ``A`` and return a ``BunchKaufman`` object. The following functions are available for ``BunchKaufman`` objects: :func:`size`\ , ``\``\ , :func:`inv`\ , :func:`issymmetric`\ , :func:`ishermitian`\ . .. [Bunch1977] J R Bunch and L Kaufman, Some stable methods for calculating inertia and solving symmetric linear systems, Mathematics of Computation 31:137 (1977), 163-179. `url <http://www.ams.org/journals/mcom/1977-31-137/S0025-5718-1977-0428694-0>`_\ . @@ -961,19 +981,58 @@ Linear algebra functions in Julia are largely implemented by calling functions f Upper triangle of a matrix. -.. function:: triu(M, k) + .. doctest:: + + julia> a = ones(4,4) + 4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + + julia> triu(a) + 4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 0.0 1.0 1.0 1.0 + 0.0 0.0 1.0 1.0 + 0.0 0.0 0.0 1.0 + +.. function:: triu(M, k::Integer) .. Docstring generated from Julia source Returns the upper triangle of ``M`` starting from the ``k``\ th superdiagonal. + .. doctest:: + + julia> a = ones(4,4) + 4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + + julia> triu(a,3) + 4×4 Array{Float64,2}: + 0.0 0.0 0.0 1.0 + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + + julia> triu(a,-3) + 4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + .. function:: triu!(M) .. Docstring generated from Julia source - Upper triangle of a matrix, overwriting ``M`` in the process. + Upper triangle of a matrix, overwriting ``M`` in the process. See also :func:`triu`\ . -.. function:: triu!(M, k) +.. function:: triu!(M, k::Integer) .. Docstring generated from Julia source @@ -985,42 +1044,108 @@ Linear algebra functions in Julia are largely implemented by calling functions f Lower triangle of a matrix. -.. function:: tril(M, k) + .. doctest:: + + julia> a = ones(4,4) + 4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + + julia> tril(a) + 4×4 Array{Float64,2}: + 1.0 0.0 0.0 0.0 + 1.0 1.0 0.0 0.0 + 1.0 1.0 1.0 0.0 + 1.0 1.0 1.0 1.0 + +.. function:: tril(M, k::Integer) .. Docstring generated from Julia source Returns the lower triangle of ``M`` starting from the ``k``\ th superdiagonal. + .. doctest:: + + julia> a = ones(4,4) + 4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + + julia> tril(a,3) + 4×4 Array{Float64,2}: + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + 1.0 1.0 1.0 1.0 + + julia> tril(a,-3) + 4×4 Array{Float64,2}: + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + 1.0 0.0 0.0 0.0 + .. function:: tril!(M) .. Docstring generated from Julia source - Lower triangle of a matrix, overwriting ``M`` in the process. + Lower triangle of a matrix, overwriting ``M`` in the process. See also :func:`tril`\ . -.. function:: tril!(M, k) +.. function:: tril!(M, k::Integer) .. Docstring generated from Julia source Returns the lower triangle of ``M`` starting from the ``k``\ th superdiagonal, overwriting ``M`` in the process. -.. function:: diagind(M[, k]) +.. function:: diagind(M, k::Integer=0) .. Docstring generated from Julia source - A ``Range`` giving the indices of the ``k``\ th diagonal of the matrix ``M``\ . + A :class:`Range` giving the indices of the ``k``\ th diagonal of the matrix ``M``\ . + + .. doctest:: + + julia> A = [1 2 3; 4 5 6; 7 8 9] + 3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + + julia> diagind(A,-1) + 2:4:6 -.. function:: diag(M[, k]) +.. function:: diag(M, k::Integer=0) .. Docstring generated from Julia source - The ``k``\ th diagonal of a matrix, as a vector. Use ``diagm`` to construct a diagonal matrix. + The ``k``\ th diagonal of a matrix, as a vector. Use :func:`diagm` to construct a diagonal matrix. -.. function:: diagm(v[, k]) + .. doctest:: + + julia> diag(A,1) + 2-element Array{Int64,1}: + 2 + 6 + +.. function:: diagm(v, k::Integer=0) .. Docstring generated from Julia source Construct a diagonal matrix and place ``v`` on the ``k``\ th diagonal. + .. doctest:: + + julia> diagm([1,2,3],1) + 4×4 Array{Int64,2}: + 0 1 0 0 + 0 0 2 0 + 0 0 0 3 + 0 0 0 0 + .. function:: scale!(A, b) scale!(b, A) @@ -1030,19 +1155,45 @@ Linear algebra functions in Julia are largely implemented by calling functions f If ``A`` is a matrix and ``b`` is a vector, then ``scale!(A,b)`` scales each column ``i`` of ``A`` by ``b[i]`` (similar to ``A*Diagonal(b)``\ ), while ``scale!(b,A)`` scales each row ``i`` of ``A`` by ``b[i]`` (similar to ``Diagonal(b)*A``\ ), again operating in-place on ``A``\ . An ``InexactError`` exception is thrown if the scaling produces a number not representable by the element type of ``A``\ , e.g. for integer types. + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> b = [1; 2] + 2-element Array{Int64,1}: + 1 + 2 + + julia> scale!(a,b) + 2×2 Array{Int64,2}: + 1 4 + 3 8 + + julia> a = [1 2; 3 4]; + + julia> b = [1; 2]; + + julia> scale!(b,a) + 2×2 Array{Int64,2}: + 1 2 + 6 8 + .. function:: Tridiagonal(dl, d, du) .. Docstring generated from Julia source Construct a tridiagonal matrix from the first subdiagonal, diagonal, and first superdiagonal, respectively. The result is of type ``Tridiagonal`` and provides efficient specialized linear solvers, but may be converted into a regular matrix with :func:`full`\ . The lengths of ``dl`` and ``du`` must be one less than the length of ``d``\ . -.. function:: rank(M) +.. function:: rank(M[, tol::Real]) .. Docstring generated from Julia source Compute the rank of a matrix. -.. function:: norm(A, [p]) +.. function:: norm(A, [p::Real=2]) .. Docstring generated from Julia source @@ -1052,7 +1203,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f For matrices, the matrix norm induced by the vector ``p``\ -norm is used, where valid values of ``p`` are ``1``\ , ``2``\ , or ``Inf``\ . (Note that for sparse matrices, ``p=2`` is currently not implemented.) Use :func:`vecnorm` to compute the Frobenius norm. -.. function:: vecnorm(A, [p]) +.. function:: vecnorm(A, [p::Real=2]) .. Docstring generated from Julia source @@ -1060,51 +1211,41 @@ Linear algebra functions in Julia are largely implemented by calling functions f For example, if ``A`` is a matrix and ``p=2``\ , then this is equivalent to the Frobenius norm. -.. function:: normalize!(v, [p=2]) +.. function:: normalize!(v, [p::Real=2]) .. Docstring generated from Julia source - Normalize the vector ``v`` in-place with respect to the ``p``\ -norm. - - Inputs: - - * ``v::AbstractVector`` - vector to be normalized - * ``p::Real`` - The ``p``\ -norm to normalize with respect to. Default: 2 - - Output: - - * ``v`` - A unit vector being the input vector, rescaled to have norm 1. The input vector is modified in-place. - - See also: - - ``normalize``\ , ``qr`` + Normalize the vector ``v`` in-place with respect to the ``p``\ -norm. See also :func:`vecnorm` and :func:`normalize`\ . -.. function:: normalize(v, [p=2]) +.. function:: normalize(v, [p::Real=2]) .. Docstring generated from Julia source - Normalize the vector ``v`` with respect to the ``p``\ -norm. + Normalize the vector ``v`` with respect to the ``p``\ -norm. See also :func:`normalize!` and :func:`vecnorm`\ . - Inputs: - - * ``v::AbstractVector`` - vector to be normalized - * ``p::Real`` - The ``p``\ -norm to normalize with respect to. Default: 2 - - Output: + .. doctest:: - * ``v`` - A unit vector being a copy of the input vector, scaled to have norm 1 + julia> a = [1,2,4]; - See also: + julia> normalize(a) + 3-element Array{Float64,1}: + 0.218218 + 0.436436 + 0.872872 - ``normalize!``\ , ``qr`` + julia> normalize(a,1) + 3-element Array{Float64,1}: + 0.142857 + 0.285714 + 0.571429 -.. function:: cond(M, [p]) +.. function:: cond(M, p::Real=2) .. Docstring generated from Julia source Condition number of the matrix ``M``\ , computed using the operator ``p``\ -norm. Valid values for ``p`` are ``1``\ , ``2`` (default), or ``Inf``\ . -.. function:: condskeel(M, [x, p]) +.. function:: condskeel(M, [x, p::Real=Inf]) .. Docstring generated from Julia source @@ -1147,7 +1288,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f Matrix inverse. -.. function:: pinv(M[, tol]) +.. function:: pinv(M[, tol::Real]) .. Docstring generated from Julia source @@ -1173,11 +1314,31 @@ Linear algebra functions in Julia are largely implemented by calling functions f Basis for nullspace of ``M``\ . -.. function:: repmat(A, n, m) +.. function:: repmat(A, m::Int, n::Int=1) .. Docstring generated from Julia source - Construct a matrix by repeating the given matrix ``n`` times in dimension 1 and ``m`` times in dimension 2. + Construct a matrix by repeating the given matrix ``m`` times in dimension 1 and ``n`` times in dimension 2. + + .. doctest:: + + julia> repmat([1, 2, 3], 2) + 6-element Array{Int64,1}: + 1 + 2 + 3 + 1 + 2 + 3 + + julia> repmat([1, 2, 3], 2, 3) + 6×3 Array{Int64,2}: + 1 1 1 + 2 2 2 + 3 3 3 + 1 1 1 + 2 2 2 + 3 3 3 .. function:: repeat(A::AbstractArray; inner=ntuple(x->1, ndims(A)), outer=ntuple(x->1, ndims(A))) @@ -1226,7 +1387,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f Perform simple linear regression using Ordinary Least Squares. Returns ``a`` and ``b`` such that ``a + b*x`` is the closest straight line to the given points ``(x, y)``\ , i.e., such that the squared error between ``y`` and ``a + b*x`` is minimized. - Examples: + **Examples:** .. code-block:: julia @@ -1239,7 +1400,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f See also: - ``\``\ , ``cov``\ , ``std``\ , ``mean`` + ``\``\ , :func:`cov`\ , :func:`std`\ , :func:`mean`\ . .. function:: expm(A) @@ -1295,6 +1456,24 @@ Linear algebra functions in Julia are largely implemented by calling functions f Test whether a matrix is symmetric. + .. doctest:: + + julia> a = [1 2; 2 -1] + 2×2 Array{Int64,2}: + 1 2 + 2 -1 + + julia> issymmetric(a) + true + + julia> b = [1 im; -im 1] + 2×2 Array{Complex{Int64},2}: + 1+0im 0+1im + 0-1im 1+0im + + julia> issymmetric(b) + false + .. function:: isposdef(A) -> Bool .. Docstring generated from Julia source @@ -1313,24 +1492,96 @@ Linear algebra functions in Julia are largely implemented by calling functions f Test whether a matrix is lower triangular. + .. doctest:: + + julia> a = [1 2; 2 -1] + 2×2 Array{Int64,2}: + 1 2 + 2 -1 + + julia> istril(a) + false + + julia> b = [1 0; -im -1] + 2×2 Array{Complex{Int64},2}: + 1+0im 0+0im + 0-1im -1+0im + + julia> istril(b) + true + .. function:: istriu(A) -> Bool .. Docstring generated from Julia source Test whether a matrix is upper triangular. + .. doctest:: + + julia> a = [1 2; 2 -1] + 2×2 Array{Int64,2}: + 1 2 + 2 -1 + + julia> istriu(a) + false + + julia> b = [1 im; 0 -1] + 2×2 Array{Complex{Int64},2}: + 1+0im 0+1im + 0+0im -1+0im + + julia> istriu(b) + true + .. function:: isdiag(A) -> Bool .. Docstring generated from Julia source Test whether a matrix is diagonal. + .. doctest:: + + julia> a = [1 2; 2 -1] + 2×2 Array{Int64,2}: + 1 2 + 2 -1 + + julia> isdiag(a) + false + + julia> b = [im 0; 0 -im] + 2×2 Array{Complex{Int64},2}: + 0+1im 0+0im + 0+0im 0-1im + + julia> isdiag(b) + true + .. function:: ishermitian(A) -> Bool .. Docstring generated from Julia source Test whether a matrix is Hermitian. + .. doctest:: + + julia> a = [1 2; 2 -1] + 2×2 Array{Int64,2}: + 1 2 + 2 -1 + + julia> ishermitian(a) + true + + julia> b = [1 im; -im 1] + 2×2 Array{Complex{Int64},2}: + 1+0im 0+1im + 0-1im 1+0im + + julia> ishermitian(b) + true + .. function:: transpose(A) .. Docstring generated from Julia source @@ -1507,11 +1758,11 @@ Linear algebra functions in Julia are largely implemented by calling functions f ``svds(A)`` is formally equivalent to calling ``eigs`` to perform implicitly restarted Lanczos tridiagonalization on the Hermitian matrix :math:`\begin{pmatrix} 0 & A^\prime \\ A & 0 \end{pmatrix}`\ , whose eigenvalues are plus and minus the singular values of :math:`A`\ . -.. function:: peakflops(n; parallel=false) +.. function:: peakflops(n::Integer=2000; parallel::Bool=false) .. Docstring generated from Julia source - ``peakflops`` computes the peak flop rate of the computer by using double precision :func:`Base.LinAlg.BLAS.gemm!`\ . By default, if no arguments are specified, it multiplies a matrix of size ``n x n``\ , where ``n = 2000``\ . If the underlying BLAS is using multiple threads, higher flop rates are realized. The number of BLAS threads can be set with ``BLAS.set_num_threads(n)``\ . + ``peakflops`` computes the peak flop rate of the computer by using double precision :func:`Base.LinAlg.BLAS.gemm!`\ . By default, if no arguments are specified, it multiplies a matrix of size ``n x n``\ , where ``n = 2000``\ . If the underlying BLAS is using multiple threads, higher flop rates are realized. The number of BLAS threads can be set with :func:`Base.LinAlg.BLAS.set_num_threads`\ . If the keyword argument ``parallel`` is set to ``true``\ , ``peakflops`` is run in parallel on all the worker processors. The flop rate of the entire parallel computer is returned. When running in parallel, only 1 BLAS thread is used. The argument ``n`` still refers to the size of the problem that is solved on each processor. diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 206f9e38733f7..97be585a2aca4 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -103,13 +103,13 @@ Mathematical Operators .. Docstring generated from Julia source - Computes ``x*y+z`` without rounding the intermediate result ``x*y``\ . On some systems this is significantly more expensive than ``x*y+z``\ . ``fma`` is used to improve accuracy in certain algorithms. See ``muladd``\ . + Computes ``x*y+z`` without rounding the intermediate result ``x*y``\ . On some systems this is significantly more expensive than ``x*y+z``\ . ``fma`` is used to improve accuracy in certain algorithms. See :func:`muladd`\ . .. function:: muladd(x, y, z) .. Docstring generated from Julia source - Combined multiply-add, computes ``x*y+z`` in an efficient manner. This may on some systems be equivalent to ``x*y+z``\ , or to ``fma(x,y,z)``\ . ``muladd`` is used to improve performance. See ``fma``\ . + Combined multiply-add, computes ``x*y+z`` in an efficient manner. This may on some systems be equivalent to ``x*y+z``\ , or to ``fma(x,y,z)``\ . ``muladd`` is used to improve performance. See :func:`fma`\ . .. function:: div(x, y) ÷(x, y) @@ -124,12 +124,22 @@ Mathematical Operators Largest integer less than or equal to ``x/y``\ . + .. doctest:: + + julia> fld(7.3,5.5) + 1.0 + .. function:: cld(x, y) .. Docstring generated from Julia source Smallest integer larger than or equal to ``x/y``\ . + .. doctest:: + + julia> cld(5.5,2.2) + 3.0 + .. function:: mod(x, y) .. Docstring generated from Julia source @@ -148,6 +158,11 @@ Mathematical Operators This function computes a floating point representation of the modulus after division by numerically exact ``2π``\ , and is therefore not exactly the same as ``mod(x,2π)``\ , which would compute the modulus of ``x`` relative to division by the floating-point number ``2π``\ . + .. doctest:: + + julia> mod2pi(9*pi/4) + 0.7853981633974481 + .. function:: rem(x, y) %(x, y) @@ -165,6 +180,14 @@ Mathematical Operators The quotient and remainder from Euclidean division. Equivalent to ``(div(x,y), rem(x,y))`` or ``(x÷y, x%y)``\ . + .. doctest:: + + julia> divrem(3,7) + (0,3) + + julia> divrem(7,3) + (2,1) + .. function:: fldmod(x, y) .. Docstring generated from Julia source @@ -762,12 +785,22 @@ Mathematical Functions Convert ``x`` from degrees to radians. + .. doctest:: + + julia> deg2rad(90) + 1.5707963267948966 + .. function:: rad2deg(x) .. Docstring generated from Julia source Convert ``x`` from radians to degrees. + .. doctest:: + + julia> rad2deg(pi) + 180.0 + .. function:: hypot(x, y) .. Docstring generated from Julia source @@ -794,6 +827,14 @@ Mathematical Functions Compute the base ``b`` logarithm of ``x``\ . Throws ``DomainError`` for negative ``Real`` arguments. + .. doctest:: + + julia> log(4,8) + 1.5 + + julia> log(4,2) + 0.5 + .. function:: log2(x) .. Docstring generated from Julia source @@ -804,7 +845,7 @@ Mathematical Functions .. Docstring generated from Julia source - Compute the logarithm of ``x`` to base 10. Throws ``DomainError`` for negative ``Real`` arguments. + Compute the logarithm of ``x`` to base 10. Throws :obj:`DomainError` for negative ``Real`` arguments. .. function:: log1p(x) @@ -818,7 +859,7 @@ Mathematical Functions .. Docstring generated from Julia source - Return ``(x,exp)`` such that ``x`` has a magnitude in the interval :math:`[1/2, 1)` or 0, and val = :math:`x \times 2^{exp}`\ . + Return ``(x,exp)`` such that ``x`` has a magnitude in the interval :math:`[1/2, 1)` or 0, and ``val`` is equal to :math:`x \times 2^{exp}`\ . .. function:: exp(x) @@ -832,6 +873,11 @@ Mathematical Functions Compute :math:`2^x`\ . + .. doctest:: + + julia> exp2(5) + 32.0 + .. function:: exp10(x) .. Docstring generated from Julia source @@ -850,6 +896,11 @@ Mathematical Functions Return a tuple (fpart,ipart) of the fractional and integral parts of a number. Both parts have the same sign as the argument. + .. doctest:: + + julia> modf(3.5) + (0.5,3.0) + .. function:: expm1(x) .. Docstring generated from Julia source @@ -1003,7 +1054,7 @@ Mathematical Functions .. Docstring generated from Julia source - Rounds (in the sense of ``round``\ ) ``x`` so that there are ``digits`` significant digits, under a base ``base`` representation, default 10. E.g., ``signif(123.456, 2)`` is ``120.0``\ , and ``signif(357.913, 4, 2)`` is ``352.0``\ . + Rounds (in the sense of :func:`round`\ ) ``x`` so that there are ``digits`` significant digits, under a base ``base`` representation, default 10. E.g., ``signif(123.456, 2)`` is ``120.0``\ , and ``signif(357.913, 4, 2)`` is ``352.0``\ . .. function:: min(x, y, ...) @@ -1172,18 +1223,28 @@ Mathematical Functions Return :math:`\sqrt{x}`\ . Throws ``DomainError`` for negative ``Real`` arguments. Use complex negative arguments instead. The prefix operator ``√`` is equivalent to ``sqrt``\ . -.. function:: isqrt(n) +.. function:: isqrt(n::Integer) .. Docstring generated from Julia source Integer square root: the largest integer ``m`` such that ``m*m <= n``\ . + .. doctest:: + + julia> isqrt(5) + 2 + .. function:: cbrt(x) .. Docstring generated from Julia source Return :math:`x^{1/3}`\ . The prefix operator ``∛`` is equivalent to ``cbrt``\ . + .. doctest:: + + julia> cbrt(big(27)) + 3.000000000000000000000000000000000000000000000000000000000000000000000000000000 + .. function:: erf(x) .. Docstring generated from Julia source @@ -1280,12 +1341,28 @@ Mathematical Functions Greatest common (positive) divisor (or zero if ``x`` and ``y`` are both zero). + .. doctest:: + + julia> gcd(6,9) + 3 + + julia> gcd(6,-9) + 3 + .. function:: lcm(x,y) .. Docstring generated from Julia source Least common (non-negative) multiple. + .. doctest:: + + julia> lcm(2,3) + 6 + + julia> lcm(-2,3) + 6 + .. function:: gcdx(x,y) .. Docstring generated from Julia source @@ -1306,24 +1383,45 @@ Mathematical Functions Bézout coefficients are *not* uniquely defined. ``gcdx`` returns the minimal Bézout coefficients that are computed by the extended Euclidean algorithm. (Ref: D. Knuth, TAoCP, 2/e, p. 325, Algorithm X.) For signed integers, these coefficients ``u`` and ``v`` are minimal in the sense that :math:`|u| < |y/d|` and :math:`|v| < |x/d|`\ . Furthermore, the signs of ``u`` and ``v`` are chosen so that ``d`` is positive. For unsigned integers, the coefficients ``u`` and ``v`` might be near their ``typemax``\ , and the identity then holds only via the unsigned integers' modulo arithmetic. -.. function:: ispow2(n) -> Bool +.. function:: ispow2(n::Integer) -> Bool .. Docstring generated from Julia source Test whether ``n`` is a power of two. -.. function:: nextpow2(n) + .. doctest:: + + julia> ispow2(4) + true + + julia> ispow2(5) + false + +.. function:: nextpow2(n::Integer) .. Docstring generated from Julia source The smallest power of two not less than ``n``\ . Returns 0 for ``n==0``\ , and returns ``-nextpow2(-n)`` for negative arguments. -.. function:: prevpow2(n) + .. doctest:: + + julia> nextpow2(16) + 16 + + julia> nextpow2(17) + 32 + +.. function:: prevpow2(n::Integer) .. Docstring generated from Julia source The largest power of two not greater than ``n``\ . Returns 0 for ``n==0``\ , and returns ``-prevpow2(-n)`` for negative arguments. + .. doctest:: + + julia> prevpow2(5) + 4 + .. function:: nextpow(a, x) .. Docstring generated from Julia source @@ -1348,7 +1446,18 @@ Mathematical Functions Take the inverse of ``x`` modulo ``m``\ : ``y`` such that :math:`x y = 1 \pmod m`\ , with :math:`div(x,y) = 0`\ . This is undefined for :math:`m = 0`\ , or if :math:`gcd(x,m) \neq 1`\ . -.. function:: powermod(x, p, m) + .. doctest:: + + julia> invmod(2,5) + 3 + + julia> invmod(2,3) + 2 + + julia> invmod(5,6) + 5 + +.. function:: powermod(x::Integer, p::Integer, m) .. Docstring generated from Julia source @@ -1578,11 +1687,11 @@ Mathematical Functions Generalized zeta function :math:`\zeta(s, z)`\ , defined by the sum :math:`\sum_{k=0}^\infty ((k+z)^2)^{-s/2}`\ , where any term with :math:`k+z=0` is excluded. For :math:`\Re z > 0`\ , this definition is equivalent to the Hurwitz zeta function :math:`\sum_{k=0}^\infty (k+z)^{-s}`\ . For :math:`z=1`\ , it yields the Riemann zeta function :math:`\zeta(s)`\ . -.. function:: ndigits(n, b = 10) +.. function:: ndigits(n::Integer, b::Integer=10) .. Docstring generated from Julia source - Compute the number of digits in number ``n`` written in base ``b``\ . + Compute the number of digits in integer ``n`` written in base ``b``\ . .. function:: widemul(x, y) @@ -1590,6 +1699,11 @@ Mathematical Functions Multiply ``x`` and ``y``\ , giving the result as a larger type. + .. doctest:: + + julia> widemul(Float32(3.), 4.) + 1.200000000000000000000000000000000000000000000000000000000000000000000000000000e+01 + .. function:: @evalpoly(z, c...) .. Docstring generated from Julia source @@ -1641,11 +1755,15 @@ Statistics Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. -.. function:: var(v[, region]) +.. function:: var(v[, region]; corrected::Bool=true, mean=nothing) .. Docstring generated from Julia source - Compute the sample variance of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sumabs2(v - mean(v)) / (length(v) - 1)``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. + Compute the sample variance of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sumabs2(v - mean(v)) / (length(v) - 1)``\ . If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . The mean ``m`` over the region may be provided. + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: varm(v, m[, region]; corrected::Bool=true) @@ -1712,7 +1830,7 @@ Statistics .. Docstring generated from Julia source - Like ``median``\ , but may overwrite the input vector. + Like :func:`median`\ , but may overwrite the input vector. .. function:: midpoints(e) @@ -2033,7 +2151,7 @@ implemented by calling functions from `FFTW Same as :func:`filt` but writes the result into the ``out`` argument, which may alias the input ``x`` to modify it in-place. -.. function:: deconv(b,a) +.. function:: deconv(b,a) -> c .. Docstring generated from Julia source diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index 1d89e90c5db58..6d4b74de91f5d 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -12,43 +12,59 @@ Standard Numeric Types Data Formats ------------ -.. function:: bin(n, [pad]) +.. function:: bin(n, pad::Int=1) .. Docstring generated from Julia source Convert an integer to a binary string, optionally specifying a number of digits to pad to. -.. function:: hex(n, [pad]) + .. doctest:: + + julia> bin(10,2) + "1010" + + julia> bin(10,8) + "00001010" + +.. function:: hex(n, pad::Int=1) .. Docstring generated from Julia source Convert an integer to a hexadecimal string, optionally specifying a number of digits to pad to. -.. function:: dec(n, [pad]) +.. function:: dec(n, pad::Int=1) .. Docstring generated from Julia source Convert an integer to a decimal string, optionally specifying a number of digits to pad to. -.. function:: oct(n, [pad]) +.. function:: oct(n, pad::Int=1) .. Docstring generated from Julia source Convert an integer to an octal string, optionally specifying a number of digits to pad to. -.. function:: base(base, n, [pad]) +.. function:: base(base::Integer, n::Integer, pad::Integer=1) .. Docstring generated from Julia source - Convert an integer to a string in the given base, optionally specifying a number of digits to pad to. + Convert an integer ``n`` to a string in the given ``base``\ , optionally specifying a number of digits to pad to. + + .. doctest:: + + julia> base(13,5,4) + "0005" -.. function:: digits([T], n, [base], [pad]) + julia> base(5,13,4) + "0023" + +.. function:: digits([T<:Integer], n::Integer, base::T=10, pad::Integer=1) .. Docstring generated from Julia source Returns an array with element type ``T`` (default ``Int``\ ) of the digits of ``n`` in the given base, optionally padded with zeros to a specified size. More significant digits are at higher indexes, such that ``n == sum([digits[k]*base^(k-1) for k=1:length(digits)])``\ . -.. function:: digits!(array, n, [base]) +.. function:: digits!(array, n::Integer, base::Integer=10) .. Docstring generated from Julia source @@ -70,13 +86,13 @@ Data Formats .. Docstring generated from Julia source - Like ``parse``\ , but returns a ``Nullable`` of the requested type. The result will be null if the string does not contain a valid number. + Like :func:`parse`\ , but returns a :obj:`Nullable` of the requested type. The result will be null if the string does not contain a valid number. .. function:: big(x) .. Docstring generated from Julia source - Convert a number to a maximum precision representation (typically ``BigInt`` or ``BigFloat``\ ). See ``BigFloat`` for information about some pitfalls with floating-point numbers. + Convert a number to a maximum precision representation (typically ``BigInt`` or ``BigFloat``\ ). See :obj:`BigFloat` for information about some pitfalls with floating-point numbers. .. function:: signed(x) @@ -134,6 +150,11 @@ Data Formats Get a hexadecimal string of the binary representation of a floating point number. + .. doctest:: + + julia> num2hex(2.2) + "400199999999999a" + .. function:: hex2num(str) .. Docstring generated from Julia source @@ -146,11 +167,34 @@ Data Formats Convert an arbitrarily long hexadecimal string to its binary representation. Returns an ``Array{UInt8,1}``\ , i.e. an array of bytes. -.. function:: bytes2hex(bin_arr::Array{UInt8, 1}) + .. doctest:: + + julia> a = hex(12345) + "3039" + + julia> hex2bytes(a) + 2-element Array{UInt8,1}: + 0x30 + 0x39 + +.. function:: bytes2hex(bin_arr::Array{UInt8, 1}) -> String .. Docstring generated from Julia source - Convert an array of bytes to its hexadecimal representation. All characters are in lower-case. Returns a ``String``\ . + Convert an array of bytes to its hexadecimal representation. All characters are in lower-case. + + .. doctest:: + + julia> a = hex(12345) + "3039" + + julia> b = hex2bytes(a) + 2-element Array{UInt8,1}: + 0x30 + 0x39 + + julia> bytes2hex(b) + "3039" General Number Functions and Constants -------------------------------------- @@ -253,7 +297,15 @@ General Number Functions and Constants .. Docstring generated from Julia source - Test whether a number is finite + Test whether a number is finite. + + .. doctest:: + + julia> isfinite(5) + true + + julia> isfinite(NaN32) + false .. function:: isinf(f) -> Bool @@ -583,13 +635,13 @@ A ``MersenneTwister`` or ``RandomDevice`` RNG can generate random numbers of the (or complex numbers of those types). Random floating point numbers are generated uniformly in :math:`[0, 1)`. As ``BigInt`` represents unbounded integers, the interval must be specified (e.g. ``rand(big(1:6))``). -.. function:: srand([rng], [seed]) +.. function:: srand([rng=GLOBAL_RNG], [seed]) .. Docstring generated from Julia source Reseed the random number generator. If a ``seed`` is provided, the RNG will give a reproducible sequence of numbers, otherwise Julia will get entropy from the system. For ``MersenneTwister``\ , the ``seed`` may be a non-negative integer, a vector of ``UInt32`` integers or a filename, in which case the seed is read from a file. ``RandomDevice`` does not support seeding. -.. function:: MersenneTwister([seed]) +.. function:: MersenneTwister(seed=0) .. Docstring generated from Julia source @@ -601,7 +653,7 @@ As ``BigInt`` represents unbounded integers, the interval must be specified (e.g Create a ``RandomDevice`` RNG object. Two such objects will always generate different streams of random numbers. -.. function:: rand([rng], [S], [dims...]) +.. function:: rand([rng=GLOBAL_RNG], [S], [dims...]) .. Docstring generated from Julia source @@ -612,43 +664,43 @@ As ``BigInt`` represents unbounded integers, the interval must be specified (e.g ``S`` defaults to ``Float64``\ . -.. function:: rand!([rng], A, [coll]) +.. function:: rand!([rng=GLOBAL_RNG], A, [coll]) .. Docstring generated from Julia source Populate the array ``A`` with random values. If the indexable collection ``coll`` is specified, the values are picked randomly from ``coll``\ . This is equivalent to ``copy!(A, rand(rng, coll, size(A)))`` or ``copy!(A, rand(rng, eltype(A), size(A)))`` but without allocating a new array. -.. function:: bitrand([rng], [dims...]) +.. function:: bitrand([rng=GLOBAL_RNG], [dims...]) .. Docstring generated from Julia source Generate a ``BitArray`` of random boolean values. -.. function:: randn([rng], [T=Float64], [dims...]) +.. function:: randn([rng=GLOBAL_RNG], [T=Float64], [dims...]) .. Docstring generated from Julia source Generate a normally-distributed random number of type ``T`` with mean 0 and standard deviation 1. Optionally generate an array of normally-distributed random numbers. The ``Base`` module currently provides an implementation for the types ``Float16``\ , ``Float32``\ , and ``Float64`` (the default). -.. function:: randn!([rng], A::AbstractArray) -> A +.. function:: randn!([rng=GLOBAL_RNG], A::AbstractArray) -> A .. Docstring generated from Julia source - Fill the array ``A`` with normally-distributed (mean 0, standard deviation 1) random numbers. Also see the ``rand`` function. + Fill the array ``A`` with normally-distributed (mean 0, standard deviation 1) random numbers. Also see the :func:`rand` function. -.. function:: randexp([rng], [T=Float64], [dims...]) +.. function:: randexp([rng=GLOBAL_RNG], [T=Float64], [dims...]) .. Docstring generated from Julia source Generate a random number of type ``T`` according to the exponential distribution with scale 1. Optionally generate an array of such random numbers. The ``Base`` module currently provides an implementation for the types ``Float16``\ , ``Float32``\ , and ``Float64`` (the default). -.. function:: randexp!([rng], A::AbstractArray) -> A +.. function:: randexp!([rng=GLOBAL_RNG], A::AbstractArray) -> A .. Docstring generated from Julia source Fill the array ``A`` with random numbers following the exponential distribution (with scale 1). -.. function:: randjump(r::MersenneTwister, jumps, [jumppoly]) -> Vector{MersenneTwister} +.. function:: randjump(r::MersenneTwister, jumps::Integer, [jumppoly::AbstractString=dSFMT.JPOLY1e21]) -> Vector{MersenneTwister} .. Docstring generated from Julia source diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 8c37e3812cb46..4cf7aa990da01 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -315,13 +315,13 @@ General Parallel Computing Support Block the current task until some event occurs, depending on the type of the argument: - * ``RemoteChannel`` : Wait for a value to become available on the specified remote channel. - * ``Future`` : Wait for a value to become available for the specified future. - * ``Channel``\ : Wait for a value to be appended to the channel. - * ``Condition``\ : Wait for ``notify`` on a condition. - * ``Process``\ : Wait for a process or process chain to exit. The ``exitcode`` field of a process can be used to determine success or failure. - * ``Task``\ : Wait for a ``Task`` to finish, returning its result value. If the task fails with an exception, the exception is propagated (re-thrown in the task that called ``wait``\ ). - * ``RawFD``\ : Wait for changes on a file descriptor (see ``poll_fd`` for keyword arguments and return code) + * :obj:`RemoteChannel` : Wait for a value to become available on the specified remote channel. + * :obj:`Future` : Wait for a value to become available for the specified future. + * :obj:`Channel`\ : Wait for a value to be appended to the channel. + * :obj:`Condition`\ : Wait for :func:`notify` on a condition. + * :obj:`Process`\ : Wait for a process or process chain to exit. The ``exitcode`` field of a process can be used to determine success or failure. + * :obj:`Task`\ : Wait for a ``Task`` to finish, returning its result value. If the task fails with an exception, the exception is propagated (re-thrown in the task that called ``wait``\ ). + * :obj:`RawFD`\ : Wait for changes on a file descriptor (see :func:`poll_fd` for keyword arguments and return code) If no argument is passed, the task blocks for an undefined period. A task can only be restarted by an explicit call to ``schedule`` or ``yieldto``\ . @@ -482,13 +482,13 @@ General Parallel Computing Support .. Docstring generated from Julia source - Creates a closure around an expression and runs it on an automatically-chosen process, returning a ``Future`` to the result. + Creates a closure around an expression and runs it on an automatically-chosen process, returning a :obj:`Future` to the result. .. function:: @spawnat .. Docstring generated from Julia source - Accepts two arguments, ``p`` and an expression. A closure is created around the expression and run asynchronously on process ``p``\ . Returns a ``Future`` to the result. + Accepts two arguments, ``p`` and an expression. A closure is created around the expression and run asynchronously on process ``p``\ . Returns a :obj:`Future` to the result. .. function:: @fetch diff --git a/doc/stdlib/sort.rst b/doc/stdlib/sort.rst index 671284119b9bc..6e20b054aea0a 100644 --- a/doc/stdlib/sort.rst +++ b/doc/stdlib/sort.rst @@ -116,25 +116,25 @@ specified via the ``lt`` keyword. Sorting Functions ----------------- -.. function:: sort!(v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) +.. function:: sort!(v; alg::Algorithm=defalg(v), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) .. Docstring generated from Julia source Sort the vector ``v`` in place. ``QuickSort`` is used by default for numeric arrays while ``MergeSort`` is used for other arrays. You can specify an algorithm to use via the ``alg`` keyword (see Sorting Algorithms for available algorithms). The ``by`` keyword lets you provide a function that will be applied to each element before comparison; the ``lt`` keyword allows providing a custom "less than" function; use ``rev=true`` to reverse the sorting order. These options are independent and can be used together in all possible combinations: if both ``by`` and ``lt`` are specified, the ``lt`` function is applied to the result of the ``by`` function; ``rev=true`` reverses whatever ordering specified via the ``by`` and ``lt`` keywords. -.. function:: sort(v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) +.. function:: sort(v; alg::Algorithm=defalg(v), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) .. Docstring generated from Julia source - Variant of ``sort!`` that returns a sorted copy of ``v`` leaving ``v`` itself unmodified. + Variant of :func:`sort!` that returns a sorted copy of ``v`` leaving ``v`` itself unmodified. -.. function:: sort(A, dim, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) +.. function:: sort(A, dim::Integer; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward, initialized::Bool=false) .. Docstring generated from Julia source - Sort a multidimensional array ``A`` along the given dimension. + Sort a multidimensional array ``A`` along the given dimension. ``lt`` defines the comparison to use. -.. function:: sortperm(v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) +.. function:: sortperm(v; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) .. Docstring generated from Julia source @@ -142,34 +142,32 @@ Sorting Functions See also :func:`sortperm!`\ . -.. function:: sortperm!(ix, v, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false,] [initialized=false]) +.. function:: sortperm!(ix, v; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward, initialized::Bool=false) .. Docstring generated from Julia source - Like ``sortperm``\ , but accepts a preallocated index vector ``ix``\ . If ``initialized`` is ``false`` (the default), ix is initialized to contain the values ``1:length(v)``\ . + Like :func:`sortperm`\ , but accepts a preallocated index vector ``ix``\ . If ``initialized`` is ``false`` (the default), ix is initialized to contain the values ``1:length(v)``\ . - See also :func:`sortperm`\ . - -.. function:: sortrows(A, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) +.. function:: sortrows(A; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) .. Docstring generated from Julia source - Sort the rows of matrix ``A`` lexicographically. + Sort the rows of matrix ``A`` lexicographically. See :func:`sort` for a description of possible keyword arguments. -.. function:: sortcols(A, [alg=<algorithm>,] [by=<transform>,] [lt=<comparison>,] [rev=false]) +.. function:: sortcols(A; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) .. Docstring generated from Julia source - Sort the columns of matrix ``A`` lexicographically. + Sort the columns of matrix ``A`` lexicographically. See :func:`sort` for a description of possible keyword arguments. Order-Related Functions ----------------------- -.. function:: issorted(v, [by=<transform>,] [lt=<comparison>,] [rev=false]) +.. function:: issorted(v, by=identity, rev:Bool=false, order::Ordering=Forward) .. Docstring generated from Julia source - Test whether a vector is in sorted order. The ``by``\ , ``lt`` and ``rev`` keywords modify what order is considered to be sorted just as they do for ``sort``\ . + Test whether a vector is in sorted order. The ``by``\ , ``lt`` and ``rev`` keywords modify what order is considered to be sorted just as they do for :func:`sort`\ . .. function:: searchsorted(a, x, [by=<transform>,] [lt=<comparison>,] [rev=false]) diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index bfb8999981544..e1b54110cfafc 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -4,7 +4,7 @@ Strings ********* -.. function:: length(s) +.. function:: length(s::AbstractString) .. Docstring generated from Julia source @@ -16,7 +16,12 @@ The number of bytes in string ``s``\ . -.. function:: *(s, t) + .. doctest:: + + julia> sizeof("❤") + 3 + +.. function:: *(s::AbstractString, t::AbstractString) .. Docstring generated from Julia source @@ -27,11 +32,11 @@ julia> "Hello " * "world" "Hello world" -.. function:: ^(s, n) +.. function:: ^(s::AbstractString, n::Integer) .. Docstring generated from Julia source - Repeat ``n`` times the string ``s``\ . The ``repeat`` function is an alias to this operator. + Repeat ``n`` times the string ``s``\ . The :func:`repeat` function is an alias to this operator. .. doctest:: @@ -42,13 +47,13 @@ .. Docstring generated from Julia source - Create a string from any values using the ``print`` function. + Create a string from any values using the :func:`print` function. .. function:: repr(x) .. Docstring generated from Julia source - Create a string from any value using the ``showall`` function. + Create a string from any value using the :func:`showall` function. .. function:: String(s::AbstractString) @@ -122,7 +127,7 @@ Create a ``Text`` object from a literal string. -.. function:: normalize_string(s, normalform::Symbol) +.. function:: normalize_string(s::AbstractString, normalform::Symbol) .. Docstring generated from Julia source @@ -143,7 +148,7 @@ For example, NFKC corresponds to the options ``compose=true, compat=true, stable=true``\ . -.. function:: graphemes(s) -> iterator over substrings of s +.. function:: graphemes(s::AbstractString) -> GraphemeIterator .. Docstring generated from Julia source @@ -161,7 +166,7 @@ Returns ``true`` if the given value is valid for that type. Types currently can be either ``Char`` or ``String``\ . Values for ``Char`` can be of type ``Char`` or ``UInt32``\ . Values for ``String`` can be of that type, or ``Vector{UInt8}``\ . -.. function:: isvalid(str, i) +.. function:: isvalid(str::AbstractString, i::Integer) .. Docstring generated from Julia source @@ -195,21 +200,31 @@ .. Docstring generated from Julia source - Return a vector of the matching substrings from eachmatch. + Return a vector of the matching substrings from :func:`eachmatch`\ . -.. function:: lpad(string, n, p) +.. function:: lpad(s, n::Integer, p::AbstractString=" ") .. Docstring generated from Julia source - Make a string at least ``n`` columns wide when printed, by padding on the left with copies of ``p``\ . + Make a string at least ``n`` columns wide when printed by padding ``s`` on the left with copies of ``p``\ . + + .. doctest:: -.. function:: rpad(string, n, p) + julia> lpad("March",10) + " March" + +.. function:: rpad(s, n::Integer, p::AbstractString=" ") .. Docstring generated from Julia source - Make a string at least ``n`` columns wide when printed, by padding on the right with copies of ``p``\ . + Make a string at least ``n`` columns wide when printed by padding ``s`` on the right with copies of ``p``\ . + + .. doctest:: + + julia> rpad("March",20) + "March " -.. function:: search(string, chars, [start]) +.. function:: search(string::AbstractString, chars::Chars, [start::Integer]) .. Docstring generated from Julia source @@ -219,151 +234,253 @@ ``search(string, 'c')`` = ``index`` such that ``string[index] == 'c'``\ , or ``0`` if unmatched. -.. function:: rsearch(string, chars, [start]) + .. doctest:: + + julia> search("Hello to the world", "z") + 0:-1 + + julia> search("JuliaLang","Julia") + 1:5 + +.. function:: rsearch(s::AbstractString, chars::Chars, [start::Integer]) .. Docstring generated from Julia source - Similar to ``search``\ , but returning the last occurrence of the given characters within the given string, searching in reverse from ``start``\ . + Similar to :func:`search`\ , but returning the last occurrence of the given characters within the given string, searching in reverse from ``start``\ . -.. function:: searchindex(string, substring, [start]) + .. doctest:: + + julia> rsearch("aaabbb","b") + 6:6 + +.. function:: searchindex(s::AbstractString, substring, [start::Integer]) .. Docstring generated from Julia source - Similar to ``search``\ , but return only the start index at which the substring is found, or ``0`` if it is not. + Similar to :func:`search`\ , but return only the start index at which the substring is found, or ``0`` if it is not. -.. function:: rsearchindex(string, substring, [start]) +.. function:: rsearchindex(s::AbstractString, substring, [start::Integer]) .. Docstring generated from Julia source - Similar to ``rsearch``\ , but return only the start index at which the substring is found, or ``0`` if it is not. + Similar to :func:`rsearch`\ , but return only the start index at which the substring is found, or ``0`` if it is not. -.. function:: contains(haystack, needle) +.. function:: contains(haystack::AbstractString, needle::AbstractString) .. Docstring generated from Julia source Determine whether the second argument is a substring of the first. + .. doctest:: + + julia> contains("JuliaLang is pretty cool!", "Julia") + true + .. function:: reverse(s::AbstractString) -> AbstractString .. Docstring generated from Julia source Reverses a string. -.. function:: replace(string, pat, r[, n]) + .. doctest:: + + julia> reverse("JuliaLang") + "gnaLailuJ" + +.. function:: replace(string::AbstractString, pat, r[, n::Integer=0]) .. Docstring generated from Julia source Search for the given pattern ``pat``\ , and replace each occurrence with ``r``\ . If ``n`` is provided, replace at most ``n`` occurrences. As with search, the second argument may be a single character, a vector or a set of characters, a string, or a regular expression. If ``r`` is a function, each occurrence is replaced with ``r(s)`` where ``s`` is the matched substring. If ``pat`` is a regular expression and ``r`` is a ``SubstitutionString``\ , then capture group references in ``r`` are replaced with the corresponding matched text. -.. function:: split(string, [chars]; limit=0, keep=true) +.. function:: split(s::AbstractString, [chars]; limit::Integer=0, keep::Bool=true) .. Docstring generated from Julia source Return an array of substrings by splitting the given string on occurrences of the given character delimiters, which may be specified in any of the formats allowed by ``search``\ 's second argument (i.e. a single character, collection of characters, string, or regular expression). If ``chars`` is omitted, it defaults to the set of all space characters, and ``keep`` is taken to be ``false``\ . The two keyword arguments are optional: they are a maximum size for the result and a flag determining whether empty fields should be kept in the result. -.. function:: rsplit(string, [chars]; limit=0, keep=true) + .. doctest:: + + julia> a = "Ma.rch" + "Ma.rch" + + julia> split(a,".") + 2-element Array{SubString{String},1}: + "Ma" + "rch" + +.. function:: rsplit(s::AbstractString, [chars]; limit::Integer=0, keep::Bool=true) .. Docstring generated from Julia source - Similar to ``split``\ , but starting from the end of the string. + Similar to :func:`split`\ , but starting from the end of the string. -.. function:: strip(string, [chars]) + .. doctest:: + + julia> a = "M.a.r.c.h" + "M.a.r.c.h" + + julia> rsplit(a,".") + 5-element Array{SubString{String},1}: + "M" + "a" + "r" + "c" + "h" + + julia> rsplit(a,".";limit=1) + 1-element Array{SubString{String},1}: + "M.a.r.c.h" + + julia> rsplit(a,".";limit=2) + 2-element Array{SubString{String},1}: + "M.a.r.c" + "h" + +.. function:: strip(s::AbstractString, [chars::Chars]) .. Docstring generated from Julia source - Return ``string`` with any leading and trailing whitespace removed. If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. + Return ``s`` with any leading and trailing whitespace removed. If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. -.. function:: lstrip(string, [chars]) +.. function:: lstrip(s::AbstractString[, chars::Chars]) .. Docstring generated from Julia source - Return ``string`` with any leading whitespace removed. If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. + Return ``s`` with any leading whitespace and delimiters removed. If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. -.. function:: rstrip(string, [chars]) +.. function:: rstrip(s::AbstractString[, chars::Chars]) .. Docstring generated from Julia source - Return ``string`` with any trailing whitespace removed. If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. + Return ``s`` with any trailing whitespace and delimiters removed. If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. + + .. doctest:: -.. function:: startswith(string, prefix) + julia> a = rpad("March",20) + "March " + + julia> rstrip(a) + "March" + +.. function:: startswith(s::AbstractString, prefix::AbstractString) .. Docstring generated from Julia source - Returns ``true`` if ``string`` starts with ``prefix``\ . If ``prefix`` is a vector or set of characters, tests whether the first character of ``string`` belongs to that set. + Returns ``true`` if ``s`` starts with ``prefix``\ . If ``prefix`` is a vector or set of characters, tests whether the first character of ``s`` belongs to that set. + + .. doctest:: + + julia> startswith("JuliaLang", "Julia") + true -.. function:: endswith(string, suffix) +.. function:: endswith(s::AbstractString, suffix::AbstractString) .. Docstring generated from Julia source - Returns ``true`` if ``string`` ends with ``suffix``\ . If ``suffix`` is a vector or set of characters, tests whether the last character of ``string`` belongs to that set. + Returns ``true`` if ``s`` ends with ``suffix``\ . If ``suffix`` is a vector or set of characters, tests whether the last character of ``s`` belongs to that set. -.. function:: uppercase(string) + .. doctest:: + + julia> endswith("Sunday", "day") + true + +.. function:: uppercase(s::AbstractString) .. Docstring generated from Julia source - Returns ``string`` with all characters converted to uppercase. + Returns ``s`` with all characters converted to uppercase. -.. function:: lowercase(string) + .. doctest:: + + julia> uppercase("Julia") + "JULIA" + +.. function:: lowercase(s::AbstractString) .. Docstring generated from Julia source - Returns ``string`` with all characters converted to lowercase. + Returns ``s`` with all characters converted to lowercase. + + .. doctest:: + + julia> lowercase("STRINGS AND THINGS") + "strings and things" -.. function:: ucfirst(string) +.. function:: ucfirst(s::AbstractString) .. Docstring generated from Julia source Returns ``string`` with the first character converted to uppercase. -.. function:: lcfirst(string) + .. doctest:: + + julia> ucfirst("python") + "Python" + +.. function:: lcfirst(s::AbstractString) .. Docstring generated from Julia source Returns ``string`` with the first character converted to lowercase. -.. function:: join(strings, delim, [last]) + .. doctest:: + + julia> lcfirst("Julia") + "julia" + +.. function:: join(io::IO, strings, delim, [last]) .. Docstring generated from Julia source - Join an array of ``strings`` into a single string, inserting the given delimiter between adjacent strings. If ``last`` is given, it will be used instead of ``delim`` between the last two strings. For example + Join an array of ``strings`` into a single string, inserting the given delimiter between adjacent strings. If ``last`` is given, it will be used instead of ``delim`` between the last two strings. For example, - .. code-block:: julia + .. doctest:: - join(["apples", "bananas", "pineapples"], ", ", " and ") == "apples, bananas and pineapples" + julia> join(["apples", "bananas", "pineapples"], ", ", " and ") + "apples, bananas and pineapples" ``strings`` can be any iterable over elements ``x`` which are convertible to strings via ``print(io::IOBuffer, x)``\ . -.. function:: chop(string) +.. function:: chop(s::AbstractString) .. Docstring generated from Julia source Remove the last character from a string. -.. function:: chomp(string) + .. doctest:: + + julia> a = string("March") + "March" + + julia> chop(a) + "Marc" + +.. function:: chomp(s::AbstractString) .. Docstring generated from Julia source Remove a single trailing newline from a string. -.. function:: ind2chr(string, i) +.. function:: ind2chr(s::AbstractString, i::Integer) .. Docstring generated from Julia source - Convert a byte index to a character index. + Convert a byte index ``i`` to a character index. -.. function:: chr2ind(string, i) +.. function:: chr2ind(s::AbstractString, i::Integer) .. Docstring generated from Julia source - Convert a character index to a byte index. + Convert a character index ``i`` to a byte index. -.. function:: nextind(str, i) +.. function:: nextind(str::AbstractString, i::Integer) .. Docstring generated from Julia source Get the next valid string index after ``i``\ . Returns a value greater than ``endof(str)`` at or after the end of the string. -.. function:: prevind(str, i) +.. function:: prevind(str::AbstractString, i::Integer) .. Docstring generated from Julia source @@ -381,12 +498,17 @@ Gives the number of columns needed to print a character. -.. function:: strwidth(s) +.. function:: strwidth(s::AbstractString) .. Docstring generated from Julia source Gives the number of columns needed to print a string. + .. doctest:: + + julia> strwidth("March") + 5 + .. function:: isalnum(c::Union{Char,AbstractString}) -> Bool .. Docstring generated from Julia source @@ -465,21 +587,29 @@ Tests whether a character is a valid hexadecimal digit, or whether this is true for all elements of a string. + .. doctest:: + + julia> isxdigit("abc") + true + + julia> isxdigit("0x9") + false + .. function:: Symbol(x...) -> Symbol .. Docstring generated from Julia source Create a ``Symbol`` by concatenating the string representations of the arguments together. -.. function:: escape_string(str::AbstractString) -> AbstractString +.. function:: escape_string([io,] str::AbstractString[, esc::AbstractString]) -> AbstractString .. Docstring generated from Julia source - General escaping of traditional C and Unicode escape sequences. + General escaping of traditional C and Unicode escape sequences. Any characters in ``esc`` are also escaped (with a backslash). See also :func:`unescape_string`\ . -.. function:: unescape_string(s::AbstractString) -> AbstractString +.. function:: unescape_string([io,] s::AbstractString) -> AbstractString .. Docstring generated from Julia source - General unescaping of traditional C and Unicode escape sequences. Reverse of :func:`escape_string`\ . See also :func:`unescape_string`\ . + General unescaping of traditional C and Unicode escape sequences. Reverse of :func:`escape_string`\ . diff --git a/doc/stdlib/test.rst b/doc/stdlib/test.rst index 4cbc2a0f9057c..dcc5c355c605e 100644 --- a/doc/stdlib/test.rst +++ b/doc/stdlib/test.rst @@ -12,7 +12,7 @@ binary install, you can run the test suite using ``Base.runtests()``. .. currentmodule:: Base -.. function:: runtests([tests=["all"] [, numcores=ceil(Integer, Sys.CPU_CORES / 2) ]]) +.. function:: runtests([tests=["all"] [, numcores=ceil(Int, Sys.CPU_CORES / 2) ]]) .. Docstring generated from Julia source From 48f7724c69f4809aa3b9896c57c717fd7337e533 Mon Sep 17 00:00:00 2001 From: Katie Hyatt <kshyatt@physics.ucsb.edu> Date: Mon, 22 Aug 2016 17:32:55 -0700 Subject: [PATCH 1095/1117] Moved docstrings to deal with aliasing --- base/abstractarray.jl | 10 ++++---- base/array.jl | 2 +- base/file.jl | 2 +- base/linalg/dense.jl | 8 +++++- base/linalg/generic.jl | 6 ++++- base/operators.jl | 20 +++++++++------ base/reduce.jl | 4 +-- base/socket.jl | 8 ++++-- base/sort.jl | 8 +++--- base/statistics.jl | 2 +- base/stream.jl | 27 +++++++++++++++++--- base/strings/io.jl | 7 ++--- base/util.jl | 13 +++++++++- doc/manual/arrays.rst | 2 +- doc/manual/stacktraces.rst | 15 +++++------ doc/stdlib/arrays.rst | 4 +-- doc/stdlib/file.rst | 2 +- doc/stdlib/io-network.rst | 52 +++++++++++++++++++++++++++++++++----- doc/stdlib/linalg.rst | 10 ++++++-- doc/stdlib/math.rst | 2 +- doc/stdlib/sort.rst | 6 ++--- doc/stdlib/strings.rst | 2 +- 22 files changed, 154 insertions(+), 58 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index cfce65719ae94..174d60905923c 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -746,11 +746,11 @@ A[iter] = 0 ``` If you supply more than one `AbstractArray` argument, `eachindex` will create an -iterable object that is fast for all arguments (a `UnitRange` if all inputs have fast -linear indexing, a [`CartesianRange`](:obj`CartesianRange`) otherwise). -If the arrays have different sizes and/or -dimensionalities, `eachindex` returns an iterable that spans the largest range along each -dimension. +iterable object that is fast for all arguments (a [`UnitRange`](:obj:`UnitRange`) +if all inputs have fast linear indexing, a [`CartesianRange`](:obj:`CartesianRange`) +otherwise). +If the arrays have different sizes and/or dimensionalities, `eachindex` returns an +iterable that spans the largest range along each dimension. """ eachindex(A::AbstractArray) = (@_inline_meta(); eachindex(linearindexing(A), A)) diff --git a/base/array.jl b/base/array.jl index 8dcd68294b1da..1bf8449e32fe1 100644 --- a/base/array.jl +++ b/base/array.jl @@ -165,7 +165,7 @@ end """ fill(x, dims) -Create an array filled with the value `x`. For example, `fill(1.0, (5,5))` returns a 10×10 +Create an array filled with the value `x`. For example, `fill(1.0, (5,5))` returns a 5×5 array of floats, with each element initialized to `1.0`. ```jldoctest diff --git a/base/file.jl b/base/file.jl index e3c69f2ff0aca..deb13c72db62b 100644 --- a/base/file.jl +++ b/base/file.jl @@ -431,7 +431,7 @@ The `walkdir` method returns an iterator that walks the directory tree of a dire The iterator returns a tuple containing `(rootpath, dirs, files)`. The directory tree can be traversed top-down or bottom-up. If `walkdir` encounters a [`SystemError`](:obj:`SystemError`) -it will raise the error by default. +it will rethrow the error by default. A custom error handling function can be provided through `onerror` keyword argument. `onerror` is called with a `SystemError` as argument. diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index 6957a1e579aa3..659a079bb6a61 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -161,6 +161,12 @@ The `k`th diagonal of a matrix, as a vector. Use [`diagm`](:func:`diagm`) to construct a diagonal matrix. ```jldoctest +julia> A = [1 2 3; 4 5 6; 7 8 9] +3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + julia> diag(A,1) 2-element Array{Int64,1}: 2 @@ -172,7 +178,7 @@ diag(A::AbstractMatrix, k::Integer=0) = A[diagind(A,k)] """ diagm(v, k::Integer=0) -Construct a diagonal matrix and place `v` on the `k`th diagonal. +Construct a matrix by placing `v` on the `k`th diagonal. ```jldoctest julia> diagm([1,2,3],1) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index ff77e554fbe92..9527d1036ae81 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -558,7 +558,11 @@ dot(x::AbstractVector, y::AbstractVector) = vecdot(x, y) """ rank(M[, tol::Real]) -Compute the rank of a matrix. +Compute the rank of a matrix by summing the singular +values of `M` with magnitude greater than `tol`. +By default, the value of `tol` is the largest +dimension of `M` multiplied by the [`eps`](:func:`eps`) +of the [`eltype`](:func:`eltype`) of `M`. """ rank(A::AbstractMatrix, tol::Real) = sum(svdvals(A) .> tol) function rank(A::AbstractMatrix) diff --git a/base/operators.jl b/base/operators.jl index 5f37a0fb1e51a..4392e9efa9d82 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -108,6 +108,7 @@ Determine whether `x` and `y` are identical, in the sense that no program could them. Compares mutable objects by address in memory, and compares immutable objects (such as numbers) by contents at the bit level. This function is sometimes called `egal`. """ +is const ≡ = is """ @@ -308,9 +309,15 @@ floating-point results for integer arguments. .>>(x::Integer,y::Integer) = x>>y .==(x::Number,y::Number) = x == y + +""" + .!=(x, y) + .≠(x,y) + +Element-wise not-equals comparison operator. +""" .!=(x::Number,y::Number) = x != y .<( x::Real,y::Real) = x < y -.<=(x::Real,y::Real) = x <= y """ .<=(x, y) @@ -318,14 +325,9 @@ floating-point results for integer arguments. Element-wise less-than-or-equals comparison operator. """ -const .≤ = .<= - -""" - .!=(x, y) - .≠(x,y) +.<=(x::Real,y::Real) = x <= y -Element-wise not-equals comparison operator. -""" +const .≤ = .<= const .≠ = .!= # Core <<, >>, and >>> take either Int or UInt as second arg. Signed shift @@ -471,6 +473,7 @@ magnitude than `y`. This value is always exact. x == div(x,y)*y + rem(x,y) ``` """ +rem const % = rem .%(x::Real, y::Real) = x%y @@ -480,6 +483,7 @@ const % = rem The quotient from Euclidean division. Computes `x/y`, truncated to an integer. """ +div const ÷ = div .÷(x::Real, y::Real) = x÷y diff --git a/base/reduce.jl b/base/reduce.jl index 2bffbe215c753..f437f38fe80c7 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -646,8 +646,6 @@ all(f::typeof(identity), itr) = ## in & contains -in(x, itr) = any(Predicate(y -> y == x), itr) - """ in(item, collection) -> Bool ∈(item,collection) -> Bool @@ -673,6 +671,8 @@ julia> 5 in a false ``` """ +in(x, itr) = any(Predicate(y -> y == x), itr) + const ∈ = in ∉(x, itr)=!∈(x, itr) ∋(itr, x)= ∈(x, itr) diff --git a/base/socket.jl b/base/socket.jl index 8e6961b6f97fb..812c33ca3c348 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -752,8 +752,12 @@ end """ listen([addr, ]port::Integer; backlog::Integer=BACKLOG_DEFAULT) -> TCPServer -Listen on port on the address specified by `addr`. By default this listens on localhost -only. To listen on all interfaces pass `IPv4(0)` or `IPv6(0)` as appropriate. +Listen on port on the address specified by `addr`. +By default this listens on `localhost` only. +To listen on all interfaces pass `IPv4(0)` or `IPv6(0)` as appropriate. +`backlog` determines how many connections can be pending (not having +called [`accept`](:func:`accept`)) before the server will begin to +reject them. The default value of `backlog` is 511. """ function listen(addr; backlog::Integer=BACKLOG_DEFAULT) sock = TCPServer() diff --git a/base/sort.jl b/base/sort.jl index 9c4d6e34a3744..95f3d43de07de 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -522,7 +522,9 @@ end """ sort(A, dim::Integer; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward, initialized::Bool=false) -Sort a multidimensional array `A` along the given dimension. `lt` defines the comparison to use. +Sort a multidimensional array `A` along the given dimension. +See [`sort!`](:func:`sort!`) for a description of possible +keyword arguments. """ function sort(A::AbstractArray, dim::Integer; alg::Algorithm=DEFAULT_UNSTABLE, @@ -559,7 +561,7 @@ end sortrows(A; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) Sort the rows of matrix `A` lexicographically. -See [`sort`](:func:`sort`) for a description of possible +See [`sort!`](:func:`sort!`) for a description of possible keyword arguments. """ function sortrows(A::AbstractMatrix; kws...) @@ -577,7 +579,7 @@ end sortcols(A; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) Sort the columns of matrix `A` lexicographically. -See [`sort`](:func:`sort`) for a description of possible +See [`sort!`](:func:`sort!`) for a description of possible keyword arguments. """ function sortcols(A::AbstractMatrix; kws...) diff --git a/base/statistics.jl b/base/statistics.jl index 864d23f63001f..e3186fc83485e 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -186,7 +186,7 @@ under the assumption that each entry of `v` is an IID drawn from that generative distribution. This computation is equivalent to calculating `sumabs2(v - mean(v)) / (length(v) - 1)`. If `corrected` is `true`, then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. -The mean `m` over the region may be provided. +The mean `mean` over the region may be provided. !!! note Julia does not ignore `NaN` values in the computation. For diff --git a/base/stream.jl b/base/stream.jl index 081d5e5c1297e..feae461f4da8a 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -975,6 +975,9 @@ Data written to [`STDOUT`](:obj:`STDOUT`) may now be read from the `rd` end of the pipe. The `wr` end is given for convenience in case the old [`STDOUT`](:obj:`STDOUT`) object was cached by the user and needs to be replaced elsewhere. + +!!! note + `stream` must be a `TTY`, a [`Pipe`](:obj:`Pipe`), or a [`TCPSocket`](:obj:`TCPSocket`). """ redirect_stdout @@ -982,6 +985,9 @@ redirect_stdout redirect_stderr([stream]) -> (rd, wr) Like [`redirect_stdout`](:func:`redirect_stdout`), but for [`STDERR`](:obj:`STDERR`). + +!!! note + `stream` must be a `TTY`, a [`Pipe`](:obj:`Pipe`), or a [`TCPSocket`](:obj:`TCPSocket`). """ redirect_stderr @@ -991,6 +997,9 @@ redirect_stderr Like [`redirect_stdout`](:func:`redirect_stdout`), but for [`STDIN`](:obj:`STDIN`). Note that the order of the return tuple is still `(rd, wr)`, i.e. data to be read from [`STDIN`](:obj:`STDIN`) may be written to `wr`. + +!!! note + `stream` must be a `TTY`, a [`Pipe`](:obj:`Pipe`), or a [`TCPSocket`](:obj:`TCPSocket`). """ redirect_stdin @@ -1011,21 +1020,33 @@ end """ redirect_stdout(f::Function, stream) -Run the function `f` while redirecting `STDOUT` to `stream`. Upon completion, `STDOUT` is restored to its prior setting. +Run the function `f` while redirecting [`STDOUT`](:obj:`STDOUT`) to `stream`. +Upon completion, [`STDOUT`](:obj:`STDOUT`) is restored to its prior setting. + +!!! note + `stream` must be a `TTY`, a [`Pipe`](:obj:`Pipe`), or a [`TCPSocket`](:obj:`TCPSocket`). """ redirect_stdout(f::Function, stream) """ redirect_stderr(f::Function, stream) -Run the function `f` while redirecting `STDERR` to `stream`. Upon completion, `STDERR` is restored to its prior setting. +Run the function `f` while redirecting [`STDERR`](:obj:`STDERR`) to `stream`. +Upon completion, [`STDERR`](:obj:`STDERR`) is restored to its prior setting. + +!!! note + `stream` must be a `TTY`, a [`Pipe`](:obj:`Pipe`), or a [`TCPSocket`](:obj:`TCPSocket`). """ redirect_stderr(f::Function, stream) """ redirect_stdin(f::Function, stream) -Run the function `f` while redirecting `STDIN` to `stream`. Upon completion, `STDIN` is restored to its prior setting. +Run the function `f` while redirecting [`STDIN`](:obj:`STDIN`) to `stream`. +Upon completion, [`STDIN`](:obj:`STDIN`) is restored to its prior setting. + +!!! note + `stream` must be a `TTY`, a [`Pipe`](:obj:`Pipe`), or a [`TCPSocket`](:obj:`TCPSocket`). """ redirect_stdin(f::Function, stream) diff --git a/base/strings/io.jl b/base/strings/io.jl index 35b27426bdffd..a99cf403309f7 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -6,8 +6,9 @@ """ print(io::IO, x) -Write (to the default output stream) a canonical (un-decorated) text -representation of a value if there is one, otherwise call [`show`](:func:`show`). +Write to `io` (or to the default output stream [`STDOUT`](:obj:`STDOUT`) +if `io` is not given) a canonical (un-decorated) text representation +of a value if there is one, otherwise call [`show`](:func:`show`). The representation used by `print` includes minimal formatting and tries to avoid Julia-specific details. """ @@ -153,7 +154,7 @@ julia> join(["apples", "bananas", "pineapples"], ", ", " and ") ``` `strings` can be any iterable over elements `x` which are convertible to strings -via `print(io::IOBuffer, x)`. +via `print(io::IOBuffer, x)`. `strings` will be printed to `io`. """ function join(io::IO, strings, delim, last) i = start(strings) diff --git a/base/util.jl b/base/util.jl index 7101958d5d24e..097140cd6998f 100644 --- a/base/util.jl +++ b/base/util.jl @@ -323,7 +323,18 @@ println_with_color(color::Symbol, msg::AbstractString...) = """ info(msg...; prefix="INFO: ") -Display an informational message. Argument `msg` is a string describing the information to be displayed. +Display an informational message. +Argument `msg` is a string describing the information to be displayed. +The `prefix` kwarg can be used to override the default prepending of +`msg`. + +```jldoctest +julia> info("hello world") +INFO: hello world + +julia> info("hello world"; prefix="MY INFO: ") +MY INFO: hello world +``` """ function info(io::IO, msg...; prefix="INFO: ") println_with_color(info_color(), io, prefix, chomp(string(msg...))) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 4c1fd9013514a..22493d7a6c506 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -517,7 +517,7 @@ the name of the function to vectorize. Here is a simple example: julia> methods(square) # 2 methods for generic function "square": - square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:914 + square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:918 square(x) at none:1 julia> square([1 2 4; 5 6 7]) diff --git a/doc/manual/stacktraces.rst b/doc/manual/stacktraces.rst index b31d1bed2ede5..32b14abef18a1 100644 --- a/doc/manual/stacktraces.rst +++ b/doc/manual/stacktraces.rst @@ -30,13 +30,10 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example example (generic function with 1 method) julia> example() - 6-element Array{StackFrame,1}: + 11-element Array{StackFrame,1}: in example() at none:1 in eval(::Module, ::Any) at boot.jl:234 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in eval(::Module, ::Any) at boot.jl:234 - in eval_user_input(::Any, ::Bool) at client.jl:117 - in _start() at client.jl:363 + ... julia> @noinline child() = stacktrace() child (generic function with 1 method) @@ -48,7 +45,7 @@ alias :obj:`StackTrace` can be used in place of ``Vector{StackFrame}``. (Example grandparent (generic function with 1 method) julia> grandparent() - 8-element Array{StackFrame,1}: + 13-element Array{StackFrame,1}: in child() at none:1 in parent() at none:1 in grandparent() at none:1 @@ -124,7 +121,7 @@ helpful in many places, the most obvious application is in error handling and de example (generic function with 1 method) julia> example() - 6-element Array{StackFrame,1}: + 11-element Array{StackFrame,1}: in example() at none:4 in eval(::Module, ::Any) at boot.jl:234 ... @@ -153,7 +150,7 @@ returns stack information for the context of the most recent exception: example (generic function with 1 method) julia> example() - 7-element Array{StackFrame,1}: + 12-element Array{StackFrame,1}: in bad_function() at none:1 in example() at none:2 ... @@ -180,7 +177,7 @@ Notice that the stack trace now indicates the appropriate line number and the mi julia> grandparent() ERROR: Whoops! - 8-element Array{StackFrame,1}: + 13-element Array{StackFrame,1}: in child() at none:1 in parent() at none:1 in grandparent() at none:3 diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 18e01cfd1d72b..3aa52ce16d202 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -110,7 +110,7 @@ Basic functions (iter.I[1],iter.I[2]) = (2,3) A[iter] = 0 - If you supply more than one ``AbstractArray`` argument, ``eachindex`` will create an iterable object that is fast for all arguments (a ``UnitRange`` if all inputs have fast linear indexing, a ```CartesianRange`` <:obj`CartesianRange`>`_ otherwise). If the arrays have different sizes and/or dimensionalities, ``eachindex`` returns an iterable that spans the largest range along each dimension. + If you supply more than one ``AbstractArray`` argument, ``eachindex`` will create an iterable object that is fast for all arguments (a :obj:`UnitRange` if all inputs have fast linear indexing, a :obj:`CartesianRange` otherwise). If the arrays have different sizes and/or dimensionalities, ``eachindex`` returns an iterable that spans the largest range along each dimension. .. function:: linearindices(A) @@ -312,7 +312,7 @@ Constructors .. Docstring generated from Julia source - Create an array filled with the value ``x``\ . For example, ``fill(1.0, (5,5))`` returns a 10×10 array of floats, with each element initialized to ``1.0``\ . + Create an array filled with the value ``x``\ . For example, ``fill(1.0, (5,5))`` returns a 5×5 array of floats, with each element initialized to ``1.0``\ . .. doctest:: diff --git a/doc/stdlib/file.rst b/doc/stdlib/file.rst index d26758e48ac99..5ea94973b686a 100644 --- a/doc/stdlib/file.rst +++ b/doc/stdlib/file.rst @@ -32,7 +32,7 @@ .. Docstring generated from Julia source - The ``walkdir`` method returns an iterator that walks the directory tree of a directory. The iterator returns a tuple containing ``(rootpath, dirs, files)``\ . The directory tree can be traversed top-down or bottom-up. If ``walkdir`` encounters a :obj:`SystemError` it will raise the error by default. A custom error handling function can be provided through ``onerror`` keyword argument. ``onerror`` is called with a ``SystemError`` as argument. + The ``walkdir`` method returns an iterator that walks the directory tree of a directory. The iterator returns a tuple containing ``(rootpath, dirs, files)``\ . The directory tree can be traversed top-down or bottom-up. If ``walkdir`` encounters a :obj:`SystemError` it will rethrow the error by default. A custom error handling function can be provided through ``onerror`` keyword argument. ``onerror`` is called with a ``SystemError`` as argument. .. code-block:: julia diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index cb648d2e7e597..20e69009a85d9 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -160,6 +160,14 @@ General I/O Read binary data from an I/O stream or file, filling in ``array``\ . +.. function:: readbytes!(stream::IOStream, b::AbstractVector{UInt8}, nb=length(b); all::Bool=true) + + .. Docstring generated from Julia source + + Read at most ``nb`` bytes from ``stream`` into ``b``\ , returning the number of bytes read. The size of ``b`` will be increased if needed (i.e. if ``nb`` is greater than ``length(b)`` and enough bytes could be read), but it will never be decreased. + + See :func:`read` for a description of the ``all`` option. + .. function:: readbytes!(stream::IO, b::AbstractVector{UInt8}, nb=length(b)) .. Docstring generated from Julia source @@ -324,11 +332,19 @@ General I/O Create a pipe to which all C and Julia level :obj:`STDOUT` output will be redirected. Returns a tuple ``(rd, wr)`` representing the pipe ends. Data written to :obj:`STDOUT` may now be read from the ``rd`` end of the pipe. The ``wr`` end is given for convenience in case the old :obj:`STDOUT` object was cached by the user and needs to be replaced elsewhere. + .. note:: + ``stream`` must be a ``TTY``\ , a :obj:`Pipe`\ , or a :obj:`TCPSocket`\ . + + .. function:: redirect_stdout(f::Function, stream) .. Docstring generated from Julia source - Run the function ``f`` while redirecting ``STDOUT`` to ``stream``\ . Upon completion, ``STDOUT`` is restored to its prior setting. + Run the function ``f`` while redirecting :obj:`STDOUT` to ``stream``\ . Upon completion, :obj:`STDOUT` is restored to its prior setting. + + .. note:: + ``stream`` must be a ``TTY``\ , a :obj:`Pipe`\ , or a :obj:`TCPSocket`\ . + .. function:: redirect_stderr([stream]) -> (rd, wr) @@ -336,11 +352,19 @@ General I/O Like :func:`redirect_stdout`\ , but for :obj:`STDERR`\ . + .. note:: + ``stream`` must be a ``TTY``\ , a :obj:`Pipe`\ , or a :obj:`TCPSocket`\ . + + .. function:: redirect_stderr(f::Function, stream) .. Docstring generated from Julia source - Run the function ``f`` while redirecting ``STDERR`` to ``stream``\ . Upon completion, ``STDERR`` is restored to its prior setting. + Run the function ``f`` while redirecting :obj:`STDERR` to ``stream``\ . Upon completion, :obj:`STDERR` is restored to its prior setting. + + .. note:: + ``stream`` must be a ``TTY``\ , a :obj:`Pipe`\ , or a :obj:`TCPSocket`\ . + .. function:: redirect_stdin([stream]) -> (rd, wr) @@ -348,11 +372,19 @@ General I/O Like :func:`redirect_stdout`\ , but for :obj:`STDIN`\ . Note that the order of the return tuple is still ``(rd, wr)``\ , i.e. data to be read from :obj:`STDIN` may be written to ``wr``\ . + .. note:: + ``stream`` must be a ``TTY``\ , a :obj:`Pipe`\ , or a :obj:`TCPSocket`\ . + + .. function:: redirect_stdin(f::Function, stream) .. Docstring generated from Julia source - Run the function ``f`` while redirecting ``STDIN`` to ``stream``\ . Upon completion, ``STDIN`` is restored to its prior setting. + Run the function ``f`` while redirecting :obj:`STDIN` to ``stream``\ . Upon completion, :obj:`STDIN` is restored to its prior setting. + + .. note:: + ``stream`` must be a ``TTY``\ , a :obj:`Pipe`\ , or a :obj:`TCPSocket`\ . + .. function:: readchomp(x) @@ -462,7 +494,7 @@ Text I/O .. Docstring generated from Julia source - Write (to the default output stream) a canonical (un-decorated) text representation of a value if there is one, otherwise call :func:`show`\ . The representation used by ``print`` includes minimal formatting and tries to avoid Julia-specific details. + Write to ``io`` (or to the default output stream :obj:`STDOUT` if ``io`` is not given) a canonical (un-decorated) text representation of a value if there is one, otherwise call :func:`show`\ . The representation used by ``print`` includes minimal formatting and tries to avoid Julia-specific details. .. function:: println(io::IO, xs...) @@ -482,7 +514,15 @@ Text I/O .. Docstring generated from Julia source - Display an informational message. Argument ``msg`` is a string describing the information to be displayed. + Display an informational message. Argument ``msg`` is a string describing the information to be displayed. The ``prefix`` kwarg can be used to override the default prepending of ``msg``\ . + + .. doctest:: + + julia> info("hello world") + INFO: hello world + + julia> info("hello world"; prefix="MY INFO: ") + MY INFO: hello world .. function:: warn(msg) @@ -889,7 +929,7 @@ Network I/O .. Docstring generated from Julia source - Listen on port on the address specified by ``addr``\ . By default this listens on localhost only. To listen on all interfaces pass ``IPv4(0)`` or ``IPv6(0)`` as appropriate. + Listen on port on the address specified by ``addr``\ . By default this listens on ``localhost`` only. To listen on all interfaces pass ``IPv4(0)`` or ``IPv6(0)`` as appropriate. ``backlog`` determines how many connections can be pending (not having called :func:`accept`\ ) before the server will begin to reject them. The default value of ``backlog`` is 511. .. function:: listen(path::AbstractString) -> PipeServer diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 1b324599d2152..f9f5116525101 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1126,6 +1126,12 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. doctest:: + julia> A = [1 2 3; 4 5 6; 7 8 9] + 3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + julia> diag(A,1) 2-element Array{Int64,1}: 2 @@ -1135,7 +1141,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Construct a diagonal matrix and place ``v`` on the ``k``\ th diagonal. + Construct a matrix by placing ``v`` on the ``k``\ th diagonal. .. doctest:: @@ -1191,7 +1197,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the rank of a matrix. + Compute the rank of a matrix by summing the singular values of ``M`` with magnitude greater than ``tol``\ . By default, the value of ``tol`` is the largest dimension of ``M`` multiplied by the :func:`eps` of the :func:`eltype` of ``M``\ . .. function:: norm(A, [p::Real=2]) diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 97be585a2aca4..2ab68c4f046a8 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1759,7 +1759,7 @@ Statistics .. Docstring generated from Julia source - Compute the sample variance of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sumabs2(v - mean(v)) / (length(v) - 1)``\ . If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . The mean ``m`` over the region may be provided. + Compute the sample variance of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sumabs2(v - mean(v)) / (length(v) - 1)``\ . If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . The mean ``mean`` over the region may be provided. .. note:: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. diff --git a/doc/stdlib/sort.rst b/doc/stdlib/sort.rst index 6e20b054aea0a..3379f52494504 100644 --- a/doc/stdlib/sort.rst +++ b/doc/stdlib/sort.rst @@ -132,7 +132,7 @@ Sorting Functions .. Docstring generated from Julia source - Sort a multidimensional array ``A`` along the given dimension. ``lt`` defines the comparison to use. + Sort a multidimensional array ``A`` along the given dimension. See :func:`sort!` for a description of possible keyword arguments. .. function:: sortperm(v; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) @@ -152,13 +152,13 @@ Sorting Functions .. Docstring generated from Julia source - Sort the rows of matrix ``A`` lexicographically. See :func:`sort` for a description of possible keyword arguments. + Sort the rows of matrix ``A`` lexicographically. See :func:`sort!` for a description of possible keyword arguments. .. function:: sortcols(A; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) .. Docstring generated from Julia source - Sort the columns of matrix ``A`` lexicographically. See :func:`sort` for a description of possible keyword arguments. + Sort the columns of matrix ``A`` lexicographically. See :func:`sort!` for a description of possible keyword arguments. Order-Related Functions ----------------------- diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index e1b54110cfafc..609c48031ea1b 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -440,7 +440,7 @@ julia> join(["apples", "bananas", "pineapples"], ", ", " and ") "apples, bananas and pineapples" - ``strings`` can be any iterable over elements ``x`` which are convertible to strings via ``print(io::IOBuffer, x)``\ . + ``strings`` can be any iterable over elements ``x`` which are convertible to strings via ``print(io::IOBuffer, x)``\ . ``strings`` will be printed to ``io``\ . .. function:: chop(s::AbstractString) From 3df0c2a26571f856c75a9aaccc697051fc67233e Mon Sep 17 00:00:00 2001 From: Katie Hyatt <kshyatt@physics.ucsb.edu> Date: Wed, 31 Aug 2016 13:35:22 -0700 Subject: [PATCH 1096/1117] More wording improvements, few more doctests --- base/expr.jl | 1 + base/file.jl | 6 +++--- base/interactiveutil.jl | 2 +- base/intfuncs.jl | 11 ++++++++--- base/io.jl | 4 ++-- base/linalg/dense.jl | 2 +- base/linalg/generic.jl | 21 +++++++++++++++++---- base/loading.jl | 7 ++++--- base/mpfr.jl | 8 ++++---- base/random.jl | 6 +++--- base/strings/basic.jl | 3 ++- base/strings/util.jl | 6 +++++- base/util.jl | 4 ++-- doc/stdlib/base.rst | 8 ++++---- doc/stdlib/file.rst | 4 ++-- doc/stdlib/io-network.rst | 6 +++--- doc/stdlib/linalg.rst | 18 ++++++++++++++---- doc/stdlib/numbers.rst | 4 ++-- doc/stdlib/strings.rst | 8 ++++---- 19 files changed, 82 insertions(+), 47 deletions(-) diff --git a/base/expr.jl b/base/expr.jl index 0b7a6cd7f4459..ce32aa0acea8a 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -50,6 +50,7 @@ copy_exprargs(x::Array{Any,1}) = Any[copy_exprs(a) for a in x] expand(x) Takes the expression `x` and returns an equivalent expression in lowered form. +See also [`code_lowered`](:func:`code_lowered`). """ expand(x::ANY) = ccall(:jl_expand, Any, (Any,), x) diff --git a/base/file.jl b/base/file.jl index deb13c72db62b..bde6502dec787 100644 --- a/base/file.jl +++ b/base/file.jl @@ -205,8 +205,8 @@ end """ cp(src::AbstractString, dst::AbstractString; remove_destination::Bool=false, follow_symlinks::Bool=false) -Copy the file, link, or directory from *src* to *dest*. `remove_destination=true` will first -remove an existing `dst`. +Copy the file, link, or directory from `src` to `dest`. +`remove_destination=true` will first remove an existing `dst`. If `follow_symlinks=false`, and `src` is a symbolic link, `dst` will be created as a symbolic link. If `follow_symlinks=true` and `src` is a symbolic link, `dst` will be a copy @@ -566,7 +566,7 @@ end """ readlink(path::AbstractString) -> AbstractString -Returns the value of a symbolic link `path`. +Returns the target location a symbolic link `path` points to. """ function readlink(path::AbstractString) req = Libc.malloc(_sizeof_uv_fs) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 91ad7d981d36f..97a1b67ea5f54 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -498,7 +498,7 @@ end # `methodswith` -- shows a list of methods using the type given """ - methodswith(typ[, module or function][, showparents::Bool=false, meths=Method[]]) + methodswith(typ[, module or function][, showparents::Bool=false]) Return an array of methods with an argument of type `typ`. diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 8d508b2793bf3..e99b50a8ea689 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -47,8 +47,6 @@ function gcd{T<:Union{Int64,UInt64,Int128,UInt128}}(a::T, b::T) r % T end -# explicit a==0 test is to handle case of lcm(0,0) correctly - """ lcm(x,y) @@ -61,7 +59,14 @@ julia> lcm(-2,3) 6 ``` """ -lcm{T<:Integer}(a::T, b::T) = a == 0 ? a : checked_abs(a * div(b, gcd(b,a))) +function lcm{T<:Integer}(a::T, b::T) + # explicit a==0 test is to handle case of lcm(0,0) correctly + if a == 0 + return a + else + return checked_abs(a * div(b, gcd(b,a))) + end +end gcd(a::Integer) = a lcm(a::Integer) = a diff --git a/base/io.jl b/base/io.jl index 6622ea84ce337..4eef9cfedd980 100644 --- a/base/io.jl +++ b/base/io.jl @@ -69,7 +69,7 @@ write(s::IO, x::UInt8) = error(typeof(s)," does not support byte I/O") """ unsafe_write(io::IO, ref, nbytes::UInt) -Copy nbytes from ref (converted to a pointer) into the IO stream object. +Copy `nbytes` from `ref` (converted to a pointer) into the `IO` object. It is recommended that subtypes `T<:IO` override the following method signature to provide more efficient implementations: @@ -86,7 +86,7 @@ end """ unsafe_read(io::IO, ref, nbytes::UInt) -Copy nbytes from the `IO` stream object into `ref` (converted to a pointer). +Copy `nbytes` from the `IO` stream object into `ref` (converted to a pointer). It is recommended that subtypes `T<:IO` override the following method signature to provide more efficient implementations: diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index 659a079bb6a61..1c5420fe60ad5 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -35,7 +35,7 @@ isposdef!{T<:BlasFloat}(A::StridedMatrix{T}, UL::Symbol) = LAPACK.potrf!(char_up """ isposdef!(A) -> Bool -Test whether a matrix is positive definite, overwriting `A` in the processes. +Test whether a matrix is positive definite, overwriting `A` in the process. """ isposdef!(A::StridedMatrix) = ishermitian(A) && isposdef!(A, :U) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 9527d1036ae81..0b226ad149878 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -558,8 +558,8 @@ dot(x::AbstractVector, y::AbstractVector) = vecdot(x, y) """ rank(M[, tol::Real]) -Compute the rank of a matrix by summing the singular -values of `M` with magnitude greater than `tol`. +Compute the rank of a matrix by counting how many singular +values of `M` have magnitude greater than `tol`. By default, the value of `tol` is the largest dimension of `M` multiplied by the [`eps`](:func:`eps`) of the [`eltype`](:func:`eltype`) of `M`. @@ -576,7 +576,17 @@ rank(x::Number) = x==0 ? 0 : 1 """ trace(M) -Matrix trace. +Matrix trace. Sums the diagonal elements of `M`. + +```jldoctest +julia> A = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> trace(A) +5 +``` """ function trace(A::AbstractMatrix) checksquare(A) @@ -594,7 +604,10 @@ inv(a::StridedMatrix) = throw(ArgumentError("argument must be a square matrix")) """ inv(M) -Matrix inverse. +Matrix inverse. Computes matrix `N` such that +`M * N = I`, where `I` is the identity matrix. +Computed by solving the left-division +`N = M \\ I`. """ function inv{T}(A::AbstractMatrix{T}) S = typeof(zero(T)/one(T)) diff --git a/base/loading.jl b/base/loading.jl index 087d73e2e2d40..5034824e6a187 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -359,7 +359,7 @@ already defined in `Main`. It can also be called directly to force reloading a m regardless of whether it has been loaded before (for example, when interactively developing libraries). -Loads a source files, in the context of the `Main` module, on every active node, searching +Loads a source file, in the context of the `Main` module, on every active node, searching standard locations for files. `require` is considered a top-level operation, so it sets the current `include` path but does not use it to search for files (see help for `include`). This function is typically used to load library code, and is implicitly called by `using` to @@ -596,8 +596,9 @@ compilecache(mod::Symbol) = compilecache(string(mod)) """ Base.compilecache(module::String) -Creates a precompiled cache file for module (see help for [`require`](:func:`require`)) and all of its -dependencies. This can be used to reduce package load times. Cache files are stored in +Creates a [precompiled cache file](:ref:`man-modules-initialization-precompilation`) for +a module and all of its dependencies. +This can be used to reduce package load times. Cache files are stored in `LOAD_CACHE_PATH[1]`, which defaults to `~/.julia/lib/VERSION`. See [Module initialization and precompilation](:ref:`Module initialization and precompilation <man-modules-initialization-precompilation>`) for important notes. diff --git a/base/mpfr.jl b/base/mpfr.jl index f4a2e2543b524..c8d28d1b32468 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -700,15 +700,15 @@ cmp(x::CdoubleMax, y::BigFloat) = -cmp(y,x) signbit(x::BigFloat) = ccall((:mpfr_signbit, :libmpfr), Int32, (Ptr{BigFloat},), &x) != 0 +function precision(x::BigFloat) # precision of an object of type BigFloat + return ccall((:mpfr_get_prec, :libmpfr), Clong, (Ptr{BigFloat},), &x) +end + """ precision(BigFloat) Get the precision (in bits) currently used for `BigFloat` arithmetic. """ -function precision(x::BigFloat) # precision of an object of type BigFloat - return ccall((:mpfr_get_prec, :libmpfr), Clong, (Ptr{BigFloat},), &x) -end - precision(::Type{BigFloat}) = DEFAULT_PRECISION[end] # precision of the type BigFloat itself """ diff --git a/base/random.jl b/base/random.jl index 06b1eae0a3bd3..fffa957242679 100644 --- a/base/random.jl +++ b/base/random.jl @@ -153,13 +153,13 @@ end """ randjump(r::MersenneTwister, jumps::Integer, [jumppoly::AbstractString=dSFMT.JPOLY1e21]) -> Vector{MersenneTwister} -Create an array of the size `jumps` of initialized `MersenneTwister` RNG objects where the -first RNG object given as a parameter and following `MersenneTwister` RNGs in the array +Create an array of the size `jumps` of initialized `MersenneTwister` RNG objects. The +first RNG object given as a parameter and following `MersenneTwister` RNGs in the array are initialized such that a state of the RNG object in the array would be moved forward (without generating numbers) from a previous RNG object array element on a particular number of steps encoded by the jump polynomial `jumppoly`. -Default jump polynomial moves forward `MersenneTwister` RNG state by 10^20 steps. +Default jump polynomial moves forward `MersenneTwister` RNG state by `10^20` steps. """ function randjump(mt::MersenneTwister, jumps::Integer, jumppoly::AbstractString) mts = MersenneTwister[] diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 04a0332e691a3..00a33f8ddc256 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -273,7 +273,8 @@ chr2ind(s::DirectIndexString, i::Integer) = begin checkbounds(s,i); i end """ ind2chr(s::AbstractString, i::Integer) -Convert a byte index `i` to a character index. +Convert a byte index `i` to a character index with +respect to string `s`. """ function ind2chr(s::AbstractString, i::Integer) s[i] # throws error if invalid diff --git a/base/strings/util.jl b/base/strings/util.jl index f0cb04c75ce9e..90f2244f77c9f 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -62,7 +62,7 @@ startswith(a::Vector{UInt8}, b::Vector{UInt8}) = """ chop(s::AbstractString) -Remove the last character from a string. +Remove the last character from `s`. ```jldoctest julia> a = string("March") @@ -106,6 +106,8 @@ const _default_delims = [' ','\t','\n','\v','\f','\r'] lstrip(s::AbstractString[, chars::Chars]) Return `s` with any leading whitespace and delimiters removed. +The default delimiters to remove are `' '`, `\\t`, `\\n`, `\\v`, +`\\f`, and `\\r`. If `chars` (a character, or vector or set of characters) is provided, instead remove characters contained in it. """ @@ -125,6 +127,8 @@ end rstrip(s::AbstractString[, chars::Chars]) Return `s` with any trailing whitespace and delimiters removed. +The default delimiters to remove are `' '`, `\\t`, `\\n`, `\\v`, +`\\f`, and `\\r`. If `chars` (a character, or vector or set of characters) is provided, instead remove characters contained in it. diff --git a/base/util.jl b/base/util.jl index 097140cd6998f..ff43543ec6f7e 100644 --- a/base/util.jl +++ b/base/util.jl @@ -325,8 +325,8 @@ println_with_color(color::Symbol, msg::AbstractString...) = Display an informational message. Argument `msg` is a string describing the information to be displayed. -The `prefix` kwarg can be used to override the default prepending of -`msg`. +The `prefix` keyword argument can be used to override the default +prepending of `msg`. ```jldoctest julia> info("hello world") diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 837c0244e441c..76f85d5b93af6 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -124,7 +124,7 @@ Getting Around This function is part of the implementation of ``using`` / ``import``\ , if a module is not already defined in ``Main``\ . It can also be called directly to force reloading a module, regardless of whether it has been loaded before (for example, when interactively developing libraries). - Loads a source files, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``\ ). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. + Loads a source file, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``\ ). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. When searching for files, ``require`` first looks for package code under ``Pkg.dir()``\ , then tries paths in the global array ``LOAD_PATH``\ . ``require`` is case-sensitive on all platforms, including those with case-insensitive filesystems like macOS and Windows. @@ -132,7 +132,7 @@ Getting Around .. Docstring generated from Julia source - Creates a precompiled cache file for module (see help for :func:`require`\ ) and all of its dependencies. This can be used to reduce package load times. Cache files are stored in ``LOAD_CACHE_PATH[1]``\ , which defaults to ``~/.julia/lib/VERSION``\ . See :ref:`Module initialization and precompilation <man-modules-initialization-precompilation>` for important notes. + Creates a :ref:`man-modules-initialization-precompilation` for a module and all of its dependencies. This can be used to reduce package load times. Cache files are stored in ``LOAD_CACHE_PATH[1]``\ , which defaults to ``~/.julia/lib/VERSION``\ . See :ref:`Module initialization and precompilation <man-modules-initialization-precompilation>` for important notes. .. function:: __precompile__(isprecompilable::Bool=true) @@ -198,7 +198,7 @@ Getting Around If ``types`` is specified, returns an array of methods whose types match. -.. function:: methodswith(typ[, module or function][, showparents::Bool=false, meths=Method[]]) +.. function:: methodswith(typ[, module or function][, showparents::Bool=false]) .. Docstring generated from Julia source @@ -1484,7 +1484,7 @@ Internals .. Docstring generated from Julia source - Takes the expression ``x`` and returns an equivalent expression in lowered form. + Takes the expression ``x`` and returns an equivalent expression in lowered form. See also :func:`code_lowered`\ . .. function:: code_lowered(f, types) diff --git a/doc/stdlib/file.rst b/doc/stdlib/file.rst index 5ea94973b686a..cd3d7b72cfa43 100644 --- a/doc/stdlib/file.rst +++ b/doc/stdlib/file.rst @@ -73,7 +73,7 @@ .. Docstring generated from Julia source - Returns the value of a symbolic link ``path``\ . + Returns the target location a symbolic link ``path`` points to. .. function:: chmod(path::AbstractString, mode::Integer; recursive::Bool=false) @@ -185,7 +185,7 @@ .. Docstring generated from Julia source - Copy the file, link, or directory from *src* to *dest*. ``remove_destination=true`` will first remove an existing ``dst``\ . + Copy the file, link, or directory from ``src`` to ``dest``\ . ``remove_destination=true`` will first remove an existing ``dst``\ . If ``follow_symlinks=false``\ , and ``src`` is a symbolic link, ``dst`` will be created as a symbolic link. If ``follow_symlinks=true`` and ``src`` is a symbolic link, ``dst`` will be a copy of the file or directory ``src`` refers to. diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 20e69009a85d9..bbbbc4c48d527 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -198,7 +198,7 @@ General I/O .. Docstring generated from Julia source - Copy nbytes from the ``IO`` stream object into ``ref`` (converted to a pointer). + Copy ``nbytes`` from the ``IO`` stream object into ``ref`` (converted to a pointer). It is recommended that subtypes ``T<:IO`` override the following method signature to provide more efficient implementations: ``unsafe_read(s::T, p::Ptr{UInt8}, n::UInt)`` @@ -206,7 +206,7 @@ General I/O .. Docstring generated from Julia source - Copy nbytes from ref (converted to a pointer) into the IO stream object. + Copy ``nbytes`` from ``ref`` (converted to a pointer) into the ``IO`` object. It is recommended that subtypes ``T<:IO`` override the following method signature to provide more efficient implementations: ``unsafe_write(s::T, p::Ptr{UInt8}, n::UInt)`` @@ -514,7 +514,7 @@ Text I/O .. Docstring generated from Julia source - Display an informational message. Argument ``msg`` is a string describing the information to be displayed. The ``prefix`` kwarg can be used to override the default prepending of ``msg``\ . + Display an informational message. Argument ``msg`` is a string describing the information to be displayed. The ``prefix`` keyword argument can be used to override the default prepending of ``msg``\ . .. doctest:: diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index f9f5116525101..1240275ee6f34 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -1197,7 +1197,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Compute the rank of a matrix by summing the singular values of ``M`` with magnitude greater than ``tol``\ . By default, the value of ``tol`` is the largest dimension of ``M`` multiplied by the :func:`eps` of the :func:`eltype` of ``M``\ . + Compute the rank of a matrix by counting how many singular values of ``M`` have magnitude greater than ``tol``\ . By default, the value of ``tol`` is the largest dimension of ``M`` multiplied by the :func:`eps` of the :func:`eltype` of ``M``\ . .. function:: norm(A, [p::Real=2]) @@ -1268,7 +1268,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Matrix trace. + Matrix trace. Sums the diagonal elements of ``M``\ . + + .. doctest:: + + julia> A = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> trace(A) + 5 .. function:: det(M) @@ -1292,7 +1302,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Matrix inverse. + Matrix inverse. Computes matrix ``N`` such that ``M * N = I``\ , where ``I`` is the identity matrix. Computed by solving the left-division ``N = M \ I``\ . .. function:: pinv(M[, tol::Real]) @@ -1490,7 +1500,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Test whether a matrix is positive definite, overwriting ``A`` in the processes. + Test whether a matrix is positive definite, overwriting ``A`` in the process. .. function:: istril(A) -> Bool diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index 6d4b74de91f5d..91420344723ef 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -704,7 +704,7 @@ As ``BigInt`` represents unbounded integers, the interval must be specified (e.g .. Docstring generated from Julia source - Create an array of the size ``jumps`` of initialized ``MersenneTwister`` RNG objects where the first RNG object given as a parameter and following ``MersenneTwister`` RNGs in the array initialized such that a state of the RNG object in the array would be moved forward (without generating numbers) from a previous RNG object array element on a particular number of steps encoded by the jump polynomial ``jumppoly``\ . + Create an array of the size ``jumps`` of initialized ``MersenneTwister`` RNG objects. The first RNG object given as a parameter and following ``MersenneTwister`` RNGs in the array are initialized such that a state of the RNG object in the array would be moved forward (without generating numbers) from a previous RNG object array element on a particular number of steps encoded by the jump polynomial ``jumppoly``\ . - Default jump polynomial moves forward ``MersenneTwister`` RNG state by 10^20 steps. + Default jump polynomial moves forward ``MersenneTwister`` RNG state by ``10^20`` steps. diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index 609c48031ea1b..13152681a55af 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -347,13 +347,13 @@ .. Docstring generated from Julia source - Return ``s`` with any leading whitespace and delimiters removed. If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. + Return ``s`` with any leading whitespace and delimiters removed. The default delimiters to remove are ``' '``\ , ``\t``\ , ``\n``\ , ``\v``\ , ``\f``\ , and ``\r``\ . If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. .. function:: rstrip(s::AbstractString[, chars::Chars]) .. Docstring generated from Julia source - Return ``s`` with any trailing whitespace and delimiters removed. If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. + Return ``s`` with any trailing whitespace and delimiters removed. The default delimiters to remove are ``' '``\ , ``\t``\ , ``\n``\ , ``\v``\ , ``\f``\ , and ``\r``\ . If ``chars`` (a character, or vector or set of characters) is provided, instead remove characters contained in it. .. doctest:: @@ -446,7 +446,7 @@ .. Docstring generated from Julia source - Remove the last character from a string. + Remove the last character from ``s``\ . .. doctest:: @@ -466,7 +466,7 @@ .. Docstring generated from Julia source - Convert a byte index ``i`` to a character index. + Convert a byte index ``i`` to a character index with respect to string ``s``\ . .. function:: chr2ind(s::AbstractString, i::Integer) From 0ab19a6e1cc2f49ed4fd1e8b25a6ce586079988a Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 13:52:30 -0700 Subject: [PATCH 1097/1117] Deprecate vectorized functions in base/special/trig.jl in favor of compact broadcast syntax. --- base/deprecated.jl | 28 ++++++++++++++++++++++++++++ base/special/trig.jl | 8 -------- test/sparsedir/sparsevector.jl | 4 ++-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 667bd10c5c4e7..48390c31403ce 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -865,4 +865,32 @@ for f in (:sin, :sinh, :sind, :asin, :asinh, :asind, @eval @deprecate $f(A::SparseMatrixCSC) $f.(A) end +# For deprecating vectorized functions in favor of compact broadcast syntax +macro dep_vectorize_1arg(S, f) + :( @deprecate $(esc(f)){T<:$(esc(S))}(x::AbstractArray{T}) $(esc(f)).(x) ) +end +macro dep_vectorize_2arg(S, f) + S = esc(S) + f = esc(f) + T1 = esc(:T1) + T2 = esc(:T2) + quote + @deprecate $f{T<:$S}(x::$S, y::AbstractArray{T}) $f.(x,y) + @deprecate $f{T<:$S}(x::AbstractArray{T}, y::$S) $f.(x,y) + @deprecate $f{T1<:$S,T2<:$S}(x::AbstractArray{T1}, y::AbstractArray{T2}) $f.(x,y) + end +end + +# Deprecate @vectorize_1arg-vectorized functions from... +for f in ( + :sinpi, :cospi, :sinc, :cosc, # base/special/trig.jl + ) + @eval @dep_vectorize_1arg Number $f +end +for f in ( + :sind, :cosd, :tand, :asind, :acosd, :atand, :asecd, :acscd, :acotd, # base/special/trig.jl + ) + @eval @dep_vectorize_1arg Real $f +end + # End deprecations scheduled for 0.6 diff --git a/base/special/trig.jl b/base/special/trig.jl index 4a4e0b94170a7..bc84f2b972554 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -284,8 +284,6 @@ function cospi{T}(z::Complex{T}) Complex(cospi(zr)*cosh(pizi), -sinpi(zr)*sinh(pizi)) end end -@vectorize_1arg Number sinpi -@vectorize_1arg Number cospi """ sinc(x) @@ -296,7 +294,6 @@ sinc(x::Number) = x==0 ? one(x) : oftype(x,sinpi(x)/(pi*x)) sinc(x::Integer) = x==0 ? one(x) : zero(x) sinc{T<:Integer}(x::Complex{T}) = sinc(float(x)) sinc(x::Real) = x==0 ? one(x) : isinf(x) ? zero(x) : sinpi(x)/(pi*x) -@vectorize_1arg Number sinc """ cosc(x) @@ -308,7 +305,6 @@ cosc(x::Number) = x==0 ? zero(x) : oftype(x,(cospi(x)-sinpi(x)/(pi*x))/x) cosc(x::Integer) = cosc(float(x)) cosc{T<:Integer}(x::Complex{T}) = cosc(float(x)) cosc(x::Real) = x==0 || isinf(x) ? zero(x) : (cospi(x)-sinpi(x)/(pi*x))/x -@vectorize_1arg Number cosc for (finv, f) in ((:sec, :cos), (:csc, :sin), (:cot, :tan), (:sech, :cosh), (:csch, :sinh), (:coth, :tanh), @@ -385,7 +381,6 @@ function sind(x::Real) return sin_kernel(y) end end -@vectorize_1arg Real sind function cosd(x::Real) if isinf(x) @@ -412,10 +407,8 @@ function cosd(x::Real) return cos_kernel(y) end end -@vectorize_1arg Real cosd tand(x::Real) = sind(x) / cosd(x) -@vectorize_1arg Real tand for (fd, f, fn) in ((:sind, :sin, "sine"), (:cosd, :cos, "cosine"), (:tand, :tan, "tangent")) name = string(fd) @@ -434,6 +427,5 @@ for (fd, f, fn) in ((:asind, :asin, "sine"), (:acosd, :acos, "cosine"), (:atand, $($name)(x) Compute the inverse $($fn) of `x`, where the output is in degrees. """ ($fd)(y) = rad2deg(($f)(y)) - @vectorize_1arg Real $fd end end diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index fc91b9ba3f11d..22d64cebd1a33 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -626,7 +626,7 @@ function check_nz2z_z2z{T}(f::Function, x::SparseVector{T}, xf::Vector{T}) isa(r, AbstractSparseVector) || error("$f(x) is not a sparse vector.") eltype(r) == R || error("$f(x) results in eltype = $(eltype(r)), expect $R") all(r.nzval .!= 0) || error("$f(x) contains zeros in nzval.") - full(r) == f(xf) || error("Incorrect results found in $f(x).") + full(r) == f.(xf) || error("Incorrect results found in $f(x).") end for f in [floor, ceil, trunc, round] @@ -647,7 +647,7 @@ function check_z2nz{T}(f::Function, x::SparseVector{T}, xf::Vector{T}) r = f(x) isa(r, Vector) || error("$f(x) is not a dense vector.") eltype(r) == R || error("$f(x) results in eltype = $(eltype(r)), expect $R") - r == f(xf) || error("Incorrect results found in $f(x).") + r == f.(xf) || error("Incorrect results found in $f(x).") end for f in [exp, exp2, exp10, log, log2, log10, From 382fda97d3510a21f4b12b078dc1094e4987ba22 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 14:53:18 -0700 Subject: [PATCH 1098/1117] Deprecate vectorized functions in base/special/log.jl in favor of compact broadcast syntax. --- base/deprecated.jl | 1 + base/linalg/diagonal.jl | 8 ++++++-- base/linalg/symmetric.jl | 12 ++++++------ base/special/log.jl | 1 - 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 48390c31403ce..e731f88473a87 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -884,6 +884,7 @@ end # Deprecate @vectorize_1arg-vectorized functions from... for f in ( :sinpi, :cospi, :sinc, :cosc, # base/special/trig.jl + :log, :log1p, # base/special/log.jl ) @eval @dep_vectorize_1arg Number $f end diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 6dd6a9b735c22..63ecbfff7582a 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -231,9 +231,9 @@ ctranspose(D::Diagonal) = conj(D) diag(D::Diagonal) = D.diag trace(D::Diagonal) = sum(D.diag) det(D::Diagonal) = prod(D.diag) -logdet{T<:Real}(D::Diagonal{T}) = sum(log(D.diag)) +logdet{T<:Real}(D::Diagonal{T}) = sum(log.(D.diag)) function logdet{T<:Complex}(D::Diagonal{T}) #Make sure branch cut is correct - x = sum(log(D.diag)) + x = sum(log.(D.diag)) -pi<imag(x)<pi ? x : real(x)+(mod2pi(imag(x)+pi)-pi)*im end # identity matrices via eye(Diagonal{type},n) @@ -247,6 +247,10 @@ for (funm, func) in ([:expm,:exp], [:sqrtm,:sqrt], [:logm,:log]) end end +expm(D::Diagonal) = Diagonal(exp(D.diag)) +logm(D::Diagonal) = Diagonal(log.(D.diag)) +sqrtm(D::Diagonal) = Diagonal(sqrt(D.diag)) + #Linear solver function A_ldiv_B!(D::Diagonal, B::StridedVecOrMat) m, n = size(B, 1), size(B, 2) diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index dead41a2b93c3..cc516377cc958 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -290,12 +290,12 @@ end function expm{T<:Real}(A::Symmetric{T}) F = eigfact(A) - return Symmetric((F.vectors * Diagonal(exp(F.values))) * F.vectors') + return Symmetric((F.vectors * Diagonal(exp.(F.values))) * F.vectors') end function expm{T}(A::Hermitian{T}) n = checksquare(A) F = eigfact(A) - retmat = (F.vectors * Diagonal(exp(F.values))) * F.vectors' + retmat = (F.vectors * Diagonal(exp.(F.values))) * F.vectors' if T <: Real return real(Hermitian(retmat)) else @@ -311,9 +311,9 @@ for (funm, func) in ([:logm,:log], [:sqrtm,:sqrt]) function ($funm){T<:Real}(A::Symmetric{T}) F = eigfact(A) if isposdef(F) - retmat = (F.vectors * Diagonal(($func)(F.values))) * F.vectors' + retmat = (F.vectors * Diagonal(($func).(F.values))) * F.vectors' else - retmat = (F.vectors * Diagonal(($func)(complex(F.values)))) * F.vectors' + retmat = (F.vectors * Diagonal(($func).(complex(F.values)))) * F.vectors' end return Symmetric(retmat) end @@ -322,7 +322,7 @@ for (funm, func) in ([:logm,:log], [:sqrtm,:sqrt]) n = checksquare(A) F = eigfact(A) if isposdef(F) - retmat = (F.vectors * Diagonal(($func)(F.values))) * F.vectors' + retmat = (F.vectors * Diagonal(($func).(F.values))) * F.vectors' if T <: Real return Hermitian(retmat) else @@ -332,7 +332,7 @@ for (funm, func) in ([:logm,:log], [:sqrtm,:sqrt]) return Hermitian(retmat) end else - retmat = (F.vectors * Diagonal(($func)(complex(F.values)))) * F.vectors' + retmat = (F.vectors * Diagonal(($func).(complex(F.values)))) * F.vectors' return retmat end end diff --git a/base/special/log.jl b/base/special/log.jl index ae642e4fd5c68..dc3473828335b 100644 --- a/base/special/log.jl +++ b/base/special/log.jl @@ -393,6 +393,5 @@ end for f in (:log,:log1p) @eval begin ($f)(x::Real) = ($f)(float(x)) - @vectorize_1arg Number $f end end From e353eebf50889fce9120ba3eda6f991fb0a2f5b5 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 14:58:40 -0700 Subject: [PATCH 1099/1117] Deprecate vectorized functions in base/special/gamma.jl in favor of compact broadcast syntax. --- base/deprecated.jl | 9 +++++++++ base/special/gamma.jl | 10 ---------- test/math.jl | 10 +++++----- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index e731f88473a87..41beca6a10935 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -885,13 +885,22 @@ end for f in ( :sinpi, :cospi, :sinc, :cosc, # base/special/trig.jl :log, :log1p, # base/special/log.jl + :gamma, :lfact, :digamma, :trigamma, :zeta, :eta,# base/special/gamma.jl ) @eval @dep_vectorize_1arg Number $f end for f in ( :sind, :cosd, :tand, :asind, :acosd, :atand, :asecd, :acscd, :acotd, # base/special/trig.jl + :invdigamma, # base/special/gamma.jl ) @eval @dep_vectorize_1arg Real $f end +# Deprecate @vectorize_2arg-vectorized functions from... +for f in ( + :polygamma, :zeta, :beta, :lbeta, # base/special/gamma.jl + ) + @eval @dep_vectorize_2arg Number $f +end + # End deprecations scheduled for 0.6 diff --git a/base/special/gamma.jl b/base/special/gamma.jl index 083ad14c30fdb..2d07215be8e94 100644 --- a/base/special/gamma.jl +++ b/base/special/gamma.jl @@ -9,7 +9,6 @@ gamma(x::Float32) = nan_dom_err(ccall((:tgammaf,libm), Float32, (Float32,), x), Compute the gamma function of `x`. """ gamma(x::Real) = gamma(float(x)) -@vectorize_1arg Number gamma function lgamma_r(x::Float64) signp = Array{Int32}(1) @@ -31,7 +30,6 @@ lgamma_r(x::Number) = lgamma(x), 1 # lgamma does not take abs for non-real x Compute the logarithmic factorial of `x` """ lfact(x::Real) = (x<=1 ? zero(float(x)) : lgamma(x+one(x))) -@vectorize_1arg Number lfact const clg_coeff = [76.18009172947146, -86.50532032941677, @@ -446,12 +444,9 @@ zeta(s::Number, z::Number) = zeta(f64(s), f64(z)) for f in (:digamma, :trigamma) @eval begin $f(z::Number) = $f(f64(z)) - @vectorize_1arg Number $f end end polygamma(m::Integer, z::Number) = polygamma(m, f64(z)) -@vectorize_2arg Number polygamma -@vectorize_2arg Number zeta # Inverse digamma function: # Implementation of fixed point algorithm described in @@ -486,7 +481,6 @@ invdigamma(x::Float32) = Float32(invdigamma(Float64(x))) Compute the inverse digamma function of `x`. """ invdigamma(x::Real) = invdigamma(Float64(x)) -@vectorize_1arg Real invdigamma function beta(x::Number, w::Number) yx, sx = lgamma_r(x) @@ -495,8 +489,6 @@ function beta(x::Number, w::Number) return exp(yx + yw - yxw) * (sx*sw*sxw) end lbeta(x::Number, w::Number) = lgamma(x)+lgamma(w)-lgamma(x+w) -@vectorize_2arg Number beta -@vectorize_2arg Number lbeta # Riemann zeta function; algorithm is based on specializing the Hurwitz # zeta function above for z==1. @@ -548,7 +540,6 @@ end zeta(x::Integer) = zeta(Float64(x)) zeta(x::Real) = oftype(float(x),zeta(Float64(x))) zeta(z::Complex) = oftype(float(z),zeta(Complex128(z))) -@vectorize_1arg Number zeta function eta(z::Union{Float64,Complex{Float64}}) δz = 1 - z @@ -567,4 +558,3 @@ end eta(x::Integer) = eta(Float64(x)) eta(x::Real) = oftype(float(x),eta(Float64(x))) eta(z::Complex) = oftype(float(z),eta(Complex128(z))) -@vectorize_1arg Number eta diff --git a/test/math.jl b/test/math.jl index 26a744b3211f0..a1547b9b7e530 100644 --- a/test/math.jl +++ b/test/math.jl @@ -543,9 +543,9 @@ end # gamma, lgamma (complex argument) if Base.Math.libm == "libopenlibm" - @test gamma(Float64[1:25;]) == gamma(1:25) + @test gamma.(Float64[1:25;]) == gamma.(1:25) else - @test gamma(Float64[1:25;]) ≈ gamma(1:25) + @test gamma.(Float64[1:25;]) ≈ gamma.(1:25) end for elty in (Float32, Float64) @test gamma(convert(elty,1/2)) ≈ convert(elty,sqrt(π)) @@ -815,9 +815,9 @@ binary_math_functions = [ for f in binary_math_functions x = y = 2 v = [f(x,y)] - @test f([x],y) == v - @test f(x,[y]) == v - @test f([x],[y]) == v + @test f.([x],y) == v + @test f.(x,[y]) == v + @test f.([x],[y]) == v end # #3024, #12822 From 56ce5ac3cb6a9591b555ab4bdd66ba5b2476b080 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 15:01:36 -0700 Subject: [PATCH 1100/1117] Deprecate vectorized functions in base/special/erf.jl in favor of compact broadcast syntax. --- base/deprecated.jl | 2 ++ base/special/erf.jl | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 41beca6a10935..27513d6c0d609 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -886,12 +886,14 @@ for f in ( :sinpi, :cospi, :sinc, :cosc, # base/special/trig.jl :log, :log1p, # base/special/log.jl :gamma, :lfact, :digamma, :trigamma, :zeta, :eta,# base/special/gamma.jl + :erfcx, :erfi, :dawson, # base/special/erf.jl ) @eval @dep_vectorize_1arg Number $f end for f in ( :sind, :cosd, :tand, :asind, :acosd, :atand, :asecd, :acscd, :acotd, # base/special/trig.jl :invdigamma, # base/special/gamma.jl + :erfinc, :erfcinv, # base/special/erf.jl ) @eval @dep_vectorize_1arg Real $f end diff --git a/base/special/erf.jl b/base/special/erf.jl index a4e8722c9801b..2e755fde73b4c 100644 --- a/base/special/erf.jl +++ b/base/special/erf.jl @@ -14,7 +14,6 @@ for f in (:erfcx, :erfi, :Dawson) ($fname)(x::Float64) = ccall(($(string("Faddeeva_",f,"_re")),openspecfun), Float64, (Float64,), x) ($fname)(x::Float32) = Float32(ccall(($(string("Faddeeva_",f,"_re")),openspecfun), Float64, (Float64,), Float64(x))) ($fname)(x::Integer) = ($fname)(float(x)) - @vectorize_1arg Number $fname end end @@ -137,7 +136,6 @@ function erfinv(x::Float32) end erfinv(x::Integer) = erfinv(float(x)) -@vectorize_1arg Real erfinv # Inverse complementary error function: use Blair tables for y = 1-x, # exploiting the greater accuracy of y (vs. x) when y is small. @@ -217,5 +215,3 @@ function erfcinv(y::Float32) end erfcinv(x::Integer) = erfcinv(float(x)) -@vectorize_1arg Real erfcinv - From 2a9e713eebffb8207c7a25c4cc535fdab84af0bc Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 15:07:30 -0700 Subject: [PATCH 1101/1117] Deprecate vectorized functions in base/special/bessel.jl in favor of compact broadcast syntax. --- base/deprecated.jl | 2 ++ base/special/bessel.jl | 14 -------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 27513d6c0d609..bf88d240282db 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -887,6 +887,7 @@ for f in ( :log, :log1p, # base/special/log.jl :gamma, :lfact, :digamma, :trigamma, :zeta, :eta,# base/special/gamma.jl :erfcx, :erfi, :dawson, # base/special/erf.jl + :airyprime, :airyai, :airyaiprime, :airybi, :airybiprime, :airy, :airyx, :besselj0, :besselj1, :bessely0, :bessely1, # base/special/bessel.jl ) @eval @dep_vectorize_1arg Number $f end @@ -901,6 +902,7 @@ end # Deprecate @vectorize_2arg-vectorized functions from... for f in ( :polygamma, :zeta, :beta, :lbeta, # base/special/gamma.jl + :airy, :airyx, :besseli, :besselix, :besselj, :besseljx, :besselk, :besselkx, :bessely, :besselyx, :besselh, :besselhx, :hankelh1, :hankelh2, :hankelh1x, :hankelh2x, # base/special/bessel.jl ) @eval @dep_vectorize_2arg Number $f end diff --git a/base/special/bessel.jl b/base/special/bessel.jl index 3151306de7c6a..bdd8068347f41 100644 --- a/base/special/bessel.jl +++ b/base/special/bessel.jl @@ -62,7 +62,6 @@ end Airy function derivative ``\\operatorname{Ai}'(x)``. """ airyprime(z) = airy(1,z) -@vectorize_1arg Number airyprime """ airyai(x) @@ -70,7 +69,6 @@ airyprime(z) = airy(1,z) Airy function ``\\operatorname{Ai}(x)``. """ airyai(z) = airy(0,z) -@vectorize_1arg Number airyai """ airyaiprime(x) @@ -78,7 +76,6 @@ airyai(z) = airy(0,z) Airy function derivative ``\\operatorname{Ai}'(x)``. """ airyaiprime(z) = airy(1,z) -@vectorize_1arg Number airyaiprime """ airybi(x) @@ -86,7 +83,6 @@ airyaiprime(z) = airy(1,z) Airy function ``\\operatorname{Bi}(x)``. """ airybi(z) = airy(2,z) -@vectorize_1arg Number airybi """ airybiprime(x) @@ -94,7 +90,6 @@ airybi(z) = airy(2,z) Airy function derivative ``\\operatorname{Bi}'(x)``. """ airybiprime(z) = airy(3,z) -@vectorize_1arg Number airybiprime function airyx(k::Integer, z::Complex128) id = Int32(k==1 || k==3) @@ -117,8 +112,6 @@ for afn in (:airy,:airyx) $afn(k::Integer, x::AbstractFloat) = real($afn(k, complex(x))) $afn(z) = $afn(0,z) - @vectorize_1arg Number $afn - @vectorize_2arg Number $afn end end """ @@ -151,7 +144,6 @@ for jy in ("j","y"), nu in (0,1) @eval begin $bjynu(x::Real) = $bjynu(float(x)) $bjynu(x::Complex) = $(Symbol("bessel",jy))($nu,x) - @vectorize_1arg Number $bjynu end end @@ -459,7 +451,6 @@ for f in ("i", "ix", "j", "jx", "k", "kx", "y", "yx") end $bfn{T<:AbstractFloat}(k::T, z::Complex{T}) = throw(MethodError($bfn,(k,z))) $bfn(nu::Float32, x::Complex64) = Complex64($bfn(Float64(nu), Complex128(x))) - @vectorize_2arg Number $bfn end end @@ -477,7 +468,6 @@ for bfn in (:besselh, :besselhx) $bfn{T<:AbstractFloat}(nu::T, k::Integer, z::Complex{T}) = throw(MethodError($bfn,(nu,k,z))) $bfn(nu::Float32, k::Integer, x::Complex64) = Complex64($bfn(Float64(nu), k, Complex128(x))) - @vectorize_2arg Number $bfn end end @@ -487,7 +477,6 @@ end Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x)``. """ hankelh1(nu, z) = besselh(nu, 1, z) -@vectorize_2arg Number hankelh1 """ hankelh2(nu, x) @@ -495,7 +484,6 @@ hankelh1(nu, z) = besselh(nu, 1, z) Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x)``. """ hankelh2(nu, z) = besselh(nu, 2, z) -@vectorize_2arg Number hankelh2 """ hankelh1x(nu, x) @@ -503,7 +491,6 @@ hankelh2(nu, z) = besselh(nu, 2, z) Scaled Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x) e^{-x i}``. """ hankelh1x(nu, z) = besselhx(nu, 1, z) -@vectorize_2arg Number hankelh1x """ hankelh2x(nu, x) @@ -511,4 +498,3 @@ hankelh1x(nu, z) = besselhx(nu, 1, z) Scaled Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x) e^{x i}``. """ hankelh2x(nu, z) = besselhx(nu, 2, z) -@vectorize_2arg Number hankelh2x From 5621af53b7488e74792331560613a77f9304fd4d Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 15:14:37 -0700 Subject: [PATCH 1102/1117] Deprecate vectorized functions in base/math.jl in favor of compact broadcast syntax. --- base/deprecated.jl | 8 ++++++++ base/irrationals.jl | 2 +- base/linalg/diagonal.jl | 4 ++-- base/markdown/GitHub/table.jl | 2 +- base/math.jl | 14 -------------- test/bitarray.jl | 8 ++++---- test/broadcast.jl | 4 ++-- test/fft.jl | 4 ++-- test/functional.jl | 2 +- test/linalg/arnoldi.jl | 2 +- test/linalg/lapack.jl | 2 +- test/math.jl | 14 +++++++------- test/offsetarray.jl | 8 ++++---- test/operators.jl | 2 +- test/reduce.jl | 2 +- test/sparsedir/sparse.jl | 7 +++++-- test/statistics.jl | 4 ++-- 17 files changed, 43 insertions(+), 46 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index bf88d240282db..1b9b7ec1f40b8 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -888,6 +888,7 @@ for f in ( :gamma, :lfact, :digamma, :trigamma, :zeta, :eta,# base/special/gamma.jl :erfcx, :erfi, :dawson, # base/special/erf.jl :airyprime, :airyai, :airyaiprime, :airybi, :airybiprime, :airy, :airyx, :besselj0, :besselj1, :bessely0, :bessely1, # base/special/bessel.jl + :cbrt, :sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :exp2, :expm1, :exp10, :sin, :cos, :tan, :asin, :acos, :acosh, :atanh, #=:log,=# :log2, :log10, :lgamma, #=:log1p,=# :sqrt, # base/math.jl ) @eval @dep_vectorize_1arg Number $f end @@ -895,6 +896,7 @@ for f in ( :sind, :cosd, :tand, :asind, :acosd, :atand, :asecd, :acscd, :acotd, # base/special/trig.jl :invdigamma, # base/special/gamma.jl :erfinc, :erfcinv, # base/special/erf.jl + :rad2deg, :deg2rad, :exponent, :significand, # base/math.jl ) @eval @dep_vectorize_1arg Real $f end @@ -903,8 +905,14 @@ end for f in ( :polygamma, :zeta, :beta, :lbeta, # base/special/gamma.jl :airy, :airyx, :besseli, :besselix, :besselj, :besseljx, :besselk, :besselkx, :bessely, :besselyx, :besselh, :besselhx, :hankelh1, :hankelh2, :hankelh1x, :hankelh2x, # base/special/bessel.jl + :log, :hypot, :atan2, # base/math.jl ) @eval @dep_vectorize_2arg Number $f end +for f in ( + :max, :min, # base/math.jl + ) + @eval @dep_vectorize_2arg Real $f +end # End deprecations scheduled for 0.6 diff --git a/base/irrationals.jl b/base/irrationals.jl index a5e091f5ca1e3..2ecf1f59dacaf 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -174,7 +174,7 @@ for T in (Irrational, Rational, Integer, Number) ^(::Irrational{:e}, x::T) = exp(x) end for T in (Range, BitArray, StridedArray, AbstractArray) - .^(::Irrational{:e}, x::T) = exp(x) + .^(::Irrational{:e}, x::T) = exp.(x) end log(::Irrational{:e}) = 1 # use 1 to correctly promote expressions like log(x)/log(e) diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 63ecbfff7582a..c1afa9343b751 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -247,9 +247,9 @@ for (funm, func) in ([:expm,:exp], [:sqrtm,:sqrt], [:logm,:log]) end end -expm(D::Diagonal) = Diagonal(exp(D.diag)) +expm(D::Diagonal) = Diagonal(exp.(D.diag)) logm(D::Diagonal) = Diagonal(log.(D.diag)) -sqrtm(D::Diagonal) = Diagonal(sqrt(D.diag)) +sqrtm(D::Diagonal) = Diagonal(sqrt.(D.diag)) #Linear solver function A_ldiv_B!(D::Diagonal, B::StridedVecOrMat) diff --git a/base/markdown/GitHub/table.jl b/base/markdown/GitHub/table.jl index 9822aed14d1fd..7aa34c6698b39 100644 --- a/base/markdown/GitHub/table.jl +++ b/base/markdown/GitHub/table.jl @@ -78,7 +78,7 @@ end mapmap(f, xss) = map(xs->map(f, xs), xss) colwidths(rows; len = length, min = 0) = - reduce(max, [min; convert(Vector{Vector{Int}}, mapmap(len, rows))]) + reduce((x,y) -> max.(x,y), [min; convert(Vector{Vector{Int}}, mapmap(len, rows))]) padding(width, twidth, a) = a == :l ? (0, twidth - width) : diff --git a/base/math.jl b/base/math.jl index 37d194ef546fc..7338d03a8b02f 100644 --- a/base/math.jl +++ b/base/math.jl @@ -146,8 +146,6 @@ julia> deg2rad(90) deg2rad(z::AbstractFloat) = z * (oftype(z, pi) / 180) rad2deg(z::Real) = rad2deg(float(z)) deg2rad(z::Real) = deg2rad(float(z)) -@vectorize_1arg Real rad2deg -@vectorize_1arg Real deg2rad log{T<:Number}(b::T, x::T) = log(x)/log(b) @@ -165,7 +163,6 @@ julia> log(4,2) ``` """ log(b::Number, x::Number) = log(promote(b,x)...) -@vectorize_2arg Number log # type specific math functions @@ -178,7 +175,6 @@ for f in (:cbrt, :sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :exp2, : ($f)(x::Float64) = ccall(($(string(f)),libm), Float64, (Float64,), x) ($f)(x::Float32) = ccall(($(string(f,"f")),libm), Float32, (Float32,), x) ($f)(x::Real) = ($f)(float(x)) - @vectorize_1arg Number $f end end @@ -230,7 +226,6 @@ end exp10(x::Float64) = 10.0^x exp10(x::Float32) = 10.0f0^x exp10(x::Integer) = exp10(float(x)) -@vectorize_1arg Number exp10 # utility for converting NaN return to DomainError @inline nan_dom_err(f, x) = isnan(f) & !isnan(x) ? throw(DomainError()) : f @@ -242,7 +237,6 @@ for f in (:sin, :cos, :tan, :asin, :acos, :acosh, :atanh, :log, :log2, :log10, ($f)(x::Float64) = nan_dom_err(ccall(($(string(f)),libm), Float64, (Float64,), x), x) ($f)(x::Float32) = nan_dom_err(ccall(($(string(f,"f")),libm), Float32, (Float32,), x), x) ($f)(x::Real) = ($f)(float(x)) - @vectorize_1arg Number $f end end @@ -256,7 +250,6 @@ Return ``\\sqrt{x}``. Throws `DomainError` for negative `Real` arguments. Use co negative arguments instead. The prefix operator `√` is equivalent to `sqrt`. """ sqrt(x::Real) = sqrt(float(x)) -@vectorize_1arg Number sqrt """ hypot(x, y) @@ -288,7 +281,6 @@ function hypot{T<:Number}(x::T, y::T) return rr end end -@vectorize_2arg Number hypot """ hypot(x...) @@ -308,16 +300,13 @@ atan2{T<:AbstractFloat}(y::T, x::T) = Base.no_op_err("atan2", T) atan2(y::Float64, x::Float64) = ccall((:atan2,libm), Float64, (Float64, Float64,), y, x) atan2(y::Float32, x::Float32) = ccall((:atan2f,libm), Float32, (Float32, Float32), y, x) -@vectorize_2arg Number atan2 max{T<:AbstractFloat}(x::T, y::T) = ifelse((y > x) | (signbit(y) < signbit(x)), ifelse(isnan(y), x, y), ifelse(isnan(x), y, x)) -@vectorize_2arg Real max min{T<:AbstractFloat}(x::T, y::T) = ifelse((y < x) | (signbit(y) > signbit(x)), ifelse(isnan(y), x, y), ifelse(isnan(x), y, x)) -@vectorize_2arg Real min minmax{T<:AbstractFloat}(x::T, y::T) = ifelse(isnan(x-y), ifelse(isnan(x), (y, y), (x, x)), ifelse((y < x) | (signbit(y) > signbit(x)), (y, x), @@ -332,7 +321,6 @@ Compute ``x \\times 2^n``. """ ldexp(x::Float64,e::Integer) = ccall((:scalbn,libm), Float64, (Float64,Int32), x, Int32(e)) ldexp(x::Float32,e::Integer) = ccall((:scalbnf,libm), Float32, (Float32,Int32), x, Int32(e)) -# TODO: vectorize ldexp """ exponent(x) -> Int @@ -353,7 +341,6 @@ function exponent{T<:AbstractFloat}(x::T) end k - exponent_bias(T) end -@vectorize_1arg Real exponent """ significand(x) @@ -386,7 +373,6 @@ function significand{T<:AbstractFloat}(x::T) xu = (xu & ~exponent_mask(T)) | exponent_one(T) reinterpret(T,xu) end -@vectorize_1arg Real significand """ frexp(val) diff --git a/test/bitarray.jl b/test/bitarray.jl index e591173a5b5bb..4f18de7d1ac4a 100644 --- a/test/bitarray.jl +++ b/test/bitarray.jl @@ -1092,8 +1092,8 @@ q[[1,3]] = true @test map(^, p, q) == map((x,y)->x^y, p, q) == p .^ q @test map(*, p, q) == map((x,y)->x*y, p, q) == p .* q -@test map(min, p, q) == map((x,y)->min(x,y), p, q) == min(p, q) -@test map(max, p, q) == map((x,y)->max(x,y), p, q) == max(p, q) +@test map(min, p, q) == map((x,y)->min(x,y), p, q) == min.(p, q) +@test map(max, p, q) == map((x,y)->max(x,y), p, q) == max.(p, q) @test map(<, p, q) == map((x,y)->x<y, p, q) == (p .< q) @test map(<=, p, q) == map((x,y)->x<=y, p, q) == (p .<= q) @@ -1117,8 +1117,8 @@ r = falses(4) @test map!(^, r, p, q) == map!((x,y)->x^y, r, p, q) == p .^ q == r @test map!(*, r, p, q) == map!((x,y)->x*y, r, p, q) == p .* q == r -@test map!(min, r, p, q) == map!((x,y)->min(x,y), r, p, q) == min(p, q) == r -@test map!(max, r, p, q) == map!((x,y)->max(x,y), r, p, q) == max(p, q) == r +@test map!(min, r, p, q) == map!((x,y)->min(x,y), r, p, q) == min.(p, q) == r +@test map!(max, r, p, q) == map!((x,y)->max(x,y), r, p, q) == max.(p, q) == r @test map!(<, r, p, q) == map!((x,y)->x<y, r, p, q) == (p .< q) == r @test map!(<=, r, p, q) == map!((x,y)->x<=y, r, p, q) == (p .<= q) == r diff --git a/test/broadcast.jl b/test/broadcast.jl index bbc2afb6d560c..fc30e1c4fec5a 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -220,7 +220,7 @@ let A = [sqrt(i)+j for i = 1:3, j=1:4] @test atan2.(log.(A), sum(A,1)) == broadcast(atan2, broadcast(log, A), sum(A, 1)) end let x = sin.(1:10) - @test atan2.((x->x+1).(x), (x->x+2).(x)) == atan2(x+1, x+2) == atan2(x.+1, x.+2) + @test atan2.((x->x+1).(x), (x->x+2).(x)) == broadcast(atan2, x+1, x+2) == broadcast(atan2, x.+1, x.+2) @test sin.(atan2.([x+1,x+2]...)) == sin.(atan2.(x+1,x+2)) @test sin.(atan2.(x, 3.7)) == broadcast(x -> sin(atan2(x,3.7)), x) @test atan2.(x, 3.7) == broadcast(x -> atan2(x,3.7), x) == broadcast(atan2, x, 3.7) @@ -235,7 +235,7 @@ end let x = sin.(1:10), a = [x] @test cos.(x) == cos.(a...) @test atan2.(x,x) == atan2.(a..., a...) == atan2.([x, x]...) - @test atan2.(x, cos.(x)) == atan2.(a..., cos.(x)) == atan2(x, cos.(a...)) == atan2(a..., cos.(a...)) + @test atan2.(x, cos.(x)) == atan2.(a..., cos.(x)) == broadcast(atan2, x, cos.(a...)) == broadcast(atan2, a..., cos.(a...)) @test ((args...)->cos(args[1])).(x) == cos.(x) == ((y,args...)->cos(y)).(x) end @test atan2.(3,4) == atan2(3,4) == (() -> atan2(3,4)).() diff --git a/test/fft.jl b/test/fft.jl index 376d7e5e32248..b55ccbe25a1cb 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -254,14 +254,14 @@ function fft_test{T<:Complex}(p::Base.DFT.Plan{T}, ntrials=4, z = zeros(T, n) i = rand(0:n-1) z[i+1] = 1 - X = exp(twopi_i*i) + X = exp.(twopi_i*i) err = norm(p*z - X, Inf) / norm(X, Inf) err <= tol || error("impulse-response error $err in $p") # time-shift: if n > 1 s = rand(1:n-1) - X = (p*x).*exp(twopi_i*s) + X = (p*x).*exp.(twopi_i*s) err = norm(p*circshift(x,s) - X, Inf) / norm(X, Inf) err <= tol || error("time-shift error $err in $p") end diff --git a/test/functional.jl b/test/functional.jl index 7846491f05153..b54e55e486764 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -7,7 +7,7 @@ # TODO: @test map!() much more thoroughly let a = [1.0, 2.0] map!(sin, a) - @test isequal(a, sin([1.0, 2.0])) + @test isequal(a, sin.([1.0, 2.0])) end # map -- ranges.jl @test isequal(map(sqrt, 1:5), [sqrt(i) for i in 1:5]) diff --git a/test/linalg/arnoldi.jl b/test/linalg/arnoldi.jl index 61bdf485ec849..48fc8d5f758ba 100644 --- a/test/linalg/arnoldi.jl +++ b/test/linalg/arnoldi.jl @@ -193,7 +193,7 @@ end debug && println("complex svds") let # complex svds test - A = sparse([1, 1, 2, 3, 4], [2, 1, 1, 3, 1], exp(im*[2.0:2:10;])) + A = sparse([1, 1, 2, 3, 4], [2, 1, 1, 3, 1], exp.(im*[2.0:2:10;])) S1 = svds(A, nsv = 2) S2 = svd(full(A)) diff --git a/test/linalg/lapack.jl b/test/linalg/lapack.jl index 51a47eace3a4a..100bb6409018f 100644 --- a/test/linalg/lapack.jl +++ b/test/linalg/lapack.jl @@ -178,7 +178,7 @@ end #gebal/gebak for elty in (Float32, Float64, Complex64, Complex128) - A = rand(elty,10,10) * Diagonal(exp10(linspace(-10,10,10))) + A = rand(elty,10,10) * Diagonal(exp10.(linspace(-10,10,10))) B = copy(A) ilo, ihi, scale = LAPACK.gebal!('S',B) Bvs = eigvecs(B) diff --git a/test/math.jl b/test/math.jl index a1547b9b7e530..fd37f255d4bb8 100644 --- a/test/math.jl +++ b/test/math.jl @@ -196,11 +196,11 @@ end TAA = rand(2,2) TAA = (TAA + TAA.')/2. STAA = Symmetric(TAA) -@test full(atanh(STAA)) == atanh(TAA) -@test full(asinh(STAA)) == asinh(TAA) -@test full(acosh(STAA+Symmetric(ones(TAA)))) == acosh(TAA+ones(TAA)) -@test full(acsch(STAA+Symmetric(ones(TAA)))) == acsch(TAA+ones(TAA)) -@test full(acoth(STAA+Symmetric(ones(TAA)))) == acoth(TAA+ones(TAA)) +@test full(atanh.(STAA)) == atanh.(TAA) +@test full(asinh.(STAA)) == asinh.(TAA) +@test full(acosh.(STAA+Symmetric(ones(TAA)))) == acosh.(TAA+ones(TAA)) +@test full(acsch.(STAA+Symmetric(ones(TAA)))) == acsch.(TAA+ones(TAA)) +@test full(acoth.(STAA+Symmetric(ones(TAA)))) == acoth.(TAA+ones(TAA)) # check exp2(::Integer) matches exp2(::Float) for ii in -2048:2048 @@ -220,8 +220,8 @@ end for T in (Int, Float64, BigFloat) @test deg2rad(T(180)) ≈ 1pi - @test deg2rad(T[45, 60]) ≈ [pi/T(4), pi/T(3)] - @test rad2deg([pi/T(4), pi/T(3)]) ≈ [45, 60] + @test deg2rad.(T[45, 60]) ≈ [pi/T(4), pi/T(3)] + @test rad2deg.([pi/T(4), pi/T(3)]) ≈ [45, 60] @test rad2deg(T(1)*pi) ≈ 180 @test rad2deg(T(1)) ≈ rad2deg(true) @test deg2rad(T(1)) ≈ deg2rad(true) diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 854712b721d02..458b6fd5ad429 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -349,10 +349,10 @@ e = eye(5) a = [e[:,1], e[:,2], e[:,3], e[:,4], e[:,5]] a1 = zeros(5) c = [ones(Complex{Float64}, 5), - exp(-2*pi*im*(0:4)/5), - exp(-4*pi*im*(0:4)/5), - exp(-6*pi*im*(0:4)/5), - exp(-8*pi*im*(0:4)/5)] + exp.(-2*pi*im*(0:4)/5), + exp.(-4*pi*im*(0:4)/5), + exp.(-6*pi*im*(0:4)/5), + exp.(-8*pi*im*(0:4)/5)] for s = -5:5 for i = 1:5 thisa = OffsetArray(a[i], (s,)) diff --git a/test/operators.jl b/test/operators.jl index 07031ed2ff81e..f42331f1a0076 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -52,6 +52,6 @@ p = 1=>:foo # issue #13144: max() with 4 or more array arguments let xs = [[i:i+4;] for i in 1:10] for n in 2:10 - @test max(xs[1:n]...) == [n:n+4;] + @test max.(xs[1:n]...) == [n:n+4;] end end diff --git a/test/reduce.jl b/test/reduce.jl index 1ca4e4f75867f..4c7135b96b17c 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -55,7 +55,7 @@ fz = float(z) @test sum(sin, [3]) == sin(3.0) a = sum(sin, z) @test a ≈ sum(sin, fz) -@test a ≈ sum(sin(fz)) +@test a ≈ sum(sin.(fz)) z = [-4, -3, 2, 5] fz = float(z) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 98c10af32dd02..df32210f91402 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -377,7 +377,7 @@ for f in (sum, prod, minimum, maximum) end # case where f(0) would throw - @test f(x->sqrt(x-1), pA+1) ≈ f(sqrt(pA)) + @test f(x->sqrt(x-1), pA+1) ≈ f(sqrt.(pA)) # these actually throw due to #10533 # @test f(x->sqrt(x-1), pA+1, 1) ≈ f(sqrt(pA), 1) # @test f(x->sqrt(x-1), pA+1, 2) ≈ f(sqrt(pA), 2) @@ -1486,9 +1486,12 @@ let @test min(A13024, B13024) == sparse([1,2,5], [1,2,5], fill(true,3)) @test typeof(min(A13024, B13024)) == SparseMatrixCSC{Bool,Int} - for op in (+, -, &, |, $, max, min) + for op in (+, -, &, |, $) @test op(A13024, B13024) == op(full(A13024), full(B13024)) end + for op in (max, min) + @test op(A13024, B13024) == op.(full(A13024), full(B13024)) + end end let A = 2. * speye(5,5) diff --git a/test/statistics.jl b/test/statistics.jl index c4f53aa38d7de..6a16cb599dbb6 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -141,8 +141,8 @@ X = [2 3 1 -1; 7 4 5 -4] @test std((1,2,3); mean=0) ≈ sqrt(7.0) @test std((1,2,3); mean=0, corrected=false) ≈ sqrt(14.0/3) -@test std([1 2 3 4 5; 6 7 8 9 10], 2) ≈ sqrt([2.5 2.5]') -@test std([1 2 3 4 5; 6 7 8 9 10], 2; corrected=false) ≈ sqrt([2.0 2.0]') +@test std([1 2 3 4 5; 6 7 8 9 10], 2) ≈ sqrt.([2.5 2.5]') +@test std([1 2 3 4 5; 6 7 8 9 10], 2; corrected=false) ≈ sqrt.([2.0 2.0]') A = Complex128[exp(i*im) for i in 1:10^4] @test varm(A,0.) ≈ sum(map(abs2,A))/(length(A)-1) From 92f8aea38919048c38e1d6f56780323a09dda1c4 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 15:18:31 -0700 Subject: [PATCH 1103/1117] Deprecate vectorized functions in base/floatfuncs.jl in favor of compact broadcast syntax. --- base/deprecated.jl | 3 + base/floatfuncs.jl | 16 ---- base/linalg/arnoldi.jl | 2 +- base/linalg/arpack.jl | 4 +- base/linalg/cholesky.jl | 2 +- base/linalg/dense.jl | 4 +- base/linalg/diagonal.jl | 6 +- base/linalg/generic.jl | 8 +- base/linalg/linalg.jl | 2 +- base/linalg/triangular.jl | 2 +- base/sparse/sparsevector.jl | 5 +- base/sysimg.jl | 5 +- examples/queens.jl | 2 +- test/arrayops.jl | 2 +- test/blas.jl | 8 +- test/linalg/arnoldi.jl | 8 +- test/linalg/bidiag.jl | 2 +- test/linalg/cholesky.jl | 6 +- test/linalg/dense.jl | 12 +-- test/linalg/diagonal.jl | 4 +- test/linalg/givens.jl | 2 +- test/linalg/schur.jl | 2 +- test/linalg/symmetric.jl | 6 +- test/linalg/triangular.jl | 6 +- test/linalg/tridiag.jl | 6 +- test/numbers.jl | 12 +-- test/reduce.jl | 8 +- test/reducedim.jl | 8 +- test/sorting.jl | 2 +- test/sparsedir/sparse.jl | 138 ++++++++++++++++++--------------- test/sparsedir/sparsevector.jl | 4 +- 31 files changed, 149 insertions(+), 148 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 1b9b7ec1f40b8..d14110821d35e 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -889,6 +889,7 @@ for f in ( :erfcx, :erfi, :dawson, # base/special/erf.jl :airyprime, :airyai, :airyaiprime, :airybi, :airybiprime, :airy, :airyx, :besselj0, :besselj1, :bessely0, :bessely1, # base/special/bessel.jl :cbrt, :sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :exp2, :expm1, :exp10, :sin, :cos, :tan, :asin, :acos, :acosh, :atanh, #=:log,=# :log2, :log10, :lgamma, #=:log1p,=# :sqrt, # base/math.jl + :abs, :abs2, :angle, :isnan, :isinf, :isfinite, # base/floatfuncs.jl ) @eval @dep_vectorize_1arg Number $f end @@ -897,6 +898,7 @@ for f in ( :invdigamma, # base/special/gamma.jl :erfinc, :erfcinv, # base/special/erf.jl :rad2deg, :deg2rad, :exponent, :significand, # base/math.jl + :trunc, :floor, :ceil, :round, # base/floatfuncs.jl ) @eval @dep_vectorize_1arg Real $f end @@ -911,6 +913,7 @@ for f in ( end for f in ( :max, :min, # base/math.jl + :copysign, :flipsign, # base/floatfuncs.jl ) @eval @dep_vectorize_2arg Real $f end diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index e6a02b176759b..c82d3b2dc169a 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -6,13 +6,11 @@ copysign(x::Float64, y::Float64) = box(Float64,copysign_float(unbox(Float64,x),u copysign(x::Float32, y::Float32) = box(Float32,copysign_float(unbox(Float32,x),unbox(Float32,y))) copysign(x::Float32, y::Real) = copysign(x, Float32(y)) copysign(x::Float64, y::Real) = copysign(x, Float64(y)) -@vectorize_2arg Real copysign flipsign(x::Float64, y::Float64) = box(Float64,xor_int(unbox(Float64,x),and_int(unbox(Float64,y),0x8000000000000000))) flipsign(x::Float32, y::Float32) = box(Float32,xor_int(unbox(Float32,x),and_int(unbox(Float32,y),0x80000000))) flipsign(x::Float32, y::Real) = flipsign(x, Float32(y)) flipsign(x::Float64, y::Real) = flipsign(x, Float64(y)) -@vectorize_2arg Real flipsign signbit(x::Float64) = signbit(reinterpret(Int64,x)) signbit(x::Float32) = signbit(reinterpret(Int32,x)) @@ -40,14 +38,6 @@ function hex2num(s::AbstractString) return box(Float64,unbox(UInt64,parse(UInt64,s,16))) end -@vectorize_1arg Number abs -@vectorize_1arg Number abs2 -@vectorize_1arg Number angle - -@vectorize_1arg Number isnan -@vectorize_1arg Number isinf -@vectorize_1arg Number isfinite - """ round([T,] x, [digits, [base]], [r::RoundingMode]) @@ -107,7 +97,6 @@ julia> round(pi, 3, 2) ``` """ round(T::Type, x) - round(x::Real, ::RoundingMode{:ToZero}) = trunc(x) round(x::Real, ::RoundingMode{:Up}) = ceil(x) round(x::Real, ::RoundingMode{:Down}) = floor(x) @@ -123,11 +112,6 @@ function round(x::AbstractFloat, ::RoundingMode{:NearestTiesUp}) end round{T<:Integer}(::Type{T}, x::AbstractFloat, r::RoundingMode) = trunc(T,round(x,r)) -@vectorize_1arg Real trunc -@vectorize_1arg Real floor -@vectorize_1arg Real ceil -@vectorize_1arg Real round - for f in (:trunc,:floor,:ceil,:round) @eval begin function ($f){T,R}(::Type{T}, x::AbstractArray{R,1}) diff --git a/base/linalg/arnoldi.jl b/base/linalg/arnoldi.jl index 7b1e9bd6036bd..b84b6f921fa60 100644 --- a/base/linalg/arnoldi.jl +++ b/base/linalg/arnoldi.jl @@ -382,7 +382,7 @@ function _svds(X; nsv::Int = 6, ritzvec::Bool = true, tol::Float64 = 0.0, maxite end ex = eigs(SVDOperator(X), I; ritzvec = ritzvec, nev = ncv, tol = tol, maxiter = maxiter, v0=padv0) ind = [1:2:ncv;] - sval = abs(ex[1][ind]) + sval = abs.(ex[1][ind]) if ritzvec # calculating singular vectors diff --git a/base/linalg/arpack.jl b/base/linalg/arpack.jl index 26a0c55e1dbb6..a76a781d670a6 100644 --- a/base/linalg/arpack.jl +++ b/base/linalg/arpack.jl @@ -114,9 +114,9 @@ function eupd_wrapper(T, n::Integer, sym::Bool, cmplx::Bool, bmat::String, select = Array{BlasInt}(ncv) info = zeros(BlasInt, 1) - dmap = abs + dmap = x->abs.(x) if iparam[7] == 3 # shift-and-invert - dmap = x->abs(1./(x-sigma)) + dmap = x->abs.(1./(x-sigma)) elseif which == "LR" || which == "LA" || which == "BE" dmap = real elseif which == "SR" || which == "SA" diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index 46e7ac367514b..2406c7e9d8009 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -426,7 +426,7 @@ function det(C::Cholesky) dd end -det(C::CholeskyPivoted) = C.rank < size(C.factors, 1) ? real(zero(eltype(C))) : prod(abs2(diag(C.factors))) +det(C::CholeskyPivoted) = C.rank < size(C.factors, 1) ? real(zero(eltype(C))) : prod(abs2.(diag(C.factors))) function logdet(C::Cholesky) dd = zero(eltype(C)) diff --git a/base/linalg/dense.jl b/base/linalg/dense.jl index 1c5420fe60ad5..e199d426c8522 100644 --- a/base/linalg/dense.jl +++ b/base/linalg/dense.jl @@ -639,7 +639,7 @@ function pinv{T}(A::StridedMatrix{T}, tol::Real) end if istril(A) if istriu(A) - maxabsA = maximum(abs(diag(A))) + maxabsA = maximum(abs.(diag(A))) B = zeros(Tout, n, m) for i = 1:min(m, n) if abs(A[i,i]) > tol*maxabsA @@ -657,7 +657,7 @@ function pinv{T}(A::StridedMatrix{T}, tol::Real) Sinv = zeros(Stype, length(SVD.S)) index = SVD.S .> tol*maximum(SVD.S) Sinv[index] = one(Stype) ./ SVD.S[index] - Sinv[find(!isfinite(Sinv))] = zero(Stype) + Sinv[find(!isfinite.(Sinv))] = zero(Stype) return SVD.Vt' * (Diagonal(Sinv) * SVD.U') end function pinv{T}(A::StridedMatrix{T}) diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index c1afa9343b751..137d44f7a3cce 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -293,7 +293,7 @@ function pinv{T}(D::Diagonal{T}) end function pinv{T}(D::Diagonal{T}, tol::Real) Di = similar(D.diag, typeof(inv(zero(T)))) - if( !isempty(D.diag) ) maxabsD = maximum(abs(D.diag)) end + if( !isempty(D.diag) ) maxabsD = maximum(abs.(D.diag)) end for i = 1:length(D.diag) if( abs(D.diag[i]) > tol*maxabsD && isfinite(inv(D.diag[i])) ) Di[i]=inv(D.diag[i]) @@ -311,10 +311,10 @@ eigvecs(D::Diagonal) = eye(D) eigfact(D::Diagonal) = Eigen(eigvals(D), eigvecs(D)) #Singular system -svdvals{T<:Number}(D::Diagonal{T}) = sort(abs(D.diag), rev = true) +svdvals{T<:Number}(D::Diagonal{T}) = sort(abs.(D.diag), rev = true) svdvals(D::Diagonal) = [svdvals(v) for v in D.diag] function svd{T<:Number}(D::Diagonal{T}) - S = abs(D.diag) + S = abs.(D.diag) piv = sortperm(S, rev = true) U = full(Diagonal(D.diag ./ S)) Up = hcat([U[:,i] for i = 1:length(D.diag)][piv]...) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 0b226ad149878..18438a18e5219 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -657,8 +657,8 @@ cond(x::Number) = x == 0 ? Inf : 1.0 cond(x::Number, p) = cond(x) #Skeel condition numbers -condskeel(A::AbstractMatrix, p::Real=Inf) = norm(abs(inv(A))*abs(A), p) -condskeel{T<:Integer}(A::AbstractMatrix{T}, p::Real=Inf) = norm(abs(inv(float(A)))*abs(A), p) +condskeel(A::AbstractMatrix, p::Real=Inf) = norm(abs.(inv(A))*abs.(A), p) +condskeel{T<:Integer}(A::AbstractMatrix{T}, p::Real=Inf) = norm(abs.(inv(float(A)))*abs.(A), p) """ condskeel(M, [x, p::Real=Inf]) @@ -675,8 +675,8 @@ vector `x`, as computed using the operator `p`-norm. This quantity is also known in the literature as the Bauer condition number, relative condition number, or componentwise relative condition number. """ -condskeel(A::AbstractMatrix, x::AbstractVector, p::Real=Inf) = norm(abs(inv(A))*abs(A)*abs(x), p) -condskeel{T<:Integer}(A::AbstractMatrix{T}, x::AbstractVector, p::Real=Inf) = norm(abs(inv(float(A)))*abs(A)*abs(x), p) +condskeel(A::AbstractMatrix, x::AbstractVector, p::Real=Inf) = norm(abs.(inv(A))*abs.(A)*abs.(x), p) +condskeel{T<:Integer}(A::AbstractMatrix{T}, x::AbstractVector, p::Real=Inf) = norm(abs.(inv(float(A)))*abs.(A)*abs.(x), p) """ issymmetric(A) -> Bool diff --git a/base/linalg/linalg.jl b/base/linalg/linalg.jl index 556020776082a..f06964f03b378 100644 --- a/base/linalg/linalg.jl +++ b/base/linalg/linalg.jl @@ -9,7 +9,7 @@ import Base: USE_BLAS64, abs, big, ceil, conj, convert, copy, copy!, copy_transp ctranspose, ctranspose!, eltype, eye, findmax, findmin, fill!, floor, full, getindex, imag, inv, isapprox, kron, ndims, parent, power_by_squaring, print_matrix, promote_rule, real, round, setindex!, show, similar, size, transpose, transpose!, - trunc + trunc, broadcast using Base: promote_op, _length # We use `_length` because of non-1 indices; releases after julia 0.5 # can go back to `length`. `_length(A)` is equivalent to `length(linearindices(A))`. diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index 5dd80ac8f8cd5..d7324ad1ce27d 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -36,7 +36,7 @@ for t in (:LowerTriangular, :UnitLowerTriangular, :UpperTriangular, real{T<:Real}(A::$t{T}) = A real{T<:Complex}(A::$t{T}) = (B = real(A.data); $t(B)) - abs(A::$t) = $t(abs(A.data)) + broadcast(::typeof(abs), A::$t) = $t(abs.(A.data)) end end diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index 781df3775fef6..898e47070f600 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -886,12 +886,13 @@ end ### Unary Map # zero-preserving functions (z->z, nz->nz) -for op in [:-, :abs, :abs2, :conj] +for op in [:abs, :abs2, :conj] @eval begin $(op)(x::AbstractSparseVector) = - SparseVector(length(x), copy(nonzeroinds(x)), $(op)(nonzeros(x))) + SparseVector(length(x), copy(nonzeroinds(x)), $(op).(nonzeros(x))) end end +-(x::AbstractSparseVector) = SparseVector(length(x), copy(nonzeroinds(x)), -(nonzeros(x))) # functions f, such that # f(x) can be zero or non-zero when x != 0 diff --git a/base/sysimg.jl b/base/sysimg.jl index 7005898b07050..755dc3393d3fc 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -294,13 +294,14 @@ include("client.jl") # misc useful functions & macros include("util.jl") +include("broadcast.jl") +importall .Broadcast + # dense linear algebra include("linalg/linalg.jl") importall .LinAlg const ⋅ = dot const × = cross -include("broadcast.jl") -importall .Broadcast # statistics include("statistics.jl") diff --git a/examples/queens.jl b/examples/queens.jl index 49ab5e1ddb0ee..b53308c502ddc 100644 --- a/examples/queens.jl +++ b/examples/queens.jl @@ -3,7 +3,7 @@ addqueen(queens::Array{Vector{Int}}, queen::Vector{Int}) = push!(copy(queens), queen) hitsany(queen::Vector{Int}, queens::Array{Vector{Int}}) = any(x->hits(queen, x), queens) -hits(a::Array{Int}, b::Array{Int}) = any(a .== b) || abs(a-b)[1] == abs(a-b)[2] +hits(a::Array{Int}, b::Array{Int}) = any(a .== b) || abs.(a-b)[1] == abs.(a-b)[2] function solve(x, y, n, d=Array{Vector{Int}}(0)) if n == 0 diff --git a/test/arrayops.jl b/test/arrayops.jl index 24a1e1b730d7f..05f23a06a76d5 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -272,7 +272,7 @@ let @test eltype(X) == Float32 @test Base.elsize(X) == sizeof(Float32) @test !isinteger(X) - @test isnan(X) == [trues(6);falses(5)] + @test isnan.(X) == [trues(6);falses(5)] @test X[7:11] == [1:5;] X = get(A, (2:4, 9:-2:-13), 0) Xv = zeros(Int, 3, 12) diff --git a/test/blas.jl b/test/blas.jl index 8b8cfeec7e25b..3f6b87821da98 100644 --- a/test/blas.jl +++ b/test/blas.jl @@ -69,7 +69,7 @@ for elty in [Float32, Float64, Complex64, Complex128] #iamax if elty <: Real x = convert(Vector{elty}, randn(n)) - @test BLAS.iamax(x) == indmax(abs(x)) + @test BLAS.iamax(x) == indmax(abs.(x)) else z = convert(Vector{elty}, complex(randn(n),randn(n))) @test BLAS.iamax(z) == indmax(map(x -> abs(real(x)) + abs(imag(x)), z)) @@ -103,10 +103,10 @@ for elty in [Float32, Float64, Complex64, Complex128] b = view(a,2:2:n,1) @test BLAS.nrm2(b) ≈ norm(b) if elty <: Real - @test BLAS.asum(b) ≈ sum(abs(b)) - @test BLAS.iamax(b) ≈ indmax(abs(b)) + @test BLAS.asum(b) ≈ sum(abs.(b)) + @test BLAS.iamax(b) ≈ indmax(abs.(b)) else - @test BLAS.asum(b) ≈ sum(abs(real(b))) + sum(abs(imag(b))) + @test BLAS.asum(b) ≈ sum(abs.(real(b))) + sum(abs.(imag(b))) @test BLAS.iamax(b) == indmax(map(x -> abs(real(x)) + abs(imag(x)), b)) end diff --git a/test/linalg/arnoldi.jl b/test/linalg/arnoldi.jl index 48fc8d5f758ba..e60f6d01dfe8f 100644 --- a/test/linalg/arnoldi.jl +++ b/test/linalg/arnoldi.jl @@ -201,13 +201,13 @@ let # complex svds test @test S1[1][:S] ≈ S2[2][1:2] ## left singular vectors - s1_left = abs(S1[1][:U][:,1:2]) - s2_left = abs(S2[1][:,1:2]) + s1_left = abs.(S1[1][:U][:,1:2]) + s2_left = abs.(S2[1][:,1:2]) @test s1_left ≈ s2_left ## right singular vectors - s1_right = abs(S1[1][:Vt][:,1:2]) - s2_right = abs(S2[3][:,1:2]) + s1_right = abs.(S1[1][:Vt][:,1:2]) + s2_right = abs.(S2[3][:,1:2]) @test s1_right ≈ s2_right ## test passing guess for Krylov vectors diff --git a/test/linalg/bidiag.jl b/test/linalg/bidiag.jl index d4d7f397f555f..be74d5fa544b3 100644 --- a/test/linalg/bidiag.jl +++ b/test/linalg/bidiag.jl @@ -65,7 +65,7 @@ for relty in (Int, Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test full(T) == diagm(dv) + diagm(ev, isupper?1:-1) @test Bidiagonal(full(T), isupper) == T @test big(T) == T - @test full(abs(T)) == abs(diagm(dv)) + abs(diagm(ev, isupper?1:-1)) + @test full(abs.(T)) == abs.(diagm(dv)) + abs.(diagm(ev, isupper?1:-1)) @test full(real(T)) == real(diagm(dv)) + real(diagm(ev, isupper?1:-1)) @test full(imag(T)) == imag(diagm(dv)) + imag(diagm(ev, isupper?1:-1)) z = zeros(elty, n) diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index bd54fd05101de..541012343232b 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -46,11 +46,11 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) #these tests were failing on 64-bit linux when inside the inner loop #for eltya = Complex64 and eltyb = Int. The E[i,j] had NaN32 elements #but only with srand(1234321) set before the loops. - E = abs(apd - r'*r) + E = abs.(apd - r'*r) for i=1:n, j=1:n @test E[i,j] <= (n+1)ε/(1-(n+1)ε)*real(sqrt(apd[i,i]*apd[j,j])) end - E = abs(apd - full(capd)) + E = abs.(apd - full(capd)) for i=1:n, j=1:n @test E[i,j] <= (n+1)ε/(1-(n+1)ε)*real(sqrt(apd[i,i]*apd[j,j])) end @@ -230,7 +230,7 @@ let apd = [5.8525753f0 + 0.0f0im -0.79540455f0 + 0.7066077f0im 0.98274714f0 + 1. -1.0568488936791578 - 0.06025820467086475im 0.12696236014017806 - 0.09853584666755086im] cholfact(apd, :L, Val{true}) \ b r = factorize(apd)[:U] - E = abs(apd - r'*r) + E = abs.(apd - r'*r) ε = eps(abs(float(one(Complex64)))) n = 10 for i=1:n, j=1:n diff --git a/test/linalg/dense.jl b/test/linalg/dense.jl index 1f7ae605fd773..b186c172f47e3 100644 --- a/test/linalg/dense.jl +++ b/test/linalg/dense.jl @@ -234,13 +234,13 @@ for elty in (Float32, Float64, BigFloat, Complex{Float32}, Complex{Float64}, Com @test norm(xs + ys,Inf) <= norm(xs,Inf) + norm(ys,Inf) # Against vectorized versions - @test norm(x,-Inf) ≈ minimum(abs(x)) - @test norm(x,-1) ≈ inv(sum(1./abs(x))) + @test norm(x,-Inf) ≈ minimum(abs.(x)) + @test norm(x,-1) ≈ inv(sum(1./abs.(x))) @test norm(x,0) ≈ sum(x .!= 0) - @test norm(x,1) ≈ sum(abs(x)) - @test norm(x) ≈ sqrt(sum(abs2(x))) - @test norm(x,3) ≈ cbrt(sum(abs(x).^3.)) - @test norm(x,Inf) ≈ maximum(abs(x)) + @test norm(x,1) ≈ sum(abs.(x)) + @test norm(x) ≈ sqrt(sum(abs2.(x))) + @test norm(x,3) ≈ cbrt(sum(abs.(x).^3.)) + @test norm(x,Inf) ≈ maximum(abs.(x)) end ## Matrix (Operator) A = ones(elty,10,10) diff --git a/test/linalg/diagonal.jl b/test/linalg/diagonal.jl index 6d87b0f97b3cb..6e056c0ce83f6 100644 --- a/test/linalg/diagonal.jl +++ b/test/linalg/diagonal.jl @@ -28,7 +28,7 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test typeof(convert(AbstractMatrix{Complex64},D)) == Diagonal{Complex64} @test full(real(D)) == real(DM) - @test full(abs(D)) == abs(DM) + @test full(abs.(D)) == abs.(DM) @test full(imag(D)) == imag(DM) @test parent(D) == d @@ -55,7 +55,7 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) for func in (expm,) @test_approx_eq_eps func(D) func(DM) n^3*eps(relty) end - @test_approx_eq_eps logm(Diagonal(abs(D.diag))) logm(abs(DM)) n^3*eps(relty) + @test_approx_eq_eps logm(Diagonal(abs.(D.diag))) logm(abs.(DM)) n^3*eps(relty) end if elty <: BlasComplex for func in (logdet, sqrtm) diff --git a/test/linalg/givens.jl b/test/linalg/givens.jl index 91eeb414bc016..864d58a358a83 100644 --- a/test/linalg/givens.jl +++ b/test/linalg/givens.jl @@ -44,7 +44,7 @@ for elty in (Float32, Float64, Complex64, Complex128) G, _ = givens(one(elty),zero(elty),11,12) @test_throws DimensionMismatch A_mul_B!(G, A) @test_throws DimensionMismatch A_mul_Bc!(A,G) - @test abs(A) ≈ abs(hessfact(Ac)[:H]) + @test abs.(A) ≈ abs.(hessfact(Ac)[:H]) @test norm(R*eye(elty, 10)) ≈ one(elty) G, _ = givens(one(elty),zero(elty),9,10) diff --git a/test/linalg/schur.jl b/test/linalg/schur.jl index 5370afd3f82f5..984f02d1a8660 100644 --- a/test/linalg/schur.jl +++ b/test/linalg/schur.jl @@ -73,7 +73,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, Int) debug && println("Reorder Generalized Schur") NS = schurfact(a1_sf, a2_sf) # Currently just testing with selecting gen eig values < 1 - select = abs2(NS[:values]) .< 1 + select = abs2.(NS[:values]) .< 1 m = sum(select) S = ordschur(NS, select) # Make sure that the new factorization stil factors matrix diff --git a/test/linalg/symmetric.jl b/test/linalg/symmetric.jl index 55f815f500fcb..452637898f361 100644 --- a/test/linalg/symmetric.jl +++ b/test/linalg/symmetric.jl @@ -105,16 +105,16 @@ let n=10 @test asym*v[:,1] ≈ d[1]*v[:,1] @test v*Diagonal(d)*v' ≈ asym @test isequal(eigvals(asym[1]), eigvals(asym[1:1,1:1])) - @test abs(eigfact(Hermitian(asym), 1:2)[:vectors]'v[:,1:2]) ≈ eye(eltya, 2) + @test abs.(eigfact(Hermitian(asym), 1:2)[:vectors]'v[:,1:2]) ≈ eye(eltya, 2) eig(Hermitian(asym), 1:2) # same result, but checks that method works - @test abs(eigfact(Hermitian(asym), d[1] - 1, (d[2] + d[3])/2)[:vectors]'v[:,1:2]) ≈ eye(eltya, 2) + @test abs.(eigfact(Hermitian(asym), d[1] - 1, (d[2] + d[3])/2)[:vectors]'v[:,1:2]) ≈ eye(eltya, 2) eig(Hermitian(asym), d[1] - 1, (d[2] + d[3])/2) # same result, but checks that method works @test eigvals(Hermitian(asym), 1:2) ≈ d[1:2] @test eigvals(Hermitian(asym), d[1] - 1, (d[2] + d[3])/2) ≈ d[1:2] @test full(eigfact(asym)) ≈ asym # relation to svdvals - @test sum(sort(abs(eigvals(Hermitian(asym))))) == sum(sort(svdvals(Hermitian(asym)))) + @test sum(sort(abs.(eigvals(Hermitian(asym))))) == sum(sort(svdvals(Hermitian(asym)))) # cond @test cond(Hermitian(asym)) ≈ cond(asym) diff --git a/test/linalg/triangular.jl b/test/linalg/triangular.jl index 4c570babc80e9..3cb032ed9aa35 100644 --- a/test/linalg/triangular.jl +++ b/test/linalg/triangular.jl @@ -144,7 +144,7 @@ for elty1 in (Float32, Float64, BigFloat, Complex64, Complex128, Complex{BigFloa # real @test full(real(A1)) == real(full(A1)) @test full(imag(A1)) == imag(full(A1)) - @test full(abs(A1)) == abs(full(A1)) + @test full(abs.(A1)) == abs.(full(A1)) # Unary operations @test full(-A1) == -full(A1) @@ -426,7 +426,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) debug && println("Test backward error [JIN 5705]") for i = 1:size(b, 2) - @test norm(abs(b[:,i] - Atri*x[:,i]), Inf) <= γ * norm(Atri, Inf) * norm(x[:,i], Inf) + @test norm(abs.(b[:,i] - Atri*x[:,i]), Inf) <= γ * norm(Atri, Inf) * norm(x[:,i], Inf) end debug && println("Solve lower triangular system") @@ -455,7 +455,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) debug && println("Test backward error [JIN 5705]") for i = 1:size(b, 2) - @test norm(abs(b[:,i] - Atri*x[:,i]), Inf) <= γ * norm(Atri, Inf) * norm(x[:,i], Inf) + @test norm(abs.(b[:,i] - Atri*x[:,i]), Inf) <= γ * norm(Atri, Inf) * norm(x[:,i], Inf) end end end diff --git a/test/linalg/tridiag.jl b/test/linalg/tridiag.jl index a5e7b8599df0c..f7152d3ae09af 100644 --- a/test/linalg/tridiag.jl +++ b/test/linalg/tridiag.jl @@ -49,10 +49,10 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) @test transpose(T) == Tridiagonal(du, d, dl) @test ctranspose(T) == Tridiagonal(conj(du), conj(d), conj(dl)) - @test abs(T) == Tridiagonal(abs(dl),abs(d),abs(du)) + @test abs.(T) == Tridiagonal(abs.(dl),abs.(d),abs.(du)) @test real(T) == Tridiagonal(real(dl),real(d),real(du)) @test imag(T) == Tridiagonal(imag(dl),imag(d),imag(du)) - @test abs(Ts) == SymTridiagonal(abs(d),abs(dl)) + @test abs.(Ts) == SymTridiagonal(abs.(d),abs.(dl)) @test real(Ts) == SymTridiagonal(real(d),real(dl)) @test imag(Ts) == SymTridiagonal(imag(d),imag(dl)) @@ -135,7 +135,7 @@ for elty in (Float32, Float64, Complex64, Complex128, Int) @inferred eig(Ts, 1.0, 2.0) D, Vecs = eig(Fs) @test DT ≈ D - @test abs(VT'Vecs) ≈ eye(elty, n) + @test abs.(VT'Vecs) ≈ eye(elty, n) @test eigvecs(Ts) == eigvecs(Fs) #call to LAPACK.stein here Test.test_approx_eq_modphase(eigvecs(Ts,eigvals(Ts)),eigvecs(Fs)) diff --git a/test/numbers.jl b/test/numbers.jl index 2094122fa4f53..4eef314f45bda 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -1707,12 +1707,12 @@ let ≈(x,y) = x==y && typeof(x)==typeof(y) for n in [0,3,255,256] r = (1:n)-div(n,2) y = t[x/4 for x in r] - @test trunc(y) ≈ t[div(i,4) for i in r] - @test floor(y) ≈ t[i>>2 for i in r] - @test ceil(y) ≈ t[(i+3)>>2 for i in r] - @test round(y) ≈ t[(i+1+isodd(i>>2))>>2 for i in r] - @test round(y,RoundNearestTiesAway) ≈ t[(i+1+(i>=0))>>2 for i in r] - @test round(y,RoundNearestTiesUp) ≈ t[(i+2)>>2 for i in r] + @test trunc.(y) ≈ t[div(i,4) for i in r] + @test floor.(y) ≈ t[i>>2 for i in r] + @test ceil.(y) ≈ t[(i+3)>>2 for i in r] + @test round.(y) ≈ t[(i+1+isodd(i>>2))>>2 for i in r] + @test broadcast(x -> round(x, RoundNearestTiesAway), y) ≈ t[(i+1+(i>=0))>>2 for i in r] + @test broadcast(x -> round(x, RoundNearestTiesUp), y) ≈ t[(i+2)>>2 for i in r] end end end diff --git a/test/reduce.jl b/test/reduce.jl index 4c7135b96b17c..9f0e91dfaf2d3 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -65,15 +65,15 @@ b = complex(randn(32), randn(32)) @test sumabs([Int8(-2)]) === Int32(2) @test sumabs(z) === 14 @test sumabs(fz) === 14.0 -@test sumabs(a) ≈ sum(abs(a)) -@test sumabs(b) ≈ sum(abs(b)) +@test sumabs(a) ≈ sum(abs.(a)) +@test sumabs(b) ≈ sum(abs.(b)) @test sumabs2(Float64[]) === 0.0 @test sumabs2([Int8(-2)]) === Int32(4) @test sumabs2(z) === 54 @test sumabs2(fz) === 54.0 -@test sumabs2(a) ≈ sum(abs2(a)) -@test sumabs2(b) ≈ sum(abs2(b)) +@test sumabs2(a) ≈ sum(abs2.(a)) +@test sumabs2(b) ≈ sum(abs2.(b)) # check variants of summation for type-stability and other issues (#6069) sum2(itr) = invoke(sum, Tuple{Any}, itr) diff --git a/test/reducedim.jl b/test/reducedim.jl index eb3c218dd00ef..dee4785878e05 100644 --- a/test/reducedim.jl +++ b/test/reducedim.jl @@ -10,10 +10,10 @@ safe_sum{T}(A::Array{T}, region) = safe_mapslices(sum, A, region) safe_prod{T}(A::Array{T}, region) = safe_mapslices(prod, A, region) safe_maximum{T}(A::Array{T}, region) = safe_mapslices(maximum, A, region) safe_minimum{T}(A::Array{T}, region) = safe_mapslices(minimum, A, region) -safe_sumabs{T}(A::Array{T}, region) = safe_mapslices(sum, abs(A), region) -safe_sumabs2{T}(A::Array{T}, region) = safe_mapslices(sum, abs2(A), region) -safe_maxabs{T}(A::Array{T}, region) = safe_mapslices(maximum, abs(A), region) -safe_minabs{T}(A::Array{T}, region) = safe_mapslices(minimum, abs(A), region) +safe_sumabs{T}(A::Array{T}, region) = safe_mapslices(sum, abs.(A), region) +safe_sumabs2{T}(A::Array{T}, region) = safe_mapslices(sum, abs2.(A), region) +safe_maxabs{T}(A::Array{T}, region) = safe_mapslices(maximum, abs.(A), region) +safe_minabs{T}(A::Array{T}, region) = safe_mapslices(minimum, abs.(A), region) Areduc = rand(3, 4, 5, 6) for region in Any[ diff --git a/test/sorting.jl b/test/sorting.jl index 1a40b9223ab51..13a666e79a59e 100644 --- a/test/sorting.jl +++ b/test/sorting.jl @@ -275,7 +275,7 @@ for n in [0:10; 100; 101; 1000; 1001] # test float sorting with NaNs s = sort(v, alg=alg, rev=rev) @test issorted(s, rev=rev) - @test reinterpret(UInt64,v[isnan(v)]) == reinterpret(UInt64,s[isnan(s)]) + @test reinterpret(UInt64,v[isnan.(v)]) == reinterpret(UInt64,s[isnan.(s)]) # test float permutation with NaNs p = sortperm(v, alg=alg, rev=rev) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index df32210f91402..0f55688106e20 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -97,7 +97,7 @@ end for i = 1:5 a = sprand(10, 5, 0.5) b = rand(5) - @test maximum(abs(a*b - full(a)*b)) < 100*eps() + @test maximum(abs.(a*b - full(a)*b)) < 100*eps() end # sparse matrix * BitArray @@ -117,91 +117,91 @@ for i = 1:5 d = randn(5) + im*randn(5) α = rand(Complex128) β = rand(Complex128) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(A_mul_B!(similar(b), a, b) - full(a)*b)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. - @test (maximum(abs(A_mul_B!(similar(c), a, c) - full(a)*c)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) - @test (maximum(abs((a'*c + d) - (full(a)'*c + d))) < 1000*eps()) - @test (maximum(abs((α*a.'*c + β*d) - (α*full(a).'*c + β*d))) < 1000*eps()) - @test (maximum(abs((a.'*c + d) - (full(a).'*c + d))) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(A_mul_B!(similar(b), a, b) - full(a)*b)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. + @test (maximum(abs.(A_mul_B!(similar(c), a, c) - full(a)*c)) < 100*eps()) # for compatibility with present matmul API. Should go away eventually. + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.((a'*c + d) - (full(a)'*c + d))) < 1000*eps()) + @test (maximum(abs.((α*a.'*c + β*d) - (α*full(a).'*c + β*d))) < 1000*eps()) + @test (maximum(abs.((a.'*c + d) - (full(a).'*c + d))) < 1000*eps()) c = randn(6) + im*randn(6) @test_throws DimensionMismatch α*a.'*c + β*c @test_throws DimensionMismatch α*a.'*ones(5) + β*c a = speye(5) + 0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2) b = randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + tril(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + tril(0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2)) b = randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2) + 0.1*im*sprandn(5, 5, 0.2)) b = randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) a = speye(5) + triu(0.1*sprandn(5, 5, 0.2)) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) a = spdiagm(randn(5)) + im*spdiagm(randn(5)) b = randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) b = randn(5,3) + im*randn(5,3) - @test (maximum(abs(a*b - full(a)*b)) < 100*eps()) - @test (maximum(abs(a'b - full(a)'b)) < 100*eps()) - @test (maximum(abs(a.'b - full(a).'b)) < 100*eps()) - @test (maximum(abs(a\b - full(a)\b)) < 1000*eps()) - @test (maximum(abs(a'\b - full(a')\b)) < 1000*eps()) - @test (maximum(abs(a.'\b - full(a.')\b)) < 1000*eps()) + @test (maximum(abs.(a*b - full(a)*b)) < 100*eps()) + @test (maximum(abs.(a'b - full(a)'b)) < 100*eps()) + @test (maximum(abs.(a.'b - full(a).'b)) < 100*eps()) + @test (maximum(abs.(a\b - full(a)\b)) < 1000*eps()) + @test (maximum(abs.(a'\b - full(a')\b)) < 1000*eps()) + @test (maximum(abs.(a.'\b - full(a.')\b)) < 1000*eps()) end end @@ -209,9 +209,9 @@ end for i = 1:5 a = sprand(10, 5, 0.7) b = sprand(5, 15, 0.3) - @test maximum(abs(a*b - full(a)*full(b))) < 100*eps() - @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:sortcols) - full(a)*full(b))) < 100*eps() - @test maximum(abs(Base.SparseArrays.spmatmul(a,b,sortindices=:doubletranspose) - full(a)*full(b))) < 100*eps() + @test maximum(abs.(a*b - full(a)*full(b))) < 100*eps() + @test maximum(abs.(Base.SparseArrays.spmatmul(a,b,sortindices=:sortcols) - full(a)*full(b))) < 100*eps() + @test maximum(abs.(Base.SparseArrays.spmatmul(a,b,sortindices=:doubletranspose) - full(a)*full(b))) < 100*eps() @test full(kron(a,b)) == kron(full(a), full(b)) @test full(kron(full(a),b)) == kron(full(a), full(b)) @test full(kron(a,full(b))) == kron(full(a), full(b)) @@ -1063,6 +1063,18 @@ end x = speye(100) @test_throws BoundsError x[-10:10] +for T in (Int, Float16, Float32, Float64, BigInt, BigFloat) + let R=rand(T[1:100;],2,2), I=rand(T[1:100;],2,2) + D = R + I*im + S = sparse(D) + @test R == real.(S) + @test I == imag.(S) + @test real.(sparse(R)) == R + @test nnz(imag.(sparse(R))) == 0 + @test abs.(S) == abs.(D) + @test abs2.(S) == abs2.(D) + end +end # issue #10407 @test maximum(spzeros(5, 5)) == 0.0 diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index 22d64cebd1a33..0aaf4975a29fe 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -559,8 +559,8 @@ let x = spv_x1, x2 = x2 = spv_x2 @test exact_equal(-x, SparseVector(8, [2, 5, 6], [-1.25, 0.75, -3.5])) # abs and abs2 - @test exact_equal(abs(x), SparseVector(8, [2, 5, 6], abs([1.25, -0.75, 3.5]))) - @test exact_equal(abs2(x), SparseVector(8, [2, 5, 6], abs2([1.25, -0.75, 3.5]))) + @test exact_equal(abs(x), SparseVector(8, [2, 5, 6], abs.([1.25, -0.75, 3.5]))) + @test exact_equal(abs2(x), SparseVector(8, [2, 5, 6], abs2.([1.25, -0.75, 3.5]))) # plus and minus xa = SparseVector(8, [1,2,5,6,7], [3.25,5.25,-0.75,-2.0,-6.0]) From b65a894f1ce4e9d1a0409a470a4ed639ee669e54 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 15:25:32 -0700 Subject: [PATCH 1104/1117] Deprecate vectorized functions in base/fastmath.jl in favor of compact broadcast syntax. --- base/deprecated.jl | 2 ++ base/fastmath.jl | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index d14110821d35e..d2ff41fab0305 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -890,6 +890,7 @@ for f in ( :airyprime, :airyai, :airyaiprime, :airybi, :airybiprime, :airy, :airyx, :besselj0, :besselj1, :bessely0, :bessely1, # base/special/bessel.jl :cbrt, :sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :exp2, :expm1, :exp10, :sin, :cos, :tan, :asin, :acos, :acosh, :atanh, #=:log,=# :log2, :log10, :lgamma, #=:log1p,=# :sqrt, # base/math.jl :abs, :abs2, :angle, :isnan, :isinf, :isfinite, # base/floatfuncs.jl + :acos_fast, :acosh_fast, :angle_fast, :asin_fast, :asinh_fast, :atan_fast, :atanh_fast, :cbrt_fast, :cis_fast, :cos_fast, :cosh_fast, :exp10_fast, :exp2_fast, :exp_fast, :expm1_fast, :lgamma_fast, :log10_fast, :log1p_fast, :log2_fast, :log_fast, :sin_fast, :sinh_fast, :sqrt_fast, :tan_fast, :tanh_fast, # base/fastmath.jl ) @eval @dep_vectorize_1arg Number $f end @@ -908,6 +909,7 @@ for f in ( :polygamma, :zeta, :beta, :lbeta, # base/special/gamma.jl :airy, :airyx, :besseli, :besselix, :besselj, :besseljx, :besselk, :besselkx, :bessely, :besselyx, :besselh, :besselhx, :hankelh1, :hankelh2, :hankelh1x, :hankelh2x, # base/special/bessel.jl :log, :hypot, :atan2, # base/math.jl + :pow_fast, :atan2_fast, :hypot_fast, :max_fast, :min_fast, :minmax_fast, # base/fastmath.jl ) @eval @dep_vectorize_2arg Number $f end diff --git a/base/fastmath.jl b/base/fastmath.jl index 903afa262a442..fd2edceb0befd 100644 --- a/base/fastmath.jl +++ b/base/fastmath.jl @@ -343,7 +343,6 @@ for f in (:acos, :acosh, :angle, :asin, :asinh, :atan, :atanh, :cbrt, f_fast = fast_op[f] @eval begin $f_fast(x) = $f(x) - @vectorize_1arg Number $f_fast end end @@ -356,7 +355,6 @@ for f in (:^, :atan2, :hypot, :max, :min, :minmax) $f_fast(x::Number, y::Number) = $f_fast(promote(x, y)...) # fall-back implementation that applies after promotion $f_fast{T<:Number}(x::T, y::T) = $f(x, y) - @vectorize_2arg Number $f_fast end end From 4508bec513080fa7347b8e26f59e0d37b393998b Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Tue, 5 Jul 2016 15:28:12 -0700 Subject: [PATCH 1105/1117] Deprecate vectorized functions in base/complex.jl in favor of compact broadcast syntax. --- base/complex.jl | 4 ---- base/deprecated.jl | 4 ++++ base/sparse/sparsematrix.jl | 2 +- test/complex.jl | 4 ++-- test/linalg/matmul.jl | 2 +- test/math.jl | 5 ++++- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/base/complex.jl b/base/complex.jl index dbfa77c91fa66..281573c624a6b 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -366,7 +366,6 @@ function cis(z::Complex) v = exp(-imag(z)) Complex(v*cos(real(z)), v*sin(real(z))) end -@vectorize_1arg Number cis """ angle(z) @@ -807,8 +806,6 @@ function round{T<:AbstractFloat, MR, MI}(z::Complex{T}, ::RoundingMode{MR}, ::Ro end round(z::Complex) = Complex(round(real(z)), round(imag(z))) -@vectorize_1arg Complex round - function round(z::Complex, digits::Integer, base::Integer=10) Complex(round(real(z), digits, base), round(imag(z), digits, base)) @@ -816,7 +813,6 @@ end float{T<:AbstractFloat}(z::Complex{T}) = z float(z::Complex) = Complex(float(real(z)), float(imag(z))) -@vectorize_1arg Complex float big{T<:AbstractFloat}(z::Complex{T}) = Complex{BigFloat}(z) big{T<:Integer}(z::Complex{T}) = Complex{BigInt}(z) diff --git a/base/deprecated.jl b/base/deprecated.jl index d2ff41fab0305..64a9f06f63558 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -891,6 +891,7 @@ for f in ( :cbrt, :sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :exp2, :expm1, :exp10, :sin, :cos, :tan, :asin, :acos, :acosh, :atanh, #=:log,=# :log2, :log10, :lgamma, #=:log1p,=# :sqrt, # base/math.jl :abs, :abs2, :angle, :isnan, :isinf, :isfinite, # base/floatfuncs.jl :acos_fast, :acosh_fast, :angle_fast, :asin_fast, :asinh_fast, :atan_fast, :atanh_fast, :cbrt_fast, :cis_fast, :cos_fast, :cosh_fast, :exp10_fast, :exp2_fast, :exp_fast, :expm1_fast, :lgamma_fast, :log10_fast, :log1p_fast, :log2_fast, :log_fast, :sin_fast, :sinh_fast, :sqrt_fast, :tan_fast, :tanh_fast, # base/fastmath.jl + :cis, # base/complex.jl ) @eval @dep_vectorize_1arg Number $f end @@ -903,6 +904,9 @@ for f in ( ) @eval @dep_vectorize_1arg Real $f end +# base/complex.jl +@dep_vectorize_1arg Complex round +@dep_vectorize_1arg Complex float # Deprecate @vectorize_2arg-vectorized functions from... for f in ( diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 87cb03485d304..d150815d7a478 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -295,7 +295,7 @@ Convert a sparse matrix or vector `S` into a dense matrix or vector. """ full -float(S::SparseMatrixCSC) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), float(copy(S.nzval))) +float(S::SparseMatrixCSC) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), float.(S.nzval)) complex(S::SparseMatrixCSC) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), complex(copy(S.nzval))) diff --git a/test/complex.jl b/test/complex.jl index 89c3c15862e6f..d6a11b76ccb25 100644 --- a/test/complex.jl +++ b/test/complex.jl @@ -884,8 +884,8 @@ end # round #8291 @test round(Complex(1.125, 0.875), 2) == Complex(1.12, 0.88) @test round(Complex(1.5, 0.5), RoundDown, RoundUp) == Complex(1.0, 1.0) -@test round([1:5;] + im) == [1:5;] + im -@test round([1:5;] + 0.5im) == [1.0:5.0;] +@test round.([1:5;] + im) == [1:5;] + im +@test round.([1:5;] + 0.5im) == [1.0:5.0;] # float #8291 @test float(Complex(1, 2)) == Complex(1.0, 2.0) diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 4755f1d9ddd64..354b2fc2e3c64 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -260,7 +260,7 @@ let AA = [1+2im 3+4im; 5+6im 7+8im], BB = [2+7im 4+1im; 3+8im 6+5im] for Atype = ["Array", "SubArray"], Btype = ["Array", "SubArray"] A = Atype == "Array" ? AA : view(AA, 1:2, 1:2) B = Btype == "Array" ? BB : view(BB, 1:2, 1:2) - @test vecdot(A,B) == dot(vec(A),vec(B)) == vecdot_(A,B) == vecdot(float(A),float(B)) + @test vecdot(A,B) == dot(vec(A),vec(B)) == vecdot_(A,B) == vecdot(float.(A),float.(B)) @test vecdot(Int[], Int[]) == 0 == vecdot_(Int[], Int[]) @test_throws MethodError vecdot(Any[], Any[]) @test_throws MethodError vecdot_(Any[], Any[]) diff --git a/test/math.jl b/test/math.jl index fd37f255d4bb8..9637a544441bf 100644 --- a/test/math.jl +++ b/test/math.jl @@ -737,9 +737,12 @@ c = 3 @test 1e-9 > errc(zeta(-3 + 99.69im), 10332.6267578711852982128675093428012860119184786399673520976+13212.8641740351391796168658602382583730208014957452167440726im) @test 1e-13 > errc(zeta(2 + 99.69im, 1.3), 0.41617652544777996034143623540420694985469543821307918291931-0.74199610821536326325073784018327392143031681111201859489991im) -for z in (1.234, 1.234 + 5.678im, [1.234, 5.678]) +for z in (1.234, 1.234 + 5.678im) @test cis(z) ≈ exp(im*z) end +let z = [1.234, 5.678] + @test cis.(z) ≈ exp.(im*z) +end # modf for elty in (Float16, Float32, Float64) From 57263cd539731ece2bf43c2d70867e3148b9ebf2 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Wed, 6 Jul 2016 15:33:07 -0700 Subject: [PATCH 1106/1117] Deprecate vectorized functions in base/dates/accessors.jl in favor of compact broadcast syntax. --- base/dates/accessors.jl | 15 --------------- base/deprecated.jl | 10 ++++++++++ test/dates/accessors.jl | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/base/dates/accessors.jl b/base/dates/accessors.jl index 4ef616a09a86d..71441a1e6fe6c 100644 --- a/base/dates/accessors.jl +++ b/base/dates/accessors.jl @@ -59,21 +59,6 @@ yearmonth(dt::TimeType) = yearmonth(days(dt)) monthday(dt::TimeType) = monthday(days(dt)) yearmonthday(dt::TimeType) = yearmonthday(days(dt)) -@vectorize_1arg TimeType year -@vectorize_1arg TimeType month -@vectorize_1arg TimeType day -@vectorize_1arg TimeType week -@vectorize_1arg DateTime hour -@vectorize_1arg DateTime minute -@vectorize_1arg DateTime second -@vectorize_1arg DateTime millisecond - -@vectorize_1arg TimeType dayofmonth -@vectorize_1arg TimeType yearmonth -@vectorize_1arg TimeType monthday -@vectorize_1arg TimeType yearmonthday - - # Documentation for exported accessors for func in (:year, :month) name = string(func) diff --git a/base/deprecated.jl b/base/deprecated.jl index 64a9f06f63558..2c69ac7f7995a 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -907,6 +907,16 @@ end # base/complex.jl @dep_vectorize_1arg Complex round @dep_vectorize_1arg Complex float +for f in ( + :year, :month, :day, :week, :dayofmonth, :yearmonth, :monthday, :yearmonthday, # base/dates/accessors.jl + ) + @eval @dep_vectorize_1arg Dates.TimeType $f +end +for f in ( + :hour, :minute, :second, :millisecond, # base/dates/accessors.jl + ) + @eval @dep_vectorize_1arg Dates.DateTime $f +end # Deprecate @vectorize_2arg-vectorized functions from... for f in ( diff --git a/test/dates/accessors.jl b/test/dates/accessors.jl index 4986d23e31b54..a2e91199d07d3 100644 --- a/test/dates/accessors.jl +++ b/test/dates/accessors.jl @@ -180,16 +180,16 @@ end # Vectorized accessors a = Dates.Date(2014,1,1) dr = [a,a,a,a,a,a,a,a,a,a] -@test Dates.year(dr) == repmat([2014],10) -@test Dates.month(dr) == repmat([1],10) -@test Dates.day(dr) == repmat([1],10) +@test Dates.year.(dr) == repmat([2014],10) +@test Dates.month.(dr) == repmat([1],10) +@test Dates.day.(dr) == repmat([1],10) a = Dates.DateTime(2014,1,1) dr = [a,a,a,a,a,a,a,a,a,a] -@test Dates.year(dr) == repmat([2014],10) -@test Dates.month(dr) == repmat([1],10) -@test Dates.day(dr) == repmat([1],10) -@test Dates.hour(dr) == repmat([0],10) -@test Dates.minute(dr) == repmat([0],10) -@test Dates.second(dr) == repmat([0],10) -@test Dates.millisecond(dr) == repmat([0],10) +@test Dates.year.(dr) == repmat([2014],10) +@test Dates.month.(dr) == repmat([1],10) +@test Dates.day.(dr) == repmat([1],10) +@test Dates.hour.(dr) == repmat([0],10) +@test Dates.minute.(dr) == repmat([0],10) +@test Dates.second.(dr) == repmat([0],10) +@test Dates.millisecond.(dr) == repmat([0],10) From 003f567beb092f2081ed18d36a0ab279186de7da Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Wed, 6 Jul 2016 15:36:06 -0700 Subject: [PATCH 1107/1117] Deprecate vectorized functions in base/dates/adjusters.jl in favor of compact broadcast syntax. --- base/dates/adjusters.jl | 12 ------------ base/deprecated.jl | 1 + 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/base/dates/adjusters.jl b/base/dates/adjusters.jl index 4836bc4dafa40..ee3f6b461d23d 100644 --- a/base/dates/adjusters.jl +++ b/base/dates/adjusters.jl @@ -42,9 +42,6 @@ function lastdayofweek end lastdayofweek(dt::Date) = Date(UTD(value(dt) + (7 - dayofweek(dt)))) lastdayofweek(dt::DateTime) = DateTime(lastdayofweek(Date(dt))) -@vectorize_1arg TimeType firstdayofweek -@vectorize_1arg TimeType lastdayofweek - """ firstdayofmonth(dt::TimeType) -> TimeType @@ -68,9 +65,6 @@ function lastdayofmonth(dt::Date) end lastdayofmonth(dt::DateTime) = DateTime(lastdayofmonth(Date(dt))) -@vectorize_1arg TimeType firstdayofmonth -@vectorize_1arg TimeType lastdayofmonth - """ firstdayofyear(dt::TimeType) -> TimeType @@ -94,9 +88,6 @@ function lastdayofyear(dt::Date) end lastdayofyear(dt::DateTime) = DateTime(lastdayofyear(Date(dt))) -@vectorize_1arg TimeType firstdayofyear -@vectorize_1arg TimeType lastdayofyear - """ firstdayofquarter(dt::TimeType) -> TimeType @@ -125,9 +116,6 @@ function lastdayofquarter(dt::Date) end lastdayofquarter(dt::DateTime) = DateTime(lastdayofquarter(Date(dt))) -@vectorize_1arg TimeType firstdayofquarter -@vectorize_1arg TimeType lastdayofquarter - # Temporal Adjusters immutable DateFunction f::Function diff --git a/base/deprecated.jl b/base/deprecated.jl index 2c69ac7f7995a..d7c2e0acd959e 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -909,6 +909,7 @@ end @dep_vectorize_1arg Complex float for f in ( :year, :month, :day, :week, :dayofmonth, :yearmonth, :monthday, :yearmonthday, # base/dates/accessors.jl + :firstdayofweek, :lastdayofweek, :firstdayofmonth, :lastdayofmonth, :firstdayofyear, :lastdayofyear, :firstdayofquarter, :lastdayofquarter, # base/dates/adjusters.jl ) @eval @dep_vectorize_1arg Dates.TimeType $f end From 2d1026fbe0f1344b574e413ac36843824c191e06 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Wed, 6 Jul 2016 15:41:03 -0700 Subject: [PATCH 1108/1117] Deprecate vectorized functions in base/dates/conversions.jl in favor of compact broadcast syntax. --- base/dates/conversions.jl | 10 ---------- base/deprecated.jl | 3 +++ test/dates/io.jl | 4 ++-- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/base/dates/conversions.jl b/base/dates/conversions.jl index a3f82c74e2604..7204fd8b18ec6 100644 --- a/base/dates/conversions.jl +++ b/base/dates/conversions.jl @@ -46,9 +46,6 @@ See `convert(Int64,dt::Date)` for inverse. """ Base.convert{R<:Real}(::Type{Date}, x::R) = Date(UTD(x)) -@vectorize_1arg DateTime Date -@vectorize_1arg Date DateTime - ### External Conversions const UNIXEPOCH = value(DateTime(1970)) #Rata Die milliseconds for 1970-01-01T00:00:00 @@ -132,10 +129,3 @@ Takes the given `DateTime` and returns the number of Julian calendar days since epoch `-4713-11-24T12:00:00` as a `Float64`. """ datetime2julian(dt::DateTime) = (value(dt) - JULIANEPOCH)/86400000.0 - -@vectorize_1arg Real unix2datetime -@vectorize_1arg DateTime datetime2unix -@vectorize_1arg Real rata2datetime -@vectorize_1arg DateTime datetime2rata -@vectorize_1arg Real julian2datetime -@vectorize_1arg DateTime datetime2julian diff --git a/base/deprecated.jl b/base/deprecated.jl index d7c2e0acd959e..47d5348d40d4b 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -901,6 +901,7 @@ for f in ( :erfinc, :erfcinv, # base/special/erf.jl :rad2deg, :deg2rad, :exponent, :significand, # base/math.jl :trunc, :floor, :ceil, :round, # base/floatfuncs.jl + :unix2datetime, :rata2datetime, :julian2datetime, # base/dates/conversions.jl ) @eval @dep_vectorize_1arg Real $f end @@ -915,9 +916,11 @@ for f in ( end for f in ( :hour, :minute, :second, :millisecond, # base/dates/accessors.jl + :Date, :datetime2unix, :datetime2rata, :datetime2julian, # base/dates/conversions.jl ) @eval @dep_vectorize_1arg Dates.DateTime $f end +@dep_vectorize_1arg Dates.Date Datetime # base/dates/conversions.jl # Deprecate @vectorize_2arg-vectorized functions from... for f in ( diff --git a/test/dates/io.jl b/test/dates/io.jl index bdcade050c16f..467d01b1f4a76 100644 --- a/test/dates/io.jl +++ b/test/dates/io.jl @@ -284,8 +284,8 @@ dr = ["2000-01-01","2000-01-02","2000-01-03","2000-01-04","2000-01-05" dr2 = [Dates.Date(2000):Dates.Date(2000,1,10);] @test Dates.Date(dr) == dr2 @test Dates.Date(dr,"yyyy-mm-dd") == dr2 -@test Dates.DateTime(dr) == Dates.DateTime(dr2) -@test Dates.DateTime(dr,"yyyy-mm-dd") == Dates.DateTime(dr2) +@test Dates.DateTime.(dr) == Dates.DateTime.(dr2) +@test Dates.DateTime(dr,"yyyy-mm-dd") == Dates.DateTime.(dr2) @test Dates.format(dr2) == dr @test Dates.format(dr2,"yyyy-mm-dd") == dr From 64c474ea0f0ff3a14893d2a4b4d811666654b546 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Wed, 6 Jul 2016 15:43:29 -0700 Subject: [PATCH 1109/1117] Deprecate vectorized functions in base/dates/query.jl in favor of compact broadcast syntax. --- base/dates/query.jl | 17 ----------------- base/deprecated.jl | 1 + 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/base/dates/query.jl b/base/dates/query.jl index 999b49bced45c..8524dec97972f 100644 --- a/base/dates/query.jl +++ b/base/dates/query.jl @@ -97,12 +97,6 @@ function daysofweekinmonth(dt::TimeType) (d in THIRTYONE) ? 5 : 4 end -@vectorize_1arg TimeType dayname -@vectorize_1arg TimeType dayabbr -@vectorize_1arg TimeType dayofweek -@vectorize_1arg TimeType dayofweekofmonth -@vectorize_1arg TimeType daysofweekinmonth - ### Months const January,February,March,April,May,June = 1,2,3,4,5,6 const July,August,September,October,November,December = 7,8,9,10,11,12 @@ -139,10 +133,6 @@ Returns the number of days in the month of `dt`. Value will be 28, 29, 30, or 31 """ daysinmonth(dt::TimeType) = ((y,m) = yearmonth(dt); return daysinmonth(y,m)) -@vectorize_1arg TimeType monthname -@vectorize_1arg TimeType monthabbr -@vectorize_1arg TimeType daysinmonth - ### Years """ isleapyear(dt::TimeType) -> Bool @@ -160,10 +150,6 @@ dayofyear(dt::TimeType) = ((y,m,d) = yearmonthday(dt); return dayofyear(y,m,d)) daysinyear(dt::TimeType) = 365 + isleapyear(dt) -@vectorize_1arg TimeType isleapyear -@vectorize_1arg TimeType dayofyear -@vectorize_1arg TimeType daysinyear - ### Quarters """ quarterofyear(dt::TimeType) -> Int @@ -182,6 +168,3 @@ const QUARTERDAYS = [0,90,181,273] Returns the day of the current quarter of `dt`. Range of value is 1:92. """ dayofquarter(dt::TimeType) = dayofyear(dt) - QUARTERDAYS[quarterofyear(dt)] - -@vectorize_1arg TimeType quarterofyear -@vectorize_1arg TimeType dayofquarter diff --git a/base/deprecated.jl b/base/deprecated.jl index 47d5348d40d4b..e4f366a85ba7a 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -911,6 +911,7 @@ end for f in ( :year, :month, :day, :week, :dayofmonth, :yearmonth, :monthday, :yearmonthday, # base/dates/accessors.jl :firstdayofweek, :lastdayofweek, :firstdayofmonth, :lastdayofmonth, :firstdayofyear, :lastdayofyear, :firstdayofquarter, :lastdayofquarter, # base/dates/adjusters.jl + :dayname, :dayabbr, :dayofweek, :dayofweekofmonth, :daysofweekinmonth, :monthname, :monthabbr, :daysinmonth, :isleapyear, :dayofyear, :daysinyear, :quarterofyear, :dayofquarter, # base/dates/query.jl ) @eval @dep_vectorize_1arg Dates.TimeType $f end From c0913ffd3aa0ba6ac8c3341881819e641ac7254f Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Thu, 7 Jul 2016 21:04:24 -0700 Subject: [PATCH 1110/1117] Clean up @vectorize_(1|2)arg deprecations. --- base/deprecated.jl | 67 ++++++++++++++++++++++++++++------------ test/sparsedir/sparse.jl | 17 ++-------- 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index e4f366a85ba7a..048f616d759b3 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -872,8 +872,6 @@ end macro dep_vectorize_2arg(S, f) S = esc(S) f = esc(f) - T1 = esc(:T1) - T2 = esc(:T2) quote @deprecate $f{T<:$S}(x::$S, y::AbstractArray{T}) $f.(x,y) @deprecate $f{T<:$S}(x::AbstractArray{T}, y::$S) $f.(x,y) @@ -883,25 +881,41 @@ end # Deprecate @vectorize_1arg-vectorized functions from... for f in ( - :sinpi, :cospi, :sinc, :cosc, # base/special/trig.jl - :log, :log1p, # base/special/log.jl - :gamma, :lfact, :digamma, :trigamma, :zeta, :eta,# base/special/gamma.jl - :erfcx, :erfi, :dawson, # base/special/erf.jl - :airyprime, :airyai, :airyaiprime, :airybi, :airybiprime, :airy, :airyx, :besselj0, :besselj1, :bessely0, :bessely1, # base/special/bessel.jl - :cbrt, :sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :exp2, :expm1, :exp10, :sin, :cos, :tan, :asin, :acos, :acosh, :atanh, #=:log,=# :log2, :log10, :lgamma, #=:log1p,=# :sqrt, # base/math.jl - :abs, :abs2, :angle, :isnan, :isinf, :isfinite, # base/floatfuncs.jl - :acos_fast, :acosh_fast, :angle_fast, :asin_fast, :asinh_fast, :atan_fast, :atanh_fast, :cbrt_fast, :cis_fast, :cos_fast, :cosh_fast, :exp10_fast, :exp2_fast, :exp_fast, :expm1_fast, :lgamma_fast, :log10_fast, :log1p_fast, :log2_fast, :log_fast, :sin_fast, :sinh_fast, :sqrt_fast, :tan_fast, :tanh_fast, # base/fastmath.jl - :cis, # base/complex.jl + # base/special/trig.jl + :sinpi, :cospi, :sinc, :cosc, + # base/special/log.jl + :log, :log1p, + # base/special/gamma.jl + :gamma, :lfact, :digamma, :trigamma, :zeta, :eta, + # base/special/erf.jl + :erfcx, :erfi, :dawson, + # base/special/bessel.jl + :airyprime, :airyai, :airyaiprime, :airybi, :airybiprime, + :airy, :airyx, :besselj0, :besselj1, :bessely0, :bessely1, + # base/math.jl + :cbrt, :sinh, :cosh, :tanh, :atan, :asinh, :exp, :erf, :erfc, :exp2, + :expm1, :exp10, :sin, :cos, :tan, :asin, :acos, :acosh, :atanh, + #=:log,=# :log2, :log10, :lgamma, #=:log1p,=# :sqrt, + # base/floatfuncs.jl + :abs, :abs2, :angle, :isnan, :isinf, :isfinite, + # base/fastmath.jl + :acos_fast, :acosh_fast, :angle_fast, :asin_fast, :asinh_fast, + :atan_fast, :atanh_fast, :cbrt_fast, :cis_fast, :cos_fast, + :cosh_fast, :exp10_fast, :exp2_fast, :exp_fast, :expm1_fast, + :lgamma_fast, :log10_fast, :log1p_fast, :log2_fast, :log_fast, + :sin_fast, :sinh_fast, :sqrt_fast, :tan_fast, :tanh_fast, + # base/complex.jl + :cis, ) @eval @dep_vectorize_1arg Number $f end for f in ( - :sind, :cosd, :tand, :asind, :acosd, :atand, :asecd, :acscd, :acotd, # base/special/trig.jl :invdigamma, # base/special/gamma.jl :erfinc, :erfcinv, # base/special/erf.jl - :rad2deg, :deg2rad, :exponent, :significand, # base/math.jl :trunc, :floor, :ceil, :round, # base/floatfuncs.jl + :rad2deg, :deg2rad, :exponent, :significand, # base/math.jl :unix2datetime, :rata2datetime, :julian2datetime, # base/dates/conversions.jl + :sind, :cosd, :tand, :asind, :acosd, :atand, :asecd, :acscd, :acotd, # base/special/trig.jl ) @eval @dep_vectorize_1arg Real $f end @@ -909,9 +923,16 @@ end @dep_vectorize_1arg Complex round @dep_vectorize_1arg Complex float for f in ( - :year, :month, :day, :week, :dayofmonth, :yearmonth, :monthday, :yearmonthday, # base/dates/accessors.jl - :firstdayofweek, :lastdayofweek, :firstdayofmonth, :lastdayofmonth, :firstdayofyear, :lastdayofyear, :firstdayofquarter, :lastdayofquarter, # base/dates/adjusters.jl - :dayname, :dayabbr, :dayofweek, :dayofweekofmonth, :daysofweekinmonth, :monthname, :monthabbr, :daysinmonth, :isleapyear, :dayofyear, :daysinyear, :quarterofyear, :dayofquarter, # base/dates/query.jl + # base/dates/accessors.jl + :year, :month, :day, :week, :dayofmonth, :yearmonth, :monthday, :yearmonthday, + # base/dates/adjusters.jl + :firstdayofweek, :lastdayofweek, :firstdayofmonth, + :lastdayofmonth, :firstdayofyear, :lastdayofyear, + :firstdayofquarter, :lastdayofquarter, + # base/dates/query.jl + :dayname, :dayabbr, :dayofweek, :dayofweekofmonth, + :daysofweekinmonth, :monthname, :monthabbr, :daysinmonth, + :isleapyear, :dayofyear, :daysinyear, :quarterofyear, :dayofquarter, ) @eval @dep_vectorize_1arg Dates.TimeType $f end @@ -925,10 +946,16 @@ end # Deprecate @vectorize_2arg-vectorized functions from... for f in ( - :polygamma, :zeta, :beta, :lbeta, # base/special/gamma.jl - :airy, :airyx, :besseli, :besselix, :besselj, :besseljx, :besselk, :besselkx, :bessely, :besselyx, :besselh, :besselhx, :hankelh1, :hankelh2, :hankelh1x, :hankelh2x, # base/special/bessel.jl - :log, :hypot, :atan2, # base/math.jl - :pow_fast, :atan2_fast, :hypot_fast, :max_fast, :min_fast, :minmax_fast, # base/fastmath.jl + # base/special/gamma.jl + :polygamma, :zeta, :beta, :lbeta, + # base/special/bessel.jl + :airy, :airyx, :besseli, :besselix, :besselj, :besseljx, + :besselk, :besselkx, :bessely, :besselyx, :besselh, + :besselhx, :hankelh1, :hankelh2, :hankelh1x, :hankelh2x, + # base/math.jl + :log, :hypot, :atan2, + # base/fastmath.jl + :pow_fast, :atan2_fast, :hypot_fast, :max_fast, :min_fast, :minmax_fast, ) @eval @dep_vectorize_2arg Number $f end diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index 0f55688106e20..42078ee09ff3b 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -497,8 +497,8 @@ let @test I == imag.(S) @test real.(sparse(R)) == R @test nnz(imag.(sparse(R))) == 0 - @test abs.(S) == abs(D) - @test abs2.(S) == abs2(D) + @test abs.(S) == abs.(D) + @test abs2.(S) == abs2.(D) end end @@ -1063,19 +1063,6 @@ end x = speye(100) @test_throws BoundsError x[-10:10] -for T in (Int, Float16, Float32, Float64, BigInt, BigFloat) - let R=rand(T[1:100;],2,2), I=rand(T[1:100;],2,2) - D = R + I*im - S = sparse(D) - @test R == real.(S) - @test I == imag.(S) - @test real.(sparse(R)) == R - @test nnz(imag.(sparse(R))) == 0 - @test abs.(S) == abs.(D) - @test abs2.(S) == abs2.(D) - end -end - # issue #10407 @test maximum(spzeros(5, 5)) == 0.0 @test minimum(spzeros(5, 5)) == 0.0 From 9b57a4bbd26a43c980bf7cedfcde8bbdb565cd52 Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Sat, 13 Aug 2016 14:07:16 -0700 Subject: [PATCH 1111/1117] Deprecate `@vectorize_1arg` and `@vectorize_2arg`. --- base/deprecated.jl | 37 +++++++++++++++++++++++++++++++++---- base/exports.jl | 2 -- base/operators.jl | 19 ------------------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 048f616d759b3..c45469dffbba8 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -867,15 +867,25 @@ end # For deprecating vectorized functions in favor of compact broadcast syntax macro dep_vectorize_1arg(S, f) - :( @deprecate $(esc(f)){T<:$(esc(S))}(x::AbstractArray{T}) $(esc(f)).(x) ) + S = esc(S) + f = esc(f) + T = esc(:T) + x = esc(:x) + AbsArr = esc(:AbstractArray) + :( @deprecate $f{$T<:$S}($x::$AbsArr{$T}) $f.($x) ) end macro dep_vectorize_2arg(S, f) S = esc(S) f = esc(f) + T1 = esc(:T1) + T2 = esc(:T2) + x = esc(:x) + y = esc(:y) + AbsArr = esc(:AbstractArray) quote - @deprecate $f{T<:$S}(x::$S, y::AbstractArray{T}) $f.(x,y) - @deprecate $f{T<:$S}(x::AbstractArray{T}, y::$S) $f.(x,y) - @deprecate $f{T1<:$S,T2<:$S}(x::AbstractArray{T1}, y::AbstractArray{T2}) $f.(x,y) + @deprecate $f{$T1<:$S}($x::$S, $y::$AbsArr{$T1}) $f.($x,$y) + @deprecate $f{$T1<:$S}($x::$AbsArr{$T1}, $y::$S) $f.($x,$y) + @deprecate $f{$T1<:$S,$T2<:$S}($x::$AbsArr{$T1}, $y::$AbsArr{$T2}) $f.($x,$y) end end @@ -966,4 +976,23 @@ for f in ( @eval @dep_vectorize_2arg Real $f end +# Deprecate @vectorize_1arg and @vectorize_2arg themselves +macro vectorize_1arg(S,f) + depwarn(string("`@vectorize_1arg` is deprecated in favor of compact broadcast syntax. ", + "Instead of `@vectorize_1arg`'ing function `f` and calling `f(arg)`, call `f.(arg)`."), + :vectorize_1arg) + quote + @dep_vectorize_1arg($(esc(S)),$(esc(f))) + end +end +macro vectorize_2arg(S,f) + depwarn(string("`@vectorize_2arg` is deprecated in favor of compact broadcast syntax. ", + "Instead of `@vectorize_2arg`'ing function `f` and calling `f(arg1, arg2)`, call ", + "`f.(arg1,arg2)`. "), :vectorize_2arg) + quote + @dep_vectorize_2arg($(esc(S)),$(esc(f))) + end +end +export @vectorize_1arg, @vectorize_2arg + # End deprecations scheduled for 0.6 diff --git a/base/exports.jl b/base/exports.jl index da164c187ed9a..ab7793502741c 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1396,8 +1396,6 @@ export @generated, @gensym, @eval, - @vectorize_1arg, - @vectorize_2arg, @deprecate, # performance annotations diff --git a/base/operators.jl b/base/operators.jl index 4392e9efa9d82..b13bad3d71ec6 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -915,25 +915,6 @@ for f in (:+, :-) end end -# vectorization - -macro vectorize_1arg(S,f) - S = esc(S); f = esc(f); T = esc(:T) - quote - ($f){$T<:$S}(x::AbstractArray{$T}) = [ ($f)(elem) for elem in x ] - end -end - -macro vectorize_2arg(S,f) - S = esc(S); f = esc(f); T1 = esc(:T1); T2 = esc(:T2) - quote - ($f){$T1<:$S, $T2<:$S}(x::($T1), y::AbstractArray{$T2}) = [ ($f)(x, z) for z in y ] - ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::($T2)) = [ ($f)(z, y) for z in x ] - ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::AbstractArray{$T2}) = - [ ($f)(xx, yy) for (xx, yy) in zip(x, y) ] - end -end - # vectorized ifelse function ifelse(c::AbstractArray{Bool}, x, y) From 90ef18559f364afc2b78f6184391a604984bb16f Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Sat, 13 Aug 2016 15:14:01 -0700 Subject: [PATCH 1112/1117] Update documentation following deprecation of macro-vectorized functions and deprecation of `@vectorize_1arg` and `@vectorize_2arg` themselves. Also add cross-referenes between max and maximum, and also between min and minimum. --- base/docs/helpdb/Base.jl | 8 ++++-- base/math.jl | 2 +- base/operators.jl | 6 ++-- doc/manual/arrays.rst | 58 ++++++-------------------------------- doc/manual/functions.rst | 1 + doc/stdlib/collections.rst | 4 +-- doc/stdlib/math.rst | 4 +-- doc/stdlib/numbers.rst | 2 +- 8 files changed, 25 insertions(+), 60 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 43c995fac199f..54188143cb8d1 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -560,7 +560,9 @@ randsubseq! """ maximum(A, dims) -Compute the maximum value of an array over the given dimensions. +Compute the maximum value of an array over the given dimensions. See also the +[`max(a,b)`](:func:`max`) function to take the maximum of two or more arguments, +which can be applied elementwise to arrays via `max.(a,b)`. """ maximum(A,dims) @@ -1531,7 +1533,9 @@ asinh """ minimum(A, dims) -Compute the minimum value of an array over the given dimensions. +Compute the minimum value of an array over the given dimensions. See also the +[`min(a,b)`](:func:`min`) function to take the minimum of two or more arguments, +which can be applied elementwise to arrays via `min.(a,b)`. """ minimum(A,dims) diff --git a/base/math.jl b/base/math.jl index 7338d03a8b02f..18497f78ea452 100644 --- a/base/math.jl +++ b/base/math.jl @@ -346,7 +346,7 @@ end significand(x) Extract the `significand(s)` (a.k.a. mantissa), in binary representation, of a -floating-point number or array. If `x` is a non-zero finite number, then the result will be +floating-point number. If `x` is a non-zero finite number, then the result will be a number of the same type on the interval ``[1,2)``. Otherwise `x` is returned. ```jldoctest diff --git a/base/operators.jl b/base/operators.jl index b13bad3d71ec6..6d9dcc66b2df6 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -218,14 +218,16 @@ cmp(x::Integer, y::Integer) = ifelse(isless(x,y), -1, ifelse(isless(y,x), 1, 0)) """ max(x, y, ...) -Return the maximum of the arguments. Operates elementwise over arrays. +Return the maximum of the arguments. See also the [`maximum`](:func:`maximum`) function +to take the maximum element from a collection. """ max(x,y) = ifelse(y < x, x, y) """ min(x, y, ...) -Return the minimum of the arguments. Operates elementwise over arrays. +Return the minimum of the arguments. See also the [`minimum`](:func:`minimum`) function +to take the minimum element from a collection. """ min(x,y) = ifelse(y < x, y, x) diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index 22493d7a6c506..93b09ab1525c3 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -474,56 +474,14 @@ operators ``/`` and ``\`` operate elementwise when the denominator is a scalar. Note that comparisons such as ``==`` operate on whole arrays, giving a single boolean answer. Use dot operators for elementwise comparisons. -The following built-in functions are also vectorized, whereby the functions act -elementwise:: - - abs abs2 angle cbrt - airy airyai airyaiprime airybi airybiprime airyprime - acos acosh asin asinh atan atan2 atanh - acsc acsch asec asech acot acoth - cos cospi cosh sin sinpi sinh tan tanh sinc cosc - csc csch sec sech cot coth - acosd asind atand asecd acscd acotd - cosd sind tand secd cscd cotd - besselh besseli besselj besselj0 besselj1 besselk bessely bessely0 bessely1 - exp erf erfc erfinv erfcinv exp2 expm1 - beta dawson digamma erfcx erfi - exponent eta zeta gamma - hankelh1 hankelh2 - ceil floor round trunc - isfinite isinf isnan - lbeta lfact lgamma - log log10 log1p log2 - copysign max min significand - sqrt hypot - -Note that there is a difference between :func:`min` and :func:`max`, which operate -elementwise over multiple array arguments, and :func:`minimum` and :func:`maximum`, which -find the smallest and largest values within an array. - -Julia provides the :func:`@vectorize_1arg` and :func:`@vectorize_2arg` -macros to automatically vectorize any function of one or two arguments -respectively. Each of these takes two arguments, namely the ``Type`` of -argument (which is usually chosen to be the most general possible) and -the name of the function to vectorize. Here is a simple example: - -.. doctest:: - - julia> square(x) = x^2 - square (generic function with 1 method) - - julia> @vectorize_1arg Number square - square (generic function with 2 methods) - - julia> methods(square) - # 2 methods for generic function "square": - square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:918 - square(x) at none:1 - - julia> square([1 2 4; 5 6 7]) - 2×3 Array{Int64,2}: - 1 4 16 - 25 36 49 +To enable convenient vectorization of mathematical and other operations, Julia provides +the compact syntax ``f.(args...)``, e.g. ``sin.(x)`` or ``min.(x,y)``, for elementwise +operations over arrays or mixtures of arrays and scalars (a :func:`broadcast` operation). +See :ref:`man-dot-vectorizing`. + +Note that there is a difference between ``max.(a,b)``, which ``broadcast``\ s :func:`max` +elementwise over ``a`` and ``b``, and ``maximum(a)``, which finds the largest value within +``a``. The same statements hold for ``min.(a,b)`` and ``minimum(a)``. .. _man-broadcasting: diff --git a/doc/manual/functions.rst b/doc/manual/functions.rst index 65e1432d06f92..3bb50d86b3bcc 100644 --- a/doc/manual/functions.rst +++ b/doc/manual/functions.rst @@ -677,6 +677,7 @@ will be fused with other nested "dot" calls. ``X .+= Y`` is equivalent to ``X .= X .+ Y`` and will eventually result in a fused in-place assignment. Similarly for ``.*=`` etcetera.) + Further Reading --------------- diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index e2cb44eea95bc..a608d8e527983 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -444,7 +444,7 @@ Iterable Collections .. Docstring generated from Julia source - Compute the maximum value of an array over the given dimensions. + Compute the maximum value of an array over the given dimensions. See also the :func:`max` function to take the maximum of two or more arguments, which can be applied elementwise to arrays via ``max.(a,b)``\ . .. function:: maximum!(r, A) @@ -470,7 +470,7 @@ Iterable Collections .. Docstring generated from Julia source - Compute the minimum value of an array over the given dimensions. + Compute the minimum value of an array over the given dimensions. See also the :func:`min` function to take the minimum of two or more arguments, which can be applied elementwise to arrays via ``min.(a,b)``\ . .. function:: minimum!(r, A) diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 2ab68c4f046a8..b8ab7cf7039b2 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1060,13 +1060,13 @@ Mathematical Functions .. Docstring generated from Julia source - Return the minimum of the arguments. Operates elementwise over arrays. + Return the minimum of the arguments. See also the :func:`minimum` function to take the minimum element from a collection. .. function:: max(x, y, ...) .. Docstring generated from Julia source - Return the maximum of the arguments. Operates elementwise over arrays. + Return the maximum of the arguments. See also the :func:`maximum` function to take the maximum element from a collection. .. function:: minmax(x, y) diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index 91420344723ef..43f3d16f40917 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -116,7 +116,7 @@ Data Formats .. Docstring generated from Julia source - Extract the ``significand(s)`` (a.k.a. mantissa), in binary representation, of a floating-point number or array. If ``x`` is a non-zero finite number, then the result will be a number of the same type on the interval :math:`[1,2)`\ . Otherwise ``x`` is returned. + Extract the ``significand(s)`` (a.k.a. mantissa), in binary representation, of a floating-point number. If ``x`` is a non-zero finite number, then the result will be a number of the same type on the interval :math:`[1,2)`\ . Otherwise ``x`` is returned. .. doctest:: From b95a124c00d885a1301b059771d5f7a7aa129d8e Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Wed, 31 Aug 2016 11:55:28 -0700 Subject: [PATCH 1113/1117] Revise a few vectorized-math methods, specifically eliminating temporaries and enabling broadcast loop fusion. --- base/linalg/arpack.jl | 2 +- base/linalg/diagonal.jl | 6 +++--- base/linalg/generic.jl | 4 ++-- base/linalg/symmetric.jl | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/linalg/arpack.jl b/base/linalg/arpack.jl index a76a781d670a6..db98510d17542 100644 --- a/base/linalg/arpack.jl +++ b/base/linalg/arpack.jl @@ -116,7 +116,7 @@ function eupd_wrapper(T, n::Integer, sym::Bool, cmplx::Bool, bmat::String, dmap = x->abs.(x) if iparam[7] == 3 # shift-and-invert - dmap = x->abs.(1./(x-sigma)) + dmap = x->abs.(1 ./ (x .- sigma)) elseif which == "LR" || which == "LA" || which == "BE" dmap = real elseif which == "SR" || which == "SA" diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 137d44f7a3cce..8f6409c04d79b 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -231,9 +231,9 @@ ctranspose(D::Diagonal) = conj(D) diag(D::Diagonal) = D.diag trace(D::Diagonal) = sum(D.diag) det(D::Diagonal) = prod(D.diag) -logdet{T<:Real}(D::Diagonal{T}) = sum(log.(D.diag)) +logdet{T<:Real}(D::Diagonal{T}) = sum(log, D.diag) function logdet{T<:Complex}(D::Diagonal{T}) #Make sure branch cut is correct - x = sum(log.(D.diag)) + x = sum(log, D.diag) -pi<imag(x)<pi ? x : real(x)+(mod2pi(imag(x)+pi)-pi)*im end # identity matrices via eye(Diagonal{type},n) @@ -311,7 +311,7 @@ eigvecs(D::Diagonal) = eye(D) eigfact(D::Diagonal) = Eigen(eigvals(D), eigvecs(D)) #Singular system -svdvals{T<:Number}(D::Diagonal{T}) = sort(abs.(D.diag), rev = true) +svdvals{T<:Number}(D::Diagonal{T}) = sort!(abs.(D.diag), rev = true) svdvals(D::Diagonal) = [svdvals(v) for v in D.diag] function svd{T<:Number}(D::Diagonal{T}) S = abs.(D.diag) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 18438a18e5219..6ecee81dc3f84 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -675,8 +675,8 @@ vector `x`, as computed using the operator `p`-norm. This quantity is also known in the literature as the Bauer condition number, relative condition number, or componentwise relative condition number. """ -condskeel(A::AbstractMatrix, x::AbstractVector, p::Real=Inf) = norm(abs.(inv(A))*abs.(A)*abs.(x), p) -condskeel{T<:Integer}(A::AbstractMatrix{T}, x::AbstractVector, p::Real=Inf) = norm(abs.(inv(float(A)))*abs.(A)*abs.(x), p) +condskeel(A::AbstractMatrix, x::AbstractVector, p::Real=Inf) = norm(abs.(inv(A))*(abs.(A)*abs.(x)), p) +condskeel{T<:Integer}(A::AbstractMatrix{T}, x::AbstractVector, p::Real=Inf) = norm(abs.(inv(float(A)))*(abs.(A)*abs.(x)), p) """ issymmetric(A) -> Bool diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index cc516377cc958..e0be275cc4fd8 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -313,7 +313,7 @@ for (funm, func) in ([:logm,:log], [:sqrtm,:sqrt]) if isposdef(F) retmat = (F.vectors * Diagonal(($func).(F.values))) * F.vectors' else - retmat = (F.vectors * Diagonal(($func).(complex(F.values)))) * F.vectors' + retmat = (F.vectors * Diagonal(($func).(complex.(F.values)))) * F.vectors' end return Symmetric(retmat) end From 7ae83080790dcb4181c853eac30d778230d9efbd Mon Sep 17 00:00:00 2001 From: Sacha Verweij <sacha@stanford.edu> Date: Wed, 31 Aug 2016 12:44:18 -0700 Subject: [PATCH 1114/1117] Remove now-redundant integer-specialized condskeel methods. --- base/linalg/generic.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 6ecee81dc3f84..87f60b29420ab 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -658,7 +658,6 @@ cond(x::Number, p) = cond(x) #Skeel condition numbers condskeel(A::AbstractMatrix, p::Real=Inf) = norm(abs.(inv(A))*abs.(A), p) -condskeel{T<:Integer}(A::AbstractMatrix{T}, p::Real=Inf) = norm(abs.(inv(float(A)))*abs.(A), p) """ condskeel(M, [x, p::Real=Inf]) @@ -676,7 +675,6 @@ This quantity is also known in the literature as the Bauer condition number, rel condition number, or componentwise relative condition number. """ condskeel(A::AbstractMatrix, x::AbstractVector, p::Real=Inf) = norm(abs.(inv(A))*(abs.(A)*abs.(x)), p) -condskeel{T<:Integer}(A::AbstractMatrix{T}, x::AbstractVector, p::Real=Inf) = norm(abs.(inv(float(A)))*(abs.(A)*abs.(x)), p) """ issymmetric(A) -> Bool From 3befe843abcf719482d26db3d44e1262d3fbf0fe Mon Sep 17 00:00:00 2001 From: Simon Kornblith <simon@simonster.com> Date: Sat, 3 Sep 2016 17:32:21 -0400 Subject: [PATCH 1115/1117] Fix docs for isdefined --- base/docs/helpdb/Base.jl | 11 ++++++++--- doc/stdlib/base.rst | 7 +++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 54188143cb8d1..54a1be788d9b9 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1679,12 +1679,17 @@ eta """ isdefined([m::Module,] s::Symbol) isdefined(object, s::Symbol) - isdefined(a::AbstractArray, index::Int) + isdefined(object, index::Int) + isdefined(a::Array, index::Int) Tests whether an assignable location is defined. The arguments can be a module and a symbol, -a composite object and field name (as a symbol), or an array and index. With a single -symbol argument, tests whether a global variable with that name is defined in +a composite object and field name (as a symbol) or index, or an `Array` and index. +With a single symbol argument, tests whether a global variable with that name is defined in `current_module()`. + +Note: For `AbstractArray`s other than `Array`, `isdefined` tests whether the given field +index is defined, not the given array index. To test whether an array index is defined, use +[`isassigned`](:func:`isassigned`). """ isdefined diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 76f85d5b93af6..572bf08bff5d3 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -349,11 +349,14 @@ All Objects .. function:: isdefined([m::Module,] s::Symbol) isdefined(object, s::Symbol) - isdefined(a::AbstractArray, index::Int) + isdefined(object, index::Int) + isdefined(a::Array, index::Int) .. Docstring generated from Julia source - Tests whether an assignable location is defined. The arguments can be a module and a symbol, a composite object and field name (as a symbol), or an array and index. With a single symbol argument, tests whether a global variable with that name is defined in ``current_module()``\ . + Tests whether an assignable location is defined. The arguments can be a module and a symbol, a composite object and field name (as a symbol) or index, or an ``Array`` and index. With a single symbol argument, tests whether a global variable with that name is defined in ``current_module()``\ . + + Note: For ``AbstractArray``\ s other than ``Array``\ , ``isdefined`` tests whether the given field index is defined, not the given array index. To test whether an array index is defined, use :func:`isassigned`\ . .. function:: convert(T, x) From 617b2a62223cd7cd0883a248a124b313bcf9d8b4 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" <stevenj@mit.edu> Date: Mon, 5 Sep 2016 02:19:48 -0400 Subject: [PATCH 1116/1117] =?UTF-8?q?don't=20throw=20DomainError=20for=20n?= =?UTF-8?q?egative=20integer=20powers=20of=20=C2=B11=20(#18342)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * don't throw DomainError for negative integer powers of ±1 * fix replutil test that was expecting 1^(-1) to fail --- base/intfuncs.jl | 4 +++- test/math.jl | 4 ++++ test/replutil.jl | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/base/intfuncs.jl b/base/intfuncs.jl index e99b50a8ea689..42ffc46046823 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -165,6 +165,8 @@ function power_by_squaring(x_, p::Integer) elseif p == 2 return x*x elseif p < 0 + x == 1 && return copy(x) + x == -1 && return iseven(p) ? one(x) : copy(x) throw(DomainError()) end t = trailing_zeros(p) + 1 @@ -185,7 +187,7 @@ function power_by_squaring(x_, p::Integer) end power_by_squaring(x::Bool, p::Unsigned) = ((p==0) | x) function power_by_squaring(x::Bool, p::Integer) - p < 0 && throw(DomainError()) + p < 0 && !x && throw(DomainError()) return (p==0) | x end diff --git a/test/math.jl b/test/math.jl index 9637a544441bf..eba7910098b4c 100644 --- a/test/math.jl +++ b/test/math.jl @@ -827,6 +827,10 @@ end @test_throws DomainError 2 ^ -2 @test_throws DomainError (-2)^(2.2) @test_throws DomainError (-2.0)^(2.2) +@test_throws DomainError false ^ -2 +@test 1 ^ -2 === (-1) ^ -2 === 1 +@test (-1) ^ -3 === -1 +@test true ^ -2 === true # issue #13748 let A = [1 2; 3 4]; B = [5 6; 7 8]; C = [9 10; 11 12] diff --git a/test/replutil.jl b/test/replutil.jl index 54e1f2a7aaf2c..642d8bfe39ff6 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -257,7 +257,7 @@ immutable TypeWithIntParam{T <: Integer} end let undefvar err_str = @except_strbt sqrt(-1) DomainError @test contains(err_str, "Try sqrt(complex(x)).") - err_str = @except_strbt 1^(-1) DomainError + err_str = @except_strbt 2^(-1) DomainError @test contains(err_str, "Cannot raise an integer x to a negative power -n") err_str = @except_strbt (-1)^0.25 DomainError @test contains(err_str, "Exponentiation yielding a complex result requires a complex argument") From a9cf2af65c730bd0994ea2de207c49eade698ac5 Mon Sep 17 00:00:00 2001 From: Massimiliano Fasi <massimiliano.fasi@gmail.com> Date: Wed, 12 Aug 2015 10:55:35 +0100 Subject: [PATCH 1117/1117] Make the algorithm for real powers of a matrix robust --- base/linalg/diagonal.jl | 8 ++++ base/linalg/triangular.jl | 87 ++++++++++++++++++--------------------- 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/base/linalg/diagonal.jl b/base/linalg/diagonal.jl index 8f6409c04d79b..9c71473a71f00 100644 --- a/base/linalg/diagonal.jl +++ b/base/linalg/diagonal.jl @@ -251,6 +251,14 @@ expm(D::Diagonal) = Diagonal(exp.(D.diag)) logm(D::Diagonal) = Diagonal(log.(D.diag)) sqrtm(D::Diagonal) = Diagonal(sqrt.(D.diag)) +# Matrix functions +^(D::Diagonal, p::Real) = Diagonal((D.diag).^p) +for (funm, func) in ([:expm,:exp], [:sqrtm,:sqrt], [:logm,:log]) + @eval begin + ($funm)(D::Diagonal) = Diagonal(($func)(D.diag)) + end +end + #Linear solver function A_ldiv_B!(D::Diagonal, B::StridedVecOrMat) m, n = size(B, 1), size(B, 2) diff --git a/base/linalg/triangular.jl b/base/linalg/triangular.jl index d7324ad1ce27d..e8b090065de72 100644 --- a/base/linalg/triangular.jl +++ b/base/linalg/triangular.jl @@ -1672,7 +1672,7 @@ powm(A::LowerTriangular, p::Real) = powm(A.', p::Real).' # Based on the code available at http://eprints.ma.man.ac.uk/1851/02/logm.zip, # Copyright (c) 2011, Awad H. Al-Mohy and Nicholas J. Higham # Julia version relicensed with permission from original authors -function logm{T<:Union{Float32,Float64,Complex{Float32},Complex{Float64}}}(A0::UpperTriangular{T}) +function logm{T<:Union{Float64,Complex{Float64}}}(A0::UpperTriangular{T}) theta = [1.586970738772063e-005, 2.313807884242979e-003, 1.938179313533253e-002, @@ -1718,7 +1718,7 @@ function logm{T<:Union{Float32,Float64,Complex{Float32},Complex{Float64}}}(A0::U scale!(2^s,Y.data) # Compute accurate diagonal and superdiagonal of log(A) - for k = 1:n-1 + @inbounds for k = 1:n-1 Ak = complex(A0[k,k]) Akp1 = complex(A0[k+1,k+1]) logAk = log(Ak) @@ -1749,7 +1749,7 @@ logm(A::LowerTriangular) = logm(A.').' # Numer. Algorithms, 59, (2012), 393–402. function sqrt_diag!(A0::UpperTriangular, A::UpperTriangular, s) n = checksquare(A0) - for i = 1:n + @inbounds for i = 1:n a = complex(A0[i,i]) if s == 0 A[i,i] = a - 1 @@ -1806,60 +1806,53 @@ function invsquaring(A0::UpperTriangular, theta) d2 = sqrt(norm(AmI^2, 1)) d3 = cbrt(norm(AmI^3, 1)) alpha2 = max(d2, d3) - foundm = false if alpha2 <= theta[2] m = alpha2<=theta[1]?1:2 - foundm = true - end - - while ~foundm - more = false - if s > s0 - d3 = cbrt(norm(AmI^3, 1)) - end - d4 = norm(AmI^4, 1)^(1/4) - alpha3 = max(d3, d4) - if alpha3 <= theta[tmax] - for j = 3:tmax - if alpha3 <= theta[j] + else + while true + more = false + if s > s0 + d3 = cbrt(norm(AmI^3, 1)) + end + d4 = norm(AmI^4, 1)^(1/4) + alpha3 = max(d3, d4) + if alpha3 <= theta[tmax] + for j = 3:tmax + if alpha3 <= theta[j] + break + end + end + if j <= 6 + m = j break + elseif alpha3 / 2 <= theta[5] && p < 2 + more = true + p = p + 1 end end - if j <= 6 - m = j - foundm = true - break - elseif alpha3 / 2 <= theta[5] && p < 2 - more = true - p = p + 1 - end - end - - if ~more - d5 = norm(AmI^5, 1)^(1/5) - alpha4 = max(d4, d5) - eta = min(alpha3, alpha4) - if eta <= theta[tmax] - j = 0 - for j = 6:tmax - if eta <= theta[j] - m = j - break + if ~more + d5 = norm(AmI^5, 1)^(1/5) + alpha4 = max(d4, d5) + eta = min(alpha3, alpha4) + if eta <= theta[tmax] + j = 0 + for j = 6:tmax + if eta <= theta[j] + m = j + break + end end + break end - foundm = true + end + if s == maxsqrt + m = tmax break end + A = sqrtm(A) + AmI = A - I + s = s + 1 end - - if s == maxsqrt - m = tmax - foundm = true - break - end - A = sqrtm(A) - AmI = A - I - s = s + 1 end # Compute accurate superdiagonal of T